def BookToList(book_id, shelf_name, action='add'): global client if action == 'remove': body = urlencode({ 'name': shelf_name, 'book_id': book_id, 'a': 'remove' }) else: body = urlencode({'name': shelf_name, 'book_id': book_id}) headers = {'Content-Type': 'application/x-www-form-urlencoded'} gr_api_sleep() try: response, content = client.request( '%s/shelf/add_to_shelf.xml' % 'https://www.goodreads.com', 'POST', body, headers) except Exception as e: logger.error("Exception in client.request: %s %s" % (type(e).__name__, traceback.format_exc())) return False, "Error in client.request: see error log" if not response['status'].startswith('2'): msg = 'Failure status: %s' % response['status'] return False, msg return True, content
def create_shelf(self, shelf='lazylibrarian'): global consumer, client, token, user_id if not lazylibrarian.CONFIG['GR_API'] or not lazylibrarian.CONFIG['GR_SECRET'] or not \ lazylibrarian.CONFIG['GR_OAUTH_TOKEN'] or not lazylibrarian.CONFIG['GR_OAUTH_SECRET']: logger.warn("Goodreads create shelf error: Please authorise first") return False, 'Unauthorised' consumer = oauth.Consumer(key=str(lazylibrarian.CONFIG['GR_API']), secret=str( lazylibrarian.CONFIG['GR_SECRET'])) token = oauth.Token(lazylibrarian.CONFIG['GR_OAUTH_TOKEN'], lazylibrarian.CONFIG['GR_OAUTH_SECRET']) client = oauth.Client(consumer, token) user_id = self.getUserId() # could also pass [featured] [exclusive_flag] [sortable_flag] all default to False body = urlencode({'user_shelf[name]': shelf.lower()}) headers = {'Content-Type': 'application/x-www-form-urlencoded'} gr_api_sleep() try: response, content = client.request( '%s/user_shelves.xml' % 'https://www.goodreads.com', 'POST', body, headers) except Exception as e: logger.error("Exception in client.request: %s %s" % (type(e).__name__, traceback.format_exc())) return False, "Error in client.request: see error log" if not response['status'].startswith('2'): msg = 'Failure status: %s' % response['status'] return False, msg return True, ''
def getShelfBooks(page, shelf_name): global client, user_id data = '${base}/review/list?format=xml&v=2&id=${user_id}&sort=author&order=a' data += '&key=${key}&page=${page}&per_page=100&shelf=${shelf_name}' owned_template = Template(data) body = urlencode({}) headers = {'Content-Type': 'application/x-www-form-urlencoded'} request_url = owned_template.substitute( base='https://www.goodreads.com', user_id=user_id, page=page, key=lazylibrarian.CONFIG['GR_API'], shelf_name=shelf_name) gr_api_sleep() try: response, content = client.request(request_url, 'GET', body, headers) except Exception as e: logger.error("Exception in client.request: %s %s" % (type(e).__name__, traceback.format_exc())) return "Error in client.request: see error log" if not response['status'].startswith('2'): logger.error('Failure status: %s for page %s' % (response['status'], page)) return content
def create_shelf(self, shelf='lazylibrarian'): global consumer, client, token, user_id if not lazylibrarian.CONFIG['GR_API'] or not lazylibrarian.CONFIG['GR_SECRET'] or not \ lazylibrarian.CONFIG['GR_OAUTH_TOKEN'] or not lazylibrarian.CONFIG['GR_OAUTH_SECRET']: logger.warn("Goodreads create shelf error: Please authorise first") return False, 'Unauthorised' consumer = oauth.Consumer(key=str(lazylibrarian.CONFIG['GR_API']), secret=str(lazylibrarian.CONFIG['GR_SECRET'])) token = oauth.Token(lazylibrarian.CONFIG['GR_OAUTH_TOKEN'], lazylibrarian.CONFIG['GR_OAUTH_SECRET']) client = oauth.Client(consumer, token) user_id = self.getUserId() # could also pass [featured] [exclusive_flag] [sortable_flag] all default to False body = urlencode({'user_shelf[name]': shelf.lower()}) headers = {'Content-Type': 'application/x-www-form-urlencoded'} gr_api_sleep() try: response, content = client.request('%s/user_shelves.xml' % 'https://www.goodreads.com', 'POST', body, headers) except Exception as e: logger.error("Exception in client.request: %s %s" % (type(e).__name__, traceback.format_exc())) return False, "Error in client.request: see error log" if not response['status'].startswith('2'): msg = 'Failure status: %s' % response['status'] return False, msg return True, ''
def get_shelf_list(self): global consumer, client, token, user_id if not lazylibrarian.CONFIG['GR_API'] or not lazylibrarian.CONFIG['GR_SECRET'] or not \ lazylibrarian.CONFIG['GR_OAUTH_TOKEN'] or not lazylibrarian.CONFIG['GR_OAUTH_SECRET']: logger.warn("Goodreads get shelf error: Please authorise first") return [] else: # # loop over each page of shelves # loop over each shelf # add shelf to list # consumer = oauth.Consumer(key=str(lazylibrarian.CONFIG['GR_API']), secret=str(lazylibrarian.CONFIG['GR_SECRET'])) token = oauth.Token(lazylibrarian.CONFIG['GR_OAUTH_TOKEN'], lazylibrarian.CONFIG['GR_OAUTH_SECRET']) client = oauth.Client(consumer, token) user_id = self.getUserId() current_page = 0 shelves = [] page_shelves = 1 while page_shelves: current_page = current_page + 1 page_shelves = 0 shelf_template = Template('${base}/shelf/list.xml?user_id=${user_id}&key=${key}&page=${page}') body = urlencode({}) headers = {'Content-Type': 'application/x-www-form-urlencoded'} request_url = shelf_template.substitute(base='https://www.goodreads.com', user_id=user_id, page=current_page, key=lazylibrarian.CONFIG['GR_API']) gr_api_sleep() try: response, content = client.request(request_url, 'GET', body, headers) except Exception as e: logger.error("Exception in client.request: %s %s" % (type(e).__name__, traceback.format_exc())) return shelves if not response['status'].startswith('2'): logger.error('Failure status: %s for page %s' % (response['status'], current_page)) if lazylibrarian.LOGLEVEL & lazylibrarian.log_grsync: logger.debug(request_url) else: xmldoc = xml.dom.minidom.parseString(content) shelf_list = xmldoc.getElementsByTagName('shelves')[0] for item in shelf_list.getElementsByTagName('user_shelf'): shelf_name = item.getElementsByTagName('name')[0].firstChild.nodeValue shelf_count = item.getElementsByTagName('book_count')[0].firstChild.nodeValue shelf_exclusive = item.getElementsByTagName('exclusive_flag')[0].firstChild.nodeValue shelves.append({'name': shelf_name, 'books': shelf_count, 'exclusive': shelf_exclusive}) page_shelves += 1 if lazylibrarian.LOGLEVEL & lazylibrarian.log_grsync: logger.debug('Shelf %s : %s: Exclusive %s' % (shelf_name, shelf_count, shelf_exclusive)) if lazylibrarian.LOGLEVEL & lazylibrarian.log_grsync: logger.debug('Found %s shelves on page %s' % (page_shelves, current_page)) logger.debug('Found %s shelves on %s page%s' % (len(shelves), current_page - 1, plural(current_page - 1))) # print shelves return shelves
def follow_author(self, authorid=None, follow=True): global consumer, client, token, user_id if not lazylibrarian.CONFIG['GR_API'] or not lazylibrarian.CONFIG['GR_SECRET'] or not \ lazylibrarian.CONFIG['GR_OAUTH_TOKEN'] or not lazylibrarian.CONFIG['GR_OAUTH_SECRET']: logger.warn( "Goodreads follow author error: Please authorise first") return False, 'Unauthorised' consumer = oauth.Consumer(key=str(lazylibrarian.CONFIG['GR_API']), secret=str( lazylibrarian.CONFIG['GR_SECRET'])) token = oauth.Token(lazylibrarian.CONFIG['GR_OAUTH_TOKEN'], lazylibrarian.CONFIG['GR_OAUTH_SECRET']) client = oauth.Client(consumer, token) user_id = self.getUserId() # follow https://www.goodreads.com/author_followings?id=AUTHOR_ID&format=xml # unfollow https://www.goodreads.com/author_followings/AUTHOR_FOLLOWING_ID?format=xml gr_api_sleep() if follow: body = urlencode({'id': authorid, 'format': 'xml'}) headers = {'Content-Type': 'application/x-www-form-urlencoded'} try: response, content = client.request( '%s/author_followings' % 'https://www.goodreads.com', 'POST', body, headers) except Exception as e: logger.error("Exception in client.request: %s %s" % (type(e).__name__, traceback.format_exc())) return False, "Error in client.request: see error log" else: body = urlencode({'format': 'xml'}) headers = {'Content-Type': 'application/x-www-form-urlencoded'} try: response, content = client.request( '%s/author_followings/%s' % ('https://www.goodreads.com', authorid), 'DELETE', body, headers) except Exception as e: logger.error("Exception in client.request: %s %s" % (type(e).__name__, traceback.format_exc())) return False, "Error in client.request: see error log" if follow and response['status'] == '422': return True, 'Already following' if response['status'].startswith('2'): if follow: return True, content.split('<id>')[1].split('</id>')[0] return True, '' return False, 'Failure status: %s' % response['status']
def getUserId(): global client, user_id gr_api_sleep() try: response, content = client.request('%s/api/auth_user' % 'https://www.goodreads.com', 'GET') except Exception as e: logger.error("Error in client.request: %s %s" % (type(e).__name__, traceback.format_exc())) return '' if not response['status'].startswith('2'): logger.error('Cannot fetch resource: %s' % response['status']) return '' userxml = xml.dom.minidom.parseString(content) user_id = userxml.getElementsByTagName('user')[0].attributes['id'].value return str(user_id)
def getUserId(): global client, user_id gr_api_sleep() try: response, content = client.request('%s/api/auth_user' % 'https://www.goodreads.com', 'GET') except Exception as e: logger.error("Error in client.request: %s %s" % (type(e).__name__, traceback.format_exc())) return '' if not response['status'].startswith('2'): logger.error('Cannot fetch userid: %s' % response['status']) return '' userxml = xml.dom.minidom.parseString(content) user_id = userxml.getElementsByTagName('user')[0].attributes['id'].value return str(user_id)
def getShelfBooks(page, shelf_name): global client, user_id data = '${base}/review/list?format=xml&v=2&id=${user_id}&sort=author&order=a' data += '&key=${key}&page=${page}&per_page=100&shelf=${shelf_name}' owned_template = Template(data) body = urlencode({}) headers = {'Content-Type': 'application/x-www-form-urlencoded'} request_url = owned_template.substitute(base='https://www.goodreads.com', user_id=user_id, page=page, key=lazylibrarian.CONFIG['GR_API'], shelf_name=shelf_name) gr_api_sleep() try: response, content = client.request(request_url, 'GET', body, headers) except Exception as e: logger.error("Exception in client.request: %s %s" % (type(e).__name__, traceback.format_exc())) return "Error in client.request: see error log" if not response['status'].startswith('2'): logger.error('Failure status: %s for page %s' % (response['status'], page)) return content
def follow_author(self, authorid=None, follow=True): global consumer, client, token, user_id if not lazylibrarian.CONFIG['GR_API'] or not lazylibrarian.CONFIG['GR_SECRET'] or not \ lazylibrarian.CONFIG['GR_OAUTH_TOKEN'] or not lazylibrarian.CONFIG['GR_OAUTH_SECRET']: logger.warn("Goodreads follow author error: Please authorise first") return False, 'Unauthorised' consumer = oauth.Consumer(key=str(lazylibrarian.CONFIG['GR_API']), secret=str(lazylibrarian.CONFIG['GR_SECRET'])) token = oauth.Token(lazylibrarian.CONFIG['GR_OAUTH_TOKEN'], lazylibrarian.CONFIG['GR_OAUTH_SECRET']) client = oauth.Client(consumer, token) user_id = self.getUserId() # follow https://www.goodreads.com/author_followings?id=AUTHOR_ID&format=xml # unfollow https://www.goodreads.com/author_followings/AUTHOR_FOLLOWING_ID?format=xml gr_api_sleep() if follow: body = urlencode({'id': authorid, 'format': 'xml'}) headers = {'Content-Type': 'application/x-www-form-urlencoded'} try: response, content = client.request('%s/author_followings' % 'https://www.goodreads.com', 'POST', body, headers) except Exception as e: logger.error("Exception in client.request: %s %s" % (type(e).__name__, traceback.format_exc())) return False, "Error in client.request: see error log" else: body = urlencode({'format': 'xml'}) headers = {'Content-Type': 'application/x-www-form-urlencoded'} try: response, content = client.request('%s/author_followings/%s' % ('https://www.goodreads.com', authorid), 'DELETE', body, headers) except Exception as e: logger.error("Exception in client.request: %s %s" % (type(e).__name__, traceback.format_exc())) return False, "Error in client.request: see error log" if follow and response['status'] == '422': return True, 'Already following' if response['status'].startswith('2'): if follow: return True, content.split('<id>')[1].split('</id>')[0] return True, '' return False, 'Failure status: %s' % response['status']
def BookToList(book_id, shelf_name, action='add'): global client if action == 'remove': body = urlencode({'name': shelf_name, 'book_id': book_id, 'a': 'remove'}) else: body = urlencode({'name': shelf_name, 'book_id': book_id}) headers = {'Content-Type': 'application/x-www-form-urlencoded'} gr_api_sleep() try: response, content = client.request('%s/shelf/add_to_shelf.xml' % 'https://www.goodreads.com', 'POST', body, headers) except Exception as e: logger.error("Exception in client.request: %s %s" % (type(e).__name__, traceback.format_exc())) return False, "Error in client.request: see error log" if not response['status'].startswith('2'): msg = 'Failure status: %s' % response['status'] return False, msg return True, content
def get_cached_request(url, useCache=True, cache="XML"): # hashfilename = hash of url # if hashfilename exists in cache and isn't too old, return its contents # if not, read url and store the result in the cache # return the result, and boolean True if source was cache # cacheLocation = cache + "Cache" cacheLocation = os.path.join(lazylibrarian.CACHEDIR, cacheLocation) if not os.path.exists(cacheLocation): os.mkdir(cacheLocation) myhash = md5_utf8(url) valid_cache = False source = None hashfilename = cacheLocation + os.path.sep + myhash + "." + cache.lower() expiry = lazylibrarian.CONFIG[ 'CACHE_AGE'] * 24 * 60 * 60 # expire cache after this many seconds if useCache and os.path.isfile(hashfilename): cache_modified_time = os.stat(hashfilename).st_mtime time_now = time.time() if cache_modified_time < time_now - expiry: # Cache entry is too old, delete it logger.debug("Expiring %s" % myhash) os.remove(hashfilename) else: valid_cache = True if valid_cache: lazylibrarian.CACHE_HIT = int(lazylibrarian.CACHE_HIT) + 1 if lazylibrarian.LOGLEVEL & lazylibrarian.log_cache: logger.debug("CacheHandler: Returning CACHED response %s for %s" % (hashfilename, url)) if cache == "JSON": try: source = json.load(open(hashfilename)) except ValueError: logger.debug("Error decoding json from %s" % hashfilename) return None, False elif cache == "XML": with open(hashfilename, "rb") as cachefile: result = cachefile.read() if result and result.startswith(b'<?xml'): try: source = ElementTree.fromstring(result) except UnicodeEncodeError: # seems sometimes the page contains utf-16 but the header says it's utf-8 try: result = result.decode('utf-16').encode('utf-8') source = ElementTree.fromstring(result) except (ElementTree.ParseError, UnicodeEncodeError, UnicodeDecodeError): logger.debug("Error parsing xml from %s" % hashfilename) source = None except ElementTree.ParseError: logger.debug("Error parsing xml from %s" % hashfilename) source = None if source is None: logger.debug("Error reading xml from %s" % hashfilename) os.remove(hashfilename) return None, False else: lazylibrarian.CACHE_MISS = int(lazylibrarian.CACHE_MISS) + 1 if cache == 'XML': gr_api_sleep() result, success = fetchURL(url, raw=True) else: result, success = fetchURL(url) if success: logger.debug("CacheHandler: Storing %s %s for %s" % (cache, myhash, url)) if cache == "JSON": try: source = json.loads(result) if not expiry: return source, False except Exception as e: logger.error("%s decoding json from %s" % (type(e).__name__, url)) logger.debug("%s : %s" % (e, result)) return None, False json.dump(source, open(hashfilename, "w")) elif cache == "XML": if result and result.startswith(b'<?xml'): try: source = ElementTree.fromstring(result) if not expiry: return source, False except UnicodeEncodeError: # sometimes we get utf-16 data labelled as utf-8 try: result = result.decode('utf-16').encode('utf-8') source = ElementTree.fromstring(result) if not expiry: return source, False except (ElementTree.ParseError, UnicodeEncodeError, UnicodeDecodeError): logger.debug("Error parsing xml from %s" % url) source = None except ElementTree.ParseError: logger.debug("Error parsing xml from %s" % url) source = None if source is not None: with open(hashfilename, "wb") as cachefile: cachefile.write(result) else: logger.debug("Error getting xml data from %s" % url) return None, False else: logger.debug("Got error response for %s: %s" % (url, result)) return None, False return source, valid_cache
def get_cached_request(url, useCache=True, cache="XML"): # hashfilename = hash of url # if hashfilename exists in cache and isn't too old, return its contents # if not, read url and store the result in the cache # return the result, and boolean True if source was cache # cacheLocation = cache + "Cache" cacheLocation = os.path.join(lazylibrarian.CACHEDIR, cacheLocation) if not os.path.exists(cacheLocation): os.mkdir(cacheLocation) myhash = md5_utf8(url) valid_cache = False source = None hashfilename = cacheLocation + os.path.sep + myhash + "." + cache.lower() expiry = lazylibrarian.CONFIG['CACHE_AGE'] * 24 * 60 * 60 # expire cache after this many seconds if useCache and os.path.isfile(hashfilename): cache_modified_time = os.stat(hashfilename).st_mtime time_now = time.time() if cache_modified_time < time_now - expiry: # Cache entry is too old, delete it logger.debug("Expiring %s" % myhash) os.remove(hashfilename) else: valid_cache = True if valid_cache: lazylibrarian.CACHE_HIT = int(lazylibrarian.CACHE_HIT) + 1 if lazylibrarian.LOGLEVEL & lazylibrarian.log_cache: logger.debug("CacheHandler: Returning CACHED response %s for %s" % (hashfilename, url)) if cache == "JSON": try: source = json.load(open(hashfilename)) except ValueError: logger.debug("Error decoding json from %s" % hashfilename) return None, False elif cache == "XML": with open(hashfilename, "rb") as cachefile: result = cachefile.read() if result and result.startswith(b'<?xml'): try: source = ElementTree.fromstring(result) except UnicodeEncodeError: # seems sometimes the page contains utf-16 but the header says it's utf-8 try: result = result.decode('utf-16').encode('utf-8') source = ElementTree.fromstring(result) except (ElementTree.ParseError, UnicodeEncodeError, UnicodeDecodeError): logger.debug("Error parsing xml from %s" % hashfilename) source = None except ElementTree.ParseError: logger.debug("Error parsing xml from %s" % hashfilename) source = None if source is None: logger.debug("Error reading xml from %s" % hashfilename) os.remove(hashfilename) return None, False else: lazylibrarian.CACHE_MISS = int(lazylibrarian.CACHE_MISS) + 1 if cache == 'XML': gr_api_sleep() result, success = fetchURL(url, raw=True) else: result, success = fetchURL(url) if success: logger.debug("CacheHandler: Storing %s %s for %s" % (cache, myhash, url)) if cache == "JSON": try: source = json.loads(result) if not expiry: return source, False except Exception as e: logger.error("%s decoding json from %s" % (type(e).__name__, url)) logger.debug("%s : %s" % (e, result)) return None, False json.dump(source, open(hashfilename, "w")) elif cache == "XML": if result and result.startswith(b'<?xml'): try: source = ElementTree.fromstring(result) if not expiry: return source, False except UnicodeEncodeError: # sometimes we get utf-16 data labelled as utf-8 try: result = result.decode('utf-16').encode('utf-8') source = ElementTree.fromstring(result) if not expiry: return source, False except (ElementTree.ParseError, UnicodeEncodeError, UnicodeDecodeError): logger.debug("Error parsing xml from %s" % url) source = None except ElementTree.ParseError: logger.debug("Error parsing xml from %s" % url) source = None if source is not None: with open(hashfilename, "wb") as cachefile: cachefile.write(result) else: logger.debug("Error getting xml data from %s" % url) return None, False else: logger.debug("Got error response for %s: %s" % (url, result)) return None, False return source, valid_cache