def BookToList(book_id, shelf_name, action='add'):
global client
if action == 'remove':
body = urlencode({
'name': shelf_name,
'book_id': book_id,
'a': 'remove'
})
else:
body = urlencode({'name': shelf_name, 'book_id': book_id})
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
gr_api_sleep()
try:
response, content = client.request(
'%s/shelf/add_to_shelf.xml' % 'https://www.goodreads.com',
'POST', body, headers)
except Exception as e:
logger.error("Exception in client.request: %s %s" %
(type(e).__name__, traceback.format_exc()))
return False, "Error in client.request: see error log"
if not response['status'].startswith('2'):
msg = 'Failure status: %s' % response['status']
return False, msg
return True, content
def create_shelf(self, shelf='lazylibrarian'):
global consumer, client, token, user_id
if not lazylibrarian.CONFIG['GR_API'] or not lazylibrarian.CONFIG['GR_SECRET'] or not \
lazylibrarian.CONFIG['GR_OAUTH_TOKEN'] or not lazylibrarian.CONFIG['GR_OAUTH_SECRET']:
logger.warn("Goodreads create shelf error: Please authorise first")
return False, 'Unauthorised'
consumer = oauth.Consumer(key=str(lazylibrarian.CONFIG['GR_API']),
secret=str(
lazylibrarian.CONFIG['GR_SECRET']))
token = oauth.Token(lazylibrarian.CONFIG['GR_OAUTH_TOKEN'],
lazylibrarian.CONFIG['GR_OAUTH_SECRET'])
client = oauth.Client(consumer, token)
user_id = self.getUserId()
# could also pass [featured] [exclusive_flag] [sortable_flag] all default to False
body = urlencode({'user_shelf[name]': shelf.lower()})
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
gr_api_sleep()
try:
response, content = client.request(
'%s/user_shelves.xml' % 'https://www.goodreads.com', 'POST',
body, headers)
except Exception as e:
logger.error("Exception in client.request: %s %s" %
(type(e).__name__, traceback.format_exc()))
return False, "Error in client.request: see error log"
if not response['status'].startswith('2'):
msg = 'Failure status: %s' % response['status']
return False, msg
return True, ''
def getShelfBooks(page, shelf_name):
global client, user_id
data = '${base}/review/list?format=xml&v=2&id=${user_id}&sort=author&order=a'
data += '&key=${key}&page=${page}&per_page=100&shelf=${shelf_name}'
owned_template = Template(data)
body = urlencode({})
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
request_url = owned_template.substitute(
base='https://www.goodreads.com',
user_id=user_id,
page=page,
key=lazylibrarian.CONFIG['GR_API'],
shelf_name=shelf_name)
gr_api_sleep()
try:
response, content = client.request(request_url, 'GET', body,
headers)
except Exception as e:
logger.error("Exception in client.request: %s %s" %
(type(e).__name__, traceback.format_exc()))
return "Error in client.request: see error log"
if not response['status'].startswith('2'):
logger.error('Failure status: %s for page %s' %
(response['status'], page))
return content
def create_shelf(self, shelf='lazylibrarian'):
global consumer, client, token, user_id
if not lazylibrarian.CONFIG['GR_API'] or not lazylibrarian.CONFIG['GR_SECRET'] or not \
lazylibrarian.CONFIG['GR_OAUTH_TOKEN'] or not lazylibrarian.CONFIG['GR_OAUTH_SECRET']:
logger.warn("Goodreads create shelf error: Please authorise first")
return False, 'Unauthorised'
consumer = oauth.Consumer(key=str(lazylibrarian.CONFIG['GR_API']),
secret=str(lazylibrarian.CONFIG['GR_SECRET']))
token = oauth.Token(lazylibrarian.CONFIG['GR_OAUTH_TOKEN'], lazylibrarian.CONFIG['GR_OAUTH_SECRET'])
client = oauth.Client(consumer, token)
user_id = self.getUserId()
# could also pass [featured] [exclusive_flag] [sortable_flag] all default to False
body = urlencode({'user_shelf[name]': shelf.lower()})
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
gr_api_sleep()
try:
response, content = client.request('%s/user_shelves.xml' % 'https://www.goodreads.com', 'POST',
body, headers)
except Exception as e:
logger.error("Exception in client.request: %s %s" % (type(e).__name__, traceback.format_exc()))
return False, "Error in client.request: see error log"
if not response['status'].startswith('2'):
msg = 'Failure status: %s' % response['status']
return False, msg
return True, ''
def get_shelf_list(self):
global consumer, client, token, user_id
if not lazylibrarian.CONFIG['GR_API'] or not lazylibrarian.CONFIG['GR_SECRET'] or not \
lazylibrarian.CONFIG['GR_OAUTH_TOKEN'] or not lazylibrarian.CONFIG['GR_OAUTH_SECRET']:
logger.warn("Goodreads get shelf error: Please authorise first")
return []
else:
#
# loop over each page of shelves
# loop over each shelf
# add shelf to list
#
consumer = oauth.Consumer(key=str(lazylibrarian.CONFIG['GR_API']),
secret=str(lazylibrarian.CONFIG['GR_SECRET']))
token = oauth.Token(lazylibrarian.CONFIG['GR_OAUTH_TOKEN'], lazylibrarian.CONFIG['GR_OAUTH_SECRET'])
client = oauth.Client(consumer, token)
user_id = self.getUserId()
current_page = 0
shelves = []
page_shelves = 1
while page_shelves:
current_page = current_page + 1
page_shelves = 0
shelf_template = Template('${base}/shelf/list.xml?user_id=${user_id}&key=${key}&page=${page}')
body = urlencode({})
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
request_url = shelf_template.substitute(base='https://www.goodreads.com', user_id=user_id,
page=current_page, key=lazylibrarian.CONFIG['GR_API'])
gr_api_sleep()
try:
response, content = client.request(request_url, 'GET', body, headers)
except Exception as e:
logger.error("Exception in client.request: %s %s" % (type(e).__name__, traceback.format_exc()))
return shelves
if not response['status'].startswith('2'):
logger.error('Failure status: %s for page %s' % (response['status'], current_page))
if lazylibrarian.LOGLEVEL & lazylibrarian.log_grsync:
logger.debug(request_url)
else:
xmldoc = xml.dom.minidom.parseString(content)
shelf_list = xmldoc.getElementsByTagName('shelves')[0]
for item in shelf_list.getElementsByTagName('user_shelf'):
shelf_name = item.getElementsByTagName('name')[0].firstChild.nodeValue
shelf_count = item.getElementsByTagName('book_count')[0].firstChild.nodeValue
shelf_exclusive = item.getElementsByTagName('exclusive_flag')[0].firstChild.nodeValue
shelves.append({'name': shelf_name, 'books': shelf_count, 'exclusive': shelf_exclusive})
page_shelves += 1
if lazylibrarian.LOGLEVEL & lazylibrarian.log_grsync:
logger.debug('Shelf %s : %s: Exclusive %s' % (shelf_name, shelf_count, shelf_exclusive))
if lazylibrarian.LOGLEVEL & lazylibrarian.log_grsync:
logger.debug('Found %s shelves on page %s' % (page_shelves, current_page))
logger.debug('Found %s shelves on %s page%s' % (len(shelves), current_page - 1, plural(current_page - 1)))
# print shelves
return shelves
def follow_author(self, authorid=None, follow=True):
global consumer, client, token, user_id
if not lazylibrarian.CONFIG['GR_API'] or not lazylibrarian.CONFIG['GR_SECRET'] or not \
lazylibrarian.CONFIG['GR_OAUTH_TOKEN'] or not lazylibrarian.CONFIG['GR_OAUTH_SECRET']:
logger.warn(
"Goodreads follow author error: Please authorise first")
return False, 'Unauthorised'
consumer = oauth.Consumer(key=str(lazylibrarian.CONFIG['GR_API']),
secret=str(
lazylibrarian.CONFIG['GR_SECRET']))
token = oauth.Token(lazylibrarian.CONFIG['GR_OAUTH_TOKEN'],
lazylibrarian.CONFIG['GR_OAUTH_SECRET'])
client = oauth.Client(consumer, token)
user_id = self.getUserId()
# follow https://www.goodreads.com/author_followings?id=AUTHOR_ID&format=xml
# unfollow https://www.goodreads.com/author_followings/AUTHOR_FOLLOWING_ID?format=xml
gr_api_sleep()
if follow:
body = urlencode({'id': authorid, 'format': 'xml'})
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
try:
response, content = client.request(
'%s/author_followings' % 'https://www.goodreads.com',
'POST', body, headers)
except Exception as e:
logger.error("Exception in client.request: %s %s" %
(type(e).__name__, traceback.format_exc()))
return False, "Error in client.request: see error log"
else:
body = urlencode({'format': 'xml'})
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
try:
response, content = client.request(
'%s/author_followings/%s' %
('https://www.goodreads.com', authorid), 'DELETE', body,
headers)
except Exception as e:
logger.error("Exception in client.request: %s %s" %
(type(e).__name__, traceback.format_exc()))
return False, "Error in client.request: see error log"
if follow and response['status'] == '422':
return True, 'Already following'
if response['status'].startswith('2'):
if follow:
return True, content.split('<id>')[1].split('</id>')[0]
return True, ''
return False, 'Failure status: %s' % response['status']
def getUserId():
global client, user_id
gr_api_sleep()
try:
response, content = client.request('%s/api/auth_user' % 'https://www.goodreads.com', 'GET')
except Exception as e:
logger.error("Error in client.request: %s %s" % (type(e).__name__, traceback.format_exc()))
return ''
if not response['status'].startswith('2'):
logger.error('Cannot fetch resource: %s' % response['status'])
return ''
userxml = xml.dom.minidom.parseString(content)
user_id = userxml.getElementsByTagName('user')[0].attributes['id'].value
return str(user_id)
def getUserId():
global client, user_id
gr_api_sleep()
try:
response, content = client.request('%s/api/auth_user' % 'https://www.goodreads.com', 'GET')
except Exception as e:
logger.error("Error in client.request: %s %s" % (type(e).__name__, traceback.format_exc()))
return ''
if not response['status'].startswith('2'):
logger.error('Cannot fetch userid: %s' % response['status'])
return ''
userxml = xml.dom.minidom.parseString(content)
user_id = userxml.getElementsByTagName('user')[0].attributes['id'].value
return str(user_id)
def getShelfBooks(page, shelf_name):
global client, user_id
data = '${base}/review/list?format=xml&v=2&id=${user_id}&sort=author&order=a'
data += '&key=${key}&page=${page}&per_page=100&shelf=${shelf_name}'
owned_template = Template(data)
body = urlencode({})
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
request_url = owned_template.substitute(base='https://www.goodreads.com', user_id=user_id, page=page,
key=lazylibrarian.CONFIG['GR_API'], shelf_name=shelf_name)
gr_api_sleep()
try:
response, content = client.request(request_url, 'GET', body, headers)
except Exception as e:
logger.error("Exception in client.request: %s %s" % (type(e).__name__, traceback.format_exc()))
return "Error in client.request: see error log"
if not response['status'].startswith('2'):
logger.error('Failure status: %s for page %s' % (response['status'], page))
return content
def follow_author(self, authorid=None, follow=True):
global consumer, client, token, user_id
if not lazylibrarian.CONFIG['GR_API'] or not lazylibrarian.CONFIG['GR_SECRET'] or not \
lazylibrarian.CONFIG['GR_OAUTH_TOKEN'] or not lazylibrarian.CONFIG['GR_OAUTH_SECRET']:
logger.warn("Goodreads follow author error: Please authorise first")
return False, 'Unauthorised'
consumer = oauth.Consumer(key=str(lazylibrarian.CONFIG['GR_API']),
secret=str(lazylibrarian.CONFIG['GR_SECRET']))
token = oauth.Token(lazylibrarian.CONFIG['GR_OAUTH_TOKEN'], lazylibrarian.CONFIG['GR_OAUTH_SECRET'])
client = oauth.Client(consumer, token)
user_id = self.getUserId()
# follow https://www.goodreads.com/author_followings?id=AUTHOR_ID&format=xml
# unfollow https://www.goodreads.com/author_followings/AUTHOR_FOLLOWING_ID?format=xml
gr_api_sleep()
if follow:
body = urlencode({'id': authorid, 'format': 'xml'})
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
try:
response, content = client.request('%s/author_followings' % 'https://www.goodreads.com', 'POST', body,
headers)
except Exception as e:
logger.error("Exception in client.request: %s %s" % (type(e).__name__, traceback.format_exc()))
return False, "Error in client.request: see error log"
else:
body = urlencode({'format': 'xml'})
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
try:
response, content = client.request('%s/author_followings/%s' % ('https://www.goodreads.com', authorid),
'DELETE', body, headers)
except Exception as e:
logger.error("Exception in client.request: %s %s" % (type(e).__name__, traceback.format_exc()))
return False, "Error in client.request: see error log"
if follow and response['status'] == '422':
return True, 'Already following'
if response['status'].startswith('2'):
if follow:
return True, content.split('<id>')[1].split('</id>')[0]
return True, ''
return False, 'Failure status: %s' % response['status']
def BookToList(book_id, shelf_name, action='add'):
global client
if action == 'remove':
body = urlencode({'name': shelf_name, 'book_id': book_id, 'a': 'remove'})
else:
body = urlencode({'name': shelf_name, 'book_id': book_id})
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
gr_api_sleep()
try:
response, content = client.request('%s/shelf/add_to_shelf.xml' % 'https://www.goodreads.com', 'POST',
body, headers)
except Exception as e:
logger.error("Exception in client.request: %s %s" % (type(e).__name__, traceback.format_exc()))
return False, "Error in client.request: see error log"
if not response['status'].startswith('2'):
msg = 'Failure status: %s' % response['status']
return False, msg
return True, content
def get_cached_request(url, useCache=True, cache="XML"):
# hashfilename = hash of url
# if hashfilename exists in cache and isn't too old, return its contents
# if not, read url and store the result in the cache
# return the result, and boolean True if source was cache
#
cacheLocation = cache + "Cache"
cacheLocation = os.path.join(lazylibrarian.CACHEDIR, cacheLocation)
if not os.path.exists(cacheLocation):
os.mkdir(cacheLocation)
myhash = md5_utf8(url)
valid_cache = False
source = None
hashfilename = cacheLocation + os.path.sep + myhash + "." + cache.lower()
expiry = lazylibrarian.CONFIG[
'CACHE_AGE'] * 24 * 60 * 60 # expire cache after this many seconds
if useCache and os.path.isfile(hashfilename):
cache_modified_time = os.stat(hashfilename).st_mtime
time_now = time.time()
if cache_modified_time < time_now - expiry:
# Cache entry is too old, delete it
logger.debug("Expiring %s" % myhash)
os.remove(hashfilename)
else:
valid_cache = True
if valid_cache:
lazylibrarian.CACHE_HIT = int(lazylibrarian.CACHE_HIT) + 1
if lazylibrarian.LOGLEVEL & lazylibrarian.log_cache:
logger.debug("CacheHandler: Returning CACHED response %s for %s" %
(hashfilename, url))
if cache == "JSON":
try:
source = json.load(open(hashfilename))
except ValueError:
logger.debug("Error decoding json from %s" % hashfilename)
return None, False
elif cache == "XML":
with open(hashfilename, "rb") as cachefile:
result = cachefile.read()
if result and result.startswith(b'<?xml'):
try:
source = ElementTree.fromstring(result)
except UnicodeEncodeError:
# seems sometimes the page contains utf-16 but the header says it's utf-8
try:
result = result.decode('utf-16').encode('utf-8')
source = ElementTree.fromstring(result)
except (ElementTree.ParseError, UnicodeEncodeError,
UnicodeDecodeError):
logger.debug("Error parsing xml from %s" %
hashfilename)
source = None
except ElementTree.ParseError:
logger.debug("Error parsing xml from %s" % hashfilename)
source = None
if source is None:
logger.debug("Error reading xml from %s" % hashfilename)
os.remove(hashfilename)
return None, False
else:
lazylibrarian.CACHE_MISS = int(lazylibrarian.CACHE_MISS) + 1
if cache == 'XML':
gr_api_sleep()
result, success = fetchURL(url, raw=True)
else:
result, success = fetchURL(url)
if success:
logger.debug("CacheHandler: Storing %s %s for %s" %
(cache, myhash, url))
if cache == "JSON":
try:
source = json.loads(result)
if not expiry:
return source, False
except Exception as e:
logger.error("%s decoding json from %s" %
(type(e).__name__, url))
logger.debug("%s : %s" % (e, result))
return None, False
json.dump(source, open(hashfilename, "w"))
elif cache == "XML":
if result and result.startswith(b'<?xml'):
try:
source = ElementTree.fromstring(result)
if not expiry:
return source, False
except UnicodeEncodeError:
# sometimes we get utf-16 data labelled as utf-8
try:
result = result.decode('utf-16').encode('utf-8')
source = ElementTree.fromstring(result)
if not expiry:
return source, False
except (ElementTree.ParseError, UnicodeEncodeError,
UnicodeDecodeError):
logger.debug("Error parsing xml from %s" % url)
source = None
except ElementTree.ParseError:
logger.debug("Error parsing xml from %s" % url)
source = None
if source is not None:
with open(hashfilename, "wb") as cachefile:
cachefile.write(result)
else:
logger.debug("Error getting xml data from %s" % url)
return None, False
else:
logger.debug("Got error response for %s: %s" % (url, result))
return None, False
return source, valid_cache
def get_cached_request(url, useCache=True, cache="XML"):
# hashfilename = hash of url
# if hashfilename exists in cache and isn't too old, return its contents
# if not, read url and store the result in the cache
# return the result, and boolean True if source was cache
#
cacheLocation = cache + "Cache"
cacheLocation = os.path.join(lazylibrarian.CACHEDIR, cacheLocation)
if not os.path.exists(cacheLocation):
os.mkdir(cacheLocation)
myhash = md5_utf8(url)
valid_cache = False
source = None
hashfilename = cacheLocation + os.path.sep + myhash + "." + cache.lower()
expiry = lazylibrarian.CONFIG['CACHE_AGE'] * 24 * 60 * 60 # expire cache after this many seconds
if useCache and os.path.isfile(hashfilename):
cache_modified_time = os.stat(hashfilename).st_mtime
time_now = time.time()
if cache_modified_time < time_now - expiry:
# Cache entry is too old, delete it
logger.debug("Expiring %s" % myhash)
os.remove(hashfilename)
else:
valid_cache = True
if valid_cache:
lazylibrarian.CACHE_HIT = int(lazylibrarian.CACHE_HIT) + 1
if lazylibrarian.LOGLEVEL & lazylibrarian.log_cache:
logger.debug("CacheHandler: Returning CACHED response %s for %s" % (hashfilename, url))
if cache == "JSON":
try:
source = json.load(open(hashfilename))
except ValueError:
logger.debug("Error decoding json from %s" % hashfilename)
return None, False
elif cache == "XML":
with open(hashfilename, "rb") as cachefile:
result = cachefile.read()
if result and result.startswith(b'<?xml'):
try:
source = ElementTree.fromstring(result)
except UnicodeEncodeError:
# seems sometimes the page contains utf-16 but the header says it's utf-8
try:
result = result.decode('utf-16').encode('utf-8')
source = ElementTree.fromstring(result)
except (ElementTree.ParseError, UnicodeEncodeError, UnicodeDecodeError):
logger.debug("Error parsing xml from %s" % hashfilename)
source = None
except ElementTree.ParseError:
logger.debug("Error parsing xml from %s" % hashfilename)
source = None
if source is None:
logger.debug("Error reading xml from %s" % hashfilename)
os.remove(hashfilename)
return None, False
else:
lazylibrarian.CACHE_MISS = int(lazylibrarian.CACHE_MISS) + 1
if cache == 'XML':
gr_api_sleep()
result, success = fetchURL(url, raw=True)
else:
result, success = fetchURL(url)
if success:
logger.debug("CacheHandler: Storing %s %s for %s" % (cache, myhash, url))
if cache == "JSON":
try:
source = json.loads(result)
if not expiry:
return source, False
except Exception as e:
logger.error("%s decoding json from %s" % (type(e).__name__, url))
logger.debug("%s : %s" % (e, result))
return None, False
json.dump(source, open(hashfilename, "w"))
elif cache == "XML":
if result and result.startswith(b'<?xml'):
try:
source = ElementTree.fromstring(result)
if not expiry:
return source, False
except UnicodeEncodeError:
# sometimes we get utf-16 data labelled as utf-8
try:
result = result.decode('utf-16').encode('utf-8')
source = ElementTree.fromstring(result)
if not expiry:
return source, False
except (ElementTree.ParseError, UnicodeEncodeError, UnicodeDecodeError):
logger.debug("Error parsing xml from %s" % url)
source = None
except ElementTree.ParseError:
logger.debug("Error parsing xml from %s" % url)
source = None
if source is not None:
with open(hashfilename, "wb") as cachefile:
cachefile.write(result)
else:
logger.debug("Error getting xml data from %s" % url)
return None, False
else:
logger.debug("Got error response for %s: %s" % (url, result))
return None, False
return source, valid_cache
