def get_book_pubdate(bookid, refresh=False):
URL = 'https://www.goodreads.com/book/show/' + bookid + '.xml?key=' + lazylibrarian.CONFIG[
'GR_API']
bookdate = "0000"
try:
rootxml, in_cache = gr_xml_request(URL, useCache=not refresh)
except Exception as e:
logger.error("%s fetching book publication date: %s" %
(type(e).__name__, str(e)))
return bookdate, False
if rootxml is None:
logger.debug("Error requesting book publication date")
return bookdate, in_cache
try:
bookdate = rootxml.find('book/work/original_publication_year').text
if bookdate is None:
bookdate = '0000'
except (KeyError, AttributeError):
logger.error(
"Error reading pubdate for GoodReads bookid %s pubdate [%s]" %
(bookid, bookdate))
logger.debug("GoodReads bookid %s pubdate [%s] %s" %
(bookid, bookdate, in_cache))
return bookdate, in_cache
def getSeriesMembers(seriesID=None):
""" Ask librarything or goodreads for details on all books in a series
order, bookname, authorname, workid, authorid
(workid and authorid are goodreads only)
Return as a list of lists """
results = []
if lazylibrarian.CONFIG['BOOK_API'] == 'GoodReads':
params = {"format": "xml", "key": lazylibrarian.CONFIG['GR_API']}
URL = 'https://www.goodreads.com/series/' + seriesID + '?' + urlencode(params)
try:
rootxml, in_cache = gr_xml_request(URL)
if rootxml is None:
logger.debug("Error requesting series %s" % seriesID)
return []
except Exception as e:
logger.error("%s finding series %s: %s" % (type(e).__name__, seriesID, str(e)))
return []
works = rootxml.find('series/series_works')
books = works.getiterator('series_work')
if books is None:
logger.warn('No books found for %s' % seriesID)
return []
for book in books:
mydict = {}
for mykey, location in [('order', 'user_position'),
('bookname', 'work/best_book/title'),
('authorname', 'work/best_book/author/name'),
('workid', 'work/id'),
('authorid', 'work/best_book/author/id')
]:
if book.find(location) is not None:
mydict[mykey] = book.find(location).text
else:
mydict[mykey] = ""
results.append([mydict['order'], mydict['bookname'], mydict['authorname'],
mydict['workid'], mydict['authorid']])
else:
data = getBookWork(None, "SeriesPage", seriesID)
if data:
try:
table = data.split('class="worksinseries"')[1].split('</table>')[0]
rows = table.split('<tr')
for row in rows:
if 'href=' in row:
booklink = row.split('href="')[1]
bookname = booklink.split('">')[1].split('<')[0]
# booklink = booklink.split('"')[0]
try:
authorlink = row.split('href="')[2]
authorname = authorlink.split('">')[1].split('<')[0]
# authorlink = authorlink.split('"')[0]
order = row.split('class="order">')[1].split('<')[0]
results.append([order, bookname, authorname, '', ''])
except IndexError:
logger.debug('Incomplete data in series table for series %s' % seriesID)
except IndexError:
if 'class="worksinseries"' in data: # error parsing, or just no series data available?
logger.debug('Error in series table for series %s' % seriesID)
return results
def getWorkSeries(bookID=None):
""" Return the series names and numbers in series for the given id as a list of tuples
For goodreads the id is a WorkID, for librarything it's a BookID """
myDB = database.DBConnection()
serieslist = []
if not bookID:
logger.error("getWorkSeries - No bookID")
return serieslist
if lazylibrarian.CONFIG['BOOK_API'] == 'GoodReads':
URL = "https://www.goodreads.com/work/"
seriesurl = URL + bookID + "/series?format=xml&key=" + lazylibrarian.CONFIG['GR_API']
rootxml, in_cache = gr_xml_request(seriesurl)
if rootxml is None:
logger.warn('Error getting XML for %s' % seriesurl)
else:
resultxml = rootxml.getiterator('series_work')
for item in resultxml:
try:
seriesname = item.find('./series/title').text
seriesname = seriesname.strip('\n').strip('\n').strip()
seriesid = item.find('./series/id').text
seriesnum = item.find('./user_position').text
except (KeyError, AttributeError):
continue
if seriesname and seriesid:
seriesname = cleanName(unaccented(seriesname), '&/')
seriesnum = cleanName(unaccented(seriesnum))
serieslist.append((seriesid, seriesnum, seriesname))
match = myDB.match('SELECT SeriesID from series WHERE SeriesName=?', (seriesname,))
if not match:
myDB.action('INSERT INTO series VALUES (?, ?, ?, ?, ?)',
(seriesid, seriesname, "Active", 0, 0))
elif match['SeriesID'] != seriesid:
myDB.action('UPDATE series SET SeriesID=? WHERE SeriesName=?', (seriesid, seriesname))
else:
work = getBookWork(bookID, "Series")
if work:
try:
slist = work.split('<h3><b>Series:')[1].split('</h3>')[0].split('<a href="/series/')
for item in slist[1:]:
try:
series = item.split('">')[1].split('</a>')[0]
if series and '(' in series:
seriesnum = series.split('(')[1].split(')')[0].strip()
series = series.split(' (')[0].strip()
else:
seriesnum = ''
series = series.strip()
seriesname = cleanName(unaccented(series), '&/')
seriesnum = cleanName(unaccented(seriesnum))
serieslist.append(('', seriesnum, seriesname))
except IndexError:
pass
except IndexError:
pass
return serieslist
def getBookAuthors(bookid):
""" Get a list of authors contributing to a book from the goodreads bookpage or the librarything bookwork file """
authorlist = []
if lazylibrarian.CONFIG['BOOK_API'] == 'GoodReads':
params = {"key": lazylibrarian.CONFIG['GR_API']}
URL = 'https://www.goodreads.com/book/show/' + bookid + '?' + urlencode(params)
try:
rootxml, in_cache = gr_xml_request(URL)
if rootxml is None:
logger.debug("Error requesting book %s" % bookid)
return []
except Exception as e:
logger.error("%s finding book %s: %s" % (type(e).__name__, bookid, str(e)))
return []
book = rootxml.find('book')
authors = book.find('authors')
anames = authors.getiterator('author')
if anames is None:
logger.warn('No authors found for %s' % bookid)
return []
for aname in anames:
author = {}
if aname.find('id') is not None:
author['id'] = aname.find('id').text
if aname.find('name') is not None:
author['name'] = aname.find('name').text
if aname.find('role') is not None:
role = aname.find('role').text
if not role:
role = ''
author['role'] = role
if author:
authorlist.append(author)
else:
data = getBookWork(bookid, "Authors")
if data:
try:
data = data.split('otherauthors_container')[1].split('</table>')[0].split('<table')[1].split('>', 1)[1]
except IndexError:
data = ''
authorlist = []
if data and 'Work?' in data:
try:
rows = data.split('<tr')
for row in rows[2:]:
author = {}
col = row.split('<td>')
author['name'] = col[1].split('">')[1].split('<')[0]
author['role'] = col[2].split('<')[0]
author['type'] = col[3].split('<')[0]
author['work'] = col[4].split('<')[0]
author['status'] = col[5].split('<')[0]
authorlist.append(author)
except IndexError:
logger.debug('Error parsing authorlist for %s' % bookid)
return authorlist
def getBookAuthors(bookid):
""" Get a list of authors contributing to a book from the goodreads bookpage or the librarything bookwork file """
authorlist = []
if lazylibrarian.CONFIG['BOOK_API'] == 'GoodReads':
params = {"key": lazylibrarian.CONFIG['GR_API']}
URL = 'https://www.goodreads.com/book/show/' + bookid + '?' + urlencode(params)
try:
rootxml, in_cache = gr_xml_request(URL)
if rootxml is None:
logger.debug("Error requesting book %s" % bookid)
return []
except Exception as e:
logger.error("%s finding book %s: %s" % (type(e).__name__, bookid, str(e)))
return []
book = rootxml.find('book')
authors = book.find('authors')
anames = authors.getiterator('author')
if anames is None:
logger.warn('No authors found for %s' % bookid)
return []
for aname in anames:
author = {}
if aname.find('id') is not None:
author['id'] = aname.find('id').text
if aname.find('name') is not None:
author['name'] = aname.find('name').text
if aname.find('role') is not None:
role = aname.find('role').text
if not role:
role = ''
author['role'] = role
if author:
authorlist.append(author)
else:
data = getBookWork(bookid, "Authors")
if data:
try:
data = data.split('otherauthors_container')[1].split('</table>')[0].split('<table')[1].split('>', 1)[1]
except IndexError:
data = ''
authorlist = []
if data and 'Work?' in data:
try:
rows = data.split('<tr')
for row in rows[2:]:
author = {}
col = row.split('<td>')
author['name'] = col[1].split('">')[1].split('<')[0]
author['role'] = col[2].split('<')[0]
author['type'] = col[3].split('<')[0]
author['work'] = col[4].split('<')[0]
author['status'] = col[5].split('<')[0]
authorlist.append(author)
except IndexError:
logger.debug('Error parsing authorlist for %s' % bookid)
return authorlist
def get_book_pubdate(bookid, refresh=False):
URL = 'https://www.goodreads.com/book/show/' + bookid + '.xml?key=' + lazylibrarian.CONFIG['GR_API']
bookdate = "0000"
try:
rootxml, in_cache = gr_xml_request(URL, useCache=not refresh)
except Exception as e:
logger.error("%s fetching book publication date: %s" % (type(e).__name__, str(e)))
return bookdate, False
if rootxml is None:
logger.debug("Error requesting book publication date")
return bookdate, in_cache
try:
bookdate = rootxml.find('book/work/original_publication_year').text
if bookdate is None:
bookdate = '0000'
except (KeyError, AttributeError):
logger.error("Error reading pubdate for GoodReads bookid %s pubdate [%s]" % (bookid, bookdate))
logger.debug("GoodReads bookid %s pubdate [%s] %s" % (bookid, bookdate, in_cache))
return bookdate, in_cache
def getWorkSeries(bookID=None):
""" Return the series names and numbers in series for the given id as a list of tuples
For goodreads the id is a WorkID, for librarything it's a BookID """
myDB = database.DBConnection()
serieslist = []
if not bookID:
logger.error("getWorkSeries - No bookID")
return serieslist
if lazylibrarian.CONFIG['BOOK_API'] == 'GoodReads':
URL = "https://www.goodreads.com/work/"
seriesurl = URL + bookID + "/series?format=xml&key=" + lazylibrarian.CONFIG[
'GR_API']
rootxml, in_cache = gr_xml_request(seriesurl)
if rootxml is None:
logger.warn('Error getting XML for %s' % seriesurl)
else:
resultxml = rootxml.getiterator('series_work')
for item in resultxml:
try:
seriesname = item.find('./series/title').text
seriesname = seriesname.strip('\n').strip('\n').strip()
seriesid = item.find('./series/id').text
seriesnum = item.find('./user_position').text
except (KeyError, AttributeError):
continue
if seriesname and seriesid:
seriesname = cleanName(unaccented(seriesname), '&/')
if seriesname:
seriesnum = cleanName(unaccented(seriesnum))
serieslist.append((seriesid, seriesnum, seriesname))
match = myDB.match(
'SELECT SeriesID from series WHERE SeriesName=?',
(seriesname, ))
if not match:
match = myDB.match(
'SELECT SeriesName from series WHERE SeriesID=?',
(seriesid, ))
if not match:
myDB.action(
'INSERT INTO series VALUES (?, ?, ?, ?, ?)',
(seriesid, seriesname, "Active", 0, 0))
else:
logger.warn(
"Name mismatch for series %s, [%s][%s]" %
(seriesid, seriesname,
match['SeriesName']))
elif match['SeriesID'] != seriesid:
myDB.action(
'UPDATE series SET SeriesID=? WHERE SeriesName=?',
(seriesid, seriesname))
else:
work = getBookWork(bookID, "Series")
if work:
try:
slist = work.split('<h3><b>Series:')[1].split(
'</h3>')[0].split('<a href="/series/')
for item in slist[1:]:
try:
series = item.split('">')[1].split('</a>')[0]
if series and '(' in series:
seriesnum = series.split('(')[1].split(
')')[0].strip()
series = series.split(' (')[0].strip()
else:
seriesnum = ''
series = series.strip()
seriesname = cleanName(unaccented(series), '&/')
seriesnum = cleanName(unaccented(seriesnum))
if seriesname:
serieslist.append(('', seriesnum, seriesname))
except IndexError:
pass
except IndexError:
pass
return serieslist
def getSeriesMembers(seriesID=None, seriesname=None):
""" Ask librarything or goodreads for details on all books in a series
order, bookname, authorname, workid, authorid
(workid and authorid are goodreads only)
Return as a list of lists """
results = []
api_hits = 0
if lazylibrarian.CONFIG['BOOK_API'] == 'GoodReads':
params = {"format": "xml", "key": lazylibrarian.CONFIG['GR_API']}
URL = 'https://www.goodreads.com/series/%s?%s' % (seriesID,
urlencode(params))
try:
rootxml, in_cache = gr_xml_request(URL)
if not in_cache:
api_hits += 1
if rootxml is None:
logger.debug("Series %s:%s not recognised at goodreads" %
(seriesID, seriesname))
return [], api_hits
except Exception as e:
logger.error("%s finding series %s: %s" %
(type(e).__name__, seriesID, str(e)))
return [], api_hits
works = rootxml.find('series/series_works')
books = works.getiterator('series_work')
if books is None:
logger.warn('No books found for %s' % seriesID)
return [], api_hits
for book in books:
mydict = {}
for mykey, location in [('order', 'user_position'),
('bookname', 'work/best_book/title'),
('authorname',
'work/best_book/author/name'),
('workid', 'work/id'),
('authorid', 'work/best_book/author/id'),
('pubyear',
'work/original_publication_year')]:
if book.find(location) is not None:
mydict[mykey] = book.find(location).text
else:
mydict[mykey] = ""
results.append([
mydict['order'], mydict['bookname'], mydict['authorname'],
mydict['workid'], mydict['authorid'], mydict['pubyear']
])
else:
api_hits = 0
data = getBookWork(None, "SeriesPage", seriesID)
if data:
try:
table = data.split('class="worksinseries"')[1].split(
'</table>')[0]
rows = table.split('<tr')
for row in rows:
if 'href=' in row:
booklink = row.split('href="')[1]
bookname = booklink.split('">')[1].split('<')[0]
# booklink = booklink.split('"')[0]
try:
authorlink = row.split('href="')[2]
authorname = authorlink.split('">')[1].split(
'<')[0]
# authorlink = authorlink.split('"')[0]
order = row.split('class="order">')[1].split(
'<')[0]
results.append(
[order, bookname, authorname, '', ''])
except IndexError:
logger.debug(
'Incomplete data in series table for series %s'
% seriesID)
except IndexError:
if 'class="worksinseries"' in data: # error parsing, or just no series data available?
logger.debug('Error in series table for series %s' %
seriesID)
return results, api_hits
def getSeriesAuthors(seriesid):
""" Get a list of authors contributing to a series
and import those authors (and their books) into the database
Return how many authors you added """
myDB = database.DBConnection()
result = myDB.match("select count(*) as counter from authors")
start = int(result['counter'])
result = myDB.match('select SeriesName from series where SeriesID=?',
(seriesid, ))
seriesname = result['SeriesName']
members, api_hits = getSeriesMembers(seriesid, seriesname)
dic = {
u'\u2018': "",
u'\u2019': "",
u'\u201c': '',
u'\u201d': '',
"'": "",
'"': ''
}
if members:
myDB = database.DBConnection()
for member in members:
# order = member[0]
bookname = member[1]
authorname = member[2]
# workid = member[3]
authorid = member[4]
# pubyear = member[5]
bookname = replace_all(bookname, dic)
if not authorid:
# goodreads gives us all the info we need, librarything/google doesn't
base_url = 'https://www.goodreads.com/search.xml?q='
params = {"key": lazylibrarian.CONFIG['GR_API']}
searchname = bookname + ' ' + authorname
searchname = cleanName(unaccented(searchname))
if PY2:
searchname = searchname.encode(lazylibrarian.SYS_ENCODING)
searchterm = quote_plus(searchname)
set_url = base_url + searchterm + '&' + urlencode(params)
try:
rootxml, in_cache = gr_xml_request(set_url)
if not in_cache:
api_hits += 1
if rootxml is None:
logger.warn('Error getting XML for %s' % searchname)
else:
resultxml = rootxml.getiterator('work')
for item in resultxml:
try:
booktitle = item.find('./best_book/title').text
booktitle = replace_all(booktitle, dic)
except (KeyError, AttributeError):
booktitle = ""
book_fuzz = fuzz.token_set_ratio(
booktitle, bookname)
if book_fuzz >= 98:
try:
author = item.find(
'./best_book/author/name').text
except (KeyError, AttributeError):
author = ""
# try:
# workid = item.find('./work/id').text
# except (KeyError, AttributeError):
# workid = ""
try:
authorid = item.find(
'./best_book/author/id').text
except (KeyError, AttributeError):
authorid = ""
logger.debug(
"Author Search found %s %s, authorid %s" %
(author, booktitle, authorid))
break
if not authorid: # try again with title only
searchname = cleanName(unaccented(bookname))
if PY2:
searchname = searchname.encode(
lazylibrarian.SYS_ENCODING)
searchterm = quote_plus(searchname)
set_url = base_url + searchterm + '&' + urlencode(
params)
rootxml, in_cache = gr_xml_request(set_url)
if not in_cache:
api_hits += 1
if rootxml is None:
logger.warn('Error getting XML for %s' %
searchname)
else:
resultxml = rootxml.getiterator('work')
for item in resultxml:
booktitle = item.find('./best_book/title').text
booktitle = replace_all(booktitle, dic)
book_fuzz = fuzz.token_set_ratio(
booktitle, bookname)
if book_fuzz >= 98:
try:
author = item.find(
'./best_book/author/name').text
except (KeyError, AttributeError):
author = ""
# try:
# workid = item.find('./work/id').text
# except (KeyError, AttributeError):
# workid = ""
try:
authorid = item.find(
'./best_book/author/id').text
except (KeyError, AttributeError):
authorid = ""
logger.debug(
"Title Search found %s %s, authorid %s"
% (author, booktitle, authorid))
break
if not authorid:
logger.warn("GoodReads doesn't know about %s %s" %
(authorname, bookname))
except Exception as e:
logger.error("Error finding goodreads results: %s %s" %
(type(e).__name__, str(e)))
if authorid:
lazylibrarian.importer.addAuthorToDB(refresh=False,
authorid=authorid)
result = myDB.match("select count(*) as counter from authors")
finish = int(result['counter'])
newauth = finish - start
logger.info("Added %s new author%s for %s" %
(newauth, plural(newauth), seriesname))
return newauth
def setWorkID(books=None):
""" Set the goodreads workid for any books that don't already have one
books is a comma separated list of bookids or if empty, select from database
Paginate requests to reduce api hits """
myDB = database.DBConnection()
pages = []
if books:
page = books
pages.append(page)
else:
cmd = "select BookID,BookName from books where WorkID='' or WorkID is null"
books = myDB.select(cmd)
if books:
counter = 0
logger.debug('Setting WorkID for %s book%s' %
(len(books), plural(len(books))))
page = ''
for book in books:
bookid = book['BookID']
if not bookid:
logger.debug("No bookid for %s" % book['BookName'])
else:
if page:
page = page + ','
page = page + bookid
counter += 1
if counter == 50:
counter = 0
pages.append(page)
page = ''
if page:
pages.append(page)
counter = 0
params = {"key": lazylibrarian.CONFIG['GR_API']}
for page in pages:
URL = 'https://www.goodreads.com/book/id_to_work_id/' + page + '?' + urlencode(
params)
try:
rootxml, in_cache = gr_xml_request(URL, useCache=False)
if rootxml is None:
logger.debug("Error requesting id_to_work_id page")
else:
resultxml = rootxml.find('work-ids')
if len(resultxml):
ids = resultxml.getiterator('item')
books = getList(page)
cnt = 0
for item in ids:
workid = item.text
if not workid:
logger.debug("No workid returned for %s" %
books[cnt])
else:
counter += 1
controlValueDict = {"BookID": books[cnt]}
newValueDict = {"WorkID": workid}
myDB.upsert("books", newValueDict,
controlValueDict)
cnt += 1
except Exception as e:
logger.error("%s parsing id_to_work_id page: %s" %
(type(e).__name__, str(e)))
msg = 'Updated %s id%s' % (counter, plural(counter))
logger.debug("setWorkID complete: " + msg)
return msg
def getSeriesAuthors(seriesid):
""" Get a list of authors contributing to a series
and import those authors (and their books) into the database
Return how many authors you added """
myDB = database.DBConnection()
result = myDB.match("select count(*) as counter from authors")
start = int(result['counter'])
result = myDB.match('select SeriesName from series where SeriesID=?', (seriesid,))
seriesname = result['SeriesName']
members = getSeriesMembers(seriesid)
dic = {u'\u2018': "", u'\u2019': "", u'\u201c': '', u'\u201d': '', "'": "", '"': ''}
if members:
myDB = database.DBConnection()
for member in members:
# order = member[0]
bookname = member[1]
authorname = member[2]
# workid = member[3]
authorid = member[4]
bookname = replace_all(bookname, dic)
if not authorid:
# goodreads gives us all the info we need, librarything/google doesn't
base_url = 'https://www.goodreads.com/search.xml?q='
params = {"key": lazylibrarian.CONFIG['GR_API']}
searchname = bookname + ' ' + authorname
searchname = cleanName(unaccented(searchname))
if PY2:
searchname = searchname.encode(lazylibrarian.SYS_ENCODING)
searchterm = quote_plus(searchname)
set_url = base_url + searchterm + '&' + urlencode(params)
try:
rootxml, in_cache = gr_xml_request(set_url)
if rootxml is None:
logger.warn('Error getting XML for %s' % searchname)
else:
resultxml = rootxml.getiterator('work')
for item in resultxml:
try:
booktitle = item.find('./best_book/title').text
booktitle = replace_all(booktitle, dic)
except (KeyError, AttributeError):
booktitle = ""
book_fuzz = fuzz.token_set_ratio(booktitle, bookname)
if book_fuzz >= 98:
try:
author = item.find('./best_book/author/name').text
except (KeyError, AttributeError):
author = ""
# try:
# workid = item.find('./work/id').text
# except (KeyError, AttributeError):
# workid = ""
try:
authorid = item.find('./best_book/author/id').text
except (KeyError, AttributeError):
authorid = ""
logger.debug("Author Search found %s %s, authorid %s" %
(author, booktitle, authorid))
break
if not authorid: # try again with title only
searchname = cleanName(unaccented(bookname))
if PY2:
searchname = searchname.encode(lazylibrarian.SYS_ENCODING)
searchterm = quote_plus(searchname)
set_url = base_url + searchterm + '&' + urlencode(params)
rootxml, in_cache = gr_xml_request(set_url)
if rootxml is None:
logger.warn('Error getting XML for %s' % searchname)
else:
resultxml = rootxml.getiterator('work')
for item in resultxml:
booktitle = item.find('./best_book/title').text
booktitle = replace_all(booktitle, dic)
book_fuzz = fuzz.token_set_ratio(booktitle, bookname)
if book_fuzz >= 98:
try:
author = item.find('./best_book/author/name').text
except (KeyError, AttributeError):
author = ""
# try:
# workid = item.find('./work/id').text
# except (KeyError, AttributeError):
# workid = ""
try:
authorid = item.find('./best_book/author/id').text
except (KeyError, AttributeError):
authorid = ""
logger.debug("Title Search found %s %s, authorid %s" %
(author, booktitle, authorid))
break
if not authorid:
logger.warn("GoodReads doesn't know about %s %s" % (authorname, bookname))
except Exception as e:
logger.error("Error finding goodreads results: %s %s" % (type(e).__name__, str(e)))
if authorid:
lazylibrarian.importer.addAuthorToDB(refresh=False, authorid=authorid)
result = myDB.match("select count(*) as counter from authors")
finish = int(result['counter'])
newauth = finish - start
logger.info("Added %s new author%s for %s" % (newauth, plural(newauth), seriesname))
return newauth
def setWorkID(books=None):
""" Set the goodreads workid for any books that don't already have one
books is a comma separated list of bookids or if empty, select from database
Paginate requests to reduce api hits """
myDB = database.DBConnection()
pages = []
if books:
page = books
pages.append(page)
else:
cmd = "select BookID,BookName from books where WorkID='' or WorkID is null"
books = myDB.select(cmd)
if books:
counter = 0
logger.debug('Setting WorkID for %s book%s' % (len(books), plural(len(books))))
page = ''
for book in books:
bookid = book['BookID']
if not bookid:
logger.debug("No bookid for %s" % book['BookName'])
else:
if page:
page = page + ','
page = page + bookid
counter += 1
if counter == 50:
counter = 0
pages.append(page)
page = ''
if page:
pages.append(page)
counter = 0
params = {"key": lazylibrarian.CONFIG['GR_API']}
for page in pages:
URL = 'https://www.goodreads.com/book/id_to_work_id/' + page + '?' + urlencode(params)
try:
rootxml, in_cache = gr_xml_request(URL, useCache=False)
if rootxml is None:
logger.debug("Error requesting id_to_work_id page")
else:
resultxml = rootxml.find('work-ids')
if len(resultxml):
ids = resultxml.getiterator('item')
books = getList(page)
cnt = 0
for item in ids:
workid = item.text
if not workid:
logger.debug("No workid returned for %s" % books[cnt])
else:
counter += 1
controlValueDict = {"BookID": books[cnt]}
newValueDict = {"WorkID": workid}
myDB.upsert("books", newValueDict, controlValueDict)
cnt += 1
except Exception as e:
logger.error("%s parsing id_to_work_id page: %s" % (type(e).__name__, str(e)))
msg = 'Updated %s id%s' % (counter, plural(counter))
logger.debug("setWorkID complete: " + msg)
return msg