def import_CSV(search_dir=None):
""" Find a csv file in the search_dir and process all the books in it,
adding authors to the database if not found
and marking the books as "Wanted"
"""
try:
if not search_dir:
msg = "Alternate Directory not configured"
logger.warn(msg)
return msg
elif not os.path.isdir(search_dir):
msg = "Alternate Directory [%s] not found" % search_dir
logger.warn(msg)
return msg
csvFile = csv_file(search_dir)
headers = None
content = {}
if not csvFile:
msg = "No CSV file found in %s" % search_dir
logger.warn(msg)
return msg
else:
logger.debug(u'Reading file %s' % csvFile)
reader = csv.reader(open(csvFile))
for row in reader:
if reader.line_num == 1:
# If we are on the first line, create the headers list from the first row
headers = row
else:
# Otherwise, the key in the content dictionary is the first item in the
# row and we can create the sub-dictionary by using the zip() function.
# we include the key in the dictionary as our exported csv files use
# bookid as the key
content[row[0]] = dict(zip(headers, row))
# We can now get to the content by using the resulting dictionary, so to see
# the list of lines, we can do: print content.keys() to get a list of keys
# To see the list of fields available for each book: print headers
if 'Author' not in headers or 'Title' not in headers:
msg = 'Invalid CSV file found %s' % csvFile
logger.warn(msg)
return msg
myDB = database.DBConnection()
bookcount = 0
authcount = 0
skipcount = 0
logger.debug(u"CSV: Found %s book%s in csv file" % (len(content.keys()), plural(len(content.keys()))))
for item in content.keys():
authorname = content[item]['Author']
if isinstance(authorname, str) and hasattr(authorname, "decode"):
authorname = authorname.decode(lazylibrarian.SYS_ENCODING)
authorname = formatAuthorName(authorname)
title = content[item]['Title']
if isinstance(title, str) and hasattr(title, "decode"):
title = title.decode(lazylibrarian.SYS_ENCODING)
authmatch = myDB.match('SELECT * FROM authors where AuthorName=?', (authorname,))
if authmatch:
logger.debug(u"CSV: Author %s found in database" % authorname)
else:
logger.debug(u"CSV: Author %s not found" % authorname)
newauthor, authorid, new = addAuthorNameToDB(author=authorname,
addbooks=lazylibrarian.CONFIG['NEWAUTHOR_BOOKS'])
if len(newauthor) and newauthor != authorname:
logger.debug("Preferred authorname changed from [%s] to [%s]" % (authorname, newauthor))
authorname = newauthor
if new:
authcount += 1
bookmatch = finditem(content[item], authorname, headers)
result = ''
if bookmatch:
authorname = bookmatch['AuthorName']
bookname = bookmatch['BookName']
bookid = bookmatch['BookID']
bookstatus = bookmatch['Status']
if bookstatus in ['Open', 'Wanted', 'Have']:
logger.info(u'Found book %s by %s, already marked as "%s"' % (bookname, authorname, bookstatus))
else: # skipped/ignored
logger.info(u'Found book %s by %s, marking as "Wanted"' % (bookname, authorname))
controlValueDict = {"BookID": bookid}
newValueDict = {"Status": "Wanted"}
myDB.upsert("books", newValueDict, controlValueDict)
bookcount += 1
else:
searchterm = "%s <ll> %s" % (title, authorname)
results = search_for(unaccented(searchterm))
if results:
result = results[0]
if result['author_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO'] \
and result['book_fuzz'] > lazylibrarian.CONFIG['MATCH_RATIO']:
logger.info("Found (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'],
result['authorname'], result['bookname']))
import_book(result['bookid'])
bookcount += 1
bookmatch = True
if not bookmatch:
msg = "Skipping book %s by %s" % (title, authorname)
if not result:
msg += ', No results returned'
logger.warn(msg)
else:
msg += ', No match found'
logger.warn(msg)
msg = "Closest match (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'],
result['authorname'], result['bookname'])
logger.warn(msg)
skipcount += 1
msg = "Added %i new author%s, marked %i book%s as 'Wanted', %i book%s not found" % \
(authcount, plural(authcount), bookcount, plural(bookcount), skipcount, plural(skipcount))
logger.info(msg)
return msg
except Exception:
msg = 'Unhandled exception in importCSV: %s' % traceback.format_exc()
logger.error(msg)
return msg
def import_CSV(search_dir=None, library='eBook'):
""" Find a csv file in the search_dir and process all the books in it,
adding authors to the database if not found
and marking the books as "Wanted"
Optionally delete the file on successful completion
"""
# noinspection PyBroadException
try:
if not search_dir:
msg = "Alternate Directory not configured"
logger.warn(msg)
return msg
elif not os.path.isdir(search_dir):
msg = "Alternate Directory [%s] not found" % search_dir
logger.warn(msg)
return msg
csvFile = csv_file(search_dir, library=library)
headers = None
myDB = database.DBConnection()
bookcount = 0
authcount = 0
skipcount = 0
total = 0
existing = 0
if not csvFile:
msg = "No %s CSV file found in %s" % (library, search_dir)
logger.warn(msg)
return msg
else:
logger.debug('Reading file %s' % csvFile)
csvreader = reader(open(csvFile, 'rU'))
for row in csvreader:
if csvreader.line_num == 1:
# If we are on the first line, create the headers list from the first row
headers = row
if 'Author' not in headers or 'Title' not in headers:
msg = 'Invalid CSV file found %s' % csvFile
logger.warn(msg)
return msg
else:
total += 1
item = dict(list(zip(headers, row)))
authorname = formatAuthorName(item['Author'])
title = makeUnicode(item['Title'])
authmatch = myDB.match('SELECT * FROM authors where AuthorName=?', (authorname,))
if authmatch:
logger.debug("CSV: Author %s found in database" % authorname)
else:
logger.debug("CSV: Author %s not found" % authorname)
newauthor, authorid, new = addAuthorNameToDB(author=authorname,
addbooks=lazylibrarian.CONFIG['NEWAUTHOR_BOOKS'])
if len(newauthor) and newauthor != authorname:
logger.debug("Preferred authorname changed from [%s] to [%s]" % (authorname, newauthor))
authorname = newauthor
if new:
authcount += 1
bookmatch = finditem(item, authorname, library=library)
result = ''
imported = ''
if bookmatch:
authorname = bookmatch['AuthorName']
bookname = bookmatch['BookName']
bookid = bookmatch['BookID']
if library == 'eBook':
bookstatus = bookmatch['Status']
else:
bookstatus = bookmatch['AudioStatus']
if bookstatus in ['Open', 'Wanted', 'Have']:
existing += 1
logger.info('Found %s %s by %s, already marked as "%s"' %
(library, bookname, authorname, bookstatus))
else: # skipped/ignored
logger.info('Found %s %s by %s, marking as "Wanted"' % (library, bookname, authorname))
controlValueDict = {"BookID": bookid}
if library == 'eBook':
newValueDict = {"Status": "Wanted"}
else:
newValueDict = {"AudioStatus": "Wanted"}
myDB.upsert("books", newValueDict, controlValueDict)
bookcount += 1
else:
searchterm = "%s <ll> %s" % (title, authorname)
results = search_for(unaccented(searchterm))
if results:
result = results[0]
if result['author_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO'] \
and result['book_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO']:
bookmatch = True
if not bookmatch: # no match on full searchterm, try splitting out subtitle
newtitle, _ = split_title(authorname, title)
if newtitle != title:
title = newtitle
searchterm = "%s <ll> %s" % (title, authorname)
results = search_for(unaccented(searchterm))
if results:
result = results[0]
if result['author_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO'] \
and result['book_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO']:
bookmatch = True
if bookmatch:
logger.info("Found (%s%% %s%%) %s: %s for %s: %s" %
(result['author_fuzz'], result['book_fuzz'],
result['authorname'], result['bookname'],
authorname, title))
if library == 'eBook':
import_book(result['bookid'], ebook="Wanted", wait=True)
else:
import_book(result['bookid'], audio="Wanted", wait=True)
imported = myDB.match('select * from books where BookID=?', (result['bookid'],))
if imported:
bookcount += 1
else:
bookmatch = False
if not bookmatch:
msg = "Skipping book %s by %s" % (title, authorname)
if not result:
msg += ', No results found'
logger.warn(msg)
elif not imported:
msg += ', Failed to import %s' % result['bookid']
logger.warn(msg)
else:
msg += ', No match found'
logger.warn(msg)
msg = "Closest match (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'],
result['authorname'], result['bookname'])
logger.warn(msg)
skipcount += 1
msg = "Found %i %s%s in csv file, %i already existing or wanted" % (total, library,
plural(total), existing)
logger.info(msg)
msg = "Added %i new author%s, marked %i %s%s as 'Wanted', %i %s%s not found" % \
(authcount, plural(authcount), bookcount, library, plural(bookcount),
skipcount, plural(skipcount), library)
logger.info(msg)
if lazylibrarian.CONFIG['DELETE_CSV']:
if skipcount == 0:
logger.info("Deleting %s on successful completion" % csvFile)
try:
os.remove(csvFile)
except OSError as why:
logger.warn('Unable to delete %s: %s' % (csvFile, why.strerror))
else:
logger.warn("Not deleting %s as not all books found" % csvFile)
if os.path.isdir(csvFile + '.fail'):
try:
shutil.rmtree(csvFile + '.fail')
except Exception as why:
logger.warn("Unable to remove %s, %s %s" % (csvFile + '.fail',
type(why).__name__, str(why)))
try:
_ = safe_move(csvFile, csvFile + '.fail')
except Exception as e:
logger.error("Unable to rename %s, %s %s" %
(csvFile, type(e).__name__, str(e)))
if not os.access(csvFile, os.R_OK):
logger.error("%s is not readable" % csvFile)
if not os.access(csvFile, os.W_OK):
logger.error("%s is not writeable" % csvFile)
parent = os.path.dirname(csvFile)
try:
with open(os.path.join(parent, 'll_temp'), 'w') as f:
f.write('test')
os.remove(os.path.join(parent, 'll_temp'))
except Exception as why:
logger.error("Directory %s is not writeable: %s" % (parent, why))
return msg
except Exception:
msg = 'Unhandled exception in importCSV: %s' % traceback.format_exc()
logger.error(msg)
return msg
def import_CSV(search_dir=None):
""" Find a csv file in the search_dir and process all the books in it,
adding authors to the database if not found
and marking the books as "Wanted"
"""
if not search_dir or os.path.isdir(search_dir) is False:
logger.warn(u"Please check Alternate Directory setting")
return False
csvFile = csv_file(search_dir)
headers = None
content = {}
if not csvFile:
logger.warn(u"No CSV file found in %s" % search_dir)
else:
logger.debug(u'Reading file %s' % csvFile)
reader = csv.reader(open(csvFile))
for row in reader:
if reader.line_num == 1:
# If we are on the first line, create the headers list from the first row
headers = row
else:
# Otherwise, the key in the content dictionary is the first item in the
# row and we can create the sub-dictionary by using the zip() function.
# we include the key in the dictionary as our exported csv files use
# bookid as the key
content[row[0]] = dict(zip(headers, row))
# We can now get to the content by using the resulting dictionary, so to see
# the list of lines, we can do:
# print content.keys() # to get a list of keys
# To see the list of fields available for each book
# print headers
if 'Author' not in headers or 'Title' not in headers:
logger.warn(u'Invalid CSV file found %s' % csvFile)
return
myDB = database.DBConnection()
bookcount = 0
authcount = 0
skipcount = 0
logger.debug(u"CSV: Found %s book%s in csv file" %
(len(content.keys()), plural(len(content.keys()))))
for item in content.keys():
authorname = content[item]['Author']
if hasattr(authorname, 'decode'):
authorname = authorname.decode(lazylibrarian.SYS_ENCODING)
authmatch = myDB.action(
'SELECT * FROM authors where AuthorName="%s"' %
(authorname)).fetchone()
if authmatch:
newauthor = False
logger.debug(u"CSV: Author %s found in database" %
(authorname))
else:
newauthor = True
logger.debug(u"CSV: Author %s not found, adding to database" %
(authorname))
addAuthorToDB(authorname)
authcount = authcount + 1
bookmatch = finditem(content[item], headers)
# if we didn't find it, maybe author info is stale
if not bookmatch and not newauthor:
addAuthorToDB(authorname, refresh=True)
bookmatch = finditem(content[item], headers)
if bookmatch:
authorname = bookmatch['AuthorName']
bookname = bookmatch['BookName']
bookid = bookmatch['BookID']
bookstatus = bookmatch['Status']
if bookstatus == 'Open' or bookstatus == 'Wanted' or bookstatus == 'Have':
logger.info(
u'Found book %s by %s, already marked as "%s"' %
(bookname, authorname, bookstatus))
else: # skipped/ignored
logger.info(u'Found book %s by %s, marking as "Wanted"' %
(bookname, authorname))
controlValueDict = {"BookID": bookid}
newValueDict = {"Status": "Wanted"}
myDB.upsert("books", newValueDict, controlValueDict)
bookcount = bookcount + 1
else:
logger.warn(u"Skipping book %s by %s, not found in database" %
(bookname, authorname))
skipcount = skipcount + 1
logger.info(
u"Added %i new author%s, marked %i book%s as 'Wanted', %i book%s not found"
% (authcount, plural(authcount), bookcount, plural(bookcount),
skipcount, plural(skipcount)))
def import_CSV(search_dir=None):
""" Find a csv file in the search_dir and process all the books in it,
adding authors to the database if not found
and marking the books as "Wanted"
"""
if not search_dir or os.path.isdir(search_dir) is False:
logger.warn(u"Please check Alternate Directory setting")
return False
csvFile = csv_file(search_dir)
headers = None
content = {}
if not csvFile:
logger.warn(u"No CSV file found in %s" % search_dir)
else:
logger.debug(u'Reading file %s' % csvFile)
reader = csv.reader(open(csvFile))
for row in reader:
if reader.line_num == 1:
# If we are on the first line, create the headers list from the first row
headers = row
else:
# Otherwise, the key in the content dictionary is the first item in the
# row and we can create the sub-dictionary by using the zip() function.
# we include the key in the dictionary as our exported csv files use
# bookid as the key
content[row[0]] = dict(zip(headers, row))
# We can now get to the content by using the resulting dictionary, so to see
# the list of lines, we can do:
# print content.keys() # to get a list of keys
# To see the list of fields available for each book
# print headers
if 'Author' not in headers or 'Title' not in headers:
logger.warn(u'Invalid CSV file found %s' % csvFile)
return
myDB = database.DBConnection()
bookcount = 0
authcount = 0
skipcount = 0
logger.debug(u"CSV: Found %s book%s in csv file" % (len(content.keys()), plural(len(content.keys()))))
for item in content.keys():
authorname = content[item]['Author']
if hasattr(authorname, 'decode'):
authorname = authorname.decode(lazylibrarian.SYS_ENCODING)
authmatch = myDB.action('SELECT * FROM authors where AuthorName="%s"' % (authorname)).fetchone()
if authmatch:
newauthor = False
logger.debug(u"CSV: Author %s found in database" % (authorname))
else:
newauthor = True
logger.debug(u"CSV: Author %s not found, adding to database" % (authorname))
addAuthorToDB(authorname)
authcount = authcount + 1
bookmatch = finditem(content[item], headers)
# if we didn't find it, maybe author info is stale
if not bookmatch and not newauthor:
addAuthorToDB(authorname, refresh=True)
bookmatch = finditem(content[item], headers)
if bookmatch:
authorname = bookmatch['AuthorName']
bookname = bookmatch['BookName']
bookid = bookmatch['BookID']
bookstatus = bookmatch['Status']
if bookstatus == 'Open' or bookstatus == 'Wanted' or bookstatus == 'Have':
logger.info(u'Found book %s by %s, already marked as "%s"' % (bookname, authorname, bookstatus))
else: # skipped/ignored
logger.info(u'Found book %s by %s, marking as "Wanted"' % (bookname, authorname))
controlValueDict = {"BookID": bookid}
newValueDict = {"Status": "Wanted"}
myDB.upsert("books", newValueDict, controlValueDict)
bookcount = bookcount + 1
else:
logger.warn(u"Skipping book %s by %s, not found in database" % (bookname, authorname))
skipcount = skipcount + 1
logger.info(u"Added %i new author%s, marked %i book%s as 'Wanted', %i book%s not found" %
(authcount, plural(authcount), bookcount, plural(bookcount), skipcount, plural(skipcount)))
def import_CSV(search_dir=None):
""" Find a csv file in the search_dir and process all the books in it,
adding authors to the database if not found
and marking the books as "Wanted"
Optionally delete the file on successful completion
"""
# noinspection PyBroadException
try:
if not search_dir:
msg = "Alternate Directory not configured"
logger.warn(msg)
return msg
elif not os.path.isdir(search_dir):
msg = "Alternate Directory [%s] not found" % search_dir
logger.warn(msg)
return msg
csvFile = csv_file(search_dir)
headers = None
myDB = database.DBConnection()
bookcount = 0
authcount = 0
skipcount = 0
total = 0
existing = 0
if not csvFile:
msg = "No CSV file found in %s" % search_dir
logger.warn(msg)
return msg
else:
logger.debug('Reading file %s' % csvFile)
csvreader = reader(open(csvFile, 'rU'))
for row in csvreader:
if csvreader.line_num == 1:
# If we are on the first line, create the headers list from the first row
headers = row
if 'Author' not in headers or 'Title' not in headers:
msg = 'Invalid CSV file found %s' % csvFile
logger.warn(msg)
return msg
else:
total += 1
item = dict(list(zip(headers, row)))
authorname = formatAuthorName(item['Author'])
title = makeUnicode(item['Title'])
authmatch = myDB.match(
'SELECT * FROM authors where AuthorName=?',
(authorname, ))
if authmatch:
logger.debug("CSV: Author %s found in database" %
authorname)
else:
logger.debug("CSV: Author %s not found" % authorname)
newauthor, authorid, new = addAuthorNameToDB(
author=authorname,
addbooks=lazylibrarian.CONFIG['NEWAUTHOR_BOOKS'])
if len(newauthor) and newauthor != authorname:
logger.debug(
"Preferred authorname changed from [%s] to [%s]"
% (authorname, newauthor))
authorname = newauthor
if new:
authcount += 1
bookmatch = finditem(item, authorname)
result = ''
imported = ''
if bookmatch:
authorname = bookmatch['AuthorName']
bookname = bookmatch['BookName']
bookid = bookmatch['BookID']
bookstatus = bookmatch['Status']
if bookstatus in ['Open', 'Wanted', 'Have']:
existing += 1
logger.info(
'Found book %s by %s, already marked as "%s"' %
(bookname, authorname, bookstatus))
else: # skipped/ignored
logger.info(
'Found book %s by %s, marking as "Wanted"' %
(bookname, authorname))
controlValueDict = {"BookID": bookid}
newValueDict = {"Status": "Wanted"}
myDB.upsert("books", newValueDict,
controlValueDict)
bookcount += 1
else:
searchterm = "%s <ll> %s" % (title, authorname)
results = search_for(unaccented(searchterm))
if results:
result = results[0]
if result['author_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO'] \
and result['book_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO']:
bookmatch = True
if not bookmatch: # no match on full searchterm, try splitting out subtitle
newtitle, _ = split_title(authorname, title)
if newtitle != title:
title = newtitle
searchterm = "%s <ll> %s" % (title, authorname)
results = search_for(unaccented(searchterm))
if results:
result = results[0]
if result['author_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO'] \
and result['book_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO']:
bookmatch = True
if bookmatch:
logger.info(
"Found (%s%% %s%%) %s: %s for %s: %s" %
(result['author_fuzz'], result['book_fuzz'],
result['authorname'], result['bookname'],
authorname, title))
import_book(result['bookid'], wait=True)
imported = myDB.match(
'select * from books where BookID=?',
(result['bookid'], ))
if imported:
bookcount += 1
else:
bookmatch = False
if not bookmatch:
msg = "Skipping book %s by %s" % (title, authorname)
if not result:
msg += ', No results found'
logger.warn(msg)
elif not imported:
msg += ', Failed to import %s' % result['bookid']
logger.warn(msg)
else:
msg += ', No match found'
logger.warn(msg)
msg = "Closest match (%s%% %s%%) %s: %s" % (
result['author_fuzz'], result['book_fuzz'],
result['authorname'], result['bookname'])
logger.warn(msg)
skipcount += 1
msg = "Found %i book%s in csv file, %i already existing or wanted" % (
total, plural(total), existing)
logger.info(msg)
msg = "Added %i new author%s, marked %i book%s as 'Wanted', %i book%s not found" % \
(authcount, plural(authcount), bookcount, plural(bookcount), skipcount, plural(skipcount))
logger.info(msg)
if lazylibrarian.CONFIG['DELETE_CSV']:
if skipcount == 0:
logger.info("Deleting %s on successful completion" %
csvFile)
try:
os.remove(csvFile)
except OSError as why:
logger.warn('Unable to delete %s: %s' %
(csvFile, why.strerror))
else:
logger.warn("Not deleting %s as not all books found" %
csvFile)
return msg
except Exception:
msg = 'Unhandled exception in importCSV: %s' % traceback.format_exc()
logger.error(msg)
return msg