def runGit(args):
# Function to execute GIT commands taking care of error logging etc
if lazylibrarian.CONFIG['GIT_PROGRAM']:
git_locations = ['"' + lazylibrarian.CONFIG['GIT_PROGRAM'] + '"']
else:
git_locations = ['git']
if platform.system().lower() == 'darwin':
git_locations.append('/usr/local/git/bin/git')
output = None
err = None
for cur_git in git_locations:
cmd = cur_git + ' ' + args
cmd = makeUnicode(cmd)
try:
logmsg(
'debug', 'Execute: "%s" with shell in %s' %
(cmd, lazylibrarian.PROG_DIR))
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=True,
cwd=lazylibrarian.PROG_DIR)
output, err = p.communicate()
output = makeUnicode(output).strip('\n')
err = makeUnicode(err)
logmsg('debug', 'Git output: [%s]' % output)
if err:
logmsg('debug', 'Git err: [%s]' % err)
except OSError:
logmsg('debug',
'Command ' + cmd + ' didn\'t work, couldn\'t find git')
continue
if 'not found' in output or "not recognized as an internal or external command" in output:
logmsg('debug', 'Unable to find git with command ' + cmd)
logmsg(
'error',
'git not found - please ensure git executable is in your PATH')
output = None
elif 'fatal:' in output or err:
logmsg(
'error',
'Git returned bad info. Are you sure this is a git installation?'
)
output = None
elif output:
break
return output, err
def _RecentAudio(self, **kwargs):
index = 0
if 'index' in kwargs:
index = check_int(kwargs['index'], 0)
myDB = database.DBConnection()
feed = {'title': 'LazyLibrarian OPDS - Recent AudioBooks', 'id': 'Recent AudioBooks', 'updated': now()}
links = []
entries = []
links.append(getLink(href=self.opdsroot, ftype='application/atom+xml; profile=opds-catalog; kind=navigation',
rel='start', title='Home'))
links.append(getLink(href='%s?cmd=RecentAudio' % self.opdsroot,
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='self'))
links.append(getLink(href='%s/opensearchbooks.xml' % self.searchroot,
ftype='application/opensearchdescription+xml', rel='search', title='Search Books'))
cmd = "select BookName,BookID,AudioLibrary,BookDate,BookImg,BookDesc,BookAdded,AuthorID from books WHERE "
if 'query' in kwargs:
cmd += "BookName LIKE '%" + kwargs['query'] + "%' AND "
cmd += "AudioStatus='Open' order by AudioLibrary DESC"
results = myDB.select(cmd)
page = results[index:(index + self.PAGE_SIZE)]
for book in page:
title = makeUnicode(book['BookName'])
entry = {'title': escape(title),
'id': escape('audio:%s' % book['BookID']),
'updated': opdstime(book['AudioLibrary']),
'href': '%s?cmd=Serve&audioid=%s' % (self.opdsroot, quote_plus(book['BookID'])),
'kind': 'acquisition',
'rel': 'file',
'type': mimeType("we_send.zip")}
if lazylibrarian.CONFIG['OPDS_METAINFO']:
author = myDB.match("SELECT AuthorName from authors WHERE AuthorID='%s'" % book['AuthorID'])
author = makeUnicode(author['AuthorName'])
entry['image'] = self.searchroot + '/' + book['BookImg']
entry['content'] = escape('%s - %s' % (title, book['BookDesc']))
entry['author'] = escape('%s' % author)
else:
entry['content'] = escape('%s (%s)' % (title, book['BookAdded']))
entries.append(entry)
if len(results) > (index + self.PAGE_SIZE):
links.append(
getLink(href='%s?cmd=RecentAudio&index=%s' % (self.opdsroot, index + self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='next'))
if index >= self.PAGE_SIZE:
links.append(
getLink(href='%s?cmd=RecentAudio&index=%s' % (self.opdsroot, index - self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='previous'))
feed['links'] = links
feed['entries'] = entries
logger.debug("Returning %s result%s" % (len(entries), plural(len(entries))))
self.data = feed
return
def _RecentAudio(self, **kwargs):
index = 0
if 'index' in kwargs:
index = check_int(kwargs['index'], 0)
myDB = database.DBConnection()
feed = {'title': 'LazyLibrarian OPDS - Recent AudioBooks', 'id': 'Recent AudioBooks', 'updated': now()}
links = []
entries = []
links.append(getLink(href=self.opdsroot, ftype='application/atom+xml; profile=opds-catalog; kind=navigation',
rel='start', title='Home'))
links.append(getLink(href='%s?cmd=RecentAudio' % self.opdsroot,
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='self'))
links.append(getLink(href='%s/opensearchbooks.xml' % self.searchroot,
ftype='application/opensearchdescription+xml', rel='search', title='Search Books'))
cmd = "select BookName,BookID,AudioLibrary,BookDate,BookImg,BookDesc,BookAdded,AuthorID from books WHERE "
if 'query' in kwargs:
cmd += "BookName LIKE '%" + kwargs['query'] + "%' AND "
cmd += "AudioStatus='Open' order by AudioLibrary DESC"
results = myDB.select(cmd)
page = results[index:(index + self.PAGE_SIZE)]
for book in page:
title = makeUnicode(book['BookName'])
entry = {'title': escape(title),
'id': escape('audio:%s' % book['BookID']),
'updated': opdstime(book['AudioLibrary']),
'href': '%s?cmd=Serve&audioid=%s' % (self.opdsroot, quote_plus(book['BookID'])),
'kind': 'acquisition',
'rel': 'file',
'type': mimeType("we_send.zip")}
if lazylibrarian.CONFIG['OPDS_METAINFO']:
author = myDB.match("SELECT AuthorName from authors WHERE AuthorID='%s'" % book['AuthorID'])
author = makeUnicode(author['AuthorName'])
entry['image'] = self.searchroot + '/' + book['BookImg']
entry['content'] = escape('%s - %s' % (title, book['BookDesc']))
entry['author'] = escape('%s' % author)
else:
entry['content'] = escape('%s (%s)' % (title, book['BookAdded']))
entries.append(entry)
if len(results) > (index + self.PAGE_SIZE):
links.append(
getLink(href='%s?cmd=RecentAudio&index=%s' % (self.opdsroot, index + self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='next'))
if index >= self.PAGE_SIZE:
links.append(
getLink(href='%s?cmd=RecentAudio&index=%s' % (self.opdsroot, index - self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='previous'))
feed['links'] = links
feed['entries'] = entries
logger.debug("Returning %s result%s" % (len(entries), plural(len(entries))))
self.data = feed
return
def id3read(filename):
if not TinyTag:
return None, None
try:
id3r = TinyTag.get(filename)
performer = id3r.artist
composer = id3r.composer
book = id3r.album
albumartist = id3r.albumartist
if performer:
performer = performer.strip()
else:
performer = ''
if composer:
composer = composer.strip()
else:
composer = ''
if book:
book = book.strip()
else:
book = ''
if albumartist:
albumartist = albumartist.strip()
else:
albumartist = ''
if lazylibrarian.LOGLEVEL & lazylibrarian.log_libsync:
logger.debug("id3r.filename [%s]" % filename)
logger.debug("id3r.performer [%s]" % performer)
logger.debug("id3r.composer [%s]" % composer)
logger.debug("id3r.album [%s]" % book)
logger.debug("id3r.albumartist [%s]" % albumartist)
if composer: # if present, should be author
author = composer
elif performer: # author, or narrator if composer == author
author = performer
elif albumartist:
author = albumartist
else:
author = None
if author and type(author) is list:
lst = ', '.join(author)
logger.debug("id3reader author list [%s]" % lst)
author = author[0] # if multiple authors, just use the first one
if author and book:
return makeUnicode(author), makeUnicode(book)
except Exception as e:
logger.error("tinytag error %s %s [%s]" % (type(e).__name__, str(e), filename))
return None, None
def runGit(args):
# Function to execute GIT commands taking care of error logging etc
if lazylibrarian.CONFIG['GIT_PROGRAM']:
git_locations = ['"' + lazylibrarian.CONFIG['GIT_PROGRAM'] + '"']
else:
git_locations = ['git']
if platform.system().lower() == 'darwin':
git_locations.append('/usr/local/git/bin/git')
output = None
err = None
for cur_git in git_locations:
cmd = cur_git + ' ' + args
cmd = makeUnicode(cmd)
try:
logmsg('debug', 'Execute: "%s" with shell in %s' % (cmd, lazylibrarian.PROG_DIR))
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
shell=True, cwd=lazylibrarian.PROG_DIR)
output, err = p.communicate()
output = makeUnicode(output).strip('\n')
err = makeUnicode(err)
logmsg('debug', 'Git output: [%s]' % output)
if err:
logmsg('debug', 'Git err: [%s]' % err)
except OSError:
logmsg('debug', 'Command ' + cmd + ' didn\'t work, couldn\'t find git')
continue
if 'not found' in output or "not recognized as an internal or external command" in output:
logmsg('debug', 'Unable to find git with command ' + cmd)
logmsg('error', 'git not found - please ensure git executable is in your PATH')
output = None
elif 'fatal:' in output or err:
logmsg('error', 'Git returned bad info. Are you sure this is a git installation?')
output = None
elif output:
break
return output, err
def _Series(self, **kwargs):
index = 0
if 'index' in kwargs:
index = check_int(kwargs['index'], 0)
myDB = database.DBConnection()
feed = {'title': 'LazyLibrarian OPDS - Series', 'id': 'Series', 'updated': now()}
links = []
entries = []
links.append(getLink(href=self.opdsroot, ftype='application/atom+xml; profile=opds-catalog; kind=navigation',
rel='start', title='Home'))
links.append(getLink(href='%s?cmd=Series' % self.opdsroot,
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='self'))
links.append(getLink(href='%s/opensearchseries.xml' % self.searchroot,
ftype='application/opensearchdescription+xml', rel='search', title='Search Series'))
cmd = "SELECT SeriesName,SeriesID,Have,Total from Series WHERE CAST(Have AS INTEGER) > 0 "
if 'query' in kwargs:
cmd += "AND SeriesName LIKE '%" + kwargs['query'] + "%' "
cmd += "order by SeriesName"
results = myDB.select(cmd)
page = results[index:(index + self.PAGE_SIZE)]
for series in page:
cmd = "SELECT books.BookID,SeriesNum from books,member where SeriesID=? "
cmd += "and books.bookid = member.bookid order by CAST(SeriesNum AS INTEGER)"
firstbook = myDB.match(cmd, (series['SeriesID'],))
if firstbook:
cmd = 'SELECT AuthorName from authors,books WHERE authors.authorid = books.authorid AND books.bookid=?'
res = myDB.match(cmd, (firstbook['BookID'],))
author = res['AuthorName']
else:
author = 'Unknown'
totalbooks = check_int(series['Total'], 0)
havebooks = check_int(series['Have'], 0)
sername = makeUnicode(series['SeriesName'])
entries.append(
{
'title': escape('%s (%s/%s) %s' % (sername, havebooks, totalbooks, author)),
'id': escape('series:%s' % series['SeriesID']),
'updated': now(),
'content': escape('%s (%s)' % (sername, havebooks)),
'href': '%s?cmd=Members&seriesid=%s' % (self.opdsroot, series['SeriesID']),
'kind': 'navigation',
'rel': 'subsection',
}
)
if len(results) > (index + self.PAGE_SIZE):
links.append(
getLink(href='%s?cmd=Series&index=%s' % (self.opdsroot, index + self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='next'))
if index >= self.PAGE_SIZE:
links.append(
getLink(href='%s?cmd=Series&index=%s' % (self.opdsroot, index - self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='previous'))
feed['links'] = links
feed['entries'] = entries
logger.debug("Returning %s series" % len(entries))
self.data = feed
return
def _Series(self, **kwargs):
index = 0
if 'index' in kwargs:
index = check_int(kwargs['index'], 0)
myDB = database.DBConnection()
feed = {'title': 'LazyLibrarian OPDS - Series', 'id': 'Series', 'updated': now()}
links = []
entries = []
links.append(getLink(href=self.opdsroot, ftype='application/atom+xml; profile=opds-catalog; kind=navigation',
rel='start', title='Home'))
links.append(getLink(href='%s?cmd=Series' % self.opdsroot,
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='self'))
links.append(getLink(href='%s/opensearchseries.xml' % self.searchroot,
ftype='application/opensearchdescription+xml', rel='search', title='Search Series'))
cmd = "SELECT SeriesName,SeriesID,Have,Total from Series WHERE CAST(Have AS INTEGER) > 0 "
if 'query' in kwargs:
cmd += "AND SeriesName LIKE '%" + kwargs['query'] + "%' "
cmd += "order by SeriesName"
results = myDB.select(cmd)
page = results[index:(index + self.PAGE_SIZE)]
for series in page:
cmd = "SELECT books.BookID,SeriesNum from books,member where SeriesID=? "
cmd += "and books.bookid = member.bookid order by CAST(SeriesNum AS INTEGER)"
firstbook = myDB.match(cmd, (series['SeriesID'],))
if firstbook:
cmd = 'SELECT AuthorName from authors,books WHERE authors.authorid = books.authorid AND books.bookid=?'
res = myDB.match(cmd, (firstbook['BookID'],))
author = res['AuthorName']
else:
author = 'Unknown'
totalbooks = check_int(series['Total'], 0)
havebooks = check_int(series['Have'], 0)
sername = makeUnicode(series['SeriesName'])
entries.append(
{
'title': escape('%s (%s/%s) %s' % (sername, havebooks, totalbooks, author)),
'id': escape('series:%s' % series['SeriesID']),
'updated': now(),
'content': escape('%s (%s)' % (sername, havebooks)),
'href': '%s?cmd=Members&seriesid=%s' % (self.opdsroot, series['SeriesID']),
'kind': 'navigation',
'rel': 'subsection',
}
)
if len(results) > (index + self.PAGE_SIZE):
links.append(
getLink(href='%s?cmd=Series&index=%s' % (self.opdsroot, index + self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='next'))
if index >= self.PAGE_SIZE:
links.append(
getLink(href='%s?cmd=Series&index=%s' % (self.opdsroot, index - self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='previous'))
feed['links'] = links
feed['entries'] = entries
logger.debug("Returning %s series" % len(entries))
self.data = feed
return
def _Magazine(self, **kwargs):
index = 0
if 'index' in kwargs:
index = check_int(kwargs['index'], 0)
myDB = database.DBConnection()
if 'magid' not in kwargs:
self.data = self._error_with_message('No Magazine Provided')
return
links = []
entries = []
title = ''
cmd = "SELECT Title,IssueID,IssueDate,IssueAcquired,IssueFile from issues "
cmd += "WHERE Title='%s' order by IssueDate DESC"
results = myDB.select(cmd % kwargs['magid'])
page = results[index:(index + self.PAGE_SIZE)]
for issue in page:
title = makeUnicode(issue['Title'])
entry = {'title': escape('%s (%s)' % (title, issue['IssueDate'])),
'id': escape('issue:%s' % issue['IssueID']),
'updated': opdstime(issue['IssueAcquired']),
'content': escape('%s - %s' % (title, issue['IssueDate'])),
'href': '%s?cmd=Serve&issueid=%s' % (self.opdsroot, quote_plus(issue['IssueID'])),
'kind': 'acquisition',
'rel': 'file',
'type': mimeType(issue['IssueFile'])}
if lazylibrarian.CONFIG['OPDS_METAINFO']:
fname = os.path.splitext(issue['IssueFile'])[0]
res = cache_img('magazine', issue['IssueID'], fname + '.jpg')
entry['image'] = self.searchroot + '/' + res[0]
entries.append(entry)
feed = {}
title = '%s (%s)' % (escape(title), len(entries))
feed['title'] = 'LazyLibrarian OPDS - %s' % title
feed['id'] = 'magazine:%s' % escape(kwargs['magid'])
feed['updated'] = now()
links.append(getLink(href=self.opdsroot, ftype='application/atom+xml; profile=opds-catalog; kind=navigation',
rel='start', title='Home'))
links.append(getLink(href='%s?cmd=Magazine&magid=%s' % (self.opdsroot, quote_plus(kwargs['magid'])),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='self'))
if len(results) > (index + self.PAGE_SIZE):
links.append(
getLink(href='%s?cmd=Magazine&magid=%s&index=%s' % (self.opdsroot, quote_plus(kwargs['magid']),
index + self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='next'))
if index >= self.PAGE_SIZE:
links.append(
getLink(href='%s?cmd=Magazine&magid=%s&index=%s' % (self.opdsroot, quote_plus(kwargs['magid']),
index - self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='previous'))
feed['links'] = links
feed['entries'] = entries
logger.debug("Returning %s issue%s" % (len(entries), plural(len(entries))))
self.data = feed
return
def _Magazine(self, **kwargs):
index = 0
if 'index' in kwargs:
index = check_int(kwargs['index'], 0)
myDB = database.DBConnection()
if 'magid' not in kwargs:
self.data = self._error_with_message('No Magazine Provided')
return
links = []
entries = []
title = ''
cmd = "SELECT Title,IssueID,IssueDate,IssueAcquired,IssueFile from issues "
cmd += "WHERE Title='%s' order by IssueDate DESC"
results = myDB.select(cmd % kwargs['magid'])
page = results[index:(index + self.PAGE_SIZE)]
for issue in page:
title = makeUnicode(issue['Title'])
entry = {'title': escape('%s (%s)' % (title, issue['IssueDate'])),
'id': escape('issue:%s' % issue['IssueID']),
'updated': opdstime(issue['IssueAcquired']),
'content': escape('%s - %s' % (title, issue['IssueDate'])),
'href': '%s?cmd=Serve&issueid=%s' % (self.opdsroot, quote_plus(issue['IssueID'])),
'kind': 'acquisition',
'rel': 'file',
'type': mimeType(issue['IssueFile'])}
if lazylibrarian.CONFIG['OPDS_METAINFO']:
fname = os.path.splitext(issue['IssueFile'])[0]
res = cache_img('magazine', issue['IssueID'], fname + '.jpg')
entry['image'] = self.searchroot + '/' + res[0]
entries.append(entry)
feed = {}
title = '%s (%s)' % (escape(title), len(entries))
feed['title'] = 'LazyLibrarian OPDS - %s' % title
feed['id'] = 'magazine:%s' % escape(kwargs['magid'])
feed['updated'] = now()
links.append(getLink(href=self.opdsroot, ftype='application/atom+xml; profile=opds-catalog; kind=navigation',
rel='start', title='Home'))
links.append(getLink(href='%s?cmd=Magazine&magid=%s' % (self.opdsroot, quote_plus(kwargs['magid'])),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='self'))
if len(results) > (index + self.PAGE_SIZE):
links.append(
getLink(href='%s?cmd=Magazine&magid=%s&index=%s' % (self.opdsroot, quote_plus(kwargs['magid']),
index + self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='next'))
if index >= self.PAGE_SIZE:
links.append(
getLink(href='%s?cmd=Magazine&magid=%s&index=%s' % (self.opdsroot, quote_plus(kwargs['magid']),
index - self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='previous'))
feed['links'] = links
feed['entries'] = entries
logger.debug("Returning %s issue%s" % (len(entries), plural(len(entries))))
self.data = feed
return
def __init__(self, name=None):
self.name = makeUnicode(name)
if not lazylibrarian.CONFIG['GB_API']:
logger.warn('No GoogleBooks API key, check config')
self.url = 'https://www.googleapis.com/books/v1/volumes?q='
self.params = {
'maxResults': 40,
'printType': 'books',
'key': lazylibrarian.CONFIG['GB_API']
}
def encode_multipart(fields, files, boundary=None):
"""Encode dict of form fields and dict of files as multipart/form-data.
Return tuple of (body_string, headers_dict). Each value in files is a dict
with required keys 'filename' and 'content', and optional 'mimetype' (if
not specified, tries to guess mime type or uses 'application/octet-stream').
"""
def escape_quote(s):
s = makeUnicode(s)
return s.replace('"', '\\"')
if boundary is None:
boundary = ''.join(random.choice(_BOUNDARY_CHARS) for _ in range(30))
lines = []
if fields:
fields = dict(
(makeBytestr(k), makeBytestr(v)) for k, v in fields.items())
for name, value in list(fields.items()):
lines.extend((
'--{0}'.format(boundary),
'Content-Disposition: form-data; name="{0}"'.format(
escape_quote(name)),
'',
makeUnicode(value),
))
if files:
for name, value in list(files.items()):
filename = value['filename']
if 'mimetype' in value:
mimetype = value['mimetype']
else:
mimetype = mimetypes.guess_type(
filename)[0] or 'application/octet-stream'
lines.extend((
'--{0}'.format(boundary),
'Content-Disposition: form-data; name="{0}"; filename="{1}"'.
format(escape_quote(name), escape_quote(filename)),
'Content-Type: {0}'.format(mimetype),
'',
value['content'],
))
lines.extend((
'--{0}--'.format(boundary),
'',
))
body = makeBytestr('\r\n'.join(lines))
headers = {
'Content-Type': 'multipart/form-data; boundary={0}'.format(boundary),
'Content-Length': str(len(body)),
}
return body, headers
def _Authors(self, **kwargs):
index = 0
if 'index' in kwargs:
index = check_int(kwargs['index'], 0)
myDB = database.DBConnection()
feed = {'title': 'LazyLibrarian OPDS - Authors', 'id': 'Authors', 'updated': now()}
links = []
entries = []
links.append(getLink(href=self.opdsroot, ftype='application/atom+xml; profile=opds-catalog; kind=navigation',
rel='start', title='Home'))
links.append(getLink(href='%s?cmd=Authors' % self.opdsroot,
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='self'))
links.append(getLink(href='%s/opensearchauthors.xml' % self.searchroot,
ftype='application/opensearchdescription+xml', rel='search', title='Search Authors'))
cmd = "SELECT AuthorName,AuthorID,HaveBooks,TotalBooks,DateAdded from Authors WHERE "
if 'query' in kwargs:
cmd += "AuthorName LIKE '%" + kwargs['query'] + "%' AND "
cmd += "CAST(HaveBooks AS INTEGER) > 0 order by AuthorName"
results = myDB.select(cmd)
page = results[index:(index + self.PAGE_SIZE)]
for author in page:
totalbooks = check_int(author['TotalBooks'], 0)
havebooks = check_int(author['HaveBooks'], 0)
lastupdated = author['DateAdded']
name = makeUnicode(author['AuthorName'])
entry = {
'title': escape('%s (%s/%s)' % (name, havebooks, totalbooks)),
'id': escape('author:%s' % author['AuthorID']),
'updated': opdstime(lastupdated),
'content': escape('%s (%s)' % (name, havebooks)),
'href': '%s?cmd=Author&authorid=%s' % (self.opdsroot, author['AuthorID']),
'author': escape('%s' % name),
'kind': 'navigation',
'rel': 'subsection',
}
# removed authorimg as it stops navigation ??
# if lazylibrarian.CONFIG['OPDS_METAINFO']:
# entry['image'] = self.searchroot + '/' + author['AuthorImg']
entries.append(entry)
if len(results) > (index + self.PAGE_SIZE):
links.append(
getLink(href='%s?cmd=Authors&index=%s' % (self.opdsroot, index + self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='next'))
if index >= self.PAGE_SIZE:
links.append(
getLink(href='%s?cmd=Authors&index=%s' % (self.opdsroot, index - self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='previous'))
feed['links'] = links
feed['entries'] = entries
logger.debug("Returning %s author%s" % (len(entries), plural(len(entries))))
self.data = feed
return
def _Authors(self, **kwargs):
index = 0
if 'index' in kwargs:
index = check_int(kwargs['index'], 0)
myDB = database.DBConnection()
feed = {'title': 'LazyLibrarian OPDS - Authors', 'id': 'Authors', 'updated': now()}
links = []
entries = []
links.append(getLink(href=self.opdsroot, ftype='application/atom+xml; profile=opds-catalog; kind=navigation',
rel='start', title='Home'))
links.append(getLink(href='%s?cmd=Authors' % self.opdsroot,
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='self'))
links.append(getLink(href='%s/opensearchauthors.xml' % self.searchroot,
ftype='application/opensearchdescription+xml', rel='search', title='Search Authors'))
cmd = "SELECT AuthorName,AuthorID,HaveBooks,TotalBooks,DateAdded from Authors WHERE "
if 'query' in kwargs:
cmd += "AuthorName LIKE '%" + kwargs['query'] + "%' AND "
cmd += "CAST(HaveBooks AS INTEGER) > 0 order by AuthorName"
results = myDB.select(cmd)
page = results[index:(index + self.PAGE_SIZE)]
for author in page:
totalbooks = check_int(author['TotalBooks'], 0)
havebooks = check_int(author['HaveBooks'], 0)
lastupdated = author['DateAdded']
name = makeUnicode(author['AuthorName'])
entry = {
'title': escape('%s (%s/%s)' % (name, havebooks, totalbooks)),
'id': escape('author:%s' % author['AuthorID']),
'updated': opdstime(lastupdated),
'content': escape('%s (%s)' % (name, havebooks)),
'href': '%s?cmd=Author&authorid=%s' % (self.opdsroot, author['AuthorID']),
'author': escape('%s' % name),
'kind': 'navigation',
'rel': 'subsection',
}
# removed authorimg as it stops navigation ??
# if lazylibrarian.CONFIG['OPDS_METAINFO']:
# entry['image'] = self.searchroot + '/' + author['AuthorImg']
entries.append(entry)
if len(results) > (index + self.PAGE_SIZE):
links.append(
getLink(href='%s?cmd=Authors&index=%s' % (self.opdsroot, index + self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='next'))
if index >= self.PAGE_SIZE:
links.append(
getLink(href='%s?cmd=Authors&index=%s' % (self.opdsroot, index - self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='previous'))
feed['links'] = links
feed['entries'] = entries
logger.debug("Returning %s author%s" % (len(entries), plural(len(entries))))
self.data = feed
return
def any_file(search_dir=None, extn=None):
# find a file with specified extension in a directory, any will do
# return full pathname of file, or empty string if none found
if search_dir is None or extn is None:
return ""
if os.path.isdir(search_dir):
for fname in os.listdir(makeBytestr(search_dir)):
fname = makeUnicode(fname)
if fname.endswith(extn):
return os.path.join(search_dir, fname)
return ""
def __init__(self, name=None):
self.name = makeUnicode(name)
if not lazylibrarian.CONFIG['GB_API']:
logger.warn('No GoogleBooks API key, check config')
self.url = 'https://www.googleapis.com/books/v1/volumes?q='
self.params = {
'maxResults': 40,
'printType': 'books',
}
if lazylibrarian.CONFIG['GB_API']:
self.params['key'] = lazylibrarian.CONFIG['GB_API']
def encode_multipart(fields, files, boundary=None):
"""Encode dict of form fields and dict of files as multipart/form-data.
Return tuple of (body_string, headers_dict). Each value in files is a dict
with required keys 'filename' and 'content', and optional 'mimetype' (if
not specified, tries to guess mime type or uses 'application/octet-stream').
"""
def escape_quote(s):
s = makeUnicode(s)
return s.replace('"', '\\"')
if boundary is None:
boundary = ''.join(random.choice(_BOUNDARY_CHARS) for _ in range(30))
lines = []
if fields:
fields = dict((makeBytestr(k), makeBytestr(v)) for k, v in fields.items())
for name, value in list(fields.items()):
lines.extend((
'--{0}'.format(boundary),
'Content-Disposition: form-data; name="{0}"'.format(escape_quote(name)),
'',
makeUnicode(value),
))
if files:
for name, value in list(files.items()):
filename = value['filename']
if 'mimetype' in value:
mimetype = value['mimetype']
else:
mimetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
lines.extend((
'--{0}'.format(boundary),
'Content-Disposition: form-data; name="{0}"; filename="{1}"'.format(
escape_quote(name), escape_quote(filename)),
'Content-Type: {0}'.format(mimetype),
'',
value['content'],
))
lines.extend((
'--{0}--'.format(boundary),
'',
))
body = makeBytestr('\r\n'.join(lines))
headers = {
'Content-Type': 'multipart/form-data; boundary={0}'.format(boundary),
'Content-Length': str(len(body)),
}
return body, headers
def _Magazines(self, **kwargs):
index = 0
if 'index' in kwargs:
index = check_int(kwargs['index'], 0)
myDB = database.DBConnection()
feed = {'title': 'LazyLibrarian OPDS - Magazines', 'id': 'Magazines', 'updated': now()}
links = []
entries = []
links.append(getLink(href=self.opdsroot, ftype='application/atom+xml; profile=opds-catalog; kind=navigation',
rel='start', title='Home'))
links.append(getLink(href='%s?cmd=Magazines' % self.opdsroot,
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='self'))
links.append(getLink(href='%s/opensearchmagazines.xml' % self.searchroot,
ftype='application/opensearchdescription+xml', rel='search', title='Search Magazines'))
cmd = 'select magazines.*,(select count(*) as counter from issues where magazines.title = issues.title)'
cmd += ' as Iss_Cnt from magazines '
if 'query' in kwargs:
cmd += "WHERE magazines.title LIKE '%" + kwargs['query'] + "%' "
cmd += 'order by magazines.title'
results = myDB.select(cmd)
page = results[index:(index + self.PAGE_SIZE)]
for mag in page:
if mag['Iss_Cnt'] > 0:
title = makeUnicode(mag['Title'])
entry = {
'title': escape('%s (%s)' % (title, mag['Iss_Cnt'])),
'id': escape('magazine:%s' % title),
'updated': opdstime(mag['LastAcquired']),
'content': escape('%s' % title),
'href': '%s?cmd=Magazine&magid=%s' % (self.opdsroot, quote_plus(title)),
'kind': 'navigation',
'rel': 'subsection',
}
# disabled cover image as it stops navigation?
# if lazylibrarian.CONFIG['OPDS_METAINFO']:
# res = cache_img('magazine', md5_utf8(mag['LatestCover']), mag['LatestCover'], refresh=True)
# entry['image'] = self.searchroot + '/' + res[0]
entries.append(entry)
if len(results) > (index + self.PAGE_SIZE):
links.append(
getLink(href='%s?cmd=Magazines&index=%s' % (self.opdsroot, index + self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='next'))
if index >= self.PAGE_SIZE:
links.append(
getLink(href='%s?cmd=Magazines&index=%s' % (self.opdsroot, index - self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='previous'))
feed['links'] = links
feed['entries'] = entries
logger.debug("Returning %s magazine%s" % (len(entries), plural(len(entries))))
self.data = feed
return
def _RecentMags(self, **kwargs):
index = 0
if 'index' in kwargs:
index = check_int(kwargs['index'], 0)
myDB = database.DBConnection()
feed = {'title': 'LazyLibrarian OPDS - Recent Magazines', 'id': 'Recent Magazines', 'updated': now()}
links = []
entries = []
links.append(getLink(href=self.opdsroot, ftype='application/atom+xml; profile=opds-catalog; kind=navigation',
rel='start', title='Home'))
links.append(getLink(href='%s?cmd=RecentMags' % self.opdsroot,
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='self'))
links.append(getLink(href='%s/opensearchmagazines.xml' % self.searchroot,
ftype='application/opensearchdescription+xml', rel='search', title='Search Magazines'))
cmd = "select Title,IssueID,IssueAcquired,IssueDate,IssueFile from issues "
cmd += "where IssueFile != '' "
if 'query' in kwargs:
cmd += "AND Title LIKE '%" + kwargs['query'] + "%' "
cmd += "order by IssueAcquired DESC"
results = myDB.select(cmd)
page = results[index:(index + self.PAGE_SIZE)]
for mag in page:
title = makeUnicode(mag['Title'])
entry = {'title': escape('%s' % mag['IssueDate']),
'id': escape('issue:%s' % mag['IssueID']),
'updated': opdstime(mag['IssueAcquired']),
'content': escape('%s - %s' % (title, mag['IssueDate'])),
'href': '%s?cmd=Serve&issueid=%s' % (self.opdsroot, quote_plus(mag['IssueID'])),
'kind': 'acquisition',
'rel': 'file',
'author': escape(title),
'type': mimeType(mag['IssueFile'])}
if lazylibrarian.CONFIG['OPDS_METAINFO']:
fname = os.path.splitext(mag['IssueFile'])[0]
res = cache_img('magazine', mag['IssueID'], fname + '.jpg')
entry['image'] = self.searchroot + '/' + res[0]
entries.append(entry)
if len(results) > (index + self.PAGE_SIZE):
links.append(
getLink(href='%s?cmd=RecentMags&index=%s' % (self.opdsroot, index + self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='next'))
if index >= self.PAGE_SIZE:
links.append(
getLink(href='%s?cmd=RecentMags&index=%s' % (self.opdsroot, index - self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='previous'))
feed['links'] = links
feed['entries'] = entries
logger.debug("Returning %s issue%s" % (len(entries), plural(len(entries))))
self.data = feed
return
def _Magazines(self, **kwargs):
index = 0
if 'index' in kwargs:
index = check_int(kwargs['index'], 0)
myDB = database.DBConnection()
feed = {'title': 'LazyLibrarian OPDS - Magazines', 'id': 'Magazines', 'updated': now()}
links = []
entries = []
links.append(getLink(href=self.opdsroot, ftype='application/atom+xml; profile=opds-catalog; kind=navigation',
rel='start', title='Home'))
links.append(getLink(href='%s?cmd=Magazines' % self.opdsroot,
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='self'))
links.append(getLink(href='%s/opensearchmagazines.xml' % self.searchroot,
ftype='application/opensearchdescription+xml', rel='search', title='Search Magazines'))
cmd = 'select magazines.*,(select count(*) as counter from issues where magazines.title = issues.title)'
cmd += ' as Iss_Cnt from magazines '
if 'query' in kwargs:
cmd += "WHERE magazines.title LIKE '%" + kwargs['query'] + "%' "
cmd += 'order by magazines.title'
results = myDB.select(cmd)
page = results[index:(index + self.PAGE_SIZE)]
for mag in page:
if mag['Iss_Cnt'] > 0:
title = makeUnicode(mag['Title'])
entry = {
'title': escape('%s (%s)' % (title, mag['Iss_Cnt'])),
'id': escape('magazine:%s' % title),
'updated': opdstime(mag['LastAcquired']),
'content': escape('%s' % title),
'href': '%s?cmd=Magazine&magid=%s' % (self.opdsroot, quote_plus(title)),
'kind': 'navigation',
'rel': 'subsection',
}
if lazylibrarian.CONFIG['OPDS_METAINFO']:
res = cache_img('magazine', md5_utf8(mag['LatestCover']), mag['LatestCover'], refresh=True)
entry['image'] = self.searchroot + '/' + res[0]
entries.append(entry)
if len(results) > (index + self.PAGE_SIZE):
links.append(
getLink(href='%s?cmd=Magazines&index=%s' % (self.opdsroot, index + self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='next'))
if index >= self.PAGE_SIZE:
links.append(
getLink(href='%s?cmd=Magazines&index=%s' % (self.opdsroot, index - self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='previous'))
feed['links'] = links
feed['entries'] = entries
logger.debug("Returning %s magazine%s" % (len(entries), plural(len(entries))))
self.data = feed
return
def _RecentMags(self, **kwargs):
index = 0
if 'index' in kwargs:
index = check_int(kwargs['index'], 0)
myDB = database.DBConnection()
feed = {'title': 'LazyLibrarian OPDS - Recent Magazines', 'id': 'Recent Magazines', 'updated': now()}
links = []
entries = []
links.append(getLink(href=self.opdsroot, ftype='application/atom+xml; profile=opds-catalog; kind=navigation',
rel='start', title='Home'))
links.append(getLink(href='%s?cmd=RecentMags' % self.opdsroot,
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='self'))
links.append(getLink(href='%s/opensearchmagazines.xml' % self.searchroot,
ftype='application/opensearchdescription+xml', rel='search', title='Search Magazines'))
cmd = "select Title,IssueID,IssueAcquired,IssueDate,IssueFile from issues "
cmd += "where IssueFile != '' "
if 'query' in kwargs:
cmd += "AND Title LIKE '%" + kwargs['query'] + "%' "
cmd += "order by IssueAcquired DESC"
results = myDB.select(cmd)
page = results[index:(index + self.PAGE_SIZE)]
for mag in page:
title = makeUnicode(mag['Title'])
entry = {'title': escape('%s' % mag['IssueDate']),
'id': escape('issue:%s' % mag['IssueID']),
'updated': opdstime(mag['IssueAcquired']),
'content': escape('%s - %s' % (title, mag['IssueDate'])),
'href': '%s?cmd=Serve&issueid=%s' % (self.opdsroot, quote_plus(mag['IssueID'])),
'kind': 'acquisition',
'rel': 'file',
'author': escape(title),
'type': mimeType(mag['IssueFile'])}
if lazylibrarian.CONFIG['OPDS_METAINFO']:
fname = os.path.splitext(mag['IssueFile'])[0]
res = cache_img('magazine', mag['IssueID'], fname + '.jpg')
entry['image'] = self.searchroot + '/' + res[0]
entries.append(entry)
if len(results) > (index + self.PAGE_SIZE):
links.append(
getLink(href='%s?cmd=RecentMags&index=%s' % (self.opdsroot, index + self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='next'))
if index >= self.PAGE_SIZE:
links.append(
getLink(href='%s?cmd=RecentMags&index=%s' % (self.opdsroot, index - self.PAGE_SIZE),
ftype='application/atom+xml; profile=opds-catalog; kind=navigation', rel='previous'))
feed['links'] = links
feed['entries'] = entries
logger.debug("Returning %s issue%s" % (len(entries), plural(len(entries))))
self.data = feed
return
def CalcTorrentHash(torrent):
# torrent could be a unicode magnet link or a bytes object torrent file contents
if makeUnicode(torrent[:6]) == 'magnet':
# torrent = makeUnicode(torrent)
hashid = re.findall('urn:btih:([\w]{32,40})', torrent)[0]
if len(hashid) == 32:
hashid = b16encode(b32decode(hashid)).lower()
else:
# noinspection PyTypeChecker
info = dict(decode(torrent))["info"] # python3 decode returns OrderedDict
hashid = sha1(encode(info)).hexdigest()
logger.debug('Torrent Hash: ' + hashid)
return hashid
def CalcTorrentHash(torrent):
# torrent could be a unicode magnet link or a bytes object torrent file contents
if makeUnicode(torrent[:6]) == 'magnet':
# torrent = makeUnicode(torrent)
hashid = re.findall('urn:btih:([\w]{32,40})', torrent)[0]
if len(hashid) == 32:
hashid = b16encode(b32decode(hashid)).lower()
else:
# noinspection PyTypeChecker
info = dict(decode(torrent))["info"] # python3 decode returns OrderedDict
hashid = sha1(encode(info)).hexdigest()
logger.debug('Torrent Hash: ' + hashid)
return hashid
def _command(self, command, args=None, content_type=None, files=None):
logger.debug('QBittorrent WebAPI Command: %s' % command)
url = self.base_url + '/' + command
data = None
headers = dict()
if files or content_type == 'multipart/form-data':
data, headers = encode_multipart(
args, files, '-------------------------acebdf13572468')
else:
if args:
data = makeBytestr(urlencode(args))
if content_type:
headers['Content-Type'] = content_type
request = Request(url, data, headers)
if lazylibrarian.CONFIG['PROXY_HOST']:
for item in getList(lazylibrarian.CONFIG['PROXY_TYPE']):
request.set_proxy(lazylibrarian.CONFIG['PROXY_HOST'], item)
request.add_header('User-Agent', USER_AGENT)
try:
response = self.opener.open(request)
try:
contentType = response.headers['content-type']
except KeyError:
contentType = ''
resp = response.read()
# some commands return json
if contentType == 'application/json':
if resp:
return json.loads(resp)
return ''
else:
# some commands return plain text
resp = makeUnicode(resp)
logger.debug("QBitTorrent returned %s" % resp)
if command == 'version/api':
return resp
# some just return Ok. or Fails.
if resp and resp != 'Ok.':
return False
# some commands return nothing but response code (always 200)
return True
except URLError as err:
logger.debug('Failed URL: %s' % url)
logger.debug('QBitTorrent webUI raised the following error: %s' %
err.reason)
return False
def book_file(search_dir=None, booktype=None):
# find a book/mag file in this directory, any book will do
# return full pathname of book/mag, or empty string if none found
if search_dir is None or booktype is None:
return ""
if search_dir and os.path.isdir(search_dir):
try:
for fname in os.listdir(makeBytestr(search_dir)):
fname = makeUnicode(fname)
if is_valid_booktype(fname, booktype=booktype):
return os.path.join(search_dir, fname)
except Exception as e:
logger.warn('Listdir error [%s]: %s %s' % (search_dir, type(e).__name__, str(e)))
return ""
def _command(self, command, args=None, content_type=None, files=None):
logger.debug('QBittorrent WebAPI Command: %s' % command)
url = self.base_url + '/' + command
data = None
headers = dict()
if files or content_type == 'multipart/form-data':
data, headers = encode_multipart(args, files, '-------------------------acebdf13572468')
else:
if args:
data = makeBytestr(urlencode(args))
if content_type:
headers['Content-Type'] = content_type
request = Request(url, data, headers)
if lazylibrarian.CONFIG['PROXY_HOST']:
for item in getList(lazylibrarian.CONFIG['PROXY_TYPE']):
request.set_proxy(lazylibrarian.CONFIG['PROXY_HOST'], item)
request.add_header('User-Agent', getUserAgent())
try:
response = self.opener.open(request)
try:
contentType = response.headers['content-type']
except KeyError:
contentType = ''
resp = response.read()
# some commands return json
if contentType == 'application/json':
if resp:
return json.loads(resp)
return ''
else:
# some commands return plain text
resp = makeUnicode(resp)
logger.debug("QBitTorrent returned %s" % resp)
if command == 'version/api':
return resp
# some just return Ok. or Fails.
if resp and resp != 'Ok.':
return False
# some commands return nothing but response code (always 200)
return True
except URLError as err:
logger.debug('Failed URL: %s' % url)
logger.debug('QBitTorrent webUI raised the following error: %s' % err.reason)
return False
def _notify(message, event, force=False):
# suppress notifications if the notifier is disabled but the notify options are checked
if not lazylibrarian.CONFIG['USE_CUSTOM'] and not force:
return False
subject = event
logger.debug('Custom Event: %s' % event)
logger.debug('Custom Message: %s' % message)
myDB = database.DBConnection()
if subject == "Test":
# grab the first entry in the book table
data = myDB.match('SELECT * from books')
else:
# message is a bookid or a magazineid
data = myDB.match('SELECT * from books where BookID=?',
(message, ))
if not data:
data = myDB.match('SELECT * from magazines where BookID=?',
(message, ))
dictionary = dict(list(zip(list(data.keys()), data)))
dictionary['Event'] = event
try:
# call the custom notifier script here, passing dictionary deconstructed as strings
if lazylibrarian.CONFIG['CUSTOM_SCRIPT']:
params = [lazylibrarian.CONFIG['CUSTOM_SCRIPT']]
for item in dictionary:
params.append(item)
if hasattr(dictionary[item], 'encode'):
params.append(dictionary[item].encode('utf-8'))
else:
params.append(str(dictionary[item]))
try:
res = subprocess.check_output(params,
stderr=subprocess.STDOUT)
return makeUnicode(res).strip()
except Exception as e:
logger.warn('Error sending command: %s' % e)
return False
else:
logger.warn('Error sending custom notification: Check config')
return False
except Exception as e:
logger.warn('Error sending custom notification: %s' % e)
return False
def _listMissingWorkpages(self):
# first the ones with no workpage
q = 'SELECT BookID from books where length(WorkPage) < 4'
res = self._dic_from_query(q)
# now the ones with an error page
cache = os.path.join(lazylibrarian.CACHEDIR, "WorkCache")
if os.path.isdir(cache):
for cached_file in os.listdir(makeBytestr(cache)):
cached_file = makeUnicode(cached_file)
target = os.path.join(cache, cached_file)
if os.path.isfile(target):
if os.path.getsize(target) < 500 and '.' in cached_file:
bookid = cached_file.split('.')[0]
res.append({"BookID": bookid})
self.data = res
def finditem(item, preferred_authorname):
"""
Try to find book matching the csv item in the database
Return database entry, or False if not found
"""
myDB = database.DBConnection()
bookmatch = ""
isbn10 = ""
isbn13 = ""
bookid = ""
bookname = item['Title']
bookname = makeUnicode(bookname)
if 'ISBN' in item:
isbn10 = item['ISBN']
if 'ISBN13' in item:
isbn13 = item['ISBN13']
if 'BookID' in item:
bookid = item['BookID']
# try to find book in our database using bookid or isbn, or if that fails, name matching
cmd = 'SELECT AuthorName,BookName,BookID,books.Status FROM books,authors where books.AuthorID = authors.AuthorID '
if bookid:
fullcmd = cmd + 'and BookID=?'
bookmatch = myDB.match(fullcmd, (bookid, ))
if not bookmatch:
if is_valid_isbn(isbn10):
fullcmd = cmd + 'and BookIsbn=?'
bookmatch = myDB.match(fullcmd, (isbn10, ))
if not bookmatch:
if is_valid_isbn(isbn13):
fullcmd = cmd + 'and BookIsbn=?'
bookmatch = myDB.match(fullcmd, (isbn13, ))
if not bookmatch:
bookid, mtype = find_book_in_db(preferred_authorname,
bookname,
ignored=False)
if bookid and mtype == "Ignored":
logger.warn(
"Book %s by %s is marked Ignored in database, importing anyway"
% (bookname, preferred_authorname))
if bookid:
fullcmd = cmd + 'and BookID=?'
bookmatch = myDB.match(fullcmd, (bookid, ))
return bookmatch
def finditem(item, preferred_authorname, library='eBook'):
"""
Try to find book matching the csv item in the database
Return database entry, or False if not found
"""
myDB = database.DBConnection()
bookmatch = ""
isbn10 = ""
isbn13 = ""
bookid = ""
bookname = item['Title']
bookname = makeUnicode(bookname)
if 'ISBN' in item:
isbn10 = item['ISBN']
if 'ISBN13' in item:
isbn13 = item['ISBN13']
if 'BookID' in item:
bookid = item['BookID']
# try to find book in our database using bookid or isbn, or if that fails, name matching
cmd = 'SELECT AuthorName,BookName,BookID,books.Status,AudioStatus,Requester,'
cmd += 'AudioRequester FROM books,authors where books.AuthorID = authors.AuthorID '
if bookid:
fullcmd = cmd + 'and BookID=?'
bookmatch = myDB.match(fullcmd, (bookid, ))
if not bookmatch:
if is_valid_isbn(isbn10):
fullcmd = cmd + 'and BookIsbn=?'
bookmatch = myDB.match(fullcmd, (isbn10, ))
if not bookmatch:
if is_valid_isbn(isbn13):
fullcmd = cmd + 'and BookIsbn=?'
bookmatch = myDB.match(fullcmd, (isbn13, ))
if not bookmatch:
bookid, mtype = find_book_in_db(preferred_authorname,
bookname,
ignored=False,
library=library)
if bookid:
fullcmd = cmd + 'and BookID=?'
bookmatch = myDB.match(fullcmd, (bookid, ))
return bookmatch
def finditem(item, preferred_authorname, library='eBook'):
"""
Try to find book matching the csv item in the database
Return database entry, or False if not found
"""
myDB = database.DBConnection()
bookmatch = ""
isbn10 = ""
isbn13 = ""
bookid = ""
bookname = item['Title']
bookname = makeUnicode(bookname)
if 'ISBN' in item:
isbn10 = item['ISBN']
if 'ISBN13' in item:
isbn13 = item['ISBN13']
if 'BookID' in item:
bookid = item['BookID']
# try to find book in our database using bookid or isbn, or if that fails, name matching
cmd = 'SELECT AuthorName,BookName,BookID,books.Status,AudioStatus,Requester,'
cmd += 'AudioRequester FROM books,authors where books.AuthorID = authors.AuthorID '
if bookid:
fullcmd = cmd + 'and BookID=?'
bookmatch = myDB.match(fullcmd, (bookid,))
if not bookmatch:
if is_valid_isbn(isbn10):
fullcmd = cmd + 'and BookIsbn=?'
bookmatch = myDB.match(fullcmd, (isbn10,))
if not bookmatch:
if is_valid_isbn(isbn13):
fullcmd = cmd + 'and BookIsbn=?'
bookmatch = myDB.match(fullcmd, (isbn13,))
if not bookmatch:
bookid, mtype = find_book_in_db(preferred_authorname, bookname, ignored=False, library=library)
if bookid:
fullcmd = cmd + 'and BookID=?'
bookmatch = myDB.match(fullcmd, (bookid,))
return bookmatch
def finditem(item, authorname, headers):
"""
Try to find book matching the csv item in the database
Return database entry, or False if not found
"""
myDB = database.DBConnection()
bookmatch = ""
isbn10 = ""
isbn13 = ""
bookid = ""
bookname = item['Title']
bookname = makeUnicode(bookname)
if 'ISBN' in headers:
isbn10 = item['ISBN']
if 'ISBN13' in headers:
isbn13 = item['ISBN13']
if 'BookID' in headers:
bookid = item['BookID']
# try to find book in our database using bookid or isbn, or if that fails, name matching
cmd = 'SELECT AuthorName,BookName,BookID,books.Status FROM books,authors where books.AuthorID = authors.AuthorID '
if bookid:
fullcmd = cmd + 'and BookID=?'
bookmatch = myDB.match(fullcmd, (bookid,))
if not bookmatch:
if is_valid_isbn(isbn10):
fullcmd = cmd + 'and BookIsbn=?'
bookmatch = myDB.match(fullcmd, (isbn10,))
if not bookmatch:
if is_valid_isbn(isbn13):
fullcmd = cmd + 'and BookIsbn=?'
bookmatch = myDB.match(fullcmd, (isbn13,))
if not bookmatch:
bookid = find_book_in_db(authorname, bookname)
if bookid:
fullcmd = cmd + 'and BookID=?'
bookmatch = myDB.match(fullcmd, (bookid,))
return bookmatch
def opf_file(search_dir=None):
if search_dir is None:
return ""
cnt = 0
res = ''
meta = ''
if os.path.isdir(search_dir):
for fname in os.listdir(makeBytestr(search_dir)):
fname = makeUnicode(fname)
if fname.endswith('.opf'):
if fname == 'metadata.opf':
meta = os.path.join(search_dir, fname)
else:
res = os.path.join(search_dir, fname)
cnt += 1
if cnt > 2 or cnt == 2 and not meta:
logger.debug("Found %d conflicting opf in %s" % (cnt, search_dir))
res = ''
elif res: # prefer bookname.opf over metadata.opf
return res
elif meta:
return meta
return res
def find_author_id(self, refresh=False):
author = self.name
author = formatAuthorName(unaccented(author))
URL = 'https://www.goodreads.com/api/author_url/' + urllib.quote(author) + \
'?' + urllib.urlencode(self.params)
# googlebooks gives us author names with long form unicode characters
author = makeUnicode(author) # ensure it's unicode
author = unicodedata.normalize('NFC', author) # normalize to short form
logger.debug("Searching for author with name: %s" % author)
authorlist = []
try:
rootxml, in_cache = get_xml_request(URL, useCache=not refresh)
except Exception as e:
logger.error("%s finding authorid: %s, %s" % (type(e).__name__, URL, str(e)))
return authorlist
if rootxml is None:
logger.debug("Error requesting authorid")
return authorlist
resultxml = rootxml.getiterator('author')
if resultxml is None:
logger.warn('No authors found with name: %s' % author)
else:
# In spite of how this looks, goodreads only returns one result, even if there are multiple matches
# we just have to hope we get the right one. eg search for "James Lovelock" returns "James E. Lovelock"
# who only has one book listed under googlebooks, the rest are under "James Lovelock"
# goodreads has all his books under "James E. Lovelock". Can't come up with a good solution yet.
# For now we'll have to let the user handle this by selecting/adding the author manually
for author in resultxml:
authorid = author.attrib.get("id")
authorlist = self.get_author_info(authorid)
return authorlist
def WWT(book=None, test=False):
errmsg = ''
provider = "WorldWideTorrents"
host = lazylibrarian.CONFIG['WWT_HOST']
if not host.startswith('http'):
host = 'http://' + host
providerurl = url_fix(host + "/torrents-search.php")
sterm = makeUnicode(book['searchterm'])
cat = 0 # 0=all, 36=ebooks, 52=mags, 56=audiobooks
if 'library' in book:
if book['library'] == 'AudioBook':
cat = 56
elif book['library'] == 'eBook':
cat = 36
elif book['library'] == 'magazine':
cat = 52
page = 0
results = []
minimumseeders = int(lazylibrarian.CONFIG['NUMBEROFSEEDERS']) - 1
next_page = True
while next_page:
params = {
"search": book['searchterm'],
"page": page,
"cat": cat
}
searchURL = providerurl + "/?%s" % urlencode(params)
next_page = False
result, success = fetchURL(searchURL)
if not success:
# might return 404 if no results, not really an error
if '404' in result:
logger.debug("No results found from %s for %s" % (provider, sterm))
success = True
elif '503' in result:
logger.warn("Cloudflare bot detection? %s: %s" % (provider, result))
logger.warn("Try unblocking %s from a browser" % providerurl)
success = True
else:
logger.debug(searchURL)
logger.debug('Error fetching data from %s: %s' % (provider, result))
errmsg = result
result = False
if test:
return success
if result:
logger.debug('Parsing results from <a href="%s">%s</a>' % (searchURL, provider))
soup = BeautifulSoup(result, 'html5lib')
rows = []
try:
tables = soup.find_all('table') # un-named table
table = tables[2]
if table:
rows = table.find_all('tr')
except IndexError: # no results table in result page
rows = []
if len(rows) > 1:
rows = rows[1:] # first row is headers
for row in rows:
td = row.find_all('td')
if len(td) > 3:
try:
title = unaccented(td[0].text)
# can return magnet or torrent or both.
magnet = ''
url = ''
mode = 'torrent'
try:
magnet = 'magnet' + str(td[0]).split('href="magnet')[1].split('"')[0]
mode = 'magnet'
except IndexError:
pass
try:
url = url_fix(host + '/download.php') + \
str(td[0]).split('href="download.php')[1].split('.torrent"')[0] + '.torrent'
mode = 'torrent'
except IndexError:
pass
if not url or (magnet and url and lazylibrarian.CONFIG['PREFER_MAGNET']):
url = magnet
mode = 'magnet'
try:
size = str(td[1].text).replace(' ', '').upper()
size = size_in_bytes(size)
except ValueError:
size = 0
try:
seeders = int(td[2].text.replace(',', ''))
except ValueError:
seeders = 0
if not url or not title:
logger.debug('Missing url or title')
elif minimumseeders < seeders:
results.append({
'bookid': book['bookid'],
'tor_prov': provider,
'tor_title': title,
'tor_url': url,
'tor_size': str(size),
'tor_type': mode,
'priority': lazylibrarian.CONFIG['WWT_DLPRIORITY']
})
logger.debug('Found %s. Size: %s' % (title, size))
next_page = True
else:
logger.debug('Found %s but %s seeder%s' % (title, seeders, plural(seeders)))
except Exception as e:
logger.error("An error occurred in the %s parser: %s" % (provider, str(e)))
logger.debug('%s: %s' % (provider, traceback.format_exc()))
page += 1
if 0 < lazylibrarian.CONFIG['MAX_PAGES'] < page:
logger.warn('Maximum results page search reached, still more results available')
next_page = False
logger.debug("Found %i result%s from %s for %s" % (len(results), plural(len(results)), provider, sterm))
return results, errmsg
def fetchURL(URL, headers=None, retry=True, raw=None):
""" Return the result of fetching a URL and True if success
Otherwise return error message and False
Return data as raw/bytes in python2 or if raw == True
On python3 default to unicode, need to set raw=True for images/data
Allow one retry on timeout by default"""
if 'googleapis' in URL:
lazylibrarian.GB_CALLS += 1
for entry in lazylibrarian.PROVIDER_BLOCKLIST:
if entry["name"] == 'googleapis':
if int(time.time()) < int(entry['resume']):
return "Blocked", False
else:
lazylibrarian.PROVIDER_BLOCKLIST.remove(entry)
lazylibrarian.GB_CALLS = 0
if raw is None:
if PY2:
raw = True
else:
raw = False
if headers is None:
# some sites insist on having a user-agent, default is to add one
# if you don't want any headers, send headers=[]
headers = {'User-Agent': getUserAgent()}
proxies = proxyList()
try:
# jackett query all indexers needs a longer timeout
# /torznab/all/api?q= or v2.0/indexers/all/results/torznab/api?q=
if '/torznab/' in URL and ('/all/' in URL or '/aggregate/' in URL):
timeout = check_int(lazylibrarian.CONFIG['HTTP_EXT_TIMEOUT'], 90)
else:
timeout = check_int(lazylibrarian.CONFIG['HTTP_TIMEOUT'], 30)
r = requests.get(URL, headers=headers, timeout=timeout, proxies=proxies)
if str(r.status_code).startswith('2'): # (200 OK etc)
if raw:
return r.content, True
try:
result = r.content.decode('utf-8')
except UnicodeDecodeError:
result = r.content.decode('latin-1')
return result, True
elif r.status_code == 403 and 'googleapis' in URL:
msg = makeUnicode(r.content)
logger.debug(msg)
# noinspection PyBroadException
try:
source = json.loads(msg)
msg = source['error']['message']
except Exception:
pass
if 'Limit Exceeded' in msg:
# how long until midnight Pacific Time when google reset the quotas
delay = seconds_to_midnight() + 28800 # PT is 8hrs behind UTC
if delay > 86400:
delay -= 86400 # no roll-over to next day
else:
# might be forbidden for a different reason where midnight might not matter
# eg "Cannot determine user location for geographically restricted operation"
delay = 3600
for entry in lazylibrarian.PROVIDER_BLOCKLIST:
if entry["name"] == 'googleapis':
lazylibrarian.PROVIDER_BLOCKLIST.remove(entry)
newentry = {"name": 'googleapis', "resume": int(time.time()) + delay, "reason": msg}
lazylibrarian.PROVIDER_BLOCKLIST.append(newentry)
# noinspection PyBroadException
try:
# noinspection PyProtectedMember
msg = requests.status_codes._codes[r.status_code][0]
except Exception:
msg = str(r.content)
return "Response status %s: %s" % (r.status_code, msg), False
except requests.exceptions.Timeout as e:
if not retry:
logger.error("fetchURL: Timeout getting response from %s" % URL)
return "Timeout %s" % str(e), False
logger.debug("fetchURL: retrying - got timeout on %s" % URL)
result, success = fetchURL(URL, headers=headers, retry=False, raw=False)
return result, success
except Exception as e:
if hasattr(e, 'reason'):
return "Exception %s: Reason: %s" % (type(e).__name__, str(e.reason)), False
return "Exception %s: %s" % (type(e).__name__, str(e)), False
def LIME(book=None, test=False):
errmsg = ''
provider = "Limetorrent"
host = lazylibrarian.CONFIG['LIME_HOST']
if not host.startswith('http'):
host = 'http://' + host
params = {"q": book['searchterm']}
providerurl = url_fix(host + "/searchrss/other")
searchURL = providerurl + "?%s" % urlencode(params)
sterm = makeUnicode(book['searchterm'])
data, success = fetchURL(searchURL)
if not success:
# may return 404 if no results, not really an error
if '404' in data:
logger.debug("No results found from %s for %s" % (provider, sterm))
success = True
else:
logger.debug(searchURL)
logger.debug('Error fetching data from %s: %s' % (provider, data))
errmsg = data
data = False
if test:
return success
results = []
minimumseeders = int(lazylibrarian.CONFIG['NUMBEROFSEEDERS']) - 1
if data:
logger.debug('Parsing results from <a href="%s">%s</a>' %
(searchURL, provider))
d = feedparser.parse(data)
if len(d.entries):
for item in d.entries:
try:
title = unaccented(item['title'])
try:
seeders = item['description']
seeders = int(
seeders.split('Seeds:')[1].split(',')[0].strip())
except (IndexError, ValueError):
seeders = 0
size = item['size']
try:
size = int(size)
except ValueError:
size = 0
url = None
for link in item['links']:
if 'x-bittorrent' in link['type']:
url = link['url']
if not url or not title:
logger.debug('No url or title found')
elif minimumseeders < int(seeders):
results.append({
'bookid':
book['bookid'],
'tor_prov':
provider,
'tor_title':
title,
'tor_url':
url,
'tor_size':
str(size),
'tor_type':
'torrent',
'priority':
lazylibrarian.CONFIG['LIME_DLPRIORITY']
})
logger.debug('Found %s. Size: %s' % (title, size))
else:
logger.debug('Found %s but %s seeder%s' %
(title, seeders, plural(seeders)))
except Exception as e:
if 'forbidden' in str(e).lower():
# may have ip based access limits
logger.error(
'Access forbidden. Please wait a while before trying %s again.'
% provider)
else:
logger.error("An error occurred in the %s parser: %s" %
(provider, str(e)))
logger.debug('%s: %s' %
(provider, traceback.format_exc()))
logger.debug("Found %i result%s from %s for %s" %
(len(results), plural(len(results)), provider, sterm))
return results, errmsg
def audioRename(bookid):
for item in ['$Part', '$Title']:
if item not in lazylibrarian.CONFIG['AUDIOBOOK_DEST_FILE']:
logger.error("Unable to audioRename, check AUDIOBOOK_DEST_FILE")
return ''
myDB = database.DBConnection()
cmd = 'select AuthorName,BookName,AudioFile from books,authors where books.AuthorID = authors.AuthorID and bookid=?'
exists = myDB.match(cmd, (bookid,))
if exists:
book_filename = exists['AudioFile']
if book_filename:
r = os.path.dirname(book_filename)
else:
logger.debug("No filename for %s in audioRename %s" % bookid)
return ''
else:
logger.debug("Invalid bookid in audioRename %s" % bookid)
return ''
if not TinyTag:
logger.warn("TinyTag library not available")
return ''
cnt = 0
parts = []
author = ''
book = ''
total = 0
audio_file = ''
for f in os.listdir(makeBytestr(r)):
f = makeUnicode(f)
if is_valid_booktype(f, booktype='audiobook'):
cnt += 1
audio_file = f
try:
id3r = TinyTag.get(os.path.join(r, f))
performer = id3r.artist
composer = id3r.composer
book = id3r.album
track = id3r.track
total = id3r.track_total
track = check_int(track, 0)
total = check_int(total, 0)
if composer: # if present, should be author
author = composer
elif performer: # author, or narrator if composer == author
author = performer
if author and book:
parts.append([track, book, author, f])
except Exception as e:
logger.error("tinytag %s %s" % (type(e).__name__, str(e)))
pass
logger.debug("%s found %s audiofile%s" % (exists['BookName'], cnt, plural(cnt)))
if cnt == 1 and not parts: # single file audiobook
parts = [1, exists['BookName'], exists['AuthorName'], audio_file]
if cnt != len(parts):
logger.warn("%s: Incorrect number of parts (found %i from %i)" % (exists['BookName'], len(parts), cnt))
return book_filename
if total and total != cnt:
logger.warn("%s: Reported %i parts, got %i" % (exists['BookName'], total, cnt))
return book_filename
# check all parts have the same author and title
if len(parts) > 1:
for part in parts:
if part[1] != book:
logger.warn("%s: Inconsistent title: [%s][%s]" % (exists['BookName'], part[1], book))
return book_filename
if part[2] != author:
logger.warn("%s: Inconsistent author: [%s][%s]" % (exists['BookName'], part[2], author))
return book_filename
# do we have any track info (value is 0 if not)
if parts[0][0] == 0:
tokmatch = ''
# try to extract part information from filename. Search for token style of part 1 in this order...
for token in [' 001.', ' 01.', ' 1.', ' 001 ', ' 01 ', ' 1 ', '01']:
if tokmatch:
break
for part in parts:
if token in part[3]:
tokmatch = token
break
if tokmatch: # we know the numbering style, get numbers for the other parts
cnt = 0
while cnt < len(parts):
cnt += 1
if tokmatch == ' 001.':
pattern = ' %s.' % str(cnt).zfill(3)
elif tokmatch == ' 01.':
pattern = ' %s.' % str(cnt).zfill(2)
elif tokmatch == ' 1.':
pattern = ' %s.' % str(cnt)
elif tokmatch == ' 001 ':
pattern = ' %s ' % str(cnt).zfill(3)
elif tokmatch == ' 01 ':
pattern = ' %s ' % str(cnt).zfill(2)
elif tokmatch == ' 1 ':
pattern = ' %s ' % str(cnt)
else:
pattern = '%s' % str(cnt).zfill(2)
# standardise numbering of the parts
for part in parts:
if pattern in part[3]:
part[0] = cnt
break
# check all parts are present
cnt = 0
found = True
while found and cnt < len(parts):
found = False
cnt += 1
for part in parts:
trk = part[0]
if trk == cnt:
found = True
break
if not found:
logger.warn("%s: No part %i found" % (exists['BookName'], cnt))
return book_filename
# if we get here, looks like we have all the parts needed to rename properly
seriesinfo = seriesInfo(bookid)
dest_path = lazylibrarian.CONFIG['EBOOK_DEST_FOLDER'].replace(
'$Author', author).replace(
'$Title', book).replace(
'$Series', seriesinfo['Full']).replace(
'$SerName', seriesinfo['Name']).replace(
'$SerNum', seriesinfo['Num']).replace(
'$$', ' ')
dest_path = ' '.join(dest_path.split()).strip()
dest_path = replace_all(dest_path, __dic__)
dest_dir = lazylibrarian.DIRECTORY('Audio')
dest_path = os.path.join(dest_dir, dest_path)
if r != dest_path:
try:
dest_path = safe_move(r, dest_path)
r = dest_path
except Exception as why:
if not os.path.isdir(dest_path):
logger.error('Unable to create directory %s: %s' % (dest_path, why))
for part in parts:
pattern = lazylibrarian.CONFIG['AUDIOBOOK_DEST_FILE']
seriesinfo = seriesInfo(bookid)
pattern = pattern.replace(
'$Author', author).replace(
'$Title', book).replace(
'$Part', str(part[0]).zfill(len(str(len(parts))))).replace(
'$Total', str(len(parts))).replace(
'$Series', seriesinfo['Full']).replace(
'$SerName', seriesinfo['Name']).replace(
'$SerNum', seriesinfo['Num']).replace(
'$$', ' ')
pattern = ' '.join(pattern.split()).strip()
n = os.path.join(r, pattern + os.path.splitext(part[3])[1])
o = os.path.join(r, part[3])
if o != n:
try:
n = safe_move(o, n)
if part[0] == 1:
book_filename = n # return part 1 of set
logger.debug('%s: audioRename [%s] to [%s]' % (exists['BookName'], o, n))
except Exception as e:
logger.error('Unable to rename [%s] to [%s] %s %s' % (o, n, type(e).__name__, str(e)))
return book_filename
def _notify(message, event, force=False):
# suppress notifications if the notifier is disabled but the notify options are checked
if not lazylibrarian.CONFIG['USE_CUSTOM'] and not force:
return False
logger.debug('Custom Event: %s' % event)
logger.debug('Custom Message: %s' % message)
myDB = database.DBConnection()
if event == "Test":
# grab the first entry in the book table and wanted table
book = myDB.match('SELECT * from books')
wanted = myDB.match('SELECT * from wanted')
else:
# message is a bookid followed by type (eBook/AudioBook)
# or a magazine title followed by it's NZBUrl
words = message.split()
ident = words[-1]
bookid = " ".join(words[:-1])
book = myDB.match('SELECT * from books where BookID=?', (bookid, ))
if not book:
book = myDB.match('SELECT * from magazines where Title=?',
(bookid, ))
if event == 'Added to Library':
wanted_status = 'Processed'
else:
wanted_status = 'Snatched'
if ident in ['eBook', 'AudioBook']:
wanted = myDB.match(
'SELECT * from wanted where BookID=? AND AuxInfo=? AND Status=?',
(bookid, ident, wanted_status))
else:
wanted = myDB.match(
'SELECT * from wanted where BookID=? AND NZBUrl=? AND Status=?',
(bookid, ident, wanted_status))
if book:
dictionary = dict(list(zip(list(book.keys()), book)))
else:
dictionary = {}
dictionary['Event'] = event
if wanted:
wanted_dictionary = dict(list(zip(list(wanted.keys()), wanted)))
for item in wanted_dictionary:
if item in ['Status', 'BookID']: # rename to avoid clash
dictionary['Wanted_' + item] = wanted_dictionary[item]
else:
dictionary[item] = wanted_dictionary[item]
try:
# call the custom notifier script here, passing dictionary deconstructed as strings
if lazylibrarian.CONFIG['CUSTOM_SCRIPT']:
params = [lazylibrarian.CONFIG['CUSTOM_SCRIPT']]
for item in dictionary:
params.append(item)
if hasattr(dictionary[item], 'encode'):
params.append(dictionary[item].encode('utf-8'))
else:
params.append(str(dictionary[item]))
try:
p = Popen(params, stdout=PIPE, stderr=PIPE)
res, err = p.communicate()
rc = p.returncode
res = makeUnicode(res)
err = makeUnicode(err)
if rc:
logger.error(
"Custom notifier returned %s: res[%s] err[%s]" %
(rc, res, err))
return False
logger.debug(res)
return True
except Exception as e:
logger.warn('Error sending command: %s' % e)
return False
else:
logger.warn('Error sending custom notification: Check config')
return False
except Exception as e:
logger.warn('Error sending custom notification: %s' % e)
return False
def TDL(book=None, test=False):
errmsg = ''
provider = "torrentdownloads"
host = lazylibrarian.CONFIG['TDL_HOST']
if not host.startswith('http'):
host = 'http://' + host
providerurl = url_fix(host)
params = {
"type": "search",
"cid": "2",
"search": book['searchterm']
}
searchURL = providerurl + "/rss.xml?%s" % urlencode(params)
sterm = makeUnicode(book['searchterm'])
data, success = fetchURL(searchURL)
if not success:
# may return 404 if no results, not really an error
if '404' in data:
logger.debug("No results found from %s for %s" % (provider, sterm))
success = True
else:
logger.debug(searchURL)
logger.debug('Error fetching data from %s: %s' % (provider, data))
errmsg = data
data = False
if test:
return success
results = []
minimumseeders = int(lazylibrarian.CONFIG['NUMBEROFSEEDERS']) - 1
if data:
logger.debug('Parsing results from <a href="%s">%s</a>' % (searchURL, provider))
d = feedparser.parse(data)
if len(d.entries):
for item in d.entries:
try:
title = item['title']
seeders = int(item['seeders'].replace(',', ''))
link = item['link']
size = int(item['size'])
url = None
try:
pubdate = item['published']
except KeyError:
pubdate = None
if link and minimumseeders < seeders:
# no point requesting the magnet link if not enough seeders
# TDL gives us a relative link
result, success = fetchURL(providerurl+link)
if success:
new_soup = BeautifulSoup(result, 'html5lib')
for link in new_soup.find_all('a'):
output = link.get('href')
if output and output.startswith('magnet'):
url = output
break
if not url or not title:
logger.debug('Missing url or title')
else:
res = {
'bookid': book['bookid'],
'tor_prov': provider,
'tor_title': title,
'tor_url': url,
'tor_size': str(size),
'tor_type': 'magnet',
'priority': lazylibrarian.CONFIG['TDL_DLPRIORITY']
}
if pubdate:
res['tor_date'] = pubdate
logger.debug('Found %s. Size: %s' % (title, size))
results.append(res)
else:
logger.debug('Found %s but %s seeder%s' % (title, seeders, plural(seeders)))
except Exception as e:
logger.error("An error occurred in the %s parser: %s" % (provider, str(e)))
logger.debug('%s: %s' % (provider, traceback.format_exc()))
logger.debug("Found %i result%s from %s for %s" % (len(results), plural(len(results)), provider, sterm))
return results, errmsg
def ZOO(book=None, test=False):
errmsg = ''
provider = "zooqle"
host = lazylibrarian.CONFIG['ZOO_HOST']
if not host.startswith('http'):
host = 'http://' + host
providerurl = url_fix(host + "/search")
params = {
"q": book['searchterm'],
"category": "books",
"fmt": "rss"
}
searchURL = providerurl + "?%s" % urlencode(params)
sterm = makeUnicode(book['searchterm'])
data, success = fetchURL(searchURL)
if not success:
# may return 404 if no results, not really an error
if '404' in data:
logger.debug("No results found from %s for %s" % (provider, sterm))
success = True
else:
logger.debug(searchURL)
logger.debug('Error fetching data from %s: %s' % (provider, data))
errmsg = data
data = False
if test:
return success
results = []
minimumseeders = int(lazylibrarian.CONFIG['NUMBEROFSEEDERS']) - 1
if data:
logger.debug('Parsing results from <a href="%s">%s</a>' % (searchURL, provider))
d = feedparser.parse(data)
if len(d.entries):
for item in d.entries:
try:
title = unaccented(item['title'])
seeders = int(item['torrent_seeds'].replace(',', ''))
link = item['links'][1]['href']
size = int(item['links'][1]['length'])
magnet = item['torrent_magneturi']
url = None
mode = 'torrent'
if link:
url = link
mode = 'torrent'
if magnet:
if not url or (url and lazylibrarian.CONFIG['PREFER_MAGNET']):
url = magnet
mode = 'magnet'
if not url or not title:
logger.debug('No url or title found')
elif minimumseeders < seeders:
results.append({
'bookid': book['bookid'],
'tor_prov': provider,
'tor_title': title,
'tor_url': url,
'tor_size': str(size),
'tor_type': mode,
'priority': lazylibrarian.CONFIG['ZOO_DLPRIORITY']
})
logger.debug('Found %s. Size: %s' % (title, size))
else:
logger.debug('Found %s but %s seeder%s' % (title, seeders, plural(seeders)))
except Exception as e:
if 'forbidden' in str(e).lower():
# looks like zooqle has ip based access limits
logger.error('Access forbidden. Please wait a while before trying %s again.' % provider)
else:
logger.error("An error occurred in the %s parser: %s" % (provider, str(e)))
logger.debug('%s: %s' % (provider, traceback.format_exc()))
logger.debug("Found %i result%s from %s for %s" % (len(results), plural(len(results)), provider, sterm))
return results, errmsg
def bookRename(bookid):
myDB = database.DBConnection()
cmd = 'select AuthorName,BookName,BookFile from books,authors where books.AuthorID = authors.AuthorID and bookid=?'
exists = myDB.match(cmd, (bookid,))
if not exists:
logger.debug("Invalid bookid in bookRename %s" % bookid)
return ''
f = exists['BookFile']
if not f:
logger.debug("No filename for %s in BookRename %s" % bookid)
return ''
r = os.path.dirname(f)
try:
# noinspection PyTypeChecker
calibreid = r.rsplit('(', 1)[1].split(')')[0]
if not calibreid.isdigit():
calibreid = ''
except IndexError:
calibreid = ''
if calibreid:
msg = '[%s] looks like a calibre directory: not renaming book' % os.path.basename(r)
logger.debug(msg)
return f
seriesinfo = seriesInfo(bookid)
dest_path = lazylibrarian.CONFIG['EBOOK_DEST_FOLDER'].replace(
'$Author', exists['AuthorName']).replace(
'$Title', exists['BookName']).replace(
'$Series', seriesinfo['Full']).replace(
'$SerName', seriesinfo['Name']).replace(
'$SerNum', seriesinfo['Num']).replace(
'$$', ' ')
dest_path = ' '.join(dest_path.split()).strip()
dest_path = replace_all(dest_path, __dic__)
dest_dir = lazylibrarian.DIRECTORY('eBook')
dest_path = os.path.join(dest_dir, dest_path)
if r != dest_path:
try:
dest_path = safe_move(r, dest_path)
r = dest_path
except Exception as why:
if not os.path.isdir(dest_path):
logger.error('Unable to create directory %s: %s' % (dest_path, why))
book_basename, prefextn = os.path.splitext(os.path.basename(f))
new_basename = lazylibrarian.CONFIG['EBOOK_DEST_FILE']
seriesinfo = seriesInfo(bookid)
new_basename = new_basename.replace(
'$Author', exists['AuthorName']).replace(
'$Title', exists['BookName']).replace(
'$Series', seriesinfo['Full']).replace(
'$SerName', seriesinfo['Name']).replace(
'$SerNum', seriesinfo['Num']).replace(
'$$', ' ')
new_basename = ' '.join(new_basename.split()).strip()
# replace all '/' not surrounded by whitespace with '_' as '/' is a directory separator
slash = new_basename.find('/')
while slash > 0:
if new_basename[slash - 1] != ' ':
if new_basename[slash + 1] != ' ':
new_basename = new_basename[:slash] + '_' + new_basename[slash + 1:]
slash = new_basename.find('/', slash + 1)
if ' / ' in new_basename: # used as a separator in goodreads omnibus
logger.warn("bookRename [%s] looks like an omnibus? Not renaming %s" % (new_basename, book_basename))
new_basename = book_basename
if book_basename != new_basename:
# only rename bookname.type, bookname.jpg, bookname.opf, not cover.jpg or metadata.opf
for fname in os.listdir(makeBytestr(r)):
fname = makeUnicode(fname)
extn = ''
if is_valid_booktype(fname, booktype='ebook'):
extn = os.path.splitext(fname)[1]
elif fname.endswith('.opf') and not fname == 'metadata.opf':
extn = '.opf'
elif fname.endswith('.jpg') and not fname == 'cover.jpg':
extn = '.jpg'
if extn:
ofname = os.path.join(r, fname)
nfname = os.path.join(r, new_basename + extn)
try:
nfname = safe_move(ofname, nfname)
logger.debug("bookRename %s to %s" % (ofname, nfname))
if ofname == exists['BookFile']: # if we renamed the preferred filetype, return new name
f = nfname
except Exception as e:
logger.error('Unable to rename [%s] to [%s] %s %s' %
(ofname, nfname, type(e).__name__, str(e)))
return f
def createMagCover(issuefile=None, refresh=False):
if not lazylibrarian.CONFIG['IMP_MAGCOVER']:
return
if issuefile is None or not os.path.isfile(issuefile):
logger.debug('No issuefile %s' % issuefile)
return
base, extn = os.path.splitext(issuefile)
if not extn:
logger.debug('Unable to create cover for %s, no extension?' % issuefile)
return
coverfile = base + '.jpg'
if os.path.isfile(coverfile):
if refresh:
os.remove(coverfile)
else:
logger.debug('Cover for %s exists' % issuefile)
return # quit if cover already exists and we didn't want to refresh
logger.debug('Creating cover for %s' % issuefile)
data = '' # result from unzip or unrar
extn = extn.lower()
if extn in ['.cbz', '.epub']:
try:
data = zipfile.ZipFile(issuefile)
except Exception as why:
logger.error("Failed to read zip file %s, %s %s" % (issuefile, type(why).__name__, str(why)))
data = ''
elif extn in ['.cbr']:
try:
# unrar will complain if the library isn't installed, needs to be compiled separately
# see https://pypi.python.org/pypi/unrar/ for instructions
# Download source from http://www.rarlab.com/rar_add.htm
# note we need LIBRARY SOURCE not a binary package
# make lib; sudo make install-lib; sudo ldconfig
# lib.unrar should then be able to find libunrar.so
from lib.unrar import rarfile
data = rarfile.RarFile(issuefile)
except Exception as why:
logger.error("Failed to read rar file %s, %s %s" % (issuefile, type(why).__name__, str(why)))
data = ''
if data:
img = None
try:
for member in data.namelist():
memlow = member.lower()
if '-00.' in memlow or '000.' in memlow or 'cover.' in memlow:
if memlow.endswith('.jpg') or memlow.endswith('.jpeg'):
img = data.read(member)
break
if img:
with open(coverfile, 'wb') as f:
if PY2:
f.write(img)
else:
f.write(img.encode())
return
else:
logger.debug("Failed to find image in %s" % issuefile)
except Exception as why:
logger.error("Failed to extract image from %s, %s %s" % (issuefile, type(why).__name__, str(why)))
elif extn == '.pdf':
generator = ""
if len(lazylibrarian.CONFIG['IMP_CONVERT']): # allow external convert to override libraries
generator = "external program: %s" % lazylibrarian.CONFIG['IMP_CONVERT']
if "gsconvert.py" in lazylibrarian.CONFIG['IMP_CONVERT']:
msg = "Use of gsconvert.py is deprecated, equivalent functionality is now built in. "
msg += "Support for gsconvert.py may be removed in a future release. See wiki for details."
logger.warn(msg)
converter = lazylibrarian.CONFIG['IMP_CONVERT']
postfix = ''
# if not os.path.isfile(converter): # full path given, or just program_name?
# converter = os.path.join(os.getcwd(), lazylibrarian.CONFIG['IMP_CONVERT'])
if 'convert' in converter and 'gs' not in converter:
# tell imagemagick to only convert first page
postfix = '[0]'
try:
params = [converter, '%s%s' % (issuefile, postfix), '%s' % coverfile]
res = subprocess.check_output(params, stderr=subprocess.STDOUT)
res = makeUnicode(res).strip()
if res:
logger.debug('%s reports: %s' % (lazylibrarian.CONFIG['IMP_CONVERT'], res))
except Exception as e:
# logger.debug(params)
logger.warn('External "convert" failed %s %s' % (type(e).__name__, str(e)))
elif platform.system() == "Windows":
GS = os.path.join(os.getcwd(), "gswin64c.exe")
generator = "local gswin64c"
if not os.path.isfile(GS):
GS = os.path.join(os.getcwd(), "gswin32c.exe")
generator = "local gswin32c"
if not os.path.isfile(GS):
params = ["where", "gswin64c"]
try:
GS = subprocess.check_output(params, stderr=subprocess.STDOUT)
GS = makeUnicode(GS).strip()
generator = "gswin64c"
except Exception as e:
logger.debug("where gswin64c failed: %s %s" % (type(e).__name__, str(e)))
if not os.path.isfile(GS):
params = ["where", "gswin32c"]
try:
GS = subprocess.check_output(params, stderr=subprocess.STDOUT)
GS = makeUnicode(GS).strip()
generator = "gswin32c"
except Exception as e:
logger.debug("where gswin32c failed: %s %s" % (type(e).__name__, str(e)))
if not os.path.isfile(GS):
logger.debug("No gswin found")
generator = "(no windows ghostscript found)"
else:
# noinspection PyBroadException
try:
params = [GS, "--version"]
res = subprocess.check_output(params, stderr=subprocess.STDOUT)
res = makeUnicode(res).strip()
logger.debug("Found %s [%s] version %s" % (generator, GS, res))
generator = "%s version %s" % (generator, res)
issuefile = issuefile.split('[')[0]
params = [GS, "-sDEVICE=jpeg", "-dNOPAUSE", "-dBATCH", "-dSAFER", "-dFirstPage=1", "-dLastPage=1",
"-dUseCropBox", "-sOutputFile=%s" % coverfile, issuefile]
res = subprocess.check_output(params, stderr=subprocess.STDOUT)
res = makeUnicode(res).strip()
if not os.path.isfile(coverfile):
logger.debug("Failed to create jpg: %s" % res)
except Exception: # as why:
logger.warn("Failed to create jpg for %s" % issuefile)
logger.debug('Exception in gswin create_cover: %s' % traceback.format_exc())
else: # not windows
try:
# noinspection PyUnresolvedReferences
from wand.image import Image
interface = "wand"
except ImportError:
try:
# No PythonMagick in python3
# noinspection PyUnresolvedReferences
import PythonMagick
interface = "pythonmagick"
except ImportError:
interface = ""
try:
if interface == 'wand':
generator = "wand interface"
with Image(filename=issuefile + '[0]') as img:
img.save(filename=coverfile)
elif interface == 'pythonmagick':
generator = "pythonmagick interface"
img = PythonMagick.Image()
# PythonMagick requires filenames to be bytestr, not unicode
if type(issuefile) is text_type:
issuefile = makeBytestr(issuefile)
if type(coverfile) is text_type:
coverfile = makeBytestr(coverfile)
img.read(issuefile + '[0]')
img.write(coverfile)
else:
GS = os.path.join(os.getcwd(), "gs")
generator = "local gs"
if not os.path.isfile(GS):
GS = ""
params = ["which", "gs"]
try:
GS = subprocess.check_output(params, stderr=subprocess.STDOUT)
GS = makeUnicode(GS).strip()
generator = GS
except Exception as e:
logger.debug("which gs failed: %s %s" % (type(e).__name__, str(e)))
if not os.path.isfile(GS):
logger.debug("Cannot find gs")
generator = "(no gs found)"
else:
params = [GS, "--version"]
res = subprocess.check_output(params, stderr=subprocess.STDOUT)
res = makeUnicode(res).strip()
logger.debug("Found gs [%s] version %s" % (GS, res))
generator = "%s version %s" % (generator, res)
issuefile = issuefile.split('[')[0]
params = [GS, "-sDEVICE=jpeg", "-dNOPAUSE", "-dBATCH", "-dSAFER", "-dFirstPage=1",
"-dLastPage=1", "-dUseCropBox", "-sOutputFile=%s" % coverfile, issuefile]
res = subprocess.check_output(params, stderr=subprocess.STDOUT)
res = makeUnicode(res).strip()
if not os.path.isfile(coverfile):
logger.debug("Failed to create jpg: %s" % res)
except Exception as e:
logger.warn("Unable to create cover for %s using %s %s" % (issuefile, type(e).__name__, generator))
logger.debug('Exception in create_cover: %s' % traceback.format_exc())
if os.path.isfile(coverfile):
setperm(coverfile)
logger.debug("Created cover for %s using %s" % (issuefile, generator))
return
# if not recognised extension or cover creation failed
try:
coverfile = safe_copy(os.path.join(lazylibrarian.PROG_DIR, 'data/images/nocover.jpg'), coverfile)
setperm(coverfile)
except Exception as why:
logger.error("Failed to copy nocover file, %s %s" % (type(why).__name__, str(why)))
return
def GEN(book=None, prov=None, test=False):
errmsg = ''
provider = "libgen.io"
if not prov:
prov = 'GEN'
host = lazylibrarian.CONFIG[prov + '_HOST']
if not host.startswith('http'):
host = 'http://' + host
search = lazylibrarian.CONFIG[prov + '_SEARCH']
if not search or not search.endswith('.php'):
search = 'search.php'
if 'index.php' not in search and 'search.php' not in search:
search = 'search.php'
if search[0] == '/':
search = search[1:]
sterm = makeUnicode(book['searchterm'])
page = 1
results = []
next_page = True
while next_page:
if 'index.php' in search:
params = {
"s": book['searchterm'],
"f_lang": "All",
"f_columns": 0,
"f_ext": "All"
}
else:
params = {
"view": "simple",
"open": 0,
"phrase": 0,
"column": "def",
"res": 100,
"req": book['searchterm']
}
if page > 1:
params['page'] = page
providerurl = url_fix(host + "/%s" % search)
searchURL = providerurl + "?%s" % urlencode(params)
next_page = False
result, success = fetchURL(searchURL)
if not success:
# may return 404 if no results, not really an error
if '404' in result:
logger.debug("No results found from %s for %s" % (provider, sterm))
success = True
elif '111' in result:
# looks like libgen has ip based access limits
logger.error('Access forbidden. Please wait a while before trying %s again.' % provider)
errmsg = result
else:
logger.debug(searchURL)
logger.debug('Error fetching page data from %s: %s' % (provider, result))
errmsg = result
result = False
if test:
return success
if result:
logger.debug('Parsing results from <a href="%s">%s</a>' % (searchURL, provider))
try:
soup = BeautifulSoup(result, 'html5lib')
rows = []
try:
table = soup.find_all('table', rules='rows')[-1] # the last table with rules=rows
if table:
rows = table.find_all('tr')
except IndexError: # no results table in result page
rows = []
if len(rows) > 1: # skip table headers
rows = rows[1:]
for row in rows:
author = ''
title = ''
size = ''
extn = ''
link = ''
td = row.find_all('td')
if 'index.php' in search and len(td) > 3:
# Foreign fiction
try:
author = formatAuthorName(td[0].text)
title = td[2].text
newsoup = BeautifulSoup(str(td[4]), 'html5lib')
data = newsoup.find('a')
if data:
link = data.get('href')
extn = td[4].text.split('(')[0].strip()
size = td[4].text.split('(')[1].split(')')[0]
size = size.upper()
except IndexError as e:
logger.debug('Error parsing libgen index.php results: %s' % str(e))
elif 'search.php' in search and len(td) > 8:
# Non-fiction
try:
author = formatAuthorName(td[1].text)
title = td[2].text
size = td[7].text.upper()
extn = td[8].text
link = ''
newsoup = BeautifulSoup(str(td[2]), 'html5lib')
for res in newsoup.find_all('a'):
output = res.get('href')
if 'md5' in output:
link = output
break
except IndexError as e:
logger.debug('Error parsing libgen search.php results; %s' % str(e))
size = size_in_bytes(size)
if link and title:
if author:
title = author.strip() + ' ' + title.strip()
if extn:
title = title + '.' + extn
if link.startswith('http'):
url = redirect_url(host, link)
else:
if "/index.php?" in link:
link = 'md5' + link.split('md5')[1]
if "/ads.php?" in link:
url = url_fix(host + "/" + link)
else:
url = url_fix(host + "/ads.php?" + link)
bookresult, success = fetchURL(url)
if not success:
logger.debug('Error fetching link data from %s: %s' % (provider, bookresult))
logger.debug(url)
url = None
else:
url = None
try:
new_soup = BeautifulSoup(bookresult, 'html5lib')
for link in new_soup.find_all('a'):
output = link.get('href')
if output:
if output.startswith('http') and '/get.php' in output:
url = output
break
elif '/get.php' in output:
url = '/get.php' + output.split('/get.php')[1]
break
elif '/download/book' in output:
url = '/download/book' + output.split('/download/book')[1]
break
if url and not url.startswith('http'):
url = url_fix(host + url)
else:
url = redirect_url(host, url)
except Exception as e:
logger.error('%s parsing bookresult for %s: %s' % (type(e).__name__, link, str(e)))
url = None
if url:
results.append({
'bookid': book['bookid'],
'tor_prov': provider + '/' + search,
'tor_title': title,
'tor_url': url,
'tor_size': str(size),
'tor_type': 'direct',
'priority': lazylibrarian.CONFIG[prov + '_DLPRIORITY']
})
logger.debug('Found %s, Size %s' % (title, size))
next_page = True
except Exception as e:
logger.error("An error occurred in the %s parser: %s" % (provider, str(e)))
logger.debug('%s: %s' % (provider, traceback.format_exc()))
page += 1
if 0 < lazylibrarian.CONFIG['MAX_PAGES'] < page:
logger.warn('Maximum results page search reached, still more results available')
next_page = False
logger.debug("Found %i result%s from %s for %s" % (len(results), plural(len(results)), provider, sterm))
return results, errmsg
def db_v14(myDB, upgradelog):
upgradelog.write("%s v14: %s\n" % (time.ctime(), "Moving image caches"))
src = lazylibrarian.CACHEDIR
try:
os.mkdir(os.path.join(src, 'author'))
except OSError as e:
if e.errno not in [17, 183]: # already exists is ok
msg = 'mkdir author cache reports: %s' % str(e)
logger.debug(msg)
upgradelog.write("%s v14: %s\n" % (time.ctime(), msg))
query = 'SELECT AuthorName, AuthorID, AuthorImg FROM authors '
query += 'WHERE AuthorImg LIKE "cache/%" '
query += 'AND AuthorImg NOT LIKE "cache/author/%"'
images = myDB.select(query)
if images:
tot = len(images)
msg = 'Moving %s author images to new location' % tot
logger.debug(msg)
upgradelog.write("%s v14: %s\n" % (time.ctime(), msg))
cnt = 0
for image in images:
cnt += 1
lazylibrarian.UPDATE_MSG = "Moving author images to new location: %s of %s" % (cnt, tot)
try:
img = image['AuthorImg']
img = img.rsplit('/', 1)[1]
srcfile = os.path.join(src, img)
if os.path.isfile(srcfile):
try:
shutil.move(srcfile, os.path.join(src, "author", img))
myDB.action(
'UPDATE authors SET AuthorImg="cache/author/?" WHERE AuthorID=?',
(img, image['AuthorID']))
except Exception as e:
logger.error("dbupgrade: %s %s" % (type(e).__name__, str(e)))
except Exception as e:
msg = 'Failed to update author image for %s: %s %s' % (image['AuthorName'], type(e).__name__, str(e))
logger.warn(msg)
upgradelog.write("%s v14: %s\n" % (time.ctime(), msg))
upgradelog.write("%s v14: %s\n" % (time.ctime(), lazylibrarian.UPDATE_MSG))
logger.debug("Author Image cache updated")
try:
os.mkdir(os.path.join(src, 'book'))
except OSError as e:
if e.errno not in [17, 183]: # already exists is ok
msg = 'mkdir book cache reports: %s' % str(e)
logger.debug(msg)
upgradelog.write("%s v14: %s\n" % (time.ctime(), msg))
query = 'SELECT BookName, BookID, BookImg FROM books '
query += 'WHERE BookImg LIKE "cache/%" '
query += 'AND BookImg NOT LIKE "cache/book/%"'
images = myDB.select(query)
if images:
tot = len(images)
msg = 'Moving %s book images to new location' % tot
upgradelog.write("%s v14: %s\n" % (time.ctime(), msg))
logger.debug(msg)
cnt = 0
for image in images:
cnt += 1
lazylibrarian.UPDATE_MSG = "Moving book images to new location: %s of %s" % (cnt, tot)
try:
img = image['BookImg']
img = img.rsplit('/', 1)[1]
srcfile = os.path.join(src, img)
if os.path.isfile(srcfile):
try:
shutil.move(srcfile, os.path.join(src, "book", img))
myDB.action('UPDATE books SET BookImg="cache/book/?" WHERE BookID=?',
(img, image['BookID']))
except Exception as e:
logger.error("dbupgrade: %s %s" % (type(e).__name__, str(e)))
except Exception as e:
logger.warn('Failed to update book image for %s: %s %s' %
(image['BookName'], type(e).__name__, str(e)))
upgradelog.write("%s v14: %s\n" % (time.ctime(), lazylibrarian.UPDATE_MSG))
logger.debug("Book Image cache updated")
# at this point there should be no more .jpg files in the root of the cachedir
# any that are still there are for books/authors deleted from database
# or magazine latest issue cover files that get copied as required
for image in os.listdir(makeBytestr(src)):
image = makeUnicode(image)
if image.endswith('.jpg'):
os.remove(os.path.join(src, image))
upgradelog.write("%s v14: complete\n" % time.ctime())
def TPB(book=None, test=False):
errmsg = ''
provider = "TPB"
host = lazylibrarian.CONFIG['TPB_HOST']
if not host.startswith('http'):
host = 'http://' + host
providerurl = url_fix(host + "/s/?")
cat = 0 # 601=ebooks, 102=audiobooks, 0=all, no mag category
if 'library' in book:
if book['library'] == 'AudioBook':
cat = 102
elif book['library'] == 'eBook':
cat = 601
elif book['library'] == 'magazine':
cat = 0
sterm = makeUnicode(book['searchterm'])
page = 0
results = []
minimumseeders = int(lazylibrarian.CONFIG['NUMBEROFSEEDERS']) - 1
next_page = True
while next_page:
params = {
"q": book['searchterm'],
"category": cat,
"page": page,
"orderby": "99"
}
searchURL = providerurl + "?%s" % urlencode(params)
next_page = False
result, success = fetchURL(searchURL)
if not success:
# may return 404 if no results, not really an error
if '404' in result:
logger.debug("No results found from %s for %s" % (provider, sterm))
success = True
else:
logger.debug(searchURL)
logger.debug('Error fetching data from %s: %s' % (provider, result))
errmsg = result
result = False
if test:
return success
if result:
logger.debug('Parsing results from <a href="%s">%s</a>' % (searchURL, provider))
soup = BeautifulSoup(result, 'html5lib')
# tpb uses a named table
table = soup.find('table', id='searchResult')
if table:
rows = table.find_all('tr')
else:
rows = []
if len(rows) > 1:
rows = rows[1:] # first row is headers
for row in rows:
td = row.find_all('td')
if len(td) > 2:
try:
new_soup = BeautifulSoup(str(td[1]), 'html5lib')
link = new_soup.find("a")
magnet = link.get("href")
title = link.text
size = td[1].text.split(', Size ')[1].split('iB')[0]
size = size.replace(' ', '')
size = size_in_bytes(size)
try:
seeders = int(td[2].text.replace(',', ''))
except ValueError:
seeders = 0
if minimumseeders < seeders:
# no point in asking for magnet link if not enough seeders
magurl = '%s/%s' % (host, magnet)
result, success = fetchURL(magurl)
if not success:
logger.debug('Error fetching url %s, %s' % (magurl, result))
else:
magnet = None
new_soup = BeautifulSoup(result, 'html5lib')
for link in new_soup.find_all('a'):
output = link.get('href')
if output and output.startswith('magnet'):
magnet = output
break
if not magnet or not title:
logger.debug('Missing magnet or title')
else:
results.append({
'bookid': book['bookid'],
'tor_prov': provider,
'tor_title': title,
'tor_url': magnet,
'tor_size': str(size),
'tor_type': 'magnet',
'priority': lazylibrarian.CONFIG['TPB_DLPRIORITY']
})
logger.debug('Found %s. Size: %s' % (title, size))
next_page = True
else:
logger.debug('Found %s but %s seeder%s' % (title, seeders, plural(seeders)))
except Exception as e:
logger.error("An error occurred in the %s parser: %s" % (provider, str(e)))
logger.debug('%s: %s' % (provider, traceback.format_exc()))
page += 1
if 0 < lazylibrarian.CONFIG['MAX_PAGES'] < page:
logger.warn('Maximum results page search reached, still more results available')
next_page = False
logger.debug("Found %i result%s from %s for %s" % (len(results), plural(len(results)), provider, sterm))
return results, errmsg
def EXTRA(book=None, test=False):
errmsg = ''
provider = "Extratorrent"
host = lazylibrarian.CONFIG['EXTRA_HOST']
if not host.startswith('http'):
host = 'http://' + host
providerurl = url_fix(host + "/rss")
params = {
"type": "search",
"s_cat": "2",
"search": book['searchterm']
}
searchURL = providerurl + "/?%s" % urlencode(params)
sterm = makeUnicode(book['searchterm'])
data, success = fetchURL(searchURL)
if not success:
# may return 404 if no results, not really an error
if '404' in data:
logger.debug("No results found from %s for %s" % (provider, sterm))
success = True
else:
logger.debug('Error fetching data from %s: %s' % (provider, data))
errmsg = data
data = False
if test:
return success
results = []
minimumseeders = int(lazylibrarian.CONFIG['NUMBEROFSEEDERS']) - 1
if data:
logger.debug('Parsing results from <a href="%s">%s</a>' % (searchURL, provider))
d = feedparser.parse(data)
if len(d.entries):
for item in d.entries:
try:
title = unaccented(item['title'])
try:
seeders = int(item['seeders'].replace(',', ''))
except ValueError:
seeders = 0
try:
size = int(item['size'])
except ValueError:
size = 0
url = None
for link in item['links']:
if 'x-bittorrent' in link['type']:
url = link['href']
if not url or not title:
logger.debug('No url or title found')
elif minimumseeders < seeders:
results.append({
'bookid': book['bookid'],
'tor_prov': provider,
'tor_title': title,
'tor_url': url,
'tor_size': str(size),
'tor_type': 'torrent',
'priority': lazylibrarian.CONFIG['EXTRA_DLPRIORITY']
})
logger.debug('Found %s. Size: %s' % (title, size))
else:
logger.debug('Found %s but %s seeder%s' % (title, seeders, plural(seeders)))
except Exception as e:
logger.error("An error occurred in the %s parser: %s" % (provider, str(e)))
logger.debug('%s: %s' % (provider, traceback.format_exc()))
logger.debug("Found %i result%s from %s for %s" % (len(results), plural(len(results)), provider, sterm))
return results, errmsg
def escape_quote(s):
s = makeUnicode(s)
return s.replace('"', '\\"')
def LIME(book=None, test=False):
errmsg = ''
provider = "Limetorrent"
host = lazylibrarian.CONFIG['LIME_HOST']
if not host.startswith('http'):
host = 'http://' + host
params = {
"q": book['searchterm']
}
providerurl = url_fix(host + "/searchrss/other")
searchURL = providerurl + "?%s" % urlencode(params)
sterm = makeUnicode(book['searchterm'])
data, success = fetchURL(searchURL)
if not success:
# may return 404 if no results, not really an error
if '404' in data:
logger.debug("No results found from %s for %s" % (provider, sterm))
success = True
else:
logger.debug(searchURL)
logger.debug('Error fetching data from %s: %s' % (provider, data))
errmsg = data
data = False
if test:
return success
results = []
minimumseeders = int(lazylibrarian.CONFIG['NUMBEROFSEEDERS']) - 1
if data:
logger.debug('Parsing results from <a href="%s">%s</a>' % (searchURL, provider))
d = feedparser.parse(data)
if len(d.entries):
for item in d.entries:
try:
title = unaccented(item['title'])
try:
seeders = item['description']
seeders = int(seeders.split('Seeds:')[1].split(' ,')[0].replace(',', '').strip())
except (IndexError, ValueError):
seeders = 0
size = item['size']
try:
size = int(size)
except ValueError:
size = 0
try:
pubdate = item['published']
except KeyError:
pubdate = None
url = None
for link in item['links']:
if 'x-bittorrent' in link['type']:
url = link['url']
if not url or not title:
logger.debug('No url or title found')
elif minimumseeders < seeders:
res = {
'bookid': book['bookid'],
'tor_prov': provider,
'tor_title': title,
'tor_url': url,
'tor_size': str(size),
'tor_type': 'torrent',
'priority': lazylibrarian.CONFIG['LIME_DLPRIORITY']
}
if pubdate:
res['tor_date'] = pubdate
results.append(res)
logger.debug('Found %s. Size: %s' % (title, size))
else:
logger.debug('Found %s but %s seeder%s' % (title, seeders, plural(seeders)))
except Exception as e:
if 'forbidden' in str(e).lower():
# may have ip based access limits
logger.error('Access forbidden. Please wait a while before trying %s again.' % provider)
else:
logger.error("An error occurred in the %s parser: %s" % (provider, str(e)))
logger.debug('%s: %s' % (provider, traceback.format_exc()))
logger.debug("Found %i result%s from %s for %s" % (len(results), plural(len(results)), provider, sterm))
return results, errmsg
def WWT(book=None, test=False):
errmsg = ''
provider = "WorldWideTorrents"
host = lazylibrarian.CONFIG['WWT_HOST']
if not host.startswith('http'):
host = 'http://' + host
providerurl = url_fix(host + "/torrents-search.php")
sterm = makeUnicode(book['searchterm'])
cat = 0 # 0=all, 36=ebooks, 52=mags, 56=audiobooks
if 'library' in book:
if book['library'] == 'AudioBook':
cat = 56
elif book['library'] == 'eBook':
cat = 36
elif book['library'] == 'magazine':
cat = 52
page = 0
results = []
minimumseeders = int(lazylibrarian.CONFIG['NUMBEROFSEEDERS']) - 1
next_page = True
while next_page:
params = {"search": book['searchterm'], "page": page, "cat": cat}
searchURL = providerurl + "/?%s" % urlencode(params)
next_page = False
result, success = fetchURL(searchURL)
if not success:
# might return 404 if no results, not really an error
if '404' in result:
logger.debug("No results found from %s for %s" %
(provider, sterm))
success = True
else:
logger.debug(searchURL)
logger.debug('Error fetching data from %s: %s' %
(provider, result))
errmsg = result
result = False
if test:
return success
if result:
logger.debug('Parsing results from <a href="%s">%s</a>' %
(searchURL, provider))
soup = BeautifulSoup(result, 'html5lib')
try:
tables = soup.find_all('table') # un-named table
table = tables[2]
if table:
rows = table.find_all('tr')
except IndexError: # no results table in result page
rows = []
if len(rows) > 1:
rows = rows[1:] # first row is headers
for row in rows:
td = row.find_all('td')
if len(td) > 3:
try:
title = unaccented(td[0].text)
# can return magnet or torrent or both.
magnet = ''
url = ''
mode = 'torrent'
try:
magnet = 'magnet' + str(
td[0]).split('href="magnet')[1].split('"')[0]
mode = 'magnet'
except IndexError:
pass
try:
url = url_fix(host + '/download.php') + \
str(td[0]).split('href="download.php')[1].split('.torrent"')[0] + '.torrent'
mode = 'torrent'
except IndexError:
pass
if not url or (magnet and url and
lazylibrarian.CONFIG['PREFER_MAGNET']):
url = magnet
mode = 'magnet'
try:
size = str(td[1].text).replace(' ',
'').upper()
mult = 1
if 'K' in size:
size = size.split('K')[0]
mult = 1024
elif 'M' in size:
size = size.split('M')[0]
mult = 1024 * 1024
elif 'G' in size:
size = size.split('G')[0]
mult = 1024 * 1024 * 1024
size = int(float(size) * mult)
except (ValueError, IndexError):
size = 0
try:
seeders = int(td[2].text)
except ValueError:
seeders = 0
if not url or not title:
logger.debug('Missing url or title')
elif minimumseeders < int(seeders):
results.append({
'bookid':
book['bookid'],
'tor_prov':
provider,
'tor_title':
title,
'tor_url':
url,
'tor_size':
str(size),
'tor_type':
mode,
'priority':
lazylibrarian.CONFIG['WWT_DLPRIORITY']
})
logger.debug('Found %s. Size: %s' % (title, size))
next_page = True
else:
logger.debug('Found %s but %s seeder%s' %
(title, seeders, plural(seeders)))
except Exception as e:
logger.error("An error occurred in the %s parser: %s" %
(provider, str(e)))
logger.debug('%s: %s' %
(provider, traceback.format_exc()))
page += 1
if 0 < lazylibrarian.CONFIG['MAX_PAGES'] < page:
logger.warn(
'Maximum results page search reached, still more results available'
)
next_page = False
logger.debug("Found %i result%s from %s for %s" %
(len(results), plural(len(results)), provider, sterm))
return results, errmsg
def TORDownloadMethod(bookid=None,
tor_title=None,
tor_url=None,
library='eBook'):
myDB = database.DBConnection()
downloadID = False
Source = ''
full_url = tor_url # keep the url as stored in "wanted" table
if tor_url and tor_url.startswith('magnet'):
torrent = tor_url # allow magnet link to write to blackhole and hash to utorrent/rtorrent
else:
# h = HTMLParser()
# tor_url = h.unescape(tor_url)
# HTMLParser is probably overkill, we only seem to get &
#
tor_url = tor_url.replace('&', '&')
if '&file=' in tor_url:
# torznab results need to be re-encoded
# had a problem with torznab utf-8 encoded strings not matching
# our utf-8 strings because of long/short form differences
url, value = tor_url.split('&file=', 1)
value = makeUnicode(value) # ensure unicode
value = unicodedata.normalize('NFC',
value) # normalize to short form
value = value.encode('unicode-escape') # then escape the result
value = value.replace(' ', '%20') # and encode any spaces
tor_url = url + '&file=' + value
# strip url back to the .torrent as some sites add parameters
if not tor_url.endswith('.torrent'):
if '.torrent' in tor_url:
tor_url = tor_url.split('.torrent')[0] + '.torrent'
headers = {'Accept-encoding': 'gzip', 'User-Agent': USER_AGENT}
proxies = proxyList()
try:
r = requests.get(tor_url,
headers=headers,
timeout=90,
proxies=proxies)
except requests.exceptions.Timeout:
logger.warn('Timeout fetching file from url: %s' % tor_url)
return False
except Exception as e:
if hasattr(e, 'reason'):
logger.warn('%s fetching file from url: %s, %s' %
(type(e).__name__, tor_url, e.reason))
else:
logger.warn('%s fetching file from url: %s, %s' %
(type(e).__name__, tor_url, str(e)))
return False
torrent = r.content
if lazylibrarian.CONFIG['TOR_DOWNLOADER_BLACKHOLE']:
Source = "BLACKHOLE"
logger.debug("Sending %s to blackhole" % tor_title)
tor_name = cleanName(tor_title).replace(' ', '_')
if tor_url and tor_url.startswith('magnet'):
if lazylibrarian.CONFIG['TOR_CONVERT_MAGNET']:
hashid = CalcTorrentHash(tor_url)
tor_name = 'meta-' + hashid + '.torrent'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'],
tor_name)
result = magnet2torrent(tor_url, tor_path)
if result is not False:
logger.debug('Magnet file saved as: %s' % tor_path)
downloadID = Source
else:
tor_name += '.magnet'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'],
tor_name)
msg = ''
try:
msg = 'Opening '
with open(tor_path, 'wb') as torrent_file:
msg += 'Writing '
if isinstance(torrent, unicode):
torrent = torrent.encode('iso-8859-1')
torrent_file.write(torrent)
msg += 'SettingPerm'
setperm(tor_path)
msg += 'Saved'
logger.debug('Magnet file saved: %s' % tor_path)
downloadID = Source
except Exception as e:
logger.debug("Failed to write magnet to file: %s %s" %
(type(e).__name__, str(e)))
logger.debug("Progress: %s" % msg)
logger.debug("Filename [%s]" % (repr(tor_path)))
return False
else:
tor_name += '.torrent'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'],
tor_name)
msg = ''
try:
msg = 'Opening '
with open(tor_path, 'wb') as torrent_file:
msg += 'Writing '
if isinstance(torrent, unicode):
torrent = torrent.encode('iso-8859-1')
torrent_file.write(torrent)
msg += 'SettingPerm '
setperm(tor_path)
msg += 'Saved'
logger.debug('Torrent file saved: %s' % tor_name)
downloadID = Source
except Exception as e:
logger.debug("Failed to write torrent to file: %s %s" %
(type(e).__name__, str(e)))
logger.debug("Progress: %s" % msg)
logger.debug("Filename [%s]" % (repr(tor_path)))
return False
if lazylibrarian.CONFIG['TOR_DOWNLOADER_UTORRENT'] and lazylibrarian.CONFIG[
'UTORRENT_HOST']:
logger.debug("Sending %s to Utorrent" % tor_title)
Source = "UTORRENT"
hashid = CalcTorrentHash(torrent)
downloadID = utorrent.addTorrent(tor_url,
hashid) # returns hash or False
if downloadID:
tor_title = utorrent.nameTorrent(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_RTORRENT'] and lazylibrarian.CONFIG[
'RTORRENT_HOST']:
logger.debug("Sending %s to rTorrent" % tor_title)
Source = "RTORRENT"
hashid = CalcTorrentHash(torrent)
downloadID = rtorrent.addTorrent(tor_url,
hashid) # returns hash or False
if downloadID:
tor_title = rtorrent.getName(downloadID)
if lazylibrarian.CONFIG[
'TOR_DOWNLOADER_QBITTORRENT'] and lazylibrarian.CONFIG[
'QBITTORRENT_HOST']:
logger.debug("Sending %s to qbittorrent" % tor_title)
Source = "QBITTORRENT"
hashid = CalcTorrentHash(torrent)
status = qbittorrent.addTorrent(tor_url,
hashid) # returns True or False
if status:
downloadID = hashid
tor_title = qbittorrent.getName(hashid)
if lazylibrarian.CONFIG[
'TOR_DOWNLOADER_TRANSMISSION'] and lazylibrarian.CONFIG[
'TRANSMISSION_HOST']:
logger.debug("Sending %s to Transmission" % tor_title)
Source = "TRANSMISSION"
downloadID = transmission.addTorrent(tor_url) # returns id or False
if downloadID:
# transmission returns it's own int, but we store hashid instead
downloadID = CalcTorrentHash(torrent)
tor_title = transmission.getTorrentFolder(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_SYNOLOGY'] and lazylibrarian.CONFIG['USE_SYNOLOGY'] and \
lazylibrarian.CONFIG['SYNOLOGY_HOST']:
logger.debug("Sending %s to Synology" % tor_title)
Source = "SYNOLOGY_TOR"
downloadID = synology.addTorrent(tor_url) # returns id or False
if downloadID:
tor_title = synology.getName(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_DELUGE'] and lazylibrarian.CONFIG[
'DELUGE_HOST']:
logger.debug("Sending %s to Deluge" % tor_title)
if not lazylibrarian.CONFIG['DELUGE_USER']:
# no username, talk to the webui
Source = "DELUGEWEBUI"
downloadID = deluge.addTorrent(tor_url) # returns hash or False
if downloadID:
tor_title = deluge.getTorrentFolder(downloadID)
else:
# have username, talk to the daemon
Source = "DELUGERPC"
client = DelugeRPCClient(lazylibrarian.CONFIG['DELUGE_HOST'],
lazylibrarian.CONFIG['DELUGE_URL_BASE'],
int(lazylibrarian.CONFIG['DELUGE_PORT']),
lazylibrarian.CONFIG['DELUGE_USER'],
lazylibrarian.CONFIG['DELUGE_PASS'])
try:
client.connect()
args = {"name": tor_title}
if tor_url.startswith('magnet'):
downloadID = client.call('core.add_torrent_magnet',
tor_url, args)
else:
downloadID = client.call('core.add_torrent_url', tor_url,
args)
if downloadID:
if lazylibrarian.CONFIG['DELUGE_LABEL']:
_ = client.call('label.set_torrent', downloadID,
lazylibrarian.CONFIG['DELUGE_LABEL'])
result = client.call('core.get_torrent_status', downloadID,
{})
# for item in result:
# logger.debug ('Deluge RPC result %s: %s' % (item, result[item]))
if 'name' in result:
tor_title = result['name']
except Exception as e:
logger.debug('DelugeRPC failed %s %s' %
(type(e).__name__, str(e)))
return False
if not Source:
logger.warn('No torrent download method is enabled, check config.')
return False
if downloadID:
if tor_title:
if downloadID.upper() in tor_title.upper():
logger.warn(
'%s: name contains hash, probably unresolved magnet' %
Source)
else:
tor_title = unaccented_str(tor_title)
# need to check against reject words list again as the name may have changed
# library = magazine eBook AudioBook to determine which reject list
# but we can't easily do the per-magazine rejects
if library == 'magazine':
reject_list = getList(lazylibrarian.CONFIG['REJECT_MAGS'])
elif library == 'eBook':
reject_list = getList(lazylibrarian.CONFIG['REJECT_WORDS'])
elif library == 'AudioBook':
reject_list = getList(lazylibrarian.CONFIG['REJECT_AUDIO'])
else:
logger.debug("Invalid library [%s] in TORDownloadMethod" %
library)
reject_list = []
rejected = False
lower_title = tor_title.lower()
for word in reject_list:
if word in lower_title:
rejected = True
logger.debug("Rejecting torrent name %s, contains %s" %
(tor_title, word))
break
if rejected:
myDB.action(
'UPDATE wanted SET status="Failed" WHERE NZBurl=?',
(full_url, ))
delete_task(Source, downloadID, True)
return False
else:
logger.debug('%s setting torrent name to [%s]' %
(Source, tor_title))
myDB.action('UPDATE wanted SET NZBtitle=? WHERE NZBurl=?',
(tor_title, full_url))
if library == 'eBook':
myDB.action('UPDATE books SET status="Snatched" WHERE BookID=?',
(bookid, ))
elif library == 'AudioBook':
myDB.action(
'UPDATE books SET audiostatus="Snatched" WHERE BookID=?',
(bookid, ))
myDB.action(
'UPDATE wanted SET status="Snatched", Source=?, DownloadID=? WHERE NZBurl=?',
(Source, downloadID, full_url))
return True
logger.error('Failed to download torrent from %s, %s' % (Source, tor_url))
myDB.action('UPDATE wanted SET status="Failed" WHERE NZBurl=?',
(full_url, ))
return False
def EXTRA(book=None, test=False):
errmsg = ''
provider = "Extratorrent"
host = lazylibrarian.CONFIG['EXTRA_HOST']
if not host.startswith('http'):
host = 'http://' + host
providerurl = url_fix(host + "/rss")
params = {"type": "search", "s_cat": "2", "search": book['searchterm']}
searchURL = providerurl + "/?%s" % urlencode(params)
sterm = makeUnicode(book['searchterm'])
data, success = fetchURL(searchURL)
if not success:
# may return 404 if no results, not really an error
if '404' in data:
logger.debug("No results found from %s for %s" % (provider, sterm))
success = True
else:
logger.debug('Error fetching data from %s: %s' % (provider, data))
errmsg = data
data = False
if test:
return success
results = []
minimumseeders = int(lazylibrarian.CONFIG['NUMBEROFSEEDERS']) - 1
if data:
logger.debug('Parsing results from <a href="%s">%s</a>' %
(searchURL, provider))
d = feedparser.parse(data)
if len(d.entries):
for item in d.entries:
try:
title = unaccented(item['title'])
try:
seeders = int(item['seeders'])
except ValueError:
seeders = 0
try:
size = int(item['size'])
except ValueError:
size = 0
url = None
for link in item['links']:
if 'x-bittorrent' in link['type']:
url = link['href']
if not url or not title:
logger.debug('No url or title found')
elif minimumseeders < int(seeders):
results.append({
'bookid':
book['bookid'],
'tor_prov':
provider,
'tor_title':
title,
'tor_url':
url,
'tor_size':
str(size),
'tor_type':
'torrent',
'priority':
lazylibrarian.CONFIG['EXTRA_DLPRIORITY']
})
logger.debug('Found %s. Size: %s' % (title, size))
else:
logger.debug('Found %s but %s seeder%s' %
(title, seeders, plural(seeders)))
except Exception as e:
logger.error("An error occurred in the %s parser: %s" %
(provider, str(e)))
logger.debug('%s: %s' % (provider, traceback.format_exc()))
logger.debug("Found %i result%s from %s for %s" %
(len(results), plural(len(results)), provider, sterm))
return results, errmsg
def KAT(book=None, test=False):
errmsg = ''
provider = "KAT"
host = lazylibrarian.CONFIG['KAT_HOST']
if not host.startswith('http'):
host = 'http://' + host
providerurl = url_fix(host + "/usearch/" + quote(book['searchterm']))
params = {"category": "books", "field": "seeders", "sorder": "desc"}
searchURL = providerurl + "/?%s" % urlencode(params)
sterm = makeUnicode(book['searchterm'])
result, success = fetchURL(searchURL)
if not success:
# seems KAT returns 404 if no results, not really an error
if '404' in result:
logger.debug("No results found from %s for %s" % (provider, sterm))
success = True
else:
logger.debug(searchURL)
logger.debug('Error fetching data from %s: %s' %
(provider, result))
errmsg = result
result = False
if test:
return success
results = []
if result:
logger.debug('Parsing results from <a href="%s">%s</a>' %
(searchURL, provider))
minimumseeders = int(lazylibrarian.CONFIG['NUMBEROFSEEDERS']) - 1
soup = BeautifulSoup(result, 'html5lib')
rows = []
try:
table = soup.find_all('table')[1] # un-named table
if table:
rows = table.find_all('tr')
except IndexError: # no results table in result page
rows = []
if len(rows) > 1:
rows = rows[1:] # first row is headers
for row in rows:
td = row.find_all('td')
if len(td) > 3:
try:
title = unaccented(td[0].text)
# kat can return magnet or torrent or both.
magnet = ''
url = ''
mode = 'torrent'
try:
magnet = 'magnet' + str(
td[0]).split('href="magnet')[1].split('"')[0]
mode = 'magnet'
except IndexError:
pass
try:
url = 'http' + str(td[0]).split('href="http')[1].split(
'.torrent?')[0] + '.torrent'
mode = 'torrent'
except IndexError:
pass
if not url or (magnet and url
and lazylibrarian.CONFIG['PREFER_MAGNET']):
url = magnet
mode = 'magnet'
try:
size = str(td[1].text).replace(' ', '').upper()
mult = 1
if 'K' in size:
size = size.split('K')[0]
mult = 1024
elif 'M' in size:
size = size.split('M')[0]
mult = 1024 * 1024
elif 'G' in size:
size = size.split('G')[0]
mult = 1024 * 1024 * 1024
size = int(float(size) * mult)
except (ValueError, IndexError):
size = 0
try:
seeders = int(td[3].text)
except ValueError:
seeders = 0
if not url or not title:
logger.debug('Missing url or title')
elif minimumseeders < int(seeders):
results.append({
'bookid':
book['bookid'],
'tor_prov':
provider,
'tor_title':
title,
'tor_url':
url,
'tor_size':
str(size),
'tor_type':
mode,
'priority':
lazylibrarian.CONFIG['KAT_DLPRIORITY']
})
logger.debug('Found %s. Size: %s' % (title, size))
else:
logger.debug('Found %s but %s seeder%s' %
(title, seeders, plural(seeders)))
except Exception as e:
logger.error("An error occurred in the %s parser: %s" %
(provider, str(e)))
logger.debug('%s: %s' % (provider, traceback.format_exc()))
logger.debug("Found %i result%s from %s for %s" %
(len(results), plural(len(results)), provider, sterm))
return results, errmsg
def TORDownloadMethod(bookid=None, tor_title=None, tor_url=None, library='eBook'):
myDB = database.DBConnection()
downloadID = False
Source = ''
full_url = tor_url # keep the url as stored in "wanted" table
if tor_url and tor_url.startswith('magnet:?'):
torrent = tor_url # allow magnet link to write to blackhole and hash to utorrent/rtorrent
elif 'magnet:?' in tor_url:
# discard any other parameters and just use the magnet link
torrent = 'magnet:?' + tor_url.split('magnet:?')[1]
else:
# h = HTMLParser()
# tor_url = h.unescape(tor_url)
# HTMLParser is probably overkill, we only seem to get &
#
tor_url = tor_url.replace('&', '&')
if '&file=' in tor_url:
# torznab results need to be re-encoded
# had a problem with torznab utf-8 encoded strings not matching
# our utf-8 strings because of long/short form differences
url, value = tor_url.split('&file=', 1)
value = makeUnicode(value) # ensure unicode
value = unicodedata.normalize('NFC', value) # normalize to short form
value = value.encode('unicode-escape') # then escape the result
value = makeUnicode(value) # ensure unicode
value = value.replace(' ', '%20') # and encode any spaces
tor_url = url + '&file=' + value
# strip url back to the .torrent as some sites add extra parameters
if not tor_url.endswith('.torrent'):
if '.torrent' in tor_url:
tor_url = tor_url.split('.torrent')[0] + '.torrent'
headers = {'Accept-encoding': 'gzip', 'User-Agent': USER_AGENT}
proxies = proxyList()
try:
r = requests.get(tor_url, headers=headers, timeout=90, proxies=proxies)
torrent = r.content
except requests.exceptions.Timeout:
logger.warn('Timeout fetching file from url: %s' % tor_url)
return False
except Exception as e:
# some jackett providers redirect internally using http 301 to a magnet link
# which requests can't handle, so throws an exception
if "magnet:?" in str(e):
torrent = 'magnet:?' + str(e).split('magnet:?')[1]. strip("'")
else:
if hasattr(e, 'reason'):
logger.warn('%s fetching file from url: %s, %s' % (type(e).__name__, tor_url, e.reason))
else:
logger.warn('%s fetching file from url: %s, %s' % (type(e).__name__, tor_url, str(e)))
return False
if lazylibrarian.CONFIG['TOR_DOWNLOADER_BLACKHOLE']:
Source = "BLACKHOLE"
logger.debug("Sending %s to blackhole" % tor_title)
tor_name = cleanName(tor_title).replace(' ', '_')
if tor_url and tor_url.startswith('magnet'):
if lazylibrarian.CONFIG['TOR_CONVERT_MAGNET']:
hashid = CalcTorrentHash(tor_url)
tor_name = 'meta-' + hashid + '.torrent'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name)
result = magnet2torrent(tor_url, tor_path)
if result is not False:
logger.debug('Magnet file saved as: %s' % tor_path)
downloadID = Source
else:
tor_name += '.magnet'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name)
msg = ''
try:
msg = 'Opening '
with open(tor_path, 'wb') as torrent_file:
msg += 'Writing '
if isinstance(torrent, text_type):
torrent = torrent.encode('iso-8859-1')
torrent_file.write(torrent)
msg += 'SettingPerm '
setperm(tor_path)
msg += 'Saved '
logger.debug('Magnet file saved: %s' % tor_path)
downloadID = Source
except Exception as e:
logger.warn("Failed to write magnet to file: %s %s" % (type(e).__name__, str(e)))
logger.debug("Progress: %s" % msg)
logger.debug("Filename [%s]" % (repr(tor_path)))
return False
else:
tor_name += '.torrent'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name)
msg = ''
try:
msg = 'Opening '
with open(tor_path, 'wb') as torrent_file:
msg += 'Writing '
if isinstance(torrent, text_type):
torrent = torrent.encode('iso-8859-1')
torrent_file.write(torrent)
msg += 'SettingPerm '
setperm(tor_path)
msg += 'Saved '
logger.debug('Torrent file saved: %s' % tor_name)
downloadID = Source
except Exception as e:
logger.warn("Failed to write torrent to file: %s %s" % (type(e).__name__, str(e)))
logger.debug("Progress: %s" % msg)
logger.debug("Filename [%s]" % (repr(tor_path)))
return False
hashid = CalcTorrentHash(torrent)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_UTORRENT'] and lazylibrarian.CONFIG['UTORRENT_HOST']:
logger.debug("Sending %s to Utorrent" % tor_title)
Source = "UTORRENT"
downloadID = utorrent.addTorrent(tor_url, hashid) # returns hash or False
if downloadID:
tor_title = utorrent.nameTorrent(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_RTORRENT'] and lazylibrarian.CONFIG['RTORRENT_HOST']:
logger.debug("Sending %s to rTorrent" % tor_title)
Source = "RTORRENT"
downloadID = rtorrent.addTorrent(tor_url, hashid) # returns hash or False
if downloadID:
tor_title = rtorrent.getName(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_QBITTORRENT'] and lazylibrarian.CONFIG['QBITTORRENT_HOST']:
logger.debug("Sending %s to qbittorrent" % tor_title)
Source = "QBITTORRENT"
if isinstance(torrent, binary_type) and torrent.startswith(b'magnet'):
status = qbittorrent.addTorrent(torrent, hashid)
elif isinstance(torrent, text_type) and torrent.startswith('magnet'):
status = qbittorrent.addTorrent(torrent, hashid)
else:
status = qbittorrent.addTorrent(tor_url, hashid) # returns True or False
if status:
downloadID = hashid
tor_title = qbittorrent.getName(hashid)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_TRANSMISSION'] and lazylibrarian.CONFIG['TRANSMISSION_HOST']:
logger.debug("Sending %s to Transmission" % tor_title)
if lazylibrarian.LOGLEVEL & lazylibrarian.log_dlcomms:
logger.debug("TORRENT %s [%s] [%s]" % (len(torrent), torrent[:20], torrent[-20:]))
Source = "TRANSMISSION"
if isinstance(torrent, binary_type) and torrent.startswith(b'magnet'):
downloadID = transmission.addTorrent(torrent) # returns id or False
elif isinstance(torrent, text_type) and torrent.startswith('magnet'):
downloadID = transmission.addTorrent(torrent)
elif torrent:
downloadID = transmission.addTorrent(None, metainfo=b64encode(torrent))
else:
downloadID = transmission.addTorrent(tor_url) # returns id or False
if downloadID:
# transmission returns it's own int, but we store hashid instead
downloadID = hashid
tor_title = transmission.getTorrentFolder(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_SYNOLOGY'] and lazylibrarian.CONFIG['USE_SYNOLOGY'] and \
lazylibrarian.CONFIG['SYNOLOGY_HOST']:
logger.debug("Sending %s to Synology" % tor_title)
Source = "SYNOLOGY_TOR"
downloadID = synology.addTorrent(tor_url) # returns id or False
if downloadID:
tor_title = synology.getName(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_DELUGE'] and lazylibrarian.CONFIG['DELUGE_HOST']:
logger.debug("Sending %s to Deluge" % tor_title)
if not lazylibrarian.CONFIG['DELUGE_USER']:
# no username, talk to the webui
Source = "DELUGEWEBUI"
if isinstance(torrent, binary_type) and torrent.startswith(b'magnet'):
downloadID = deluge.addTorrent(torrent)
elif isinstance(torrent, text_type) and torrent.startswith('magnet'):
downloadID = deluge.addTorrent(torrent)
elif torrent:
downloadID = deluge.addTorrent(tor_title, data=b64encode(torrent))
else:
downloadID = deluge.addTorrent(tor_url) # can be link or magnet, returns hash or False
if downloadID:
tor_title = deluge.getTorrentFolder(downloadID)
else:
# have username, talk to the daemon
Source = "DELUGERPC"
client = DelugeRPCClient(lazylibrarian.CONFIG['DELUGE_HOST'],
int(lazylibrarian.CONFIG['DELUGE_PORT']),
lazylibrarian.CONFIG['DELUGE_USER'],
lazylibrarian.CONFIG['DELUGE_PASS'])
try:
client.connect()
args = {"name": tor_title}
if tor_url.startswith('magnet'):
downloadID = client.call('core.add_torrent_magnet', tor_url, args)
elif isinstance(torrent, binary_type) and torrent.startswith(b'magnet'):
downloadID = client.call('core.add_torrent_magnet', torrent, args)
elif isinstance(torrent, text_type) and torrent.startswith('magnet'):
downloadID = client.call('core.add_torrent_magnet', torrent, args)
elif torrent:
downloadID = client.call('core.add_torrent_file', tor_title, b64encode(torrent), args)
else:
downloadID = client.call('core.add_torrent_url', tor_url, args)
if downloadID:
if lazylibrarian.CONFIG['DELUGE_LABEL']:
_ = client.call('label.set_torrent', downloadID, lazylibrarian.CONFIG['DELUGE_LABEL'].lower())
result = client.call('core.get_torrent_status', downloadID, {})
# for item in result:
# logger.debug ('Deluge RPC result %s: %s' % (item, result[item]))
if 'name' in result:
tor_title = result['name']
except Exception as e:
logger.error('DelugeRPC failed %s %s' % (type(e).__name__, str(e)))
return False
if not Source:
logger.warn('No torrent download method is enabled, check config.')
return False
if downloadID:
if tor_title:
if downloadID.upper() in tor_title.upper():
logger.warn('%s: name contains hash, probably unresolved magnet' % Source)
else:
tor_title = unaccented_str(tor_title)
# need to check against reject words list again as the name may have changed
# library = magazine eBook AudioBook to determine which reject list
# but we can't easily do the per-magazine rejects
if library == 'magazine':
reject_list = getList(lazylibrarian.CONFIG['REJECT_MAGS'])
elif library == 'eBook':
reject_list = getList(lazylibrarian.CONFIG['REJECT_WORDS'])
elif library == 'AudioBook':
reject_list = getList(lazylibrarian.CONFIG['REJECT_AUDIO'])
else:
logger.debug("Invalid library [%s] in TORDownloadMethod" % library)
reject_list = []
rejected = False
lower_title = tor_title.lower()
for word in reject_list:
if word in lower_title:
rejected = True
logger.debug("Rejecting torrent name %s, contains %s" % (tor_title, word))
break
if rejected:
myDB.action('UPDATE wanted SET status="Failed" WHERE NZBurl=?', (full_url,))
delete_task(Source, downloadID, True)
return False
else:
logger.debug('%s setting torrent name to [%s]' % (Source, tor_title))
myDB.action('UPDATE wanted SET NZBtitle=? WHERE NZBurl=?', (tor_title, full_url))
if library == 'eBook':
myDB.action('UPDATE books SET status="Snatched" WHERE BookID=?', (bookid,))
elif library == 'AudioBook':
myDB.action('UPDATE books SET audiostatus="Snatched" WHERE BookID=?', (bookid,))
myDB.action('UPDATE wanted SET status="Snatched", Source=?, DownloadID=? WHERE NZBurl=?',
(Source, downloadID, full_url))
return True
logger.error('Failed to download torrent from %s, %s' % (Source, tor_url))
myDB.action('UPDATE wanted SET status="Failed" WHERE NZBurl=?', (full_url,))
return False
def TPB(book=None, test=False):
errmsg = ''
provider = "TPB"
host = lazylibrarian.CONFIG['TPB_HOST']
if not host.startswith('http'):
host = 'http://' + host
providerurl = url_fix(host + "/s/?")
cat = 0 # 601=ebooks, 102=audiobooks, 0=all, no mag category
if 'library' in book:
if book['library'] == 'AudioBook':
cat = 102
elif book['library'] == 'eBook':
cat = 601
elif book['library'] == 'magazine':
cat = 0
sterm = makeUnicode(book['searchterm'])
page = 0
results = []
minimumseeders = int(lazylibrarian.CONFIG['NUMBEROFSEEDERS']) - 1
next_page = True
while next_page:
params = {
"q": book['searchterm'],
"category": cat,
"page": page,
"orderby": "99"
}
searchURL = providerurl + "?%s" % urlencode(params)
next_page = False
result, success = fetchURL(searchURL)
if not success:
# may return 404 if no results, not really an error
if '404' in result:
logger.debug("No results found from %s for %s" %
(provider, sterm))
success = True
else:
logger.debug(searchURL)
logger.debug('Error fetching data from %s: %s' %
(provider, result))
errmsg = result
result = False
if test:
return success
if result:
logger.debug('Parsing results from <a href="%s">%s</a>' %
(searchURL, provider))
soup = BeautifulSoup(result, 'html5lib')
# tpb uses a named table
table = soup.find('table', id='searchResult')
if table:
rows = table.find_all('tr')
else:
rows = []
if len(rows) > 1:
rows = rows[1:] # first row is headers
for row in rows:
td = row.find_all('td')
if len(td) > 2:
try:
new_soup = BeautifulSoup(str(td[1]), 'html5lib')
link = new_soup.find("a")
magnet = link.get("href")
title = link.text
size = td[1].text.split(', Size ')[1].split('iB')[0]
size = size.replace(' ', '')
mult = 1
try:
if 'K' in size:
size = size.split('K')[0]
mult = 1024
elif 'M' in size:
size = size.split('M')[0]
mult = 1024 * 1024
elif 'G' in size:
size = size.split('G')[0]
mult = 1024 * 1024 * 1024
size = int(float(size) * mult)
except (ValueError, IndexError):
size = 0
try:
seeders = int(td[2].text)
except ValueError:
seeders = 0
if minimumseeders < int(seeders):
# no point in asking for magnet link if not enough seeders
magurl = '%s/%s' % (host, magnet)
result, success = fetchURL(magurl)
if not success:
logger.debug('Error fetching url %s, %s' %
(magurl, result))
else:
magnet = None
new_soup = BeautifulSoup(result, 'html5lib')
for link in new_soup.find_all('a'):
output = link.get('href')
if output and output.startswith('magnet'):
magnet = output
break
if not magnet or not title:
logger.debug('Missing magnet or title')
else:
results.append({
'bookid':
book['bookid'],
'tor_prov':
provider,
'tor_title':
title,
'tor_url':
magnet,
'tor_size':
str(size),
'tor_type':
'magnet',
'priority':
lazylibrarian.CONFIG['TPB_DLPRIORITY']
})
logger.debug('Found %s. Size: %s' %
(title, size))
next_page = True
else:
logger.debug('Found %s but %s seeder%s' %
(title, seeders, plural(seeders)))
except Exception as e:
logger.error("An error occurred in the %s parser: %s" %
(provider, str(e)))
logger.debug('%s: %s' %
(provider, traceback.format_exc()))
page += 1
if 0 < lazylibrarian.CONFIG['MAX_PAGES'] < page:
logger.warn(
'Maximum results page search reached, still more results available'
)
next_page = False
logger.debug("Found %i result%s from %s for %s" %
(len(results), plural(len(results)), provider, sterm))
return results, errmsg
def import_CSV(search_dir=None, library='eBook'):
""" Find a csv file in the search_dir and process all the books in it,
adding authors to the database if not found
and marking the books as "Wanted"
Optionally delete the file on successful completion
"""
# noinspection PyBroadException
try:
if not search_dir:
msg = "Alternate Directory not configured"
logger.warn(msg)
return msg
elif not os.path.isdir(search_dir):
msg = "Alternate Directory [%s] not found" % search_dir
logger.warn(msg)
return msg
csvFile = csv_file(search_dir, library=library)
headers = None
myDB = database.DBConnection()
bookcount = 0
authcount = 0
skipcount = 0
total = 0
existing = 0
if not csvFile:
msg = "No %s CSV file found in %s" % (library, search_dir)
logger.warn(msg)
return msg
else:
logger.debug('Reading file %s' % csvFile)
csvreader = reader(open(csvFile, 'rU'))
for row in csvreader:
if csvreader.line_num == 1:
# If we are on the first line, create the headers list from the first row
headers = row
if 'Author' not in headers or 'Title' not in headers:
msg = 'Invalid CSV file found %s' % csvFile
logger.warn(msg)
return msg
else:
total += 1
item = dict(list(zip(headers, row)))
authorname = formatAuthorName(item['Author'])
title = makeUnicode(item['Title'])
authmatch = myDB.match('SELECT * FROM authors where AuthorName=?', (authorname,))
if authmatch:
logger.debug("CSV: Author %s found in database" % authorname)
else:
logger.debug("CSV: Author %s not found" % authorname)
newauthor, authorid, new = addAuthorNameToDB(author=authorname,
addbooks=lazylibrarian.CONFIG['NEWAUTHOR_BOOKS'])
if len(newauthor) and newauthor != authorname:
logger.debug("Preferred authorname changed from [%s] to [%s]" % (authorname, newauthor))
authorname = newauthor
if new:
authcount += 1
bookmatch = finditem(item, authorname, library=library)
result = ''
imported = ''
if bookmatch:
authorname = bookmatch['AuthorName']
bookname = bookmatch['BookName']
bookid = bookmatch['BookID']
if library == 'eBook':
bookstatus = bookmatch['Status']
else:
bookstatus = bookmatch['AudioStatus']
if bookstatus in ['Open', 'Wanted', 'Have']:
existing += 1
logger.info('Found %s %s by %s, already marked as "%s"' %
(library, bookname, authorname, bookstatus))
else: # skipped/ignored
logger.info('Found %s %s by %s, marking as "Wanted"' % (library, bookname, authorname))
controlValueDict = {"BookID": bookid}
if library == 'eBook':
newValueDict = {"Status": "Wanted"}
else:
newValueDict = {"AudioStatus": "Wanted"}
myDB.upsert("books", newValueDict, controlValueDict)
bookcount += 1
else:
searchterm = "%s <ll> %s" % (title, authorname)
results = search_for(unaccented(searchterm))
if results:
result = results[0]
if result['author_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO'] \
and result['book_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO']:
bookmatch = True
if not bookmatch: # no match on full searchterm, try splitting out subtitle
newtitle, _ = split_title(authorname, title)
if newtitle != title:
title = newtitle
searchterm = "%s <ll> %s" % (title, authorname)
results = search_for(unaccented(searchterm))
if results:
result = results[0]
if result['author_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO'] \
and result['book_fuzz'] >= lazylibrarian.CONFIG['MATCH_RATIO']:
bookmatch = True
if bookmatch:
logger.info("Found (%s%% %s%%) %s: %s for %s: %s" %
(result['author_fuzz'], result['book_fuzz'],
result['authorname'], result['bookname'],
authorname, title))
if library == 'eBook':
import_book(result['bookid'], ebook="Wanted", wait=True)
else:
import_book(result['bookid'], audio="Wanted", wait=True)
imported = myDB.match('select * from books where BookID=?', (result['bookid'],))
if imported:
bookcount += 1
else:
bookmatch = False
if not bookmatch:
msg = "Skipping book %s by %s" % (title, authorname)
if not result:
msg += ', No results found'
logger.warn(msg)
elif not imported:
msg += ', Failed to import %s' % result['bookid']
logger.warn(msg)
else:
msg += ', No match found'
logger.warn(msg)
msg = "Closest match (%s%% %s%%) %s: %s" % (result['author_fuzz'], result['book_fuzz'],
result['authorname'], result['bookname'])
logger.warn(msg)
skipcount += 1
msg = "Found %i %s%s in csv file, %i already existing or wanted" % (total, library,
plural(total), existing)
logger.info(msg)
msg = "Added %i new author%s, marked %i %s%s as 'Wanted', %i %s%s not found" % \
(authcount, plural(authcount), bookcount, library, plural(bookcount),
skipcount, plural(skipcount), library)
logger.info(msg)
if lazylibrarian.CONFIG['DELETE_CSV']:
if skipcount == 0:
logger.info("Deleting %s on successful completion" % csvFile)
try:
os.remove(csvFile)
except OSError as why:
logger.warn('Unable to delete %s: %s' % (csvFile, why.strerror))
else:
logger.warn("Not deleting %s as not all books found" % csvFile)
if os.path.isdir(csvFile + '.fail'):
try:
shutil.rmtree(csvFile + '.fail')
except Exception as why:
logger.warn("Unable to remove %s, %s %s" % (csvFile + '.fail',
type(why).__name__, str(why)))
try:
_ = safe_move(csvFile, csvFile + '.fail')
except Exception as e:
logger.error("Unable to rename %s, %s %s" %
(csvFile, type(e).__name__, str(e)))
if not os.access(csvFile, os.R_OK):
logger.error("%s is not readable" % csvFile)
if not os.access(csvFile, os.W_OK):
logger.error("%s is not writeable" % csvFile)
parent = os.path.dirname(csvFile)
try:
with open(os.path.join(parent, 'll_temp'), 'w') as f:
f.write('test')
os.remove(os.path.join(parent, 'll_temp'))
except Exception as why:
logger.error("Directory %s is not writeable: %s" % (parent, why))
return msg
except Exception:
msg = 'Unhandled exception in importCSV: %s' % traceback.format_exc()
logger.error(msg)
return msg
def ZOO(book=None, test=False):
errmsg = ''
provider = "zooqle"
host = lazylibrarian.CONFIG['ZOO_HOST']
if not host.startswith('http'):
host = 'http://' + host
providerurl = url_fix(host + "/search")
params = {"q": book['searchterm'], "category": "books", "fmt": "rss"}
searchURL = providerurl + "?%s" % urlencode(params)
sterm = makeUnicode(book['searchterm'])
data, success = fetchURL(searchURL)
if not success:
# may return 404 if no results, not really an error
if '404' in data:
logger.debug("No results found from %s for %s" % (provider, sterm))
success = True
else:
logger.debug(searchURL)
logger.debug('Error fetching data from %s: %s' % (provider, data))
errmsg = data
data = False
if test:
return success
results = []
minimumseeders = int(lazylibrarian.CONFIG['NUMBEROFSEEDERS']) - 1
if data:
logger.debug('Parsing results from <a href="%s">%s</a>' %
(searchURL, provider))
d = feedparser.parse(data)
if len(d.entries):
for item in d.entries:
try:
title = unaccented(item['title'])
seeders = int(item['torrent_seeds'])
link = item['links'][1]['href']
size = int(item['links'][1]['length'])
magnet = item['torrent_magneturi']
url = None
mode = 'torrent'
if link:
url = link
mode = 'torrent'
if magnet:
if not url or (url and
lazylibrarian.CONFIG['PREFER_MAGNET']):
url = magnet
mode = 'magnet'
if not url or not title:
logger.debug('No url or title found')
elif minimumseeders < int(seeders):
results.append({
'bookid':
book['bookid'],
'tor_prov':
provider,
'tor_title':
title,
'tor_url':
url,
'tor_size':
str(size),
'tor_type':
mode,
'priority':
lazylibrarian.CONFIG['ZOO_DLPRIORITY']
})
logger.debug('Found %s. Size: %s' % (title, size))
else:
logger.debug('Found %s but %s seeder%s' %
(title, seeders, plural(seeders)))
except Exception as e:
if 'forbidden' in str(e).lower():
# looks like zooqle has ip based access limits
logger.error(
'Access forbidden. Please wait a while before trying %s again.'
% provider)
else:
logger.error("An error occurred in the %s parser: %s" %
(provider, str(e)))
logger.debug('%s: %s' %
(provider, traceback.format_exc()))
logger.debug("Found %i result%s from %s for %s" %
(len(results), plural(len(results)), provider, sterm))
return results, errmsg
def restore_table(table, savedir=None, status=None):
myDB = database.DBConnection()
# noinspection PyBroadException
try:
columns = myDB.select('PRAGMA table_info(%s)' % table)
if not columns: # no such table
logger.warn("No such table [%s]" % table)
return 0
if not os.path.isdir(savedir):
savedir = lazylibrarian.DATADIR
headers = ''
label = table
if status:
label += '_%s' % status
csvFile = os.path.join(savedir, "%s.csv" % label)
logger.debug('Reading file %s' % csvFile)
csvreader = reader(open(csvFile, 'rU'))
count = 0
for row in csvreader:
if csvreader.line_num == 1:
headers = row
else:
item = dict(list(zip(headers, row)))
if table == 'magazines':
controlValueDict = {"Title": makeUnicode(item['Title'])}
newValueDict = {"Regex": makeUnicode(item['Regex']),
"Reject": makeUnicode(item['Reject']),
"Status": item['Status'],
"MagazineAdded": item['MagazineAdded'],
"IssueStatus": item['IssueStatus'],
"CoverPage": item['CoverPage']}
myDB.upsert("magazines", newValueDict, controlValueDict)
count += 1
elif table == 'users':
controlValueDict = {"UserID": item['UserID']}
newValueDict = {"UserName": item['UserName'],
"Password": item['Password'],
"Email": item['Email'],
"Name": item['Name'],
"Perms": item['Perms'],
"HaveRead": item['HaveRead'],
"ToRead": item['ToRead'],
"CalibreRead": item['CalibreRead'],
"CalibreToRead": item['CalibreToRead'],
"BookType": item['BookType']
}
myDB.upsert("users", newValueDict, controlValueDict)
count += 1
else:
logger.error("Invalid table [%s]" % table)
return 0
msg = "Imported %s item%s from %s" % (count, plural(count), csvFile)
logger.info(msg)
return count
except Exception:
msg = 'Unhandled exception in restore_table: %s' % traceback.format_exc()
logger.error(msg)
return 0
def TDL(book=None, test=False):
errmsg = ''
provider = "torrentdownloads"
host = lazylibrarian.CONFIG['TDL_HOST']
if not host.startswith('http'):
host = 'http://' + host
providerurl = url_fix(host)
params = {"type": "search", "cid": "2", "search": book['searchterm']}
searchURL = providerurl + "/rss.xml?%s" % urlencode(params)
sterm = makeUnicode(book['searchterm'])
data, success = fetchURL(searchURL)
if not success:
# may return 404 if no results, not really an error
if '404' in data:
logger.debug("No results found from %s for %s" % (provider, sterm))
success = True
else:
logger.debug(searchURL)
logger.debug('Error fetching data from %s: %s' % (provider, data))
errmsg = data
data = False
if test:
return success
results = []
minimumseeders = int(lazylibrarian.CONFIG['NUMBEROFSEEDERS']) - 1
if data:
logger.debug('Parsing results from <a href="%s">%s</a>' %
(searchURL, provider))
d = feedparser.parse(data)
if len(d.entries):
for item in d.entries:
try:
title = item['title']
seeders = int(item['seeders'])
link = item['link']
size = int(item['size'])
url = None
if link and minimumseeders < int(seeders):
# no point requesting the magnet link if not enough seeders
# TDL gives us a relative link
result, success = fetchURL(providerurl + link)
if success:
new_soup = BeautifulSoup(result, 'html5lib')
for link in new_soup.find_all('a'):
output = link.get('href')
if output and output.startswith('magnet'):
url = output
break
if not url or not title:
logger.debug('Missing url or title')
else:
results.append({
'bookid':
book['bookid'],
'tor_prov':
provider,
'tor_title':
title,
'tor_url':
url,
'tor_size':
str(size),
'tor_type':
'magnet',
'priority':
lazylibrarian.CONFIG['TDL_DLPRIORITY']
})
logger.debug('Found %s. Size: %s' % (title, size))
else:
logger.debug('Found %s but %s seeder%s' %
(title, seeders, plural(seeders)))
except Exception as e:
logger.error("An error occurred in the %s parser: %s" %
(provider, str(e)))
logger.debug('%s: %s' % (provider, traceback.format_exc()))
logger.debug("Found %i result%s from %s for %s" %
(len(results), plural(len(results)), provider, sterm))
return results, errmsg
def KAT(book=None, test=False):
errmsg = ''
provider = "KAT"
host = lazylibrarian.CONFIG['KAT_HOST']
if not host.startswith('http'):
host = 'http://' + host
providerurl = url_fix(host + "/usearch/" + quote(book['searchterm']))
params = {
"category": "books",
"field": "seeders",
"sorder": "desc"
}
searchURL = providerurl + "/?%s" % urlencode(params)
sterm = makeUnicode(book['searchterm'])
result, success = fetchURL(searchURL)
if not success:
# seems KAT returns 404 if no results, not really an error
if '404' in result:
logger.debug("No results found from %s for %s" % (provider, sterm))
success = True
else:
logger.debug(searchURL)
logger.debug('Error fetching data from %s: %s' % (provider, result))
errmsg = result
result = False
if test:
return success
results = []
if result:
logger.debug('Parsing results from <a href="%s">%s</a>' % (searchURL, provider))
minimumseeders = int(lazylibrarian.CONFIG['NUMBEROFSEEDERS']) - 1
soup = BeautifulSoup(result, 'html5lib')
rows = []
try:
table = soup.find_all('table')[1] # un-named table
if table:
rows = table.find_all('tr')
except IndexError: # no results table in result page
rows = []
if len(rows) > 1:
rows = rows[1:] # first row is headers
for row in rows:
td = row.find_all('td')
if len(td) > 3:
try:
title = unaccented(td[0].text)
# kat can return magnet or torrent or both.
magnet = ''
url = ''
mode = 'torrent'
try:
magnet = 'magnet' + str(td[0]).split('href="magnet')[1].split('"')[0]
mode = 'magnet'
except IndexError:
pass
try:
url = 'http' + str(td[0]).split('href="http')[1].split('.torrent?')[0] + '.torrent'
mode = 'torrent'
except IndexError:
pass
if not url or (magnet and url and lazylibrarian.CONFIG['PREFER_MAGNET']):
url = magnet
mode = 'magnet'
try:
size = str(td[1].text).replace(' ', '').upper()
size = size_in_bytes(size)
except ValueError:
size = 0
try:
seeders = int(td[3].text.replace(',', ''))
except ValueError:
seeders = 0
if not url or not title:
logger.debug('Missing url or title')
elif minimumseeders < seeders:
results.append({
'bookid': book['bookid'],
'tor_prov': provider,
'tor_title': title,
'tor_url': url,
'tor_size': str(size),
'tor_type': mode,
'priority': lazylibrarian.CONFIG['KAT_DLPRIORITY']
})
logger.debug('Found %s. Size: %s' % (title, size))
else:
logger.debug('Found %s but %s seeder%s' % (title, seeders, plural(seeders)))
except Exception as e:
logger.error("An error occurred in the %s parser: %s" % (provider, str(e)))
logger.debug('%s: %s' % (provider, traceback.format_exc()))
logger.debug("Found %i result%s from %s for %s" % (len(results), plural(len(results)), provider, sterm))
return results, errmsg