def _action(self, params, body=None, content_type=None):
# noinspection PyTypeChecker
url = self.base_url + '/gui/' + '?token=' + self.token + '&' + urlencode(
params)
request = Request(url)
if lazylibrarian.CONFIG['PROXY_HOST']:
for item in getList(lazylibrarian.CONFIG['PROXY_TYPE']):
request.set_proxy(lazylibrarian.CONFIG['PROXY_HOST'], item)
request.add_header('User-Agent', getUserAgent())
if body:
if PY2:
request.add_data(body)
else:
request.data(body)
request.add_header('Content-length', len(body))
if content_type:
request.add_header('Content-type', content_type)
try:
response = self.opener.open(request)
return response.code, json.loads(response.read())
except HTTPError as err:
logger.debug('URL: %s' % url)
logger.debug('uTorrent webUI raised the following error: ' +
str(err))
def _command(self, command, args=None, content_type=None, files=None):
logger.debug('QBittorrent WebAPI Command: %s' % command)
url = self.base_url + '/' + command
data = None
headers = dict()
if files or content_type == 'multipart/form-data':
data, headers = encode_multipart(
args, files, '-------------------------acebdf13572468')
else:
if args:
data = makeBytestr(urlencode(args))
if content_type:
headers['Content-Type'] = content_type
request = Request(url, data, headers)
if lazylibrarian.CONFIG['PROXY_HOST']:
for item in getList(lazylibrarian.CONFIG['PROXY_TYPE']):
request.set_proxy(lazylibrarian.CONFIG['PROXY_HOST'], item)
request.add_header('User-Agent', getUserAgent())
try:
response = self.opener.open(request)
try:
contentType = response.headers['content-type']
except KeyError:
contentType = ''
resp = response.read()
# some commands return json
if contentType == 'application/json':
if resp:
return json.loads(resp)
return ''
else:
# some commands return plain text
resp = makeUnicode(resp)
logger.debug("QBitTorrent returned %s" % resp)
if command == 'version/api':
return resp
# some just return Ok. or Fails.
if resp and resp != 'Ok.':
return False
# some commands return nothing but response code (always 200)
return True
except URLError as err:
logger.debug('Failed URL: %s' % url)
logger.debug('QBitTorrent webUI raised the following error: %s' %
err.reason)
return False
def _command(self, command, args=None, content_type=None, files=None):
logger.debug('QBittorrent WebAPI Command: %s' % command)
url = self.base_url + '/' + command
data = None
headers = dict()
if files or content_type == 'multipart/form-data':
data, headers = encode_multipart(args, files, '-------------------------acebdf13572468')
else:
if args:
data = makeBytestr(urlencode(args))
if content_type:
headers['Content-Type'] = content_type
request = Request(url, data, headers)
if lazylibrarian.CONFIG['PROXY_HOST']:
for item in getList(lazylibrarian.CONFIG['PROXY_TYPE']):
request.set_proxy(lazylibrarian.CONFIG['PROXY_HOST'], item)
request.add_header('User-Agent', getUserAgent())
try:
response = self.opener.open(request)
try:
contentType = response.headers['content-type']
except KeyError:
contentType = ''
resp = response.read()
# some commands return json
if contentType == 'application/json':
if resp:
return json.loads(resp)
return ''
else:
# some commands return plain text
resp = makeUnicode(resp)
logger.debug("QBitTorrent returned %s" % resp)
if command == 'version/api':
return resp
# some just return Ok. or Fails.
if resp and resp != 'Ok.':
return False
# some commands return nothing but response code (always 200)
return True
except URLError as err:
logger.debug('Failed URL: %s' % url)
logger.debug('QBitTorrent webUI raised the following error: %s' % err.reason)
return False
def _action(self, params, body=None, content_type=None):
# noinspection PyTypeChecker
url = self.base_url + '/gui/' + '?token=' + self.token + '&' + urlencode(params)
request = Request(url)
if lazylibrarian.CONFIG['PROXY_HOST']:
for item in getList(lazylibrarian.CONFIG['PROXY_TYPE']):
request.set_proxy(lazylibrarian.CONFIG['PROXY_HOST'], item)
request.add_header('User-Agent', getUserAgent())
if body:
if PY2:
request.add_data(body)
else:
request.data(body)
request.add_header('Content-length', len(body))
if content_type:
request.add_header('Content-type', content_type)
try:
response = self.opener.open(request)
return response.code, json.loads(response.read())
except HTTPError as err:
logger.debug('URL: %s' % url)
logger.debug('uTorrent webUI raised the following error: ' + str(err))
def TORDownloadMethod(bookid=None,
tor_title=None,
tor_url=None,
library='eBook'):
myDB = database.DBConnection()
downloadID = False
Source = ''
torrent = ''
full_url = tor_url # keep the url as stored in "wanted" table
if 'magnet:?' in tor_url:
# discard any other parameters and just use the magnet link
tor_url = 'magnet:?' + tor_url.split('magnet:?')[1]
else:
# h = HTMLParser()
# tor_url = h.unescape(tor_url)
# HTMLParser is probably overkill, we only seem to get &
#
tor_url = tor_url.replace('&', '&')
if '&file=' in tor_url:
# torznab results need to be re-encoded
# had a problem with torznab utf-8 encoded strings not matching
# our utf-8 strings because of long/short form differences
url, value = tor_url.split('&file=', 1)
value = makeUnicode(value) # ensure unicode
value = unicodedata.normalize('NFC',
value) # normalize to short form
value = value.encode('unicode-escape') # then escape the result
value = makeUnicode(value) # ensure unicode
value = value.replace(' ', '%20') # and encode any spaces
tor_url = url + '&file=' + value
# strip url back to the .torrent as some sites add extra parameters
if not tor_url.endswith('.torrent') and '.torrent' in tor_url:
tor_url = tor_url.split('.torrent')[0] + '.torrent'
headers = {'Accept-encoding': 'gzip', 'User-Agent': getUserAgent()}
proxies = proxyList()
try:
logger.debug("Fetching %s" % tor_url)
r = requests.get(tor_url,
headers=headers,
timeout=90,
proxies=proxies)
if str(r.status_code).startswith('2'):
torrent = r.content
if not len(torrent):
res = "Got empty response for %s" % tor_url
logger.warn(res)
return False, res
elif len(torrent) < 100:
res = "Only got %s bytes for %s" % (len(torrent), tor_url)
logger.warn(res)
return False, res
else:
logger.debug("Got %s bytes for %s" %
(len(torrent), tor_url))
else:
res = "Got a %s response for %s" % (r.status_code, tor_url)
logger.warn(res)
return False, res
except requests.exceptions.Timeout:
res = 'Timeout fetching file from url: %s' % tor_url
logger.warn(res)
return False, res
except Exception as e:
# some jackett providers redirect internally using http 301 to a magnet link
# which requests can't handle, so throws an exception
logger.debug("Requests exception: %s" % str(e))
if "magnet:?" in str(e):
tor_url = 'magnet:?' + str(e).split('magnet:?')[1].strip("'")
logger.debug("Redirecting to %s" % tor_url)
else:
if hasattr(e, 'reason'):
res = '%s fetching file from url: %s, %s' % (
type(e).__name__, tor_url, e.reason)
else:
res = '%s fetching file from url: %s, %s' % (
type(e).__name__, tor_url, str(e))
logger.warn(res)
return False, res
if not torrent and not tor_url.startswith('magnet:?'):
res = "No magnet or data, cannot continue"
logger.warn(res)
return False, res
if lazylibrarian.CONFIG['TOR_DOWNLOADER_BLACKHOLE']:
Source = "BLACKHOLE"
logger.debug("Sending %s to blackhole" % tor_title)
tor_name = cleanName(tor_title).replace(' ', '_')
if tor_url and tor_url.startswith('magnet'):
if lazylibrarian.CONFIG['TOR_CONVERT_MAGNET']:
hashid = calculate_torrent_hash(tor_url)
if not hashid:
hashid = tor_name
tor_name = 'meta-' + hashid + '.torrent'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'],
tor_name)
result = magnet2torrent(tor_url, tor_path)
if result is not False:
logger.debug('Magnet file saved as: %s' % tor_path)
downloadID = Source
else:
tor_name += '.magnet'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'],
tor_name)
msg = ''
try:
msg = 'Opening '
with open(tor_path, 'wb') as torrent_file:
msg += 'Writing '
if isinstance(torrent, text_type):
torrent = torrent.encode('iso-8859-1')
torrent_file.write(torrent)
msg += 'SettingPerm '
setperm(tor_path)
msg += 'Saved '
logger.debug('Magnet file saved: %s' % tor_path)
downloadID = Source
except Exception as e:
res = "Failed to write magnet to file: %s %s" % (
type(e).__name__, str(e))
logger.warn(res)
logger.debug("Progress: %s Filename [%s]" %
(msg, repr(tor_path)))
return False, res
else:
tor_name += '.torrent'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'],
tor_name)
msg = ''
try:
msg = 'Opening '
with open(tor_path, 'wb') as torrent_file:
msg += 'Writing '
if isinstance(torrent, text_type):
torrent = torrent.encode('iso-8859-1')
torrent_file.write(torrent)
msg += 'SettingPerm '
setperm(tor_path)
msg += 'Saved '
logger.debug('Torrent file saved: %s' % tor_name)
downloadID = Source
except Exception as e:
res = "Failed to write torrent to file: %s %s" % (
type(e).__name__, str(e))
logger.warn(res)
logger.debug("Progress: %s Filename [%s]" %
(msg, repr(tor_path)))
return False, res
hashid = calculate_torrent_hash(tor_url, torrent)
if not hashid:
res = "Unable to calculate torrent hash from url/data"
logger.error(res)
logger.debug("url: %s" % tor_url)
logger.debug("data: %s" % makeUnicode(str(torrent[:50])))
return False, res
if lazylibrarian.CONFIG['TOR_DOWNLOADER_UTORRENT'] and lazylibrarian.CONFIG[
'UTORRENT_HOST']:
logger.debug("Sending %s to Utorrent" % tor_title)
Source = "UTORRENT"
downloadID, res = utorrent.addTorrent(tor_url,
hashid) # returns hash or False
if downloadID:
tor_title = utorrent.nameTorrent(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_RTORRENT'] and lazylibrarian.CONFIG[
'RTORRENT_HOST']:
logger.debug("Sending %s to rTorrent" % tor_title)
Source = "RTORRENT"
if torrent:
logger.debug("Sending %s data to rTorrent" % tor_title)
downloadID, res = rtorrent.addTorrent(tor_title,
hashid,
data=torrent)
else:
logger.debug("Sending %s url to rTorrent" % tor_title)
downloadID, res = rtorrent.addTorrent(
tor_url, hashid) # returns hash or False
if downloadID:
tor_title = rtorrent.getName(downloadID)
if lazylibrarian.CONFIG[
'TOR_DOWNLOADER_QBITTORRENT'] and lazylibrarian.CONFIG[
'QBITTORRENT_HOST']:
Source = "QBITTORRENT"
if torrent:
logger.debug("Sending %s data to qBittorrent" % tor_title)
status, res = qbittorrent.addFile(torrent, hashid, tor_title)
else:
logger.debug("Sending %s url to qBittorrent" % tor_title)
status, res = qbittorrent.addTorrent(
tor_url, hashid) # returns True or False
if status:
downloadID = hashid
tor_title = qbittorrent.getName(hashid)
if lazylibrarian.CONFIG[
'TOR_DOWNLOADER_TRANSMISSION'] and lazylibrarian.CONFIG[
'TRANSMISSION_HOST']:
Source = "TRANSMISSION"
if torrent:
logger.debug("Sending %s data to Transmission" % tor_title)
# transmission needs b64encoded metainfo to be unicode, not bytes
downloadID, res = transmission.addTorrent(None,
metainfo=makeUnicode(
b64encode(torrent)))
else:
logger.debug("Sending %s url to Transmission" % tor_title)
downloadID, res = transmission.addTorrent(
tor_url) # returns id or False
if downloadID:
# transmission returns it's own int, but we store hashid instead
downloadID = hashid
tor_title = transmission.getTorrentFolder(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_SYNOLOGY'] and lazylibrarian.CONFIG['USE_SYNOLOGY'] and \
lazylibrarian.CONFIG['SYNOLOGY_HOST']:
logger.debug("Sending %s url to Synology" % tor_title)
Source = "SYNOLOGY_TOR"
downloadID, res = synology.addTorrent(tor_url) # returns id or False
if downloadID:
tor_title = synology.getName(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_DELUGE'] and lazylibrarian.CONFIG[
'DELUGE_HOST']:
if not lazylibrarian.CONFIG['DELUGE_USER']:
# no username, talk to the webui
Source = "DELUGEWEBUI"
if torrent:
logger.debug("Sending %s data to Deluge" % tor_title)
downloadID, res = deluge.addTorrent(tor_title,
data=b64encode(torrent))
else:
logger.debug("Sending %s url to Deluge" % tor_title)
downloadID, res = deluge.addTorrent(
tor_url) # can be link or magnet, returns hash or False
if downloadID:
tor_title = deluge.getTorrentFolder(downloadID)
else:
return False, res
else:
# have username, talk to the daemon
Source = "DELUGERPC"
client = DelugeRPCClient(lazylibrarian.CONFIG['DELUGE_HOST'],
int(lazylibrarian.CONFIG['DELUGE_PORT']),
lazylibrarian.CONFIG['DELUGE_USER'],
lazylibrarian.CONFIG['DELUGE_PASS'])
try:
client.connect()
args = {"name": tor_title}
if tor_url.startswith('magnet'):
res = "Sending %s magnet to DelugeRPC" % tor_title
logger.debug(res)
downloadID = client.call('core.add_torrent_magnet',
tor_url, args)
elif torrent:
res = "Sending %s data to DelugeRPC" % tor_title
logger.debug(res)
downloadID = client.call('core.add_torrent_file',
tor_title, b64encode(torrent),
args)
else:
res = "Sending %s url to DelugeRPC" % tor_title
logger.debug(res)
downloadID = client.call('core.add_torrent_url', tor_url,
args)
if downloadID:
if lazylibrarian.CONFIG['DELUGE_LABEL']:
_ = client.call(
'label.set_torrent', downloadID,
lazylibrarian.CONFIG['DELUGE_LABEL'].lower())
result = client.call('core.get_torrent_status', downloadID,
{})
if 'name' in result:
tor_title = result['name']
else:
res += ' failed'
logger.error(res)
return False, res
except Exception as e:
res = 'DelugeRPC failed %s %s' % (type(e).__name__, str(e))
logger.error(res)
return False, res
if not Source:
res = 'No torrent download method is enabled, check config.'
logger.warn(res)
return False, res
if downloadID:
if tor_title:
if downloadID.upper() in tor_title.upper():
logger.warn(
'%s: name contains hash, probably unresolved magnet' %
Source)
else:
tor_title = unaccented_str(tor_title)
# need to check against reject words list again as the name may have changed
# library = magazine eBook AudioBook to determine which reject list
# but we can't easily do the per-magazine rejects
if library == 'Magazine':
reject_list = getList(lazylibrarian.CONFIG['REJECT_MAGS'],
',')
elif library == 'eBook':
reject_list = getList(lazylibrarian.CONFIG['REJECT_WORDS'],
',')
elif library == 'AudioBook':
reject_list = getList(lazylibrarian.CONFIG['REJECT_AUDIO'],
',')
else:
logger.debug("Invalid library [%s] in TORDownloadMethod" %
library)
reject_list = []
rejected = False
lower_title = tor_title.lower()
for word in reject_list:
if word in lower_title:
rejected = "Rejecting torrent name %s, contains %s" % (
tor_title, word)
logger.debug(rejected)
break
if not rejected:
rejected = check_contents(Source, downloadID, library,
tor_title)
if rejected:
myDB.action(
'UPDATE wanted SET status="Failed",DLResult=? WHERE NZBurl=?',
(rejected, full_url))
delete_task(Source, downloadID, True)
return False
else:
logger.debug('%s setting torrent name to [%s]' %
(Source, tor_title))
myDB.action('UPDATE wanted SET NZBtitle=? WHERE NZBurl=?',
(tor_title, full_url))
if library == 'eBook':
myDB.action('UPDATE books SET status="Snatched" WHERE BookID=?',
(bookid, ))
elif library == 'AudioBook':
myDB.action(
'UPDATE books SET audiostatus="Snatched" WHERE BookID=?',
(bookid, ))
myDB.action(
'UPDATE wanted SET status="Snatched", Source=?, DownloadID=? WHERE NZBurl=?',
(Source, downloadID, full_url))
return True, ''
res = 'Failed to send torrent to %s' % Source
logger.error(res)
return False, res
def DirectDownloadMethod(bookid=None,
dl_title=None,
dl_url=None,
library='eBook'):
myDB = database.DBConnection()
Source = "DIRECT"
logger.debug("Starting Direct Download for [%s]" % dl_title)
proxies = proxyList()
headers = {'Accept-encoding': 'gzip', 'User-Agent': getUserAgent()}
try:
r = requests.get(dl_url, headers=headers, timeout=90, proxies=proxies)
except requests.exceptions.Timeout:
res = 'Timeout fetching file from url: %s' % dl_url
logger.warn(res)
return False, res
except Exception as e:
if hasattr(e, 'reason'):
res = '%s fetching file from url: %s, %s' % (type(e).__name__,
dl_url, e.reason)
else:
res = '%s fetching file from url: %s, %s' % (type(e).__name__,
dl_url, str(e))
logger.warn(res)
return False, res
if not str(r.status_code).startswith('2'):
res = "Got a %s response for %s" % (r.status_code, dl_url)
logger.debug(res)
return False, res
elif len(r.content) < 1000:
res = "Only got %s bytes for %s" % (len(r.content), dl_title)
logger.debug(res)
return False, res
else:
extn = ''
basename = ''
if ' ' in dl_title:
basename, extn = dl_title.rsplit(
' ', 1) # last word is often the extension - but not always...
if extn and extn in getList(lazylibrarian.CONFIG['EBOOK_TYPE']):
dl_title = '.'.join(dl_title.rsplit(' ', 1))
elif magic:
mtype = magic.from_buffer(r.content)
if 'EPUB' in mtype:
extn = 'epub'
elif 'Mobipocket' in mtype: # also true for azw and azw3, does it matter?
extn = 'mobi'
elif 'PDF' in mtype:
extn = 'pdf'
else:
logger.debug("magic reports %s" % mtype)
basename = dl_title
else:
logger.warn("Don't know the filetype for %s" % dl_title)
basename = dl_title
logger.debug("File download got %s bytes for %s" %
(len(r.content), dl_title))
destdir = os.path.join(lazylibrarian.DIRECTORY('Download'), basename)
# destdir = os.path.join(lazylibrarian.DIRECTORY('Download'), '%s LL.(%s)' % (basename, bookid))
if not os.path.isdir(destdir):
_ = mymakedirs(destdir)
try:
hashid = dl_url.split("md5=")[1].split("&")[0]
except IndexError:
hashid = sha1(bencode(dl_url)).hexdigest()
destfile = os.path.join(destdir, basename + '.' + extn)
if os.name == 'nt': #Windows has max path length of 256
destfile = '\\\\?\\' + destfile
try:
with open(destfile, 'wb') as bookfile:
bookfile.write(r.content)
setperm(destfile)
downloadID = hashid
except Exception as e:
res = "%s writing book to %s, %s" % (type(e).__name__, destfile, e)
logger.error(res)
return False, res
if downloadID:
logger.debug('File %s has been downloaded from %s' %
(dl_title, dl_url))
if library == 'eBook':
myDB.action('UPDATE books SET status="Snatched" WHERE BookID=?',
(bookid, ))
elif library == 'AudioBook':
myDB.action(
'UPDATE books SET audiostatus="Snatched" WHERE BookID=?',
(bookid, ))
myDB.action(
'UPDATE wanted SET status="Snatched", Source=?, DownloadID=? WHERE NZBurl=?',
(Source, downloadID, dl_url))
return True, ''
else:
res = 'Failed to download file @ <a href="%s">%s</a>' % (dl_url,
dl_url)
logger.error(res)
return False, res
def update():
if lazylibrarian.CONFIG['INSTALL_TYPE'] == 'win':
logmsg('info', 'Windows .exe updating not supported yet.')
return False
elif lazylibrarian.CONFIG['INSTALL_TYPE'] == 'package':
logmsg('info', 'Please use your package manager to update')
return False
elif lazylibrarian.CONFIG['INSTALL_TYPE'] == 'git':
branch = getCurrentGitBranch()
_, _ = runGit('stash clear')
output, err = runGit('pull origin ' + branch)
if not output:
logmsg('error', 'Couldn\'t download latest version')
return False
for line in output.split('\n'):
if 'Already up-to-date.' in line:
logmsg('info', 'No update available: ' + str(output))
return False
elif 'Aborting' in line or 'local changes' in line:
logmsg('error', 'Unable to update: ' + str(output))
return False
# Update version.txt and timestamp
updateVersionFile(lazylibrarian.CONFIG['LATEST_VERSION'])
lazylibrarian.CONFIG['GIT_UPDATED'] = str(int(time.time()))
return True
elif lazylibrarian.CONFIG['INSTALL_TYPE'] == 'source':
if 'gitlab' in lazylibrarian.CONFIG['GIT_HOST']:
tar_download_url = 'https://%s/%s/%s/-/archive/%s/%s-%s.tar.gz' % (
lazylibrarian.CONFIG['GIT_HOST'],
lazylibrarian.CONFIG['GIT_USER'],
lazylibrarian.CONFIG['GIT_REPO'],
lazylibrarian.CONFIG['GIT_BRANCH'],
lazylibrarian.CONFIG['GIT_REPO'],
lazylibrarian.CONFIG['GIT_BRANCH'])
else:
tar_download_url = 'https://%s/%s/%s/tarball/%s' % (
lazylibrarian.CONFIG['GIT_HOST'],
lazylibrarian.CONFIG['GIT_USER'],
lazylibrarian.CONFIG['GIT_REPO'],
lazylibrarian.CONFIG['GIT_BRANCH'])
update_dir = os.path.join(lazylibrarian.PROG_DIR, 'update')
try:
logmsg('info', 'Downloading update from: ' + tar_download_url)
headers = {'User-Agent': getUserAgent()}
if 'gitlab' in lazylibrarian.CONFIG['GIT_HOST']:
headers['Private-Token'] = '_G8Shnw1-xEWsXPi8fB_'
proxies = proxyList()
timeout = check_int(lazylibrarian.CONFIG['HTTP_TIMEOUT'], 30)
r = requests.get(tar_download_url,
timeout=timeout,
headers=headers,
proxies=proxies)
except requests.exceptions.Timeout:
logmsg('error',
"Timeout retrieving new version from " + tar_download_url)
return False
except Exception as e:
if hasattr(e, 'reason'):
errmsg = e.reason
else:
errmsg = str(e)
logmsg(
'error', "Unable to retrieve new version from " +
tar_download_url + ", can't update: %s" % errmsg)
return False
download_name = r.url.split('/')[-1]
tar_download_path = os.path.join(lazylibrarian.PROG_DIR, download_name)
# Save tar to disk
with open(tar_download_path, 'wb') as f:
f.write(r.content)
# Extract the tar to update folder
logmsg('info', 'Extracting file ' + tar_download_path)
try:
with tarfile.open(tar_download_path) as tar:
tar.extractall(update_dir)
except Exception as e:
logger.error('Failed to unpack tarfile %s (%s): %s' %
(type(e).__name__, tar_download_path, str(e)))
return False
# Delete the tar.gz
logmsg('info', 'Deleting file ' + tar_download_path)
os.remove(tar_download_path)
# Find update dir name
update_dir_contents = [
x for x in os.listdir(update_dir)
if os.path.isdir(os.path.join(update_dir, x))
]
if len(update_dir_contents) != 1:
logmsg(
'error', "Invalid update data, update failed: " +
str(update_dir_contents))
return False
content_dir = os.path.join(update_dir, update_dir_contents[0])
# walk temp folder and move files to main folder
for rootdir, dirnames, filenames in os.walk(content_dir):
rootdir = rootdir[len(content_dir) + 1:]
for curfile in filenames:
old_path = os.path.join(content_dir, rootdir, curfile)
new_path = os.path.join(lazylibrarian.PROG_DIR, rootdir,
curfile)
if os.path.isfile(new_path):
os.remove(new_path)
os.renames(old_path, new_path)
# Update version.txt and timestamp
updateVersionFile(lazylibrarian.CONFIG['LATEST_VERSION'])
lazylibrarian.CONFIG['GIT_UPDATED'] = str(int(time.time()))
return True
else:
logmsg('error', "Cannot perform update - Install Type not set")
return False
def getCommitDifferenceFromGit():
# See how many commits behind we are
# Takes current latest version value and tries to diff it with the latest version in the current branch.
commit_list = ''
commits = -1
if lazylibrarian.CONFIG['LATEST_VERSION'] == 'Not_Available_From_Git':
commits = 0 # don't report a commit diff as we don't know anything
commit_list = 'Unable to get latest version from %s' % lazylibrarian.CONFIG[
'GIT_HOST']
logmsg('info', commit_list)
elif lazylibrarian.CONFIG['CURRENT_VERSION'] and commits != 0:
if 'gitlab' in lazylibrarian.CONFIG['GIT_HOST']:
url = 'https://%s/api/v4/projects/%s%%2F%s/repository/compare?from=%s&to=%s' % (
lazylibrarian.CONFIG['GIT_HOST'],
lazylibrarian.CONFIG['GIT_USER'],
lazylibrarian.CONFIG['GIT_REPO'],
lazylibrarian.CONFIG['CURRENT_VERSION'],
lazylibrarian.CONFIG['LATEST_VERSION'])
else:
url = 'https://api.%s/repos/%s/%s/compare/%s...%s' % (
lazylibrarian.CONFIG['GIT_HOST'],
lazylibrarian.CONFIG['GIT_USER'],
lazylibrarian.CONFIG['GIT_REPO'],
lazylibrarian.CONFIG['CURRENT_VERSION'],
lazylibrarian.CONFIG['LATEST_VERSION'])
logmsg('debug',
'Check for differences between local & repo by [%s]' % url)
try:
headers = {'User-Agent': getUserAgent()}
if 'gitlab' in lazylibrarian.CONFIG['GIT_HOST']:
headers['Private-Token'] = '_G8Shnw1-xEWsXPi8fB_'
proxies = proxyList()
timeout = check_int(lazylibrarian.CONFIG['HTTP_TIMEOUT'], 30)
r = requests.get(url,
timeout=timeout,
headers=headers,
proxies=proxies)
git = r.json()
# for gitlab, commits = len(git['commits']) no status/ahead/behind
if 'gitlab' in lazylibrarian.CONFIG['GIT_HOST']:
if 'commits' in git:
commits = len(git['commits'])
msg = 'Git: Total Commits [%s]' % commits
logmsg('debug', msg)
else:
logmsg(
'warn',
'Could not get difference status from git: %s' %
str(git))
if commits > 0:
for item in git['commits']:
commit_list = "%s\n%s" % (item['title'], commit_list)
else:
if 'total_commits' in git:
commits = int(git['total_commits'])
msg = 'Git: Status [%s] - Ahead [%s] - Behind [%s] - Total Commits [%s]' % (
git['status'], git['ahead_by'], git['behind_by'],
git['total_commits'])
logmsg('debug', msg)
else:
logmsg(
'warn',
'Could not get difference status from git: %s' %
str(git))
if commits > 0:
for item in git['commits']:
commit_list = "%s\n%s" % (item['commit']['message'],
commit_list)
except Exception as e:
logmsg(
'warn', 'Could not get difference status from git: %s' %
type(e).__name__)
if commits > 1:
logmsg('info',
'New version is available. You are %s commits behind' % commits)
elif commits == 1:
logmsg('info', 'New version is available. You are one commit behind')
elif commits == 0:
logmsg('info', 'Lazylibrarian is up to date')
else:
logmsg(
'info',
'Unknown version of lazylibrarian. Run the updater to identify your version'
)
return commits, commit_list
def getLatestVersion_FromGit():
# Don't call directly, use getLatestVersion as wrapper.
latest_version = 'Unknown'
# Can only work for non Windows driven installs, so check install type
if lazylibrarian.CONFIG['INSTALL_TYPE'] == 'win':
logmsg('debug', 'Error - should not be called under a windows install')
latest_version = 'WINDOWS INSTALL'
else:
# check current branch value of the local git repo as folks may pull from a branch not master
branch = lazylibrarian.CONFIG['GIT_BRANCH']
if branch == 'InvalidBranch':
logmsg('debug',
'Failed to get a valid branch name from local repo')
else:
if branch == 'Package': # check packages against master
branch = 'master'
# Get the latest commit available from git
if 'gitlab' in lazylibrarian.CONFIG['GIT_HOST']:
url = 'https://%s/api/v4/projects/%s%%2F%s/repository/branches/%s' % (
lazylibrarian.CONFIG['GIT_HOST'],
lazylibrarian.CONFIG['GIT_USER'],
lazylibrarian.CONFIG['GIT_REPO'], branch)
else:
url = 'https://api.%s/repos/%s/%s/commits/%s' % (
lazylibrarian.CONFIG['GIT_HOST'],
lazylibrarian.CONFIG['GIT_USER'],
lazylibrarian.CONFIG['GIT_REPO'], branch)
logmsg(
'debug',
'Retrieving latest version information from git command=[%s]' %
url)
timestamp = check_int(lazylibrarian.CONFIG['GIT_UPDATED'], 0)
age = ''
if timestamp:
# timestring for 'If-Modified-Since' needs to be english short day/month names and in gmt
# we already have english month names stored in MONTHNAMES[] but need capitalising
# so use hard coded versions here instead
DAYNAMES = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
MONNAMES = [
'', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug',
'Sep', 'Oct', 'Nov', 'Dec'
]
tm = time.gmtime(timestamp)
age = "%s, %02d %s %04d %02d:%02d:%02d GMT" % (
DAYNAMES[tm.tm_wday], tm.tm_mday, MONNAMES[tm.tm_mon],
tm.tm_year, tm.tm_hour, tm.tm_min, tm.tm_sec)
try:
headers = {'User-Agent': getUserAgent()}
if 'gitlab' in lazylibrarian.CONFIG['GIT_HOST']:
headers['Private-Token'] = '_G8Shnw1-xEWsXPi8fB_'
if age:
logmsg('debug', 'Checking if modified since %s' % age)
headers.update({'If-Modified-Since': age})
proxies = proxyList()
timeout = check_int(lazylibrarian.CONFIG['HTTP_TIMEOUT'], 30)
r = requests.get(url,
timeout=timeout,
headers=headers,
proxies=proxies)
if str(r.status_code).startswith('2'):
git = r.json()
if 'gitlab' in lazylibrarian.CONFIG['GIT_HOST']:
latest_version = git['commit']['id']
else:
latest_version = git['sha']
logmsg(
'debug',
'Branch [%s] Latest Version has been set to [%s]' %
(branch, latest_version))
elif str(r.status_code) == '304':
latest_version = lazylibrarian.CONFIG['CURRENT_VERSION']
logmsg('debug',
'Not modified, currently on Latest Version')
else:
logmsg('warn', 'Could not get the latest commit from git')
logmsg('debug',
'git latest version returned %s' % r.status_code)
latest_version = 'Not_Available_From_Git'
except Exception as e:
logmsg('warn', 'Could not get the latest commit from git')
logmsg('debug',
'git %s for %s: %s' % (type(e).__name__, url, str(e)))
latest_version = 'Not_Available_From_Git'
return latest_version
def fetchURL(URL, headers=None, retry=True, raw=None):
""" Return the result of fetching a URL and True if success
Otherwise return error message and False
Return data as raw/bytes in python2 or if raw == True
On python3 default to unicode, need to set raw=True for images/data
Allow one retry on timeout by default"""
if 'googleapis' in URL:
lazylibrarian.GB_CALLS += 1
for entry in lazylibrarian.PROVIDER_BLOCKLIST:
if entry["name"] == 'googleapis':
if int(time.time()) < int(entry['resume']):
return "Blocked", False
else:
lazylibrarian.PROVIDER_BLOCKLIST.remove(entry)
lazylibrarian.GB_CALLS = 0
if raw is None:
if PY2:
raw = True
else:
raw = False
if headers is None:
# some sites insist on having a user-agent, default is to add one
# if you don't want any headers, send headers=[]
headers = {'User-Agent': getUserAgent()}
proxies = proxyList()
try:
# jackett query all indexers needs a longer timeout
# /torznab/all/api?q= or v2.0/indexers/all/results/torznab/api?q=
if '/torznab/' in URL and ('/all/' in URL or '/aggregate/' in URL):
timeout = check_int(lazylibrarian.CONFIG['HTTP_EXT_TIMEOUT'], 90)
else:
timeout = check_int(lazylibrarian.CONFIG['HTTP_TIMEOUT'], 30)
r = requests.get(URL, headers=headers, timeout=timeout, proxies=proxies)
if str(r.status_code).startswith('2'): # (200 OK etc)
if raw:
return r.content, True
try:
result = r.content.decode('utf-8')
except UnicodeDecodeError:
result = r.content.decode('latin-1')
return result, True
elif r.status_code == 403 and 'googleapis' in URL:
msg = makeUnicode(r.content)
logger.debug(msg)
# noinspection PyBroadException
try:
source = json.loads(msg)
msg = source['error']['message']
except Exception:
pass
if 'Limit Exceeded' in msg:
# how long until midnight Pacific Time when google reset the quotas
delay = seconds_to_midnight() + 28800 # PT is 8hrs behind UTC
if delay > 86400:
delay -= 86400 # no roll-over to next day
else:
# might be forbidden for a different reason where midnight might not matter
# eg "Cannot determine user location for geographically restricted operation"
delay = 3600
for entry in lazylibrarian.PROVIDER_BLOCKLIST:
if entry["name"] == 'googleapis':
lazylibrarian.PROVIDER_BLOCKLIST.remove(entry)
newentry = {"name": 'googleapis', "resume": int(time.time()) + delay, "reason": msg}
lazylibrarian.PROVIDER_BLOCKLIST.append(newentry)
# noinspection PyBroadException
try:
# noinspection PyProtectedMember
msg = requests.status_codes._codes[r.status_code][0]
except Exception:
msg = str(r.content)
return "Response status %s: %s" % (r.status_code, msg), False
except requests.exceptions.Timeout as e:
if not retry:
logger.error("fetchURL: Timeout getting response from %s" % URL)
return "Timeout %s" % str(e), False
logger.debug("fetchURL: retrying - got timeout on %s" % URL)
result, success = fetchURL(URL, headers=headers, retry=False, raw=False)
return result, success
except Exception as e:
if hasattr(e, 'reason'):
return "Exception %s: Reason: %s" % (type(e).__name__, str(e.reason)), False
return "Exception %s: %s" % (type(e).__name__, str(e)), False
def update():
if lazylibrarian.CONFIG['INSTALL_TYPE'] == 'win':
logmsg('info', 'Windows .exe updating not supported yet.')
return False
elif lazylibrarian.CONFIG['INSTALL_TYPE'] == 'package':
logmsg('info', 'Please use your package manager to update')
return False
elif lazylibrarian.CONFIG['INSTALL_TYPE'] == 'git':
branch = getCurrentGitBranch()
_, _ = runGit('stash clear')
output, err = runGit('pull origin ' + branch)
if not output:
logmsg('error', 'Couldn\'t download latest version')
return False
for line in output.split('\n'):
if 'Already up-to-date.' in line:
logmsg('info', 'No update available: ' + str(output))
return False
elif 'Aborting' in line or 'local changes' in line:
logmsg('error', 'Unable to update: ' + str(output))
return False
# Update version.txt and timestamp
updateVersionFile(lazylibrarian.CONFIG['LATEST_VERSION'])
lazylibrarian.CONFIG['GIT_UPDATED'] = str(int(time.time()))
return True
elif lazylibrarian.CONFIG['INSTALL_TYPE'] == 'source':
if 'gitlab' in lazylibrarian.CONFIG['GIT_HOST']:
tar_download_url = 'https://%s/%s/%s/-/archive/%s/%s-%s.tar.gz' % (
lazylibrarian.GITLAB_TOKEN, lazylibrarian.CONFIG['GIT_USER'],
lazylibrarian.CONFIG['GIT_REPO'], lazylibrarian.CONFIG['GIT_BRANCH'],
lazylibrarian.CONFIG['GIT_REPO'], lazylibrarian.CONFIG['GIT_BRANCH'])
else:
tar_download_url = 'https://%s/%s/%s/tarball/%s' % (
lazylibrarian.CONFIG['GIT_HOST'], lazylibrarian.CONFIG['GIT_USER'],
lazylibrarian.CONFIG['GIT_REPO'], lazylibrarian.CONFIG['GIT_BRANCH'])
update_dir = os.path.join(lazylibrarian.PROG_DIR, 'update')
try:
logmsg('info', 'Downloading update from: ' + tar_download_url)
headers = {'User-Agent': getUserAgent()}
proxies = proxyList()
timeout = check_int(lazylibrarian.CONFIG['HTTP_TIMEOUT'], 30)
r = requests.get(tar_download_url, timeout=timeout, headers=headers, proxies=proxies)
except requests.exceptions.Timeout:
logmsg('error', "Timeout retrieving new version from " + tar_download_url)
return False
except Exception as e:
if hasattr(e, 'reason'):
errmsg = e.reason
else:
errmsg = str(e)
logmsg('error',
"Unable to retrieve new version from " + tar_download_url + ", can't update: %s" % errmsg)
return False
download_name = r.url.split('/')[-1]
tar_download_path = os.path.join(lazylibrarian.PROG_DIR, download_name)
# Save tar to disk
with open(tar_download_path, 'wb') as f:
f.write(r.content)
# Extract the tar to update folder
logmsg('info', 'Extracting file ' + tar_download_path)
try:
with tarfile.open(tar_download_path) as tar:
tar.extractall(update_dir)
except Exception as e:
logger.error('Failed to unpack tarfile %s (%s): %s' % (type(e).__name__, tar_download_path, str(e)))
return False
# Delete the tar.gz
logmsg('info', 'Deleting file ' + tar_download_path)
os.remove(tar_download_path)
# Find update dir name
update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))]
if len(update_dir_contents) != 1:
logmsg('error', "Invalid update data, update failed: " + str(update_dir_contents))
return False
content_dir = os.path.join(update_dir, update_dir_contents[0])
# walk temp folder and move files to main folder
for rootdir, dirnames, filenames in os.walk(content_dir):
rootdir = rootdir[len(content_dir) + 1:]
for curfile in filenames:
old_path = os.path.join(content_dir, rootdir, curfile)
new_path = os.path.join(lazylibrarian.PROG_DIR, rootdir, curfile)
if os.path.isfile(new_path):
os.remove(new_path)
os.renames(old_path, new_path)
# Update version.txt and timestamp
updateVersionFile(lazylibrarian.CONFIG['LATEST_VERSION'])
lazylibrarian.CONFIG['GIT_UPDATED'] = str(int(time.time()))
return True
else:
logmsg('error', "Cannot perform update - Install Type not set")
return False
def getCommitDifferenceFromGit():
# See how many commits behind we are
# Takes current latest version value and tries to diff it with the latest version in the current branch.
commit_list = ''
commits = -1
if lazylibrarian.CONFIG['LATEST_VERSION'] == 'Not_Available_From_Git':
commits = 0 # don't report a commit diff as we don't know anything
commit_list = 'Unable to get latest version from %s' % lazylibrarian.CONFIG['GIT_HOST']
logmsg('info', commit_list)
elif lazylibrarian.CONFIG['CURRENT_VERSION'] and commits != 0:
if 'gitlab' in lazylibrarian.CONFIG['GIT_HOST']:
url = 'https://%s/api/v4/projects/%s%%2F%s/repository/compare?from=%s&to=%s' % (
lazylibrarian.GITLAB_TOKEN, lazylibrarian.CONFIG['GIT_USER'],
lazylibrarian.CONFIG['GIT_REPO'], lazylibrarian.CONFIG['CURRENT_VERSION'],
lazylibrarian.CONFIG['LATEST_VERSION'])
else:
url = 'https://api.%s/repos/%s/%s/compare/%s...%s' % (
lazylibrarian.CONFIG['GIT_HOST'], lazylibrarian.CONFIG['GIT_USER'],
lazylibrarian.CONFIG['GIT_REPO'], lazylibrarian.CONFIG['CURRENT_VERSION'],
lazylibrarian.CONFIG['LATEST_VERSION'])
logmsg('debug', 'Check for differences between local & repo by [%s]' % url)
try:
headers = {'User-Agent': getUserAgent()}
proxies = proxyList()
timeout = check_int(lazylibrarian.CONFIG['HTTP_TIMEOUT'], 30)
r = requests.get(url, timeout=timeout, headers=headers, proxies=proxies)
git = r.json()
# for gitlab, commits = len(git['commits']) no status/ahead/behind
if 'gitlab' in lazylibrarian.CONFIG['GIT_HOST']:
if 'commits' in git:
commits = len(git['commits'])
msg = 'Git: Total Commits [%s]' % commits
logmsg('debug', msg)
else:
logmsg('warn', 'Could not get difference status from git: %s' % str(git))
if commits > 0:
for item in git['commits']:
commit_list = "%s\n%s" % (item['title'], commit_list)
else:
if 'total_commits' in git:
commits = int(git['total_commits'])
msg = 'Git: Status [%s] - Ahead [%s] - Behind [%s] - Total Commits [%s]' % (
git['status'], git['ahead_by'], git['behind_by'], git['total_commits'])
logmsg('debug', msg)
else:
logmsg('warn', 'Could not get difference status from git: %s' % str(git))
if commits > 0:
for item in git['commits']:
commit_list = "%s\n%s" % (item['commit']['message'], commit_list)
except Exception as e:
logmsg('warn', 'Could not get difference status from git: %s' % type(e).__name__)
if commits > 1:
logmsg('info', 'New version is available. You are %s commits behind' % commits)
elif commits == 1:
logmsg('info', 'New version is available. You are one commit behind')
elif commits == 0:
logmsg('info', 'Lazylibrarian is up to date')
else:
logmsg('info', 'Unknown version of lazylibrarian. Run the updater to identify your version')
return commits, commit_list
def getLatestVersion_FromGit():
# Don't call directly, use getLatestVersion as wrapper.
latest_version = 'Unknown'
# Can only work for non Windows driven installs, so check install type
if lazylibrarian.CONFIG['INSTALL_TYPE'] == 'win':
logmsg('debug', 'Error - should not be called under a windows install')
latest_version = 'WINDOWS INSTALL'
else:
# check current branch value of the local git repo as folks may pull from a branch not master
branch = lazylibrarian.CONFIG['GIT_BRANCH']
if branch == 'InvalidBranch':
logmsg('debug', 'Failed to get a valid branch name from local repo')
else:
if branch == 'Package': # check packages against master
branch = 'master'
# Get the latest commit available from git
if 'gitlab' in lazylibrarian.CONFIG['GIT_HOST']:
url = 'https://%s/api/v4/projects/%s%%2F%s/repository/branches/%s' % (
lazylibrarian.GITLAB_TOKEN, lazylibrarian.CONFIG['GIT_USER'],
lazylibrarian.CONFIG['GIT_REPO'], branch)
else:
url = 'https://api.%s/repos/%s/%s/commits/%s' % (
lazylibrarian.CONFIG['GIT_HOST'], lazylibrarian.CONFIG['GIT_USER'],
lazylibrarian.CONFIG['GIT_REPO'], branch)
logmsg('debug',
'Retrieving latest version information from git command=[%s]' % url)
timestamp = check_int(lazylibrarian.CONFIG['GIT_UPDATED'], 0)
age = ''
if timestamp:
# timestring for 'If-Modified-Since' needs to be english short day/month names and in gmt
# we already have english month names stored in MONTHNAMES[] but need capitalising
# so use hard coded versions here instead
DAYNAMES = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
MONNAMES = ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
tm = time.gmtime(timestamp)
age = "%s, %02d %s %04d %02d:%02d:%02d GMT" % (DAYNAMES[tm.tm_wday], tm.tm_mday,
MONNAMES[tm.tm_mon], tm.tm_year, tm.tm_hour,
tm.tm_min, tm.tm_sec)
try:
headers = {'User-Agent': getUserAgent()}
if age:
logmsg('debug', 'Checking if modified since %s' % age)
headers.update({'If-Modified-Since': age})
proxies = proxyList()
timeout = check_int(lazylibrarian.CONFIG['HTTP_TIMEOUT'], 30)
r = requests.get(url, timeout=timeout, headers=headers, proxies=proxies)
if str(r.status_code).startswith('2'):
git = r.json()
if 'gitlab' in lazylibrarian.CONFIG['GIT_HOST']:
latest_version = git['commit']['id']
else:
latest_version = git['sha']
logmsg('debug', 'Branch [%s] Latest Version has been set to [%s]' % (
branch, latest_version))
elif str(r.status_code) == '304':
latest_version = lazylibrarian.CONFIG['CURRENT_VERSION']
logmsg('debug', 'Not modified, currently on Latest Version')
else:
logmsg('warn', 'Could not get the latest commit from git')
logmsg('debug', 'git latest version returned %s' % r.status_code)
latest_version = 'Not_Available_From_Git'
except Exception as e:
logmsg('warn', 'Could not get the latest commit from git')
logmsg('debug', 'git %s for %s: %s' % (type(e).__name__, url, str(e)))
latest_version = 'Not_Available_From_Git'
return latest_version
def TORDownloadMethod(bookid=None, tor_title=None, tor_url=None, library='eBook'):
myDB = database.DBConnection()
downloadID = False
Source = ''
torrent = ''
full_url = tor_url # keep the url as stored in "wanted" table
if 'magnet:?' in tor_url:
# discard any other parameters and just use the magnet link
tor_url = 'magnet:?' + tor_url.split('magnet:?')[1]
else:
# h = HTMLParser()
# tor_url = h.unescape(tor_url)
# HTMLParser is probably overkill, we only seem to get &
#
tor_url = tor_url.replace('&', '&')
if '&file=' in tor_url:
# torznab results need to be re-encoded
# had a problem with torznab utf-8 encoded strings not matching
# our utf-8 strings because of long/short form differences
url, value = tor_url.split('&file=', 1)
value = makeUnicode(value) # ensure unicode
value = unicodedata.normalize('NFC', value) # normalize to short form
value = value.encode('unicode-escape') # then escape the result
value = makeUnicode(value) # ensure unicode
value = value.replace(' ', '%20') # and encode any spaces
tor_url = url + '&file=' + value
# strip url back to the .torrent as some sites add extra parameters
if not tor_url.endswith('.torrent') and '.torrent' in tor_url:
tor_url = tor_url.split('.torrent')[0] + '.torrent'
headers = {'Accept-encoding': 'gzip', 'User-Agent': getUserAgent()}
proxies = proxyList()
try:
logger.debug("Fetching %s" % tor_url)
r = requests.get(tor_url, headers=headers, timeout=90, proxies=proxies)
if str(r.status_code).startswith('2'):
torrent = r.content
if not len(torrent):
res = "Got empty response for %s" % tor_url
logger.warn(res)
return False, res
elif len(torrent) < 100:
res = "Only got %s bytes for %s" % (len(torrent), tor_url)
logger.warn(res)
return False, res
else:
logger.debug("Got %s bytes for %s" % (len(torrent), tor_url))
else:
res = "Got a %s response for %s" % (r.status_code, tor_url)
logger.warn(res)
return False, res
except requests.exceptions.Timeout:
res = 'Timeout fetching file from url: %s' % tor_url
logger.warn(res)
return False, res
except Exception as e:
# some jackett providers redirect internally using http 301 to a magnet link
# which requests can't handle, so throws an exception
logger.debug("Requests exception: %s" % str(e))
if "magnet:?" in str(e):
tor_url = 'magnet:?' + str(e).split('magnet:?')[1].strip("'")
logger.debug("Redirecting to %s" % tor_url)
else:
if hasattr(e, 'reason'):
res = '%s fetching file from url: %s, %s' % (type(e).__name__, tor_url, e.reason)
else:
res = '%s fetching file from url: %s, %s' % (type(e).__name__, tor_url, str(e))
logger.warn(res)
return False, res
if not torrent and not tor_url.startswith('magnet:?'):
res = "No magnet or data, cannot continue"
logger.warn(res)
return False, res
if lazylibrarian.CONFIG['TOR_DOWNLOADER_BLACKHOLE']:
Source = "BLACKHOLE"
logger.debug("Sending %s to blackhole" % tor_title)
tor_name = cleanName(tor_title).replace(' ', '_')
if tor_url and tor_url.startswith('magnet'):
if lazylibrarian.CONFIG['TOR_CONVERT_MAGNET']:
hashid = calculate_torrent_hash(tor_url)
if not hashid:
hashid = tor_name
tor_name = 'meta-' + hashid + '.torrent'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name)
result = magnet2torrent(tor_url, tor_path)
if result is not False:
logger.debug('Magnet file saved as: %s' % tor_path)
downloadID = Source
else:
tor_name += '.magnet'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name)
msg = ''
try:
msg = 'Opening '
with open(tor_path, 'wb') as torrent_file:
msg += 'Writing '
if isinstance(torrent, text_type):
torrent = torrent.encode('iso-8859-1')
torrent_file.write(torrent)
msg += 'SettingPerm '
setperm(tor_path)
msg += 'Saved '
logger.debug('Magnet file saved: %s' % tor_path)
downloadID = Source
except Exception as e:
res = "Failed to write magnet to file: %s %s" % (type(e).__name__, str(e))
logger.warn(res)
logger.debug("Progress: %s Filename [%s]" % (msg, repr(tor_path)))
return False, res
else:
tor_name += '.torrent'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name)
msg = ''
try:
msg = 'Opening '
with open(tor_path, 'wb') as torrent_file:
msg += 'Writing '
if isinstance(torrent, text_type):
torrent = torrent.encode('iso-8859-1')
torrent_file.write(torrent)
msg += 'SettingPerm '
setperm(tor_path)
msg += 'Saved '
logger.debug('Torrent file saved: %s' % tor_name)
downloadID = Source
except Exception as e:
res = "Failed to write torrent to file: %s %s" % (type(e).__name__, str(e))
logger.warn(res)
logger.debug("Progress: %s Filename [%s]" % (msg, repr(tor_path)))
return False, res
hashid = calculate_torrent_hash(tor_url, torrent)
if not hashid:
res = "Unable to calculate torrent hash from url/data"
logger.error(res)
logger.debug("url: %s" % tor_url)
logger.debug("data: %s" % makeUnicode(str(torrent[:50])))
return False, res
if lazylibrarian.CONFIG['TOR_DOWNLOADER_UTORRENT'] and lazylibrarian.CONFIG['UTORRENT_HOST']:
logger.debug("Sending %s to Utorrent" % tor_title)
Source = "UTORRENT"
downloadID, res = utorrent.addTorrent(tor_url, hashid) # returns hash or False
if downloadID:
tor_title = utorrent.nameTorrent(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_RTORRENT'] and lazylibrarian.CONFIG['RTORRENT_HOST']:
logger.debug("Sending %s to rTorrent" % tor_title)
Source = "RTORRENT"
if torrent:
logger.debug("Sending %s data to rTorrent" % tor_title)
downloadID, res = rtorrent.addTorrent(tor_title, hashid, data=torrent)
else:
logger.debug("Sending %s url to rTorrent" % tor_title)
downloadID, res = rtorrent.addTorrent(tor_url, hashid) # returns hash or False
if downloadID:
tor_title = rtorrent.getName(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_QBITTORRENT'] and lazylibrarian.CONFIG['QBITTORRENT_HOST']:
Source = "QBITTORRENT"
if torrent:
logger.debug("Sending %s data to qBittorrent" % tor_title)
status, res = qbittorrent.addFile(torrent, hashid, tor_title)
else:
logger.debug("Sending %s url to qBittorrent" % tor_title)
status, res = qbittorrent.addTorrent(tor_url, hashid) # returns True or False
if status:
downloadID = hashid
tor_title = qbittorrent.getName(hashid)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_TRANSMISSION'] and lazylibrarian.CONFIG['TRANSMISSION_HOST']:
Source = "TRANSMISSION"
if torrent:
logger.debug("Sending %s data to Transmission" % tor_title)
# transmission needs b64encoded metainfo to be unicode, not bytes
downloadID, res = transmission.addTorrent(None, metainfo=makeUnicode(b64encode(torrent)))
else:
logger.debug("Sending %s url to Transmission" % tor_title)
downloadID, res = transmission.addTorrent(tor_url) # returns id or False
if downloadID:
# transmission returns it's own int, but we store hashid instead
downloadID = hashid
tor_title = transmission.getTorrentFolder(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_SYNOLOGY'] and lazylibrarian.CONFIG['USE_SYNOLOGY'] and \
lazylibrarian.CONFIG['SYNOLOGY_HOST']:
logger.debug("Sending %s url to Synology" % tor_title)
Source = "SYNOLOGY_TOR"
downloadID, res = synology.addTorrent(tor_url) # returns id or False
if downloadID:
tor_title = synology.getName(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_DELUGE'] and lazylibrarian.CONFIG['DELUGE_HOST']:
if not lazylibrarian.CONFIG['DELUGE_USER']:
# no username, talk to the webui
Source = "DELUGEWEBUI"
if torrent:
logger.debug("Sending %s data to Deluge" % tor_title)
downloadID, res = deluge.addTorrent(tor_title, data=b64encode(torrent))
else:
logger.debug("Sending %s url to Deluge" % tor_title)
downloadID, res = deluge.addTorrent(tor_url) # can be link or magnet, returns hash or False
if downloadID:
tor_title = deluge.getTorrentFolder(downloadID)
else:
return False, res
else:
# have username, talk to the daemon
Source = "DELUGERPC"
client = DelugeRPCClient(lazylibrarian.CONFIG['DELUGE_HOST'],
int(lazylibrarian.CONFIG['DELUGE_PORT']),
lazylibrarian.CONFIG['DELUGE_USER'],
lazylibrarian.CONFIG['DELUGE_PASS'])
try:
client.connect()
args = {"name": tor_title}
if tor_url.startswith('magnet'):
res = "Sending %s magnet to DelugeRPC" % tor_title
logger.debug(res)
downloadID = client.call('core.add_torrent_magnet', tor_url, args)
elif torrent:
res = "Sending %s data to DelugeRPC" % tor_title
logger.debug(res)
downloadID = client.call('core.add_torrent_file', tor_title, b64encode(torrent), args)
else:
res = "Sending %s url to DelugeRPC" % tor_title
logger.debug(res)
downloadID = client.call('core.add_torrent_url', tor_url, args)
if downloadID:
if lazylibrarian.CONFIG['DELUGE_LABEL']:
_ = client.call('label.set_torrent', downloadID, lazylibrarian.CONFIG['DELUGE_LABEL'].lower())
result = client.call('core.get_torrent_status', downloadID, {})
if 'name' in result:
tor_title = result['name']
else:
res += ' failed'
logger.error(res)
return False, res
except Exception as e:
res = 'DelugeRPC failed %s %s' % (type(e).__name__, str(e))
logger.error(res)
return False, res
if not Source:
res = 'No torrent download method is enabled, check config.'
logger.warn(res)
return False, res
if downloadID:
if tor_title:
if downloadID.upper() in tor_title.upper():
logger.warn('%s: name contains hash, probably unresolved magnet' % Source)
else:
tor_title = unaccented_str(tor_title)
# need to check against reject words list again as the name may have changed
# library = magazine eBook AudioBook to determine which reject list
# but we can't easily do the per-magazine rejects
if library == 'Magazine':
reject_list = getList(lazylibrarian.CONFIG['REJECT_MAGS'], ',')
elif library == 'eBook':
reject_list = getList(lazylibrarian.CONFIG['REJECT_WORDS'], ',')
elif library == 'AudioBook':
reject_list = getList(lazylibrarian.CONFIG['REJECT_AUDIO'], ',')
else:
logger.debug("Invalid library [%s] in TORDownloadMethod" % library)
reject_list = []
rejected = False
lower_title = tor_title.lower()
for word in reject_list:
if word in lower_title:
rejected = "Rejecting torrent name %s, contains %s" % (tor_title, word)
logger.debug(rejected)
break
if not rejected:
rejected = check_contents(Source, downloadID, library, tor_title)
if rejected:
myDB.action('UPDATE wanted SET status="Failed",DLResult=? WHERE NZBurl=?',
(rejected, full_url))
delete_task(Source, downloadID, True)
return False
else:
logger.debug('%s setting torrent name to [%s]' % (Source, tor_title))
myDB.action('UPDATE wanted SET NZBtitle=? WHERE NZBurl=?', (tor_title, full_url))
if library == 'eBook':
myDB.action('UPDATE books SET status="Snatched" WHERE BookID=?', (bookid,))
elif library == 'AudioBook':
myDB.action('UPDATE books SET audiostatus="Snatched" WHERE BookID=?', (bookid,))
myDB.action('UPDATE wanted SET status="Snatched", Source=?, DownloadID=? WHERE NZBurl=?',
(Source, downloadID, full_url))
return True, ''
res = 'Failed to send torrent to %s' % Source
logger.error(res)
return False, res
def DirectDownloadMethod(bookid=None, dl_title=None, dl_url=None, library='eBook'):
myDB = database.DBConnection()
Source = "DIRECT"
logger.debug("Starting Direct Download for [%s]" % dl_title)
proxies = proxyList()
headers = {'Accept-encoding': 'gzip', 'User-Agent': getUserAgent()}
try:
r = requests.get(dl_url, headers=headers, timeout=90, proxies=proxies)
except requests.exceptions.Timeout:
res = 'Timeout fetching file from url: %s' % dl_url
logger.warn(res)
return False, res
except Exception as e:
if hasattr(e, 'reason'):
res = '%s fetching file from url: %s, %s' % (type(e).__name__, dl_url, e.reason)
else:
res = '%s fetching file from url: %s, %s' % (type(e).__name__, dl_url, str(e))
logger.warn(res)
return False, res
if not str(r.status_code).startswith('2'):
res = "Got a %s response for %s" % (r.status_code, dl_url)
logger.debug(res)
return False, res
elif len(r.content) < 1000:
res = "Only got %s bytes for %s" % (len(r.content), dl_title)
logger.debug(res)
return False, res
else:
extn = ''
basename = ''
if ' ' in dl_title:
basename, extn = dl_title.rsplit(' ', 1) # last word is often the extension - but not always...
if extn and extn in getList(lazylibrarian.CONFIG['EBOOK_TYPE']):
dl_title = '.'.join(dl_title.rsplit(' ', 1))
elif magic:
mtype = magic.from_buffer(r.content)
if 'EPUB' in mtype:
extn = 'epub'
elif 'Mobipocket' in mtype: # also true for azw and azw3, does it matter?
extn = 'mobi'
elif 'PDF' in mtype:
extn = 'pdf'
else:
logger.debug("magic reports %s" % mtype)
basename = dl_title
else:
logger.warn("Don't know the filetype for %s" % dl_title)
basename = dl_title
logger.debug("File download got %s bytes for %s" % (len(r.content), dl_title))
destdir = os.path.join(lazylibrarian.DIRECTORY('Download'), basename)
# destdir = os.path.join(lazylibrarian.DIRECTORY('Download'), '%s LL.(%s)' % (basename, bookid))
if not os.path.isdir(destdir):
_ = mymakedirs(destdir)
try:
hashid = dl_url.split("md5=")[1].split("&")[0]
except IndexError:
hashid = sha1(bencode(dl_url)).hexdigest()
destfile = os.path.join(destdir, basename + '.' + extn)
if os.name == 'nt': #Windows has max path length of 256
destfile = '\\\\?\\' + destfile
try:
with open(destfile, 'wb') as bookfile:
bookfile.write(r.content)
setperm(destfile)
downloadID = hashid
except Exception as e:
res = "%s writing book to %s, %s" % (type(e).__name__, destfile, e)
logger.error(res)
return False, res
if downloadID:
logger.debug('File %s has been downloaded from %s' % (dl_title, dl_url))
if library == 'eBook':
myDB.action('UPDATE books SET status="Snatched" WHERE BookID=?', (bookid,))
elif library == 'AudioBook':
myDB.action('UPDATE books SET audiostatus="Snatched" WHERE BookID=?', (bookid,))
myDB.action('UPDATE wanted SET status="Snatched", Source=?, DownloadID=? WHERE NZBurl=?',
(Source, downloadID, dl_url))
return True, ''
else:
res = 'Failed to download file @ <a href="%s">%s</a>' % (dl_url, dl_url)
logger.error(res)
return False, res
def fetchURL(URL, headers=None, retry=True, raw=None):
""" Return the result of fetching a URL and True if success
Otherwise return error message and False
Return data as raw/bytes in python2 or if raw == True
On python3 default to unicode, need to set raw=True for images/data
Allow one retry on timeout by default"""
if 'googleapis' in URL:
lazylibrarian.GB_CALLS += 1
for entry in lazylibrarian.PROVIDER_BLOCKLIST:
if entry["name"] == 'googleapis':
if int(time.time()) < int(entry['resume']):
return "Blocked", False
else:
lazylibrarian.PROVIDER_BLOCKLIST.remove(entry)
lazylibrarian.GB_CALLS = 0
if raw is None:
if PY2:
raw = True
else:
raw = False
if headers is None:
# some sites insist on having a user-agent, default is to add one
# if you don't want any headers, send headers=[]
headers = {'User-Agent': getUserAgent()}
proxies = proxyList()
try:
# jackett query all indexers needs a longer timeout
# /torznab/all/api?q= or v2.0/indexers/all/results/torznab/api?q=
if '/torznab/' in URL and ('/all/' in URL or '/aggregate/' in URL):
timeout = check_int(lazylibrarian.CONFIG['HTTP_EXT_TIMEOUT'], 90)
else:
timeout = check_int(lazylibrarian.CONFIG['HTTP_TIMEOUT'], 30)
r = requests.get(URL,
headers=headers,
timeout=timeout,
proxies=proxies)
if str(r.status_code).startswith('2'): # (200 OK etc)
if raw:
return r.content, True
try:
result = r.content.decode('utf-8')
except UnicodeDecodeError:
result = r.content.decode('latin-1')
return result, True
elif r.status_code == 403 and 'googleapis' in URL:
msg = makeUnicode(r.content)
logger.debug(msg)
# noinspection PyBroadException
try:
source = json.loads(msg)
msg = source['error']['message']
except Exception:
pass
if 'Limit Exceeded' in msg:
# how long until midnight Pacific Time when google reset the quotas
delay = seconds_to_midnight() + 28800 # PT is 8hrs behind UTC
if delay > 86400:
delay -= 86400 # no roll-over to next day
else:
# might be forbidden for a different reason where midnight might not matter
# eg "Cannot determine user location for geographically restricted operation"
delay = 3600
for entry in lazylibrarian.PROVIDER_BLOCKLIST:
if entry["name"] == 'googleapis':
lazylibrarian.PROVIDER_BLOCKLIST.remove(entry)
newentry = {
"name": 'googleapis',
"resume": int(time.time()) + delay,
"reason": msg
}
lazylibrarian.PROVIDER_BLOCKLIST.append(newentry)
# noinspection PyBroadException
try:
# noinspection PyProtectedMember
msg = requests.status_codes._codes[r.status_code][0]
except Exception:
msg = str(r.content)
return "Response status %s: %s" % (r.status_code, msg), False
except requests.exceptions.Timeout as e:
if not retry:
logger.error("fetchURL: Timeout getting response from %s" % URL)
return "Timeout %s" % str(e), False
logger.debug("fetchURL: retrying - got timeout on %s" % URL)
result, success = fetchURL(URL,
headers=headers,
retry=False,
raw=False)
return result, success
except Exception as e:
if hasattr(e, 'reason'):
return "Exception %s: Reason: %s" % (type(e).__name__, str(
e.reason)), False
return "Exception %s: %s" % (type(e).__name__, str(e)), False