def get_torrents_status(stalled_for=None, progress={}):
''' Get torrents and calculate status
Returns list
'''
conf = core.CONFIG['Downloader']['Torrent']['DelugeRPC']
logging.info('Get torrents from DelugeRPC')
host = conf['host']
port = conf['port']
user = conf['user']
password = conf['pass']
client = DelugeRPCClient(host, port, user, password)
try:
client.connect()
torrents = []
now = int(datetime.timestamp(datetime.now()))
fields = [
'hash', 'state', 'name', 'last_seen_complete',
'time_since_download', 'total_payload_download'
]
for id, torrent in client.call('core.get_torrents_status', {
'id': list(progress.keys())
}, fields).items():
# deluge return empty hash for every requested hash, even when it's missing
if not torrent:
continue
logging.info(torrent)
data = {
'hash': torrent[b'hash'].decode(),
'status': torrent[b'state'].lower().decode(),
'name': torrent[b'name'].decode()
}
if data['status'] == 'downloading' and stalled_for:
if b'last_seen_complete' in torrent and b'time_since_download' in torrent:
if now > torrent['last_seen_complete'] + stalled_for * 3600 and \
now > torrent['time_since_download'] + stalled_for * 3600:
#data['status'] = 'stalled'
logging.info('torrent {} detected as stalled'.format(
data['name']))
elif data['hash'] in progress:
data['progress'] = torrent[b'total_payload_download']
torrent_progress = progress[data['hash']]
if data['progress'] == torrent_progress['progress'] and \
now > torrent_progress['time'] + stalled_for * 3600:
#data['status'] = 'stalled'
logging.info('torrent {} detected as stalled'.format(
data['name']))
logging.info(data)
torrents.append(data)
return torrents
except Exception as e:
logging.error('Unable to list torrents from DelugeRPC.', exc_info=True)
return []
def test_connection(config):
''' Tests connectivity to deluge daemon rpc
config: dict of deluge server information
Tests if we can open a socket to the rpc and creates DelugeRPC.client if successful
Returns Bool True on success or str error message on failure
'''
logging.info('Testing connection to DelugeRPC')
host = config['host']
port = config['port']
user = config['user']
password = config['pass']
client = DelugeRPCClient(host, port, user, password)
try:
error = client.connect()
if error:
return '{}.'.format(error)
except Exception as e:
logging.error('Unable to connect to Deluge RPC.', exc_info=True)
return str(e)
else:
return True
def test_connection(data):
''' Tests connectivity to deluge daemon rpc
data: dict of deluge server information
Tests if we can open a socket to the rpc
Return True on success or str error message on failure
'''
host = data['delugerpchost']
port = int(data['delugerpcport'])
user = data['delugerpcuser']
password = data['delugerpcpass']
client = DelugeRPCClient(host, port, user, password)
try:
error = client.connect()
if error:
return '{}.'.format(error)
except Exception, e:
return str(e)
def add_torrent(data):
''' Adds torrent or magnet to Deluge
data: dict of torrrent/magnet information
Returns dict {'response': True, 'download_id': 'id'}
{'response': False, 'error': 'exception'}
'''
conf = core.CONFIG['Downloader']['Torrent']['DelugeRPC']
host = conf['host']
port = conf['port']
user = conf['user']
password = conf['pass']
client = DelugeRPCClient(host, port, user, password)
try:
error = client.connect()
if error:
return {'response': False, 'error': error}
except Exception as e:
logging.error('Deluge Add Torrent.', exc_info=True)
return {'response': False, 'error': str(e)}
try:
def_download_path = client.call(
'core.get_config')[b'download_location'].decode('utf-8')
except Exception as e:
logging.error('Unable to get download path.', exc_info=True)
return {'response': False, 'error': 'Unable to get download path.'}
download_path = '{}/{}'.format(def_download_path, conf['category'])
priority_keys = {'Normal': 0, 'High': 128, 'Max': 255}
options = {}
options['add_paused'] = conf['addpaused']
options['download_location'] = download_path
options['priority'] = priority_keys[conf['priority']]
if data['type'] == 'magnet':
try:
download_id = client.call('core.add_torrent_magnet',
data['torrentfile'],
options).decode('utf-8')
return {'response': True, 'downloadid': download_id}
except Exception as e:
logging.error('Unable to send magnet.', exc_info=True)
return {'response': False, 'error': str(e)}
elif data['type'] == 'torrent':
try:
download_id = client.call('core.add_torrent_url',
data['torrentfile'],
options).decode('utf-8')
return {'response': True, 'downloadid': download_id}
except Exception as e:
logging.error('Unable to send magnet.', exc_info=True)
return {'response': False, 'error': str(e)}
return
def cancel_download(downloadid):
''' Cancels download in client
downloadid: int download id
Returns bool
'''
logging.info('Cancelling DelugeRPC download # {}'.format(downloadid))
conf = core.CONFIG['Downloader']['Torrent']['DelugeRPC']
host = conf['host']
port = conf['port']
user = conf['user']
password = conf['pass']
client = DelugeRPCClient(host, port, user, password)
try:
client.connect()
return client.call('core.remove_torrent', downloadid, True)
except Exception as e:
logging.error('Unable to cancel download.', exc_info=True)
return False
def add_torrent(data):
''' Adds torrent or magnet to Deluge
data: dict of torrrent/magnet information
Returns dict {'response': True, 'download_id': 'id'}
{'response': False, 'error': 'exception'}
'''
conf = core.CONFIG['Downloader']['Torrent']['DelugeRPC']
host = conf['host']
port = conf['port']
user = conf['user']
password = conf['pass']
client = DelugeRPCClient(host, port, user, password)
try:
error = client.connect()
if error:
return {'response': False, 'error': error}
except Exception, e:
return {'response': False, 'error': str(e)[1:-1]}
def add_torrent(data):
''' Adds torrent or magnet to Deluge
data: dict of torrrent/magnet information
Returns dict {'response': 'true', 'download_id': 'id'}
{'response': 'false', 'error': 'exception'}
'''
conf = core.CONFIG['DelugeRPC']
host = conf['delugerpchost']
port = int(conf['delugerpcport'])
user = conf['delugerpcuser']
password = conf['delugerpcpass']
client = DelugeRPCClient(host, port, user, password)
try:
error = client.connect()
if error:
return {'response': 'false', 'error': error}
except Exception, e:
return {'response': 'false', 'error': str(e)}
def delete_task(Source, DownloadID, remove_data):
try:
if Source == "SABNZBD":
sabnzbd.SABnzbd(DownloadID, 'delete', remove_data)
elif Source == "NZBGET":
nzbget.deleteNZB(DownloadID, remove_data)
elif Source == "UTORRENT":
utorrent.removeTorrent(DownloadID, remove_data)
elif Source == "RTORRENT":
rtorrent.removeTorrent(DownloadID, remove_data)
elif Source == "QBITTORRENT":
qbittorrent.removeTorrent(DownloadID, remove_data)
elif Source == "TRANSMISSION":
transmission.removeTorrent(DownloadID, remove_data)
elif Source == "SYNOLOGY_TOR" or Source == "SYNOLOGY_NZB":
synology.removeTorrent(DownloadID, remove_data)
elif Source == "DELUGEWEBUI":
deluge.removeTorrent(DownloadID, remove_data)
elif Source == "DELUGERPC":
client = DelugeRPCClient(lazylibrarian.DELUGE_HOST,
int(lazylibrarian.DELUGE_PORT),
lazylibrarian.DELUGE_USER,
lazylibrarian.DELUGE_PASS)
try:
client.connect()
client.call('core.remove_torrent', DownloadID, remove_data)
except Exception as e:
logger.debug('DelugeRPC failed %s' % str(e))
return False
else:
logger.debug("Unknown source [%s] in delete_task" % Source)
return False
return True
except Exception as e:
logger.debug("Failed to delete task %s from %s: %s" % (DownloadID, Source, str(e)))
return False
def TORDownloadMethod(bookid=None, tor_prov=None, tor_title=None, tor_url=None):
myDB = database.DBConnection()
downloadID = False
Source = ''
full_url = tor_url # keep the url as stored in "wanted" table
if tor_url and tor_url.startswith('magnet'):
torrent = tor_url # allow magnet link to write to blackhole and hash to utorrent/rtorrent
else:
if '&file=' in tor_url:
# torznab results need to be re-encoded
# had a problem with torznab utf-8 encoded strings not matching
# our utf-8 strings because of long/short form differences
url = tor_url.split('&file=')[0]
value = tor_url.split('&file=')[1]
if isinstance(value, str):
value = value.decode('utf-8') # make unicode
value = unicodedata.normalize('NFC', value) # normalize to short form
value = value.encode('unicode-escape') # then escape the result
value = value.replace(' ', '%20') # and encode any spaces
tor_url = url + '&file=' + value
# strip url back to the .torrent as some sites add parameters
if not tor_url.endswith('.torrent'):
if '.torrent' in tor_url:
tor_url = tor_url.split('.torrent')[0] + '.torrent'
request = urllib2.Request(ur'%s' % tor_url)
if lazylibrarian.PROXY_HOST:
request.set_proxy(lazylibrarian.PROXY_HOST, lazylibrarian.PROXY_TYPE)
request.add_header('Accept-encoding', 'gzip')
request.add_header('User-Agent', USER_AGENT)
try:
response = urllib2.urlopen(request, timeout=90)
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO(response.read())
f = gzip.GzipFile(fileobj=buf)
torrent = f.read()
else:
torrent = response.read()
except (socket.timeout) as e:
logger.warn('Timeout fetching torrent from url: %s' % tor_url)
return False
except (urllib2.URLError) as e:
logger.warn('Error fetching torrent from url: %s, %s' % (tor_url, e.reason))
return False
if lazylibrarian.TOR_DOWNLOADER_BLACKHOLE:
Source = "BLACKHOLE"
logger.debug("Sending %s to blackhole" % tor_title)
tor_name = cleanName(tor_title).replace(' ', '_')
tor_title = None
if tor_url and tor_url.startswith('magnet'):
if lazylibrarian.TOR_CONVERT_MAGNET:
hashid = CalcTorrentHash(tor_url)
tor_name = 'meta-' + hashid + '.torrent'
tor_path = os.path.join(lazylibrarian.TORRENT_DIR, tor_name)
result = magnet2torrent(tor_url, tor_path)
if result is not False:
logger.debug('Magnet file saved as: %s' % tor_path)
downloadID = Source
else:
tor_name = tor_name + '.magnet'
tor_path = os.path.join(lazylibrarian.TORRENT_DIR, tor_name)
with open(tor_path, 'wb') as torrent_file:
torrent_file.write(torrent)
logger.debug('Magnet file saved: %s' % tor_path)
setperm(tor_path)
downloadID = Source
else:
tor_name = tor_name + '.torrent'
tor_path = os.path.join(lazylibrarian.TORRENT_DIR, tor_name)
with open(tor_path, 'wb') as torrent_file:
torrent_file.write(torrent)
setperm(tor_path)
logger.debug('Torrent file saved: %s' % tor_name)
downloadID = Source
if (lazylibrarian.TOR_DOWNLOADER_UTORRENT and lazylibrarian.UTORRENT_HOST):
logger.debug("Sending %s to Utorrent" % tor_title)
Source = "UTORRENT"
hashid = CalcTorrentHash(torrent)
downloadID = utorrent.addTorrent(tor_url, hashid) # returns hash or False
if downloadID:
tor_title = utorrent.nameTorrent(downloadID)
if (lazylibrarian.TOR_DOWNLOADER_RTORRENT and lazylibrarian.RTORRENT_HOST):
logger.debug("Sending %s to rTorrent" % tor_title)
Source = "RTORRENT"
hashid = CalcTorrentHash(torrent)
downloadID = rtorrent.addTorrent(tor_url, hashid) # returns hash or False
if downloadID:
tor_title = rtorrent.getName(downloadID)
if (lazylibrarian.TOR_DOWNLOADER_QBITTORRENT and lazylibrarian.QBITTORRENT_HOST):
logger.debug("Sending %s to qbittorrent" % tor_title)
Source = "QBITTORRENT"
hashid = CalcTorrentHash(torrent)
status = qbittorrent.addTorrent(tor_url) # returns hash or False
if status:
downloadID = hashid
tor_title = qbittorrent.getName(hashid)
else:
logger.debug("qbittorrent returned: %s" % str(response))
if (lazylibrarian.TOR_DOWNLOADER_TRANSMISSION and lazylibrarian.TRANSMISSION_HOST):
logger.debug("Sending %s to Transmission" % tor_title)
Source = "TRANSMISSION"
downloadID = transmission.addTorrent(tor_url) # returns id or False
if downloadID:
# transmission returns it's own int, but we store hashid instead
downloadID = CalcTorrentHash(torrent)
tor_title = transmission.getTorrentFolder(downloadID)
if (lazylibrarian.TOR_DOWNLOADER_SYNOLOGY and lazylibrarian.USE_SYNOLOGY and lazylibrarian.SYNOLOGY_HOST):
logger.debug("Sending %s to Synology" % tor_title)
Source = "SYNOLOGY_TOR"
downloadID = synology.addTorrent(tor_url) # returns id or False
if downloadID:
tor_title = synology.getName(downloadID)
if (lazylibrarian.TOR_DOWNLOADER_DELUGE and lazylibrarian.DELUGE_HOST):
logger.debug("Sending %s to Deluge" % tor_title)
if not lazylibrarian.DELUGE_USER:
# no username, talk to the webui
Source = "DELUGEWEBUI"
downloadID = deluge.addTorrent(tor_url) # returns hash or False
if downloadID:
tor_title = deluge.getTorrentFolder(downloadID)
else:
# have username, talk to the daemon
Source = "DELUGERPC"
client = DelugeRPCClient(lazylibrarian.DELUGE_HOST,
int(lazylibrarian.DELUGE_PORT),
lazylibrarian.DELUGE_USER,
lazylibrarian.DELUGE_PASS)
try:
client.connect()
args = {"name": tor_title}
if tor_url.startswith('magnet'):
downloadID = client.call('core.add_torrent_magnet', tor_url, args)
else:
downloadID = client.call('core.add_torrent_url', tor_url, args)
if downloadID:
if lazylibrarian.DELUGE_LABEL:
labelled = client.call('label.set_torrent', downloadID, lazylibrarian.DELUGE_LABEL)
result = client.call('core.get_torrent_status', downloadID, {})
#for item in result:
# logger.debug ('Deluge RPC result %s: %s' % (item, result[item]))
if 'name' in result:
tor_title = result['name']
except Exception as e:
logger.debug('DelugeRPC failed %s' % str(e))
return False
if not Source:
logger.warn('No torrent download method is enabled, check config.')
return False
if downloadID:
myDB.action('UPDATE books SET status = "Snatched" WHERE BookID="%s"' % bookid)
myDB.action('UPDATE wanted SET status = "Snatched", Source = "%s", DownloadID = "%s" WHERE NZBurl="%s"' %
(Source, downloadID, full_url))
if tor_title:
if downloadID.upper() in tor_title.upper():
logger.warn('%s: name contains hash, probably unresolved magnet' % Source)
else:
tor_title = unaccented_str(tor_title)
logger.debug('%s setting torrent name to [%s]' % (Source, tor_title))
myDB.action('UPDATE wanted SET NZBtitle = "%s" WHERE NZBurl="%s"' % (tor_title, full_url))
return True
else:
logger.error(u'Failed to download torrent from %s, %s' % (Source, tor_url))
myDB.action('UPDATE wanted SET status = "Failed" WHERE NZBurl="%s"' % full_url)
return False
def TORDownloadMethod(bookid=None,
tor_title=None,
tor_url=None,
library='eBook'):
myDB = database.DBConnection()
downloadID = False
Source = ''
full_url = tor_url # keep the url as stored in "wanted" table
if tor_url and tor_url.startswith('magnet'):
torrent = tor_url # allow magnet link to write to blackhole and hash to utorrent/rtorrent
else:
# h = HTMLParser()
# tor_url = h.unescape(tor_url)
# HTMLParser is probably overkill, we only seem to get &
#
tor_url = tor_url.replace('&', '&')
if '&file=' in tor_url:
# torznab results need to be re-encoded
# had a problem with torznab utf-8 encoded strings not matching
# our utf-8 strings because of long/short form differences
url, value = tor_url.split('&file=', 1)
value = makeUnicode(value) # ensure unicode
value = unicodedata.normalize('NFC',
value) # normalize to short form
value = value.encode('unicode-escape') # then escape the result
value = value.replace(' ', '%20') # and encode any spaces
tor_url = url + '&file=' + value
# strip url back to the .torrent as some sites add parameters
if not tor_url.endswith('.torrent'):
if '.torrent' in tor_url:
tor_url = tor_url.split('.torrent')[0] + '.torrent'
headers = {'Accept-encoding': 'gzip', 'User-Agent': USER_AGENT}
proxies = proxyList()
try:
r = requests.get(tor_url,
headers=headers,
timeout=90,
proxies=proxies)
except requests.exceptions.Timeout:
logger.warn('Timeout fetching file from url: %s' % tor_url)
return False
except Exception as e:
if hasattr(e, 'reason'):
logger.warn('%s fetching file from url: %s, %s' %
(type(e).__name__, tor_url, e.reason))
else:
logger.warn('%s fetching file from url: %s, %s' %
(type(e).__name__, tor_url, str(e)))
return False
torrent = r.content
if lazylibrarian.CONFIG['TOR_DOWNLOADER_BLACKHOLE']:
Source = "BLACKHOLE"
logger.debug("Sending %s to blackhole" % tor_title)
tor_name = cleanName(tor_title).replace(' ', '_')
if tor_url and tor_url.startswith('magnet'):
if lazylibrarian.CONFIG['TOR_CONVERT_MAGNET']:
hashid = CalcTorrentHash(tor_url)
tor_name = 'meta-' + hashid + '.torrent'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'],
tor_name)
result = magnet2torrent(tor_url, tor_path)
if result is not False:
logger.debug('Magnet file saved as: %s' % tor_path)
downloadID = Source
else:
tor_name += '.magnet'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'],
tor_name)
msg = ''
try:
msg = 'Opening '
with open(tor_path, 'wb') as torrent_file:
msg += 'Writing '
if isinstance(torrent, unicode):
torrent = torrent.encode('iso-8859-1')
torrent_file.write(torrent)
msg += 'SettingPerm'
setperm(tor_path)
msg += 'Saved'
logger.debug('Magnet file saved: %s' % tor_path)
downloadID = Source
except Exception as e:
logger.debug("Failed to write magnet to file: %s %s" %
(type(e).__name__, str(e)))
logger.debug("Progress: %s" % msg)
logger.debug("Filename [%s]" % (repr(tor_path)))
return False
else:
tor_name += '.torrent'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'],
tor_name)
msg = ''
try:
msg = 'Opening '
with open(tor_path, 'wb') as torrent_file:
msg += 'Writing '
if isinstance(torrent, unicode):
torrent = torrent.encode('iso-8859-1')
torrent_file.write(torrent)
msg += 'SettingPerm '
setperm(tor_path)
msg += 'Saved'
logger.debug('Torrent file saved: %s' % tor_name)
downloadID = Source
except Exception as e:
logger.debug("Failed to write torrent to file: %s %s" %
(type(e).__name__, str(e)))
logger.debug("Progress: %s" % msg)
logger.debug("Filename [%s]" % (repr(tor_path)))
return False
if lazylibrarian.CONFIG['TOR_DOWNLOADER_UTORRENT'] and lazylibrarian.CONFIG[
'UTORRENT_HOST']:
logger.debug("Sending %s to Utorrent" % tor_title)
Source = "UTORRENT"
hashid = CalcTorrentHash(torrent)
downloadID = utorrent.addTorrent(tor_url,
hashid) # returns hash or False
if downloadID:
tor_title = utorrent.nameTorrent(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_RTORRENT'] and lazylibrarian.CONFIG[
'RTORRENT_HOST']:
logger.debug("Sending %s to rTorrent" % tor_title)
Source = "RTORRENT"
hashid = CalcTorrentHash(torrent)
downloadID = rtorrent.addTorrent(tor_url,
hashid) # returns hash or False
if downloadID:
tor_title = rtorrent.getName(downloadID)
if lazylibrarian.CONFIG[
'TOR_DOWNLOADER_QBITTORRENT'] and lazylibrarian.CONFIG[
'QBITTORRENT_HOST']:
logger.debug("Sending %s to qbittorrent" % tor_title)
Source = "QBITTORRENT"
hashid = CalcTorrentHash(torrent)
status = qbittorrent.addTorrent(tor_url,
hashid) # returns True or False
if status:
downloadID = hashid
tor_title = qbittorrent.getName(hashid)
if lazylibrarian.CONFIG[
'TOR_DOWNLOADER_TRANSMISSION'] and lazylibrarian.CONFIG[
'TRANSMISSION_HOST']:
logger.debug("Sending %s to Transmission" % tor_title)
Source = "TRANSMISSION"
downloadID = transmission.addTorrent(tor_url) # returns id or False
if downloadID:
# transmission returns it's own int, but we store hashid instead
downloadID = CalcTorrentHash(torrent)
tor_title = transmission.getTorrentFolder(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_SYNOLOGY'] and lazylibrarian.CONFIG['USE_SYNOLOGY'] and \
lazylibrarian.CONFIG['SYNOLOGY_HOST']:
logger.debug("Sending %s to Synology" % tor_title)
Source = "SYNOLOGY_TOR"
downloadID = synology.addTorrent(tor_url) # returns id or False
if downloadID:
tor_title = synology.getName(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_DELUGE'] and lazylibrarian.CONFIG[
'DELUGE_HOST']:
logger.debug("Sending %s to Deluge" % tor_title)
if not lazylibrarian.CONFIG['DELUGE_USER']:
# no username, talk to the webui
Source = "DELUGEWEBUI"
downloadID = deluge.addTorrent(tor_url) # returns hash or False
if downloadID:
tor_title = deluge.getTorrentFolder(downloadID)
else:
# have username, talk to the daemon
Source = "DELUGERPC"
client = DelugeRPCClient(lazylibrarian.CONFIG['DELUGE_HOST'],
lazylibrarian.CONFIG['DELUGE_URL_BASE'],
int(lazylibrarian.CONFIG['DELUGE_PORT']),
lazylibrarian.CONFIG['DELUGE_USER'],
lazylibrarian.CONFIG['DELUGE_PASS'])
try:
client.connect()
args = {"name": tor_title}
if tor_url.startswith('magnet'):
downloadID = client.call('core.add_torrent_magnet',
tor_url, args)
else:
downloadID = client.call('core.add_torrent_url', tor_url,
args)
if downloadID:
if lazylibrarian.CONFIG['DELUGE_LABEL']:
_ = client.call('label.set_torrent', downloadID,
lazylibrarian.CONFIG['DELUGE_LABEL'])
result = client.call('core.get_torrent_status', downloadID,
{})
# for item in result:
# logger.debug ('Deluge RPC result %s: %s' % (item, result[item]))
if 'name' in result:
tor_title = result['name']
except Exception as e:
logger.debug('DelugeRPC failed %s %s' %
(type(e).__name__, str(e)))
return False
if not Source:
logger.warn('No torrent download method is enabled, check config.')
return False
if downloadID:
if tor_title:
if downloadID.upper() in tor_title.upper():
logger.warn(
'%s: name contains hash, probably unresolved magnet' %
Source)
else:
tor_title = unaccented_str(tor_title)
# need to check against reject words list again as the name may have changed
# library = magazine eBook AudioBook to determine which reject list
# but we can't easily do the per-magazine rejects
if library == 'magazine':
reject_list = getList(lazylibrarian.CONFIG['REJECT_MAGS'])
elif library == 'eBook':
reject_list = getList(lazylibrarian.CONFIG['REJECT_WORDS'])
elif library == 'AudioBook':
reject_list = getList(lazylibrarian.CONFIG['REJECT_AUDIO'])
else:
logger.debug("Invalid library [%s] in TORDownloadMethod" %
library)
reject_list = []
rejected = False
lower_title = tor_title.lower()
for word in reject_list:
if word in lower_title:
rejected = True
logger.debug("Rejecting torrent name %s, contains %s" %
(tor_title, word))
break
if rejected:
myDB.action(
'UPDATE wanted SET status="Failed" WHERE NZBurl=?',
(full_url, ))
delete_task(Source, downloadID, True)
return False
else:
logger.debug('%s setting torrent name to [%s]' %
(Source, tor_title))
myDB.action('UPDATE wanted SET NZBtitle=? WHERE NZBurl=?',
(tor_title, full_url))
if library == 'eBook':
myDB.action('UPDATE books SET status="Snatched" WHERE BookID=?',
(bookid, ))
elif library == 'AudioBook':
myDB.action(
'UPDATE books SET audiostatus="Snatched" WHERE BookID=?',
(bookid, ))
myDB.action(
'UPDATE wanted SET status="Snatched", Source=?, DownloadID=? WHERE NZBurl=?',
(Source, downloadID, full_url))
return True
logger.error('Failed to download torrent from %s, %s' % (Source, tor_url))
myDB.action('UPDATE wanted SET status="Failed" WHERE NZBurl=?',
(full_url, ))
return False
def TORDownloadMethod(bookid=None,
tor_title=None,
tor_url=None,
library='eBook'):
myDB = database.DBConnection()
downloadID = False
Source = ''
torrent = ''
full_url = tor_url # keep the url as stored in "wanted" table
if 'magnet:?' in tor_url:
# discard any other parameters and just use the magnet link
tor_url = 'magnet:?' + tor_url.split('magnet:?')[1]
else:
# h = HTMLParser()
# tor_url = h.unescape(tor_url)
# HTMLParser is probably overkill, we only seem to get &
#
tor_url = tor_url.replace('&', '&')
if '&file=' in tor_url:
# torznab results need to be re-encoded
# had a problem with torznab utf-8 encoded strings not matching
# our utf-8 strings because of long/short form differences
url, value = tor_url.split('&file=', 1)
value = makeUnicode(value) # ensure unicode
value = unicodedata.normalize('NFC',
value) # normalize to short form
value = value.encode('unicode-escape') # then escape the result
value = makeUnicode(value) # ensure unicode
value = value.replace(' ', '%20') # and encode any spaces
tor_url = url + '&file=' + value
# strip url back to the .torrent as some sites add extra parameters
if not tor_url.endswith('.torrent') and '.torrent' in tor_url:
tor_url = tor_url.split('.torrent')[0] + '.torrent'
headers = {'Accept-encoding': 'gzip', 'User-Agent': getUserAgent()}
proxies = proxyList()
try:
logger.debug("Fetching %s" % tor_url)
r = requests.get(tor_url,
headers=headers,
timeout=90,
proxies=proxies)
if str(r.status_code).startswith('2'):
torrent = r.content
if not len(torrent):
res = "Got empty response for %s" % tor_url
logger.warn(res)
return False, res
elif len(torrent) < 100:
res = "Only got %s bytes for %s" % (len(torrent), tor_url)
logger.warn(res)
return False, res
else:
logger.debug("Got %s bytes for %s" %
(len(torrent), tor_url))
else:
res = "Got a %s response for %s" % (r.status_code, tor_url)
logger.warn(res)
return False, res
except requests.exceptions.Timeout:
res = 'Timeout fetching file from url: %s' % tor_url
logger.warn(res)
return False, res
except Exception as e:
# some jackett providers redirect internally using http 301 to a magnet link
# which requests can't handle, so throws an exception
logger.debug("Requests exception: %s" % str(e))
if "magnet:?" in str(e):
tor_url = 'magnet:?' + str(e).split('magnet:?')[1].strip("'")
logger.debug("Redirecting to %s" % tor_url)
else:
if hasattr(e, 'reason'):
res = '%s fetching file from url: %s, %s' % (
type(e).__name__, tor_url, e.reason)
else:
res = '%s fetching file from url: %s, %s' % (
type(e).__name__, tor_url, str(e))
logger.warn(res)
return False, res
if not torrent and not tor_url.startswith('magnet:?'):
res = "No magnet or data, cannot continue"
logger.warn(res)
return False, res
if lazylibrarian.CONFIG['TOR_DOWNLOADER_BLACKHOLE']:
Source = "BLACKHOLE"
logger.debug("Sending %s to blackhole" % tor_title)
tor_name = cleanName(tor_title).replace(' ', '_')
if tor_url and tor_url.startswith('magnet'):
if lazylibrarian.CONFIG['TOR_CONVERT_MAGNET']:
hashid = calculate_torrent_hash(tor_url)
if not hashid:
hashid = tor_name
tor_name = 'meta-' + hashid + '.torrent'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'],
tor_name)
result = magnet2torrent(tor_url, tor_path)
if result is not False:
logger.debug('Magnet file saved as: %s' % tor_path)
downloadID = Source
else:
tor_name += '.magnet'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'],
tor_name)
msg = ''
try:
msg = 'Opening '
with open(tor_path, 'wb') as torrent_file:
msg += 'Writing '
if isinstance(torrent, text_type):
torrent = torrent.encode('iso-8859-1')
torrent_file.write(torrent)
msg += 'SettingPerm '
setperm(tor_path)
msg += 'Saved '
logger.debug('Magnet file saved: %s' % tor_path)
downloadID = Source
except Exception as e:
res = "Failed to write magnet to file: %s %s" % (
type(e).__name__, str(e))
logger.warn(res)
logger.debug("Progress: %s Filename [%s]" %
(msg, repr(tor_path)))
return False, res
else:
tor_name += '.torrent'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'],
tor_name)
msg = ''
try:
msg = 'Opening '
with open(tor_path, 'wb') as torrent_file:
msg += 'Writing '
if isinstance(torrent, text_type):
torrent = torrent.encode('iso-8859-1')
torrent_file.write(torrent)
msg += 'SettingPerm '
setperm(tor_path)
msg += 'Saved '
logger.debug('Torrent file saved: %s' % tor_name)
downloadID = Source
except Exception as e:
res = "Failed to write torrent to file: %s %s" % (
type(e).__name__, str(e))
logger.warn(res)
logger.debug("Progress: %s Filename [%s]" %
(msg, repr(tor_path)))
return False, res
hashid = calculate_torrent_hash(tor_url, torrent)
if not hashid:
res = "Unable to calculate torrent hash from url/data"
logger.error(res)
logger.debug("url: %s" % tor_url)
logger.debug("data: %s" % makeUnicode(str(torrent[:50])))
return False, res
if lazylibrarian.CONFIG['TOR_DOWNLOADER_UTORRENT'] and lazylibrarian.CONFIG[
'UTORRENT_HOST']:
logger.debug("Sending %s to Utorrent" % tor_title)
Source = "UTORRENT"
downloadID, res = utorrent.addTorrent(tor_url,
hashid) # returns hash or False
if downloadID:
tor_title = utorrent.nameTorrent(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_RTORRENT'] and lazylibrarian.CONFIG[
'RTORRENT_HOST']:
logger.debug("Sending %s to rTorrent" % tor_title)
Source = "RTORRENT"
if torrent:
logger.debug("Sending %s data to rTorrent" % tor_title)
downloadID, res = rtorrent.addTorrent(tor_title,
hashid,
data=torrent)
else:
logger.debug("Sending %s url to rTorrent" % tor_title)
downloadID, res = rtorrent.addTorrent(
tor_url, hashid) # returns hash or False
if downloadID:
tor_title = rtorrent.getName(downloadID)
if lazylibrarian.CONFIG[
'TOR_DOWNLOADER_QBITTORRENT'] and lazylibrarian.CONFIG[
'QBITTORRENT_HOST']:
Source = "QBITTORRENT"
if torrent:
logger.debug("Sending %s data to qBittorrent" % tor_title)
status, res = qbittorrent.addFile(torrent, hashid, tor_title)
else:
logger.debug("Sending %s url to qBittorrent" % tor_title)
status, res = qbittorrent.addTorrent(
tor_url, hashid) # returns True or False
if status:
downloadID = hashid
tor_title = qbittorrent.getName(hashid)
if lazylibrarian.CONFIG[
'TOR_DOWNLOADER_TRANSMISSION'] and lazylibrarian.CONFIG[
'TRANSMISSION_HOST']:
Source = "TRANSMISSION"
if torrent:
logger.debug("Sending %s data to Transmission" % tor_title)
# transmission needs b64encoded metainfo to be unicode, not bytes
downloadID, res = transmission.addTorrent(None,
metainfo=makeUnicode(
b64encode(torrent)))
else:
logger.debug("Sending %s url to Transmission" % tor_title)
downloadID, res = transmission.addTorrent(
tor_url) # returns id or False
if downloadID:
# transmission returns it's own int, but we store hashid instead
downloadID = hashid
tor_title = transmission.getTorrentFolder(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_SYNOLOGY'] and lazylibrarian.CONFIG['USE_SYNOLOGY'] and \
lazylibrarian.CONFIG['SYNOLOGY_HOST']:
logger.debug("Sending %s url to Synology" % tor_title)
Source = "SYNOLOGY_TOR"
downloadID, res = synology.addTorrent(tor_url) # returns id or False
if downloadID:
tor_title = synology.getName(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_DELUGE'] and lazylibrarian.CONFIG[
'DELUGE_HOST']:
if not lazylibrarian.CONFIG['DELUGE_USER']:
# no username, talk to the webui
Source = "DELUGEWEBUI"
if torrent:
logger.debug("Sending %s data to Deluge" % tor_title)
downloadID, res = deluge.addTorrent(tor_title,
data=b64encode(torrent))
else:
logger.debug("Sending %s url to Deluge" % tor_title)
downloadID, res = deluge.addTorrent(
tor_url) # can be link or magnet, returns hash or False
if downloadID:
tor_title = deluge.getTorrentFolder(downloadID)
else:
return False, res
else:
# have username, talk to the daemon
Source = "DELUGERPC"
client = DelugeRPCClient(lazylibrarian.CONFIG['DELUGE_HOST'],
int(lazylibrarian.CONFIG['DELUGE_PORT']),
lazylibrarian.CONFIG['DELUGE_USER'],
lazylibrarian.CONFIG['DELUGE_PASS'])
try:
client.connect()
args = {"name": tor_title}
if tor_url.startswith('magnet'):
res = "Sending %s magnet to DelugeRPC" % tor_title
logger.debug(res)
downloadID = client.call('core.add_torrent_magnet',
tor_url, args)
elif torrent:
res = "Sending %s data to DelugeRPC" % tor_title
logger.debug(res)
downloadID = client.call('core.add_torrent_file',
tor_title, b64encode(torrent),
args)
else:
res = "Sending %s url to DelugeRPC" % tor_title
logger.debug(res)
downloadID = client.call('core.add_torrent_url', tor_url,
args)
if downloadID:
if lazylibrarian.CONFIG['DELUGE_LABEL']:
_ = client.call(
'label.set_torrent', downloadID,
lazylibrarian.CONFIG['DELUGE_LABEL'].lower())
result = client.call('core.get_torrent_status', downloadID,
{})
if 'name' in result:
tor_title = result['name']
else:
res += ' failed'
logger.error(res)
return False, res
except Exception as e:
res = 'DelugeRPC failed %s %s' % (type(e).__name__, str(e))
logger.error(res)
return False, res
if not Source:
res = 'No torrent download method is enabled, check config.'
logger.warn(res)
return False, res
if downloadID:
if tor_title:
if downloadID.upper() in tor_title.upper():
logger.warn(
'%s: name contains hash, probably unresolved magnet' %
Source)
else:
tor_title = unaccented_str(tor_title)
# need to check against reject words list again as the name may have changed
# library = magazine eBook AudioBook to determine which reject list
# but we can't easily do the per-magazine rejects
if library == 'Magazine':
reject_list = getList(lazylibrarian.CONFIG['REJECT_MAGS'],
',')
elif library == 'eBook':
reject_list = getList(lazylibrarian.CONFIG['REJECT_WORDS'],
',')
elif library == 'AudioBook':
reject_list = getList(lazylibrarian.CONFIG['REJECT_AUDIO'],
',')
else:
logger.debug("Invalid library [%s] in TORDownloadMethod" %
library)
reject_list = []
rejected = False
lower_title = tor_title.lower()
for word in reject_list:
if word in lower_title:
rejected = "Rejecting torrent name %s, contains %s" % (
tor_title, word)
logger.debug(rejected)
break
if not rejected:
rejected = check_contents(Source, downloadID, library,
tor_title)
if rejected:
myDB.action(
'UPDATE wanted SET status="Failed",DLResult=? WHERE NZBurl=?',
(rejected, full_url))
delete_task(Source, downloadID, True)
return False
else:
logger.debug('%s setting torrent name to [%s]' %
(Source, tor_title))
myDB.action('UPDATE wanted SET NZBtitle=? WHERE NZBurl=?',
(tor_title, full_url))
if library == 'eBook':
myDB.action('UPDATE books SET status="Snatched" WHERE BookID=?',
(bookid, ))
elif library == 'AudioBook':
myDB.action(
'UPDATE books SET audiostatus="Snatched" WHERE BookID=?',
(bookid, ))
myDB.action(
'UPDATE wanted SET status="Snatched", Source=?, DownloadID=? WHERE NZBurl=?',
(Source, downloadID, full_url))
return True, ''
res = 'Failed to send torrent to %s' % Source
logger.error(res)
return False, res
def TORDownloadMethod(bookid=None, tor_prov=None, tor_title=None, tor_url=None):
myDB = database.DBConnection()
download = False
full_url = tor_url # keep the url as stored in "wanted" table
if (
lazylibrarian.TOR_DOWNLOADER_DELUGE
or lazylibrarian.TOR_DOWNLOADER_UTORRENT
or lazylibrarian.TOR_DOWNLOADER_QBITTORRENT
or lazylibrarian.TOR_DOWNLOADER_BLACKHOLE
or lazylibrarian.TOR_DOWNLOADER_TRANSMISSION
):
if tor_url.startswith("magnet"):
torrent = tor_url # allow magnet link to write to blackhole and hash to utorrent
else:
if "&file=" in tor_url:
# torznab results need to be re-encoded
# had a problem with torznab utf-8 encoded strings not matching
# our utf-8 strings because of long/short form differences
url = tor_url.split("&file=")[0]
value = tor_url.split("&file=")[1]
if isinstance(value, str):
value = value.decode("utf-8") # make unicode
value = unicodedata.normalize("NFC", value) # normalize to short form
value = value.encode("unicode-escape") # then escape the result
value = value.replace(" ", "%20") # and encode any spaces
tor_url = url + "&file=" + value
# strip url back to the .torrent as some sites add parameters
if "?" in tor_url:
tor_url = tor_url.split("?")[0]
request = urllib2.Request(ur"%s" % tor_url)
if lazylibrarian.PROXY_HOST:
request.set_proxy(lazylibrarian.PROXY_HOST, lazylibrarian.PROXY_TYPE)
request.add_header("Accept-encoding", "gzip")
request.add_header("User-Agent", common.USER_AGENT)
# PAB removed this, KAT serves us html instead of torrent if this header is sent
# if tor_prov == 'KAT':
# host = lazylibrarian.KAT_HOST
# if not str(host)[:4] == "http":
# host = 'http://' + host
# request.add_header('Referer', host)
try:
response = urllib2.urlopen(request, timeout=90)
if response.info().get("Content-Encoding") == "gzip":
buf = StringIO(response.read())
f = gzip.GzipFile(fileobj=buf)
torrent = f.read()
else:
torrent = response.read()
except urllib2.URLError as e:
logger.warn("Error fetching torrent from url: " + tor_url + " %s" % e.reason)
return False
if lazylibrarian.TOR_DOWNLOADER_BLACKHOLE:
tor_title = common.removeDisallowedFilenameChars(tor_title)
logger.debug("Sending %s to blackhole" % tor_title)
tor_name = str.replace(str(tor_title), " ", "_")
if tor_url.startswith("magnet"):
tor_name = tor_name + ".magnet"
else:
tor_name = tor_name + ".torrent"
tor_path = os.path.join(lazylibrarian.TORRENT_DIR, tor_name)
with open(tor_path, "wb") as torrent_file:
torrent_file.write(torrent)
logger.debug("Torrent file saved: %s" % tor_title)
download = True
if lazylibrarian.TOR_DOWNLOADER_UTORRENT:
logger.debug("Sending %s to Utorrent" % tor_title)
hash = CalcTorrentHash(torrent)
download = utorrent.addTorrent(tor_url, hash)
if lazylibrarian.TOR_DOWNLOADER_QBITTORRENT:
logger.debug("Sending %s to qbittorrent" % tor_title)
download = qbittorrent.addTorrent(tor_url)
if lazylibrarian.TOR_DOWNLOADER_TRANSMISSION:
logger.debug("Sending %s to Transmission" % tor_title)
download = transmission.addTorrent(tor_url)
if lazylibrarian.TOR_DOWNLOADER_DELUGE:
logger.debug("Sending %s to Deluge" % tor_title)
client = DelugeRPCClient(
lazylibrarian.DELUGE_HOST,
int(lazylibrarian.DELUGE_PORT),
lazylibrarian.DELUGE_USER,
lazylibrarian.DELUGE_PASS,
)
if lazylibrarian.DELUGE_USER and lazylibrarian.DELUGE_PASS:
client.connect()
download = client.call("core.add_torrent_url", tor_url, {"name": tor_title})
logger.debug("Deluge return value: %s" % download)
else:
logger.warn("Need user & pass for deluge, check config.")
else:
logger.warn("No torrent download method is enabled, check config.")
return False
if download:
logger.debug(u"Torrent file has been downloaded from %s" % tor_url)
myDB.action('UPDATE books SET status = "Snatched" WHERE BookID="%s"' % bookid)
myDB.action('UPDATE wanted SET status = "Snatched" WHERE NZBurl="%s"' % full_url)
return True
else:
logger.error(u'Failed to download torrent @ <a href="%s">%s</a>' % (full_url, tor_url))
myDB.action('UPDATE wanted SET status = "Failed" WHERE NZBurl="%s"' % full_url)
return False
def processDir(reset=False):
try:
threadname = threading.currentThread().name
if "Thread-" in threadname:
threading.currentThread().name = "POSTPROCESS"
processpath = lazylibrarian.DIRECTORY('Download')
logger.debug('Checking [%s] for files to post process' % processpath)
try:
downloads = os.listdir(processpath)
except OSError as why:
logger.error('Could not access [%s] directory [%s]' % (processpath, why.strerror))
return
myDB = database.DBConnection()
snatched = myDB.select('SELECT * from wanted WHERE Status="Snatched"')
if len(snatched) == 0:
logger.info('Nothing marked as snatched.')
scheduleJob(action='Stop', target='processDir')
return
if len(downloads) == 0:
logger.info('No downloads are found. Nothing to process yet.')
return
logger.info("Checking %s download%s for %s snatched file%s" %
(len(downloads), plural(len(downloads)), len(snatched), plural(len(snatched))))
ppcount = 0
for book in snatched:
# if torrent, see if we can get current status from the downloader as the name
# may have been changed once magnet resolved, or download started or completed
# depending on torrent downloader. Usenet doesn't change the name. We like usenet.
torrentname = ''
try:
logger.debug("%s was sent to %s" % (book['NZBtitle'], book['Source']))
if book['Source'] == 'TRANSMISSION':
torrentname = transmission.getTorrentFolder(book['DownloadID'])
elif book['Source'] == 'UTORRENT':
torrentname = utorrent.nameTorrent(book['DownloadID'])
elif book['Source'] == 'RTORRENT':
torrentname = rtorrent.getName(book['DownloadID'])
elif book['Source'] == 'QBITTORRENT':
torrentname = qbittorrent.getName(book['DownloadID'])
elif book['Source'] == 'SYNOLOGY_TOR':
torrentname = synology.getName(book['DownloadID'])
elif book['Source'] == 'DELUGEWEBUI':
torrentname = deluge.getTorrentFolder(book['DownloadID'])
elif book['Source'] == 'DELUGERPC':
client = DelugeRPCClient(lazylibrarian.DELUGE_HOST,
int(lazylibrarian.DELUGE_PORT),
lazylibrarian.DELUGE_USER,
lazylibrarian.DELUGE_PASS)
try:
client.connect()
result = client.call('core.get_torrent_status', book['DownloadID'], {})
# for item in result:
# logger.debug ('Deluge RPC result %s: %s' % (item, result[item]))
if 'name' in result:
torrentname = unaccented_str(result['name'])
except Exception as e:
logger.debug('DelugeRPC failed %s' % str(e))
except Exception as e:
logger.debug("Failed to get updated torrent name from %s for %s: %s" %
(book['Source'], book['DownloadID'], str(e)))
matchtitle = unaccented_str(book['NZBtitle'])
if torrentname and torrentname != matchtitle:
logger.debug("%s Changing [%s] to [%s]" % (book['Source'], matchtitle, torrentname))
myDB.action('UPDATE wanted SET NZBtitle = "%s" WHERE NZBurl = "%s"' % (torrentname, book['NZBurl']))
matchtitle = torrentname
# here we could also check percentage downloaded or eta or status?
# If downloader says it hasn't completed, no need to look for it.
matches = []
logger.info('Looking for %s in %s' % (matchtitle, processpath))
for fname in downloads:
# skip if failed before or incomplete torrents, or incomplete btsync
extn = os.path.splitext(fname)[1]
if extn not in ['.fail', '.part', '.bts', '.!ut']:
# This is to get round differences in torrent filenames.
# Usenet is ok, but Torrents aren't always returned with the name we searched for
# We ask the torrent downloader for the torrent name, but don't always get an answer
# so we try to do a "best match" on the name, there might be a better way...
if isinstance(fname, str):
matchname = fname.decode(lazylibrarian.SYS_ENCODING)
else:
matchname = fname
if ' LL.(' in matchname:
matchname = matchname.split(' LL.(')[0]
match = 0
if matchtitle:
if ' LL.(' in matchtitle:
matchtitle = matchtitle.split(' LL.(')[0]
match = fuzz.token_set_ratio(matchtitle, matchname)
if match and match >= lazylibrarian.DLOAD_RATIO:
fname = matchname
if os.path.isfile(os.path.join(processpath, fname)):
# handle single file downloads here. Book/mag file in download root.
# move the file into it's own subdirectory so we don't move/delete things that aren't ours
logger.debug('filename [%s] is a file' % os.path.join(processpath, fname))
if is_valid_booktype(fname, booktype="book") \
or is_valid_booktype(fname, booktype="mag"):
logger.debug('filename [%s] is a valid book/mag' % os.path.join(processpath, fname))
if bts_file(processpath):
logger.debug("Skipping %s, found a .bts file" % processpath)
else:
fname = os.path.splitext(fname)[0]
dirname = os.path.join(processpath, fname)
if not os.path.exists(dirname):
try:
os.makedirs(dirname)
setperm(dirname)
except OSError as why:
logger.debug('Failed to create directory %s, %s' % (dirname, why.strerror))
if os.path.exists(dirname):
# move the book and any related files too
# ie other book formats, or opf, jpg with same title
# can't move metadata.opf or cover.jpg or similar
# as can't be sure they are ours
# not sure if we need a new listdir here, or whether we can use the old one
list_dir = os.listdir(processpath)
for ourfile in list_dir:
if ourfile.startswith(fname):
if is_valid_booktype(ourfile, booktype="book") \
or is_valid_booktype(ourfile, booktype="mag") \
or os.path.splitext(ourfile)[1].lower() in ['.opf', '.jpg']:
try:
if lazylibrarian.DESTINATION_COPY:
shutil.copyfile(os.path.join(processpath, ourfile),
os.path.join(dirname, ourfile))
setperm(os.path.join(dirname, ourfile))
else:
shutil.move(os.path.join(processpath, ourfile),
os.path.join(dirname, ourfile))
setperm(os.path.join(dirname, ourfile))
except Exception as why:
logger.debug("Failed to copy/move file %s to %s, %s" %
(ourfile, dirname, str(why)))
pp_path = os.path.join(processpath, fname)
if os.path.isdir(pp_path):
logger.debug('Found folder (%s%%) %s for %s' % (match, pp_path, matchtitle))
if not os.listdir(pp_path):
logger.debug("Skipping %s, folder is empty" % pp_path)
elif bts_file(pp_path):
logger.debug("Skipping %s, found a .bts file" % pp_path)
else:
matches.append([match, pp_path, book])
else:
pp_path = os.path.join(processpath, fname)
matches.append([match, pp_path, book]) # so we can report closest match
else:
logger.debug('Skipping %s' % fname)
match = 0
if matches:
highest = max(matches, key=lambda x: x[0])
match = highest[0]
pp_path = highest[1]
book = highest[2]
if match and match >= lazylibrarian.DLOAD_RATIO:
logger.debug(u'Found match (%s%%): %s for %s' % (match, pp_path, book['NZBtitle']))
data = myDB.match('SELECT * from books WHERE BookID="%s"' % book['BookID'])
if data: # it's a book
logger.debug(u'Processing book %s' % book['BookID'])
authorname = data['AuthorName']
bookname = data['BookName']
if 'windows' in platform.system().lower() and '/' in lazylibrarian.EBOOK_DEST_FOLDER:
logger.warn('Please check your EBOOK_DEST_FOLDER setting')
lazylibrarian.EBOOK_DEST_FOLDER = lazylibrarian.EBOOK_DEST_FOLDER.replace('/', '\\')
# Default destination path, should be allowed change per config file.
dest_path = lazylibrarian.EBOOK_DEST_FOLDER.replace('$Author', authorname).replace(
'$Title', bookname)
global_name = lazylibrarian.EBOOK_DEST_FILE.replace('$Author', authorname).replace(
'$Title', bookname)
global_name = unaccented(global_name)
# dest_path = authorname+'/'+bookname
# global_name = bookname + ' - ' + authorname
# Remove characters we don't want in the filename BEFORE adding to DESTINATION_DIR
# as windows drive identifiers have colon, eg c: but no colons allowed elsewhere?
dic = {'<': '', '>': '', '...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's',
' + ': ' ', '"': '', ',': '', '*': '', ':': '', ';': '', '\'': ''}
dest_path = unaccented_str(replace_all(dest_path, dic))
dest_path = os.path.join(processpath, dest_path).encode(lazylibrarian.SYS_ENCODING)
else:
data = myDB.match('SELECT * from magazines WHERE Title="%s"' % book['BookID'])
if data: # it's a magazine
logger.debug(u'Processing magazine %s' % book['BookID'])
# AuxInfo was added for magazine release date, normally housed in 'magazines' but if multiple
# files are downloading, there will be an error in post-processing, trying to go to the
# same directory.
mostrecentissue = data['IssueDate'] # keep for processing issues arriving out of order
# Remove characters we don't want in the filename before (maybe) adding to DESTINATION_DIR
# as windows drive identifiers have colon, eg c: but no colons allowed elsewhere?
dic = {'<': '', '>': '', '...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's',
' + ': ' ', '"': '', ',': '', '*': '', ':': '', ';': '', '\'': ''}
mag_name = unaccented_str(replace_all(book['BookID'], dic))
# book auxinfo is a cleaned date, eg 2015-01-01
dest_path = lazylibrarian.MAG_DEST_FOLDER.replace(
'$IssueDate', book['AuxInfo']).replace('$Title', mag_name)
if lazylibrarian.MAG_RELATIVE:
if dest_path[0] not in '._':
dest_path = '_' + dest_path
dest_path = os.path.join(processpath, dest_path).encode(
lazylibrarian.SYS_ENCODING)
else:
dest_path = dest_path.encode(lazylibrarian.SYS_ENCODING)
authorname = None
bookname = None
global_name = lazylibrarian.MAG_DEST_FILE.replace('$IssueDate', book['AuxInfo']).replace(
'$Title', mag_name)
global_name = unaccented(global_name)
else: # not recognised
logger.debug('Nothing in database matching "%s"' % book['BookID'])
continue
else:
logger.debug("Snatched %s %s is not in download directory" % (book['NZBmode'], book['NZBtitle']))
if match:
logger.debug(u'Closest match (%s%%): %s' % (match, pp_path))
#for match in matches:
# logger.info('Match: %s%% %s' % (match[0], match[1]))
continue
processBook = processDestination(pp_path, dest_path, authorname, bookname, global_name)
if processBook:
logger.debug("Processing %s, %s" % (global_name, book['NZBurl']))
# update nzbs, only update the snatched ones in case multiple matches for same book / magazine issue
controlValueDict = {"BookID": book['BookID'], "NZBurl": book['NZBurl'], "Status": "Snatched"}
newValueDict = {"Status": "Processed", "NZBDate": now()} # say when we processed it
myDB.upsert("wanted", newValueDict, controlValueDict)
if bookname:
# it's a book, if None it's a magazine
if len(lazylibrarian.IMP_CALIBREDB):
logger.debug('Calibre should have created the extras for us')
else:
processExtras(myDB, dest_path, global_name, data)
else:
# update mags
controlValueDict = {"Title": book['BookID']}
if mostrecentissue:
if mostrecentissue.isdigit() and str(book['AuxInfo']).isdigit():
older = int(mostrecentissue) > int(book['AuxInfo']) # issuenumber
else:
older = mostrecentissue > book['AuxInfo'] # YYYY-MM-DD
else:
older = False
if older: # check this in case processing issues arriving out of order
newValueDict = {"LastAcquired": today(), "IssueStatus": "Open"}
else:
newValueDict = {"IssueDate": book['AuxInfo'], "LastAcquired": today(),
"IssueStatus": "Open"}
myDB.upsert("magazines", newValueDict, controlValueDict)
# dest_path is where we put the magazine after processing, but we don't have the full filename
# so look for any "book" in that directory
dest_file = book_file(dest_path, booktype='mag')
controlValueDict = {"Title": book['BookID'], "IssueDate": book['AuxInfo']}
newValueDict = {"IssueAcquired": today(),
"IssueFile": dest_file,
"IssueID": create_id("%s %s" % (book['BookID'], book['AuxInfo']))
}
myDB.upsert("issues", newValueDict, controlValueDict)
# create a thumbnail cover for the new issue
create_cover(dest_file)
# calibre or ll copied/moved the files we want, now delete source files
to_delete = True
if book['NZBmode'] in ['torrent', 'magnet']:
# Only delete torrents if we don't want to keep seeding
if lazylibrarian.KEEP_SEEDING:
logger.warn('%s is seeding %s %s' % (book['Source'], book['NZBmode'], book['NZBtitle']))
to_delete = False
else:
# ask downloader to delete the torrent, but not the files
# we may delete them later, depending on other settings
if book['DownloadID'] != "unknown":
logger.debug('Removing %s from %s' % (book['NZBtitle'], book['Source'].lower()))
delete_task(book['Source'], book['DownloadID'], False)
else:
logger.warn("Unable to remove %s from %s, no DownloadID" %
(book['NZBtitle'], book['Source'].lower()))
if to_delete:
# only delete the files if not in download root dir and if DESTINATION_COPY not set
if not lazylibrarian.DESTINATION_COPY and (pp_path != processpath):
if os.path.isdir(pp_path):
# calibre might have already deleted it?
try:
shutil.rmtree(pp_path)
except Exception as why:
logger.debug("Unable to remove %s, %s" % (pp_path, str(why)))
logger.info('Successfully processed: %s' % global_name)
ppcount = ppcount + 1
notify_download("%s from %s at %s" % (global_name, book['NZBprov'], now()))
else:
logger.error('Postprocessing for %s has failed.' % global_name)
logger.error('Warning - Residual files remain in %s.fail' % pp_path)
controlValueDict = {"NZBurl": book['NZBurl'], "Status": "Snatched"}
newValueDict = {"Status": "Failed", "NZBDate": now()}
myDB.upsert("wanted", newValueDict, controlValueDict)
# if it's a book, reset status so we try for a different version
# if it's a magazine, user can select a different one from pastissues table
if bookname:
myDB.action('UPDATE books SET status = "Wanted" WHERE BookID="%s"' % book['BookID'])
# at this point, as it failed we should move it or it will get postprocessed
# again (and fail again)
try:
os.rename(pp_path, pp_path + '.fail')
except Exception as e:
logger.debug("Unable to rename %s, %s" % (pp_path, str(e)))
downloads = os.listdir(processpath) # check in case we processed/deleted some above
for directory in downloads:
dname, extn = os.path.splitext(directory)
if "LL.(" in dname and extn not in ['.fail', '.part', '.bts', '.!ut']:
bookID = str(directory).split("LL.(")[1].split(")")[0]
logger.debug("Book with id: " + str(bookID) + " found in download directory")
pp_path = os.path.join(processpath, directory)
if os.path.isfile(pp_path):
pp_path = os.path.join(processpath)
if (os.path.isdir(pp_path)):
if import_book(pp_path, bookID):
ppcount = ppcount + 1
if ppcount == 0:
logger.info('No snatched books/mags have been found')
else:
logger.info('%s book%s/mag%s processed.' % (ppcount, plural(ppcount), plural(ppcount)))
# Now check for any that are still marked snatched...
if lazylibrarian.TASK_AGE:
snatched = myDB.select('SELECT * from wanted WHERE Status="Snatched"')
if len(snatched) > 0:
for snatch in snatched:
# FUTURE: we could check percentage downloaded or eta?
# if percentage is increasing, it's just slow
try:
when_snatched = time.strptime(snatch['NZBdate'], '%Y-%m-%d %H:%M:%S')
when_snatched = time.mktime(when_snatched)
diff = time.time() - when_snatched # time difference in seconds
except:
diff = 0
hours = int(diff / 3600)
if hours >= lazylibrarian.TASK_AGE:
logger.warn('%s was sent to %s %s hours ago, deleting failed task' %
(snatch['NZBtitle'], snatch['Source'].lower(), hours))
# change status to "Failed", and ask downloader to delete task and files
if snatch['BookID'] != 'unknown':
myDB.action('UPDATE wanted SET Status="Failed" WHERE BookID="%s"' % snatch['BookID'])
myDB.action('UPDATE books SET status = "Wanted" WHERE BookID="%s"' % snatch['BookID'])
delete_task(snatch['Source'], snatch['DownloadID'], True)
if reset:
scheduleJob(action='Restart', target='processDir')
except Exception as e:
logger.error('Unhandled exception in processDir: %s' % traceback.format_exc())
def TORDownloadMethod(bookid=None,
tor_prov=None,
tor_title=None,
tor_url=None):
myDB = database.DBConnection()
download = False
full_url = tor_url # keep the url as stored in "wanted" table
if (lazylibrarian.TOR_DOWNLOADER_DELUGE
or lazylibrarian.TOR_DOWNLOADER_UTORRENT
or lazylibrarian.TOR_DOWNLOADER_QBITTORRENT
or lazylibrarian.TOR_DOWNLOADER_BLACKHOLE
or lazylibrarian.TOR_DOWNLOADER_TRANSMISSION):
if tor_url and tor_url.startswith('magnet'):
torrent = tor_url # allow magnet link to write to blackhole and hash to utorrent
else:
if '&file=' in tor_url:
# torznab results need to be re-encoded
# had a problem with torznab utf-8 encoded strings not matching
# our utf-8 strings because of long/short form differences
url = tor_url.split('&file=')[0]
value = tor_url.split('&file=')[1]
if isinstance(value, str):
value = value.decode('utf-8') # make unicode
value = unicodedata.normalize('NFC',
value) # normalize to short form
value = value.encode(
'unicode-escape') # then escape the result
value = value.replace(' ', '%20') # and encode any spaces
tor_url = url + '&file=' + value
# strip url back to the .torrent as some sites add parameters
if not tor_url.endswith('.torrent'):
if '.torrent' in tor_url:
tor_url = tor_url.split('.torrent')[0] + '.torrent'
request = urllib2.Request(ur'%s' % tor_url)
if lazylibrarian.PROXY_HOST:
request.set_proxy(lazylibrarian.PROXY_HOST,
lazylibrarian.PROXY_TYPE)
request.add_header('Accept-encoding', 'gzip')
request.add_header('User-Agent', USER_AGENT)
# PAB removed this, KAT serves us html instead of torrent if this header is sent
# if tor_prov == 'KAT':
# host = lazylibrarian.KAT_HOST
# if not str(host)[:4] == "http":
# host = 'http://' + host
# request.add_header('Referer', host)
try:
response = urllib2.urlopen(request, timeout=90)
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO(response.read())
f = gzip.GzipFile(fileobj=buf)
torrent = f.read()
else:
torrent = response.read()
except (urllib2.URLError, socket.timeout) as e:
logger.warn('Error fetching torrent from url: %s, %s' %
(tor_url, e))
return False
if lazylibrarian.TOR_DOWNLOADER_BLACKHOLE:
tor_title = cleanName(tor_title)
logger.debug("Sending %s to blackhole" % tor_title)
tor_name = str.replace(str(tor_title), ' ', '_')
if tor_url and tor_url.startswith('magnet'):
tor_name = tor_name + '.magnet'
else:
tor_name = tor_name + '.torrent'
tor_path = os.path.join(lazylibrarian.TORRENT_DIR, tor_name)
with open(tor_path, 'wb') as torrent_file:
torrent_file.write(torrent)
logger.debug('Torrent file saved: %s' % tor_title)
download = True
if (lazylibrarian.TOR_DOWNLOADER_UTORRENT
and lazylibrarian.UTORRENT_HOST):
logger.debug("Sending %s to Utorrent" % tor_title)
hash = CalcTorrentHash(torrent)
download = utorrent.addTorrent(tor_url, hash)
if (lazylibrarian.TOR_DOWNLOADER_QBITTORRENT
and lazylibrarian.QBITTORRENT_HOST):
logger.debug("Sending %s to qbittorrent" % tor_title)
download = qbittorrent.addTorrent(tor_url)
if (lazylibrarian.TOR_DOWNLOADER_TRANSMISSION
and lazylibrarian.TRANSMISSION_HOST):
logger.debug("Sending %s to Transmission" % tor_title)
download = transmission.addTorrent(tor_url)
if (lazylibrarian.TOR_DOWNLOADER_DELUGE and lazylibrarian.DELUGE_HOST):
logger.debug("Sending %s to Deluge" % tor_title)
if not lazylibrarian.DELUGE_USER:
# no username, talk to the webui
download = deluge.addTorrent(tor_url)
else:
# have username, talk to the daemon
client = DelugeRPCClient(lazylibrarian.DELUGE_HOST,
int(lazylibrarian.DELUGE_PORT),
lazylibrarian.DELUGE_USER,
lazylibrarian.DELUGE_PASS)
client.connect()
args = {"name": tor_title}
download = client.call('core.add_torrent_url', tor_url, args)
logger.debug('Deluge torrent_id: %s' % download)
if download and lazylibrarian.DELUGE_LABEL:
labelled = client.call('label.set_torrent', download,
lazylibrarian.DELUGE_LABEL)
logger.debug('Deluge label returned: %s' % labelled)
else:
logger.warn('No torrent download method is enabled, check config.')
return False
if download:
logger.debug(u'Torrent id %s has been downloaded from %s' %
(download, tor_url))
myDB.action('UPDATE books SET status = "Snatched" WHERE BookID="%s"' %
bookid)
myDB.action('UPDATE wanted SET status = "Snatched" WHERE NZBurl="%s"' %
full_url)
return True
else:
logger.error(u'Failed to download torrent @ <a href="%s">%s</a>' %
(full_url, tor_url))
myDB.action('UPDATE wanted SET status = "Failed" WHERE NZBurl="%s"' %
full_url)
return False
def TORDownloadMethod(bookid=None, tor_title=None, tor_url=None):
myDB = database.DBConnection()
downloadID = False
Source = ''
full_url = tor_url # keep the url as stored in "wanted" table
if tor_url and tor_url.startswith('magnet'):
torrent = tor_url # allow magnet link to write to blackhole and hash to utorrent/rtorrent
else:
# h = HTMLParser()
# tor_url = h.unescape(tor_url)
# HTMLParser is probably overkill, we only seem to get &
#
tor_url = tor_url.replace('&', '&')
if '&file=' in tor_url:
# torznab results need to be re-encoded
# had a problem with torznab utf-8 encoded strings not matching
# our utf-8 strings because of long/short form differences
url, value = tor_url.split('&file=', 1)
if isinstance(value, str):
value = value.decode('utf-8') # make unicode
value = unicodedata.normalize('NFC', value) # normalize to short form
value = value.encode('unicode-escape') # then escape the result
value = value.replace(' ', '%20') # and encode any spaces
tor_url = url + '&file=' + value
# strip url back to the .torrent as some sites add parameters
if not tor_url.endswith('.torrent'):
if '.torrent' in tor_url:
tor_url = tor_url.split('.torrent')[0] + '.torrent'
request = urllib2.Request(ur'%s' % tor_url)
if lazylibrarian.CONFIG['PROXY_HOST']:
request.set_proxy(lazylibrarian.CONFIG['PROXY_HOST'], lazylibrarian.CONFIG['PROXY_TYPE'])
request.add_header('Accept-encoding', 'gzip')
request.add_header('User-Agent', USER_AGENT)
try:
response = urllib2.urlopen(request, timeout=90)
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO(response.read())
f = gzip.GzipFile(fileobj=buf)
torrent = f.read()
else:
torrent = response.read()
except socket.timeout:
logger.warn('Timeout fetching torrent from url: %s' % tor_url)
return False
except urllib2.URLError as e:
logger.warn('Error fetching torrent from url: %s, %s' % (tor_url, e.reason))
return False
except ValueError as e:
logger.warn('Error, invalid url: [%s] %s' % (full_url, str(e)))
return False
if lazylibrarian.CONFIG['TOR_DOWNLOADER_BLACKHOLE']:
Source = "BLACKHOLE"
logger.debug("Sending %s to blackhole" % tor_title)
tor_name = cleanName(tor_title).replace(' ', '_')
if tor_url and tor_url.startswith('magnet'):
if lazylibrarian.CONFIG['TOR_CONVERT_MAGNET']:
hashid = CalcTorrentHash(tor_url)
tor_name = 'meta-' + hashid + '.torrent'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name)
result = magnet2torrent(tor_url, tor_path)
if result is not False:
logger.debug('Magnet file saved as: %s' % tor_path)
downloadID = Source
else:
tor_name += '.magnet'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name)
try:
with open(tor_path, 'wb') as torrent_file:
torrent_file.write(torrent)
logger.debug('Magnet file saved: %s' % tor_path)
setperm(tor_path)
downloadID = Source
except Exception as e:
logger.debug("Failed to write magnet to file %s, %s" % (tor_path, str(e)))
return False
else:
tor_name += '.torrent'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name)
try:
with open(tor_path, 'wb') as torrent_file:
torrent_file.write(torrent)
setperm(tor_path)
logger.debug('Torrent file saved: %s' % tor_name)
downloadID = Source
except Exception as e:
logger.debug("Failed to write torrent to file %s, %s" % (tor_path, str(e)))
return False
if lazylibrarian.CONFIG['TOR_DOWNLOADER_UTORRENT'] and lazylibrarian.CONFIG['UTORRENT_HOST']:
logger.debug("Sending %s to Utorrent" % tor_title)
Source = "UTORRENT"
hashid = CalcTorrentHash(torrent)
downloadID = utorrent.addTorrent(tor_url, hashid) # returns hash or False
if downloadID:
tor_title = utorrent.nameTorrent(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_RTORRENT'] and lazylibrarian.CONFIG['RTORRENT_HOST']:
logger.debug("Sending %s to rTorrent" % tor_title)
Source = "RTORRENT"
hashid = CalcTorrentHash(torrent)
downloadID = rtorrent.addTorrent(tor_url, hashid) # returns hash or False
if downloadID:
tor_title = rtorrent.getName(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_QBITTORRENT'] and lazylibrarian.CONFIG['QBITTORRENT_HOST']:
logger.debug("Sending %s to qbittorrent" % tor_title)
Source = "QBITTORRENT"
hashid = CalcTorrentHash(torrent)
status = qbittorrent.addTorrent(tor_url) # returns hash or False
if status:
downloadID = hashid
tor_title = qbittorrent.getName(hashid)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_TRANSMISSION'] and lazylibrarian.CONFIG['TRANSMISSION_HOST']:
logger.debug("Sending %s to Transmission" % tor_title)
Source = "TRANSMISSION"
downloadID = transmission.addTorrent(tor_url) # returns id or False
if downloadID:
# transmission returns it's own int, but we store hashid instead
downloadID = CalcTorrentHash(torrent)
tor_title = transmission.getTorrentFolder(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_SYNOLOGY'] and lazylibrarian.CONFIG['USE_SYNOLOGY'] and lazylibrarian.CONFIG['SYNOLOGY_HOST']:
logger.debug("Sending %s to Synology" % tor_title)
Source = "SYNOLOGY_TOR"
downloadID = synology.addTorrent(tor_url) # returns id or False
if downloadID:
tor_title = synology.getName(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_DELUGE'] and lazylibrarian.CONFIG['DELUGE_HOST']:
logger.debug("Sending %s to Deluge" % tor_title)
if not lazylibrarian.CONFIG['DELUGE_USER']:
# no username, talk to the webui
Source = "DELUGEWEBUI"
downloadID = deluge.addTorrent(tor_url) # returns hash or False
if downloadID:
tor_title = deluge.getTorrentFolder(downloadID)
else:
# have username, talk to the daemon
Source = "DELUGERPC"
client = DelugeRPCClient(lazylibrarian.CONFIG['DELUGE_HOST'],
int(lazylibrarian.CONFIG['DELUGE_PORT']),
lazylibrarian.CONFIG['DELUGE_USER'],
lazylibrarian.CONFIG['DELUGE_PASS'])
try:
client.connect()
args = {"name": tor_title}
if tor_url.startswith('magnet'):
downloadID = client.call('core.add_torrent_magnet', tor_url, args)
else:
downloadID = client.call('core.add_torrent_url', tor_url, args)
if downloadID:
if lazylibrarian.CONFIG['DELUGE_LABEL']:
_ = client.call('label.set_torrent', downloadID, lazylibrarian.CONFIG['DELUGE_LABEL'])
result = client.call('core.get_torrent_status', downloadID, {})
# for item in result:
# logger.debug ('Deluge RPC result %s: %s' % (item, result[item]))
if 'name' in result:
tor_title = result['name']
except Exception as e:
logger.debug('DelugeRPC failed %s' % str(e))
return False
if not Source:
logger.warn('No torrent download method is enabled, check config.')
return False
if downloadID:
myDB.action('UPDATE books SET status = "Snatched" WHERE BookID="%s"' % bookid)
myDB.action('UPDATE wanted SET status = "Snatched", Source = "%s", DownloadID = "%s" WHERE NZBurl="%s"' %
(Source, downloadID, full_url))
if tor_title:
if downloadID.upper() in tor_title.upper():
logger.warn('%s: name contains hash, probably unresolved magnet' % Source)
else:
tor_title = unaccented_str(tor_title)
logger.debug('%s setting torrent name to [%s]' % (Source, tor_title))
myDB.action('UPDATE wanted SET NZBtitle = "%s" WHERE NZBurl="%s"' % (tor_title, full_url))
return True
else:
logger.error(u'Failed to download torrent from %s, %s' % (Source, tor_url))
myDB.action('UPDATE wanted SET status = "Failed" WHERE NZBurl="%s"' % full_url)
return False
def TORDownloadMethod(bookid=None, tor_title=None, tor_url=None, library='eBook'):
myDB = database.DBConnection()
downloadID = False
Source = ''
full_url = tor_url # keep the url as stored in "wanted" table
if tor_url and tor_url.startswith('magnet:?'):
torrent = tor_url # allow magnet link to write to blackhole and hash to utorrent/rtorrent
elif 'magnet:?' in tor_url:
# discard any other parameters and just use the magnet link
torrent = 'magnet:?' + tor_url.split('magnet:?')[1]
else:
# h = HTMLParser()
# tor_url = h.unescape(tor_url)
# HTMLParser is probably overkill, we only seem to get &
#
tor_url = tor_url.replace('&', '&')
if '&file=' in tor_url:
# torznab results need to be re-encoded
# had a problem with torznab utf-8 encoded strings not matching
# our utf-8 strings because of long/short form differences
url, value = tor_url.split('&file=', 1)
value = makeUnicode(value) # ensure unicode
value = unicodedata.normalize('NFC', value) # normalize to short form
value = value.encode('unicode-escape') # then escape the result
value = makeUnicode(value) # ensure unicode
value = value.replace(' ', '%20') # and encode any spaces
tor_url = url + '&file=' + value
# strip url back to the .torrent as some sites add extra parameters
if not tor_url.endswith('.torrent'):
if '.torrent' in tor_url:
tor_url = tor_url.split('.torrent')[0] + '.torrent'
headers = {'Accept-encoding': 'gzip', 'User-Agent': USER_AGENT}
proxies = proxyList()
try:
r = requests.get(tor_url, headers=headers, timeout=90, proxies=proxies)
torrent = r.content
except requests.exceptions.Timeout:
logger.warn('Timeout fetching file from url: %s' % tor_url)
return False
except Exception as e:
# some jackett providers redirect internally using http 301 to a magnet link
# which requests can't handle, so throws an exception
if "magnet:?" in str(e):
torrent = 'magnet:?' + str(e).split('magnet:?')[1]. strip("'")
else:
if hasattr(e, 'reason'):
logger.warn('%s fetching file from url: %s, %s' % (type(e).__name__, tor_url, e.reason))
else:
logger.warn('%s fetching file from url: %s, %s' % (type(e).__name__, tor_url, str(e)))
return False
if lazylibrarian.CONFIG['TOR_DOWNLOADER_BLACKHOLE']:
Source = "BLACKHOLE"
logger.debug("Sending %s to blackhole" % tor_title)
tor_name = cleanName(tor_title).replace(' ', '_')
if tor_url and tor_url.startswith('magnet'):
if lazylibrarian.CONFIG['TOR_CONVERT_MAGNET']:
hashid = CalcTorrentHash(tor_url)
tor_name = 'meta-' + hashid + '.torrent'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name)
result = magnet2torrent(tor_url, tor_path)
if result is not False:
logger.debug('Magnet file saved as: %s' % tor_path)
downloadID = Source
else:
tor_name += '.magnet'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name)
msg = ''
try:
msg = 'Opening '
with open(tor_path, 'wb') as torrent_file:
msg += 'Writing '
if isinstance(torrent, text_type):
torrent = torrent.encode('iso-8859-1')
torrent_file.write(torrent)
msg += 'SettingPerm '
setperm(tor_path)
msg += 'Saved '
logger.debug('Magnet file saved: %s' % tor_path)
downloadID = Source
except Exception as e:
logger.warn("Failed to write magnet to file: %s %s" % (type(e).__name__, str(e)))
logger.debug("Progress: %s" % msg)
logger.debug("Filename [%s]" % (repr(tor_path)))
return False
else:
tor_name += '.torrent'
tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name)
msg = ''
try:
msg = 'Opening '
with open(tor_path, 'wb') as torrent_file:
msg += 'Writing '
if isinstance(torrent, text_type):
torrent = torrent.encode('iso-8859-1')
torrent_file.write(torrent)
msg += 'SettingPerm '
setperm(tor_path)
msg += 'Saved '
logger.debug('Torrent file saved: %s' % tor_name)
downloadID = Source
except Exception as e:
logger.warn("Failed to write torrent to file: %s %s" % (type(e).__name__, str(e)))
logger.debug("Progress: %s" % msg)
logger.debug("Filename [%s]" % (repr(tor_path)))
return False
hashid = CalcTorrentHash(torrent)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_UTORRENT'] and lazylibrarian.CONFIG['UTORRENT_HOST']:
logger.debug("Sending %s to Utorrent" % tor_title)
Source = "UTORRENT"
downloadID = utorrent.addTorrent(tor_url, hashid) # returns hash or False
if downloadID:
tor_title = utorrent.nameTorrent(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_RTORRENT'] and lazylibrarian.CONFIG['RTORRENT_HOST']:
logger.debug("Sending %s to rTorrent" % tor_title)
Source = "RTORRENT"
downloadID = rtorrent.addTorrent(tor_url, hashid) # returns hash or False
if downloadID:
tor_title = rtorrent.getName(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_QBITTORRENT'] and lazylibrarian.CONFIG['QBITTORRENT_HOST']:
logger.debug("Sending %s to qbittorrent" % tor_title)
Source = "QBITTORRENT"
if isinstance(torrent, binary_type) and torrent.startswith(b'magnet'):
status = qbittorrent.addTorrent(torrent, hashid)
elif isinstance(torrent, text_type) and torrent.startswith('magnet'):
status = qbittorrent.addTorrent(torrent, hashid)
else:
status = qbittorrent.addTorrent(tor_url, hashid) # returns True or False
if status:
downloadID = hashid
tor_title = qbittorrent.getName(hashid)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_TRANSMISSION'] and lazylibrarian.CONFIG['TRANSMISSION_HOST']:
logger.debug("Sending %s to Transmission" % tor_title)
if lazylibrarian.LOGLEVEL & lazylibrarian.log_dlcomms:
logger.debug("TORRENT %s [%s] [%s]" % (len(torrent), torrent[:20], torrent[-20:]))
Source = "TRANSMISSION"
if isinstance(torrent, binary_type) and torrent.startswith(b'magnet'):
downloadID = transmission.addTorrent(torrent) # returns id or False
elif isinstance(torrent, text_type) and torrent.startswith('magnet'):
downloadID = transmission.addTorrent(torrent)
elif torrent:
downloadID = transmission.addTorrent(None, metainfo=b64encode(torrent))
else:
downloadID = transmission.addTorrent(tor_url) # returns id or False
if downloadID:
# transmission returns it's own int, but we store hashid instead
downloadID = hashid
tor_title = transmission.getTorrentFolder(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_SYNOLOGY'] and lazylibrarian.CONFIG['USE_SYNOLOGY'] and \
lazylibrarian.CONFIG['SYNOLOGY_HOST']:
logger.debug("Sending %s to Synology" % tor_title)
Source = "SYNOLOGY_TOR"
downloadID = synology.addTorrent(tor_url) # returns id or False
if downloadID:
tor_title = synology.getName(downloadID)
if lazylibrarian.CONFIG['TOR_DOWNLOADER_DELUGE'] and lazylibrarian.CONFIG['DELUGE_HOST']:
logger.debug("Sending %s to Deluge" % tor_title)
if not lazylibrarian.CONFIG['DELUGE_USER']:
# no username, talk to the webui
Source = "DELUGEWEBUI"
if isinstance(torrent, binary_type) and torrent.startswith(b'magnet'):
downloadID = deluge.addTorrent(torrent)
elif isinstance(torrent, text_type) and torrent.startswith('magnet'):
downloadID = deluge.addTorrent(torrent)
elif torrent:
downloadID = deluge.addTorrent(tor_title, data=b64encode(torrent))
else:
downloadID = deluge.addTorrent(tor_url) # can be link or magnet, returns hash or False
if downloadID:
tor_title = deluge.getTorrentFolder(downloadID)
else:
# have username, talk to the daemon
Source = "DELUGERPC"
client = DelugeRPCClient(lazylibrarian.CONFIG['DELUGE_HOST'],
int(lazylibrarian.CONFIG['DELUGE_PORT']),
lazylibrarian.CONFIG['DELUGE_USER'],
lazylibrarian.CONFIG['DELUGE_PASS'])
try:
client.connect()
args = {"name": tor_title}
if tor_url.startswith('magnet'):
downloadID = client.call('core.add_torrent_magnet', tor_url, args)
elif isinstance(torrent, binary_type) and torrent.startswith(b'magnet'):
downloadID = client.call('core.add_torrent_magnet', torrent, args)
elif isinstance(torrent, text_type) and torrent.startswith('magnet'):
downloadID = client.call('core.add_torrent_magnet', torrent, args)
elif torrent:
downloadID = client.call('core.add_torrent_file', tor_title, b64encode(torrent), args)
else:
downloadID = client.call('core.add_torrent_url', tor_url, args)
if downloadID:
if lazylibrarian.CONFIG['DELUGE_LABEL']:
_ = client.call('label.set_torrent', downloadID, lazylibrarian.CONFIG['DELUGE_LABEL'].lower())
result = client.call('core.get_torrent_status', downloadID, {})
# for item in result:
# logger.debug ('Deluge RPC result %s: %s' % (item, result[item]))
if 'name' in result:
tor_title = result['name']
except Exception as e:
logger.error('DelugeRPC failed %s %s' % (type(e).__name__, str(e)))
return False
if not Source:
logger.warn('No torrent download method is enabled, check config.')
return False
if downloadID:
if tor_title:
if downloadID.upper() in tor_title.upper():
logger.warn('%s: name contains hash, probably unresolved magnet' % Source)
else:
tor_title = unaccented_str(tor_title)
# need to check against reject words list again as the name may have changed
# library = magazine eBook AudioBook to determine which reject list
# but we can't easily do the per-magazine rejects
if library == 'magazine':
reject_list = getList(lazylibrarian.CONFIG['REJECT_MAGS'])
elif library == 'eBook':
reject_list = getList(lazylibrarian.CONFIG['REJECT_WORDS'])
elif library == 'AudioBook':
reject_list = getList(lazylibrarian.CONFIG['REJECT_AUDIO'])
else:
logger.debug("Invalid library [%s] in TORDownloadMethod" % library)
reject_list = []
rejected = False
lower_title = tor_title.lower()
for word in reject_list:
if word in lower_title:
rejected = True
logger.debug("Rejecting torrent name %s, contains %s" % (tor_title, word))
break
if rejected:
myDB.action('UPDATE wanted SET status="Failed" WHERE NZBurl=?', (full_url,))
delete_task(Source, downloadID, True)
return False
else:
logger.debug('%s setting torrent name to [%s]' % (Source, tor_title))
myDB.action('UPDATE wanted SET NZBtitle=? WHERE NZBurl=?', (tor_title, full_url))
if library == 'eBook':
myDB.action('UPDATE books SET status="Snatched" WHERE BookID=?', (bookid,))
elif library == 'AudioBook':
myDB.action('UPDATE books SET audiostatus="Snatched" WHERE BookID=?', (bookid,))
myDB.action('UPDATE wanted SET status="Snatched", Source=?, DownloadID=? WHERE NZBurl=?',
(Source, downloadID, full_url))
return True
logger.error('Failed to download torrent from %s, %s' % (Source, tor_url))
myDB.action('UPDATE wanted SET status="Failed" WHERE NZBurl=?', (full_url,))
return False
def TORDownloadMethod(bookid=None, tor_prov=None, tor_title=None, tor_url=None):
myDB = database.DBConnection()
download = False
full_url = tor_url # keep the url as stored in "wanted" table
if (lazylibrarian.TOR_DOWNLOADER_DELUGE or
lazylibrarian.TOR_DOWNLOADER_UTORRENT or
lazylibrarian.TOR_DOWNLOADER_QBITTORRENT or
lazylibrarian.TOR_DOWNLOADER_BLACKHOLE or
lazylibrarian.TOR_DOWNLOADER_TRANSMISSION):
if tor_url.startswith('magnet'):
torrent = tor_url # allow magnet link to write to blackhole and hash to utorrent
else:
if '&file=' in tor_url:
# torznab results need to be re-encoded
# had a problem with torznab utf-8 encoded strings not matching
# our utf-8 strings because of long/short form differences
url = tor_url.split('&file=')[0]
value = tor_url.split('&file=')[1]
if isinstance(value, str):
value = value.decode('utf-8') # make unicode
value = unicodedata.normalize('NFC', value) # normalize to short form
value = value.encode('unicode-escape') # then escape the result
value = value.replace(' ', '%20') # and encode any spaces
tor_url = url + '&file=' + value
# strip url back to the .torrent as some sites add parameters
if '?' in tor_url:
tor_url = tor_url.split('?')[0]
request = urllib2.Request(ur'%s' % tor_url)
if lazylibrarian.PROXY_HOST:
request.set_proxy(lazylibrarian.PROXY_HOST, lazylibrarian.PROXY_TYPE)
request.add_header('Accept-encoding', 'gzip')
request.add_header('User-Agent', common.USER_AGENT)
# PAB removed this, KAT serves us html instead of torrent if this header is sent
# if tor_prov == 'KAT':
# host = lazylibrarian.KAT_HOST
# if not str(host)[:4] == "http":
# host = 'http://' + host
# request.add_header('Referer', host)
try:
response = urllib2.urlopen(request, timeout=90)
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO(response.read())
f = gzip.GzipFile(fileobj=buf)
torrent = f.read()
else:
torrent = response.read()
except (urllib2.URLError, socket.timeout) as e:
logger.warn('Error fetching torrent from url: %s, %s' % (tor_url, e))
return False
if lazylibrarian.TOR_DOWNLOADER_BLACKHOLE:
tor_title = common.removeDisallowedFilenameChars(tor_title)
logger.debug("Sending %s to blackhole" % tor_title)
tor_name = str.replace(str(tor_title), ' ', '_')
if tor_url.startswith('magnet'):
tor_name = tor_name + '.magnet'
else:
tor_name = tor_name + '.torrent'
tor_path = os.path.join(lazylibrarian.TORRENT_DIR, tor_name)
with open(tor_path, 'wb') as torrent_file:
torrent_file.write(torrent)
logger.debug('Torrent file saved: %s' % tor_title)
download = True
if (lazylibrarian.TOR_DOWNLOADER_UTORRENT and lazylibrarian.UTORRENT_HOST):
logger.debug("Sending %s to Utorrent" % tor_title)
hash = CalcTorrentHash(torrent)
download = utorrent.addTorrent(tor_url, hash)
if (lazylibrarian.TOR_DOWNLOADER_QBITTORRENT and lazylibrarian.QBITTORRENT_HOST):
logger.debug("Sending %s to qbittorrent" % tor_title)
download = qbittorrent.addTorrent(tor_url)
if (lazylibrarian.TOR_DOWNLOADER_TRANSMISSION and lazylibrarian.TRANSMISSION_HOST):
logger.debug("Sending %s to Transmission" % tor_title)
download = transmission.addTorrent(tor_url)
if (lazylibrarian.TOR_DOWNLOADER_DELUGE and lazylibrarian.DELUGE_HOST):
logger.debug("Sending %s to Deluge" % tor_title)
if not lazylibrarian.DELUGE_USER:
# no username, talk to the webui
download = deluge.addTorrent(tor_url)
else:
# have username, talk to the daemon
client = DelugeRPCClient(lazylibrarian.DELUGE_HOST,
int(lazylibrarian.DELUGE_PORT),
lazylibrarian.DELUGE_USER,
lazylibrarian.DELUGE_PASS)
client.connect()
args = {"name": tor_title}
download = client.call('core.add_torrent_url', tor_url, args)
logger.debug('Deluge torrent_id: %s' % download)
if download and lazylibrarian.DELUGE_LABEL:
labelled = client.call('label.set_torrent', download, lazylibrarian.DELUGE_LABEL)
logger.debug('Deluge label returned: %s' % labelled)
else:
logger.warn('No torrent download method is enabled, check config.')
return False
if download:
logger.debug(u'Torrent id %s has been downloaded from %s' % (download, tor_url))
myDB.action('UPDATE books SET status = "Snatched" WHERE BookID="%s"' % bookid)
myDB.action('UPDATE wanted SET status = "Snatched" WHERE NZBurl="%s"' % full_url)
return True
else:
logger.error(u'Failed to download torrent @ <a href="%s">%s</a>' % (full_url, tor_url))
myDB.action('UPDATE wanted SET status = "Failed" WHERE NZBurl="%s"' % full_url)
return False
def DownloadMethod(bookid=None, tor_prov=None, tor_title=None, tor_url=None):
myDB = database.DBConnection()
download = False
if (lazylibrarian.USE_TOR) and (
lazylibrarian.TOR_DOWNLOADER_DELUGE
or lazylibrarian.TOR_DOWNLOADER_UTORRENT
or lazylibrarian.TOR_DOWNLOADER_BLACKHOLE
or lazylibrarian.TOR_DOWNLOADER_TRANSMISSION
):
request = urllib2.Request(tor_url)
if lazylibrarian.PROXY_HOST:
request.set_proxy(lazylibrarian.PROXY_HOST, lazylibrarian.PROXY_TYPE)
request.add_header("Accept-encoding", "gzip")
request.add_header("User-Agent", common.USER_AGENT)
if tor_prov == "KAT":
request.add_header("Referer", "http://kat.ph/")
response = urllib2.urlopen(request)
if response.info().get("Content-Encoding") == "gzip":
buf = StringIO(response.read())
f = gzip.GzipFile(fileobj=buf)
torrent = f.read()
else:
torrent = response.read()
if lazylibrarian.TOR_DOWNLOADER_BLACKHOLE:
logger.info("Torrent blackhole")
tor_title = common.removeDisallowedFilenameChars(tor_title)
tor_name = str.replace(str(tor_title), " ", "_") + ".torrent"
tor_path = os.path.join(lazylibrarian.TORRENT_DIR, tor_name)
torrent_file = open(tor_path, "wb")
torrent_file.write(torrent)
torrent_file.close()
logger.info("Torrent file saved: %s" % tor_title)
download = True
if lazylibrarian.TOR_DOWNLOADER_UTORRENT:
logger.info("Utorrent")
hash = CalcTorrentHash(torrent)
download = utorrent.addTorrent(tor_url, hash)
if lazylibrarian.TOR_DOWNLOADER_TRANSMISSION:
logger.info("Transmission")
download = transmission.addTorrent(tor_url)
if lazylibrarian.TOR_DOWNLOADER_DELUGE:
client = DelugeRPCClient(
lazylibrarian.DELUGE_HOST,
int(lazylibrarian.DELUGE_PORT),
lazylibrarian.DELUGE_USER,
lazylibrarian.DELUGE_PASS,
)
client.connect()
download = client.call("add_torrent_url", tor_url, {"name": tor_title})
logger.info("Deluge return value: %s" % download)
else:
logger.error("No torrent download method is enabled, check config.")
return False
if download:
logger.debug("Torrent file has been downloaded from " + str(tor_url))
myDB.action('UPDATE books SET status = "Snatched" WHERE BookID=?', [bookid])
myDB.action('UPDATE wanted SET status = "Snatched" WHERE NZBurl=?', [tor_url])
else:
logger.error(u'Failed to download torrent @ <a href="%s">%s</a>' % (tor_url, tor_url))
myDB.action('UPDATE wanted SET status = "Failed" WHERE NZBurl=?', [tor_url])
def add_torrent(torrent):
''' Adds torrent or magnet to Deluge
torrent: dict of torrrent/magnet information
Returns dict {'response': True, 'downloadid': 'id'}
{'response': False, 'error': 'exception'}
'''
logging.info('Sending torrent {} to DelugeRPC.'.format(torrent['title']))
conf = core.CONFIG['Downloader']['Torrent']['DelugeRPC']
host = conf['host']
port = conf['port']
user = conf['user']
password = conf['pass']
client = DelugeRPCClient(host, port, user, password)
try:
error = client.connect()
if error:
return {'response': False, 'error': error}
except Exception as e:
logging.error('Deluge Add Torrent.', exc_info=True)
return {'response': False, 'error': str(e)}
try:
def_download_path = client.call(
'core.get_config')[b'download_location'].decode('utf-8')
except Exception as e:
logging.error('Unable to get download path.', exc_info=True)
return {'response': False, 'error': 'Unable to get download path.'}
download_path = '{}/{}'.format(def_download_path, conf['category'])
priority_keys = {
'Low': 64,
'Normal': 128,
'High': 255,
}
options = {}
options['add_paused'] = conf['addpaused']
options['download_location'] = download_path
options['priority'] = priority_keys[conf['priority']]
ratio_limit = conf.get('seedratiolimit', '')
if ratio_limit != '':
options['stop_at_ratio'] = True
options['stop_ratio'] = ratio_limit
elif ratio_limit == -1:
torrent['options']['stop_at_ratio'] = False
if conf.get('removetorrents'):
options['remove_at_ratio'] = True
if torrent['type'] == 'magnet':
try:
downloadid = client.call('core.add_torrent_magnet',
torrent['torrentfile'],
options).decode('utf-8')
except Exception as e:
logging.error('Unable to send magnet.', exc_info=True)
return {'response': False, 'error': str(e)}
elif torrent['type'] == 'torrent':
try:
downloadid = (client.call('core.add_torrent_url',
torrent['torrentfile'], options)
or b'').decode('utf-8')
except Exception as e:
logging.error('Unable to send torrent.', exc_info=True)
return {'response': False, 'error': str(e)}
else:
return {
'response': False,
'error': 'Invalid torrent type {}'.format(torrent['type'])
}
_set_label(downloadid, conf['category'], client)
return {'response': True, 'downloadid': downloadid}
def TORDownloadMethod(bookid=None, tor_prov=None, tor_title=None, tor_url=None):
myDB = database.DBConnection()
download = False
full_url = tor_url # keep the url as stored in "wanted" table
if (lazylibrarian.USE_TOR) and (lazylibrarian.TOR_DOWNLOADER_DELUGE or lazylibrarian.TOR_DOWNLOADER_UTORRENT
or lazylibrarian.TOR_DOWNLOADER_BLACKHOLE or lazylibrarian.TOR_DOWNLOADER_TRANSMISSION):
if tor_url.startswith('magnet'):
torrent = tor_url # allow magnet link to write to blackhole and hash to utorrent
else:
if '&file=' in tor_url: # torznab results need to be re-encoded
url = tor_url.split('&file=')[0]
value = tor_url.split('&file=')[1]
if isinstance(value, str):
value = value.decode('utf-8') # make unicode
value = unicodedata.normalize('NFC', value) # normalize to short form
value = value.encode('unicode-escape') # then escape the result
value = value.replace(' ', '%20') # and encode any spaces
tor_url = url + '&file=' + value
request = urllib2.Request(ur'%s' % tor_url)
if lazylibrarian.PROXY_HOST:
request.set_proxy(lazylibrarian.PROXY_HOST, lazylibrarian.PROXY_TYPE)
request.add_header('Accept-encoding', 'gzip')
request.add_header('User-Agent', common.USER_AGENT)
if tor_prov == 'KAT':
host = lazylibrarian.KAT_HOST
if not str(host)[:4] == "http":
host = 'http://' + host
request.add_header('Referer', host)
try:
response = urllib2.urlopen(request)
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO(response.read())
f = gzip.GzipFile(fileobj=buf)
torrent = f.read()
else:
torrent = response.read()
except urllib2.URLError as e:
logger.warn('Error fetching torrent from url: ' + tor_url + ' %s' % e.reason)
return
# strip url back to the .torrent for passing to downloaders
# deluge needs it stripping, transmission doesn't mind
# not sure about utorrent
if '?' in tor_url:
tor_url = tor_url.split('?')[0]
if (lazylibrarian.TOR_DOWNLOADER_BLACKHOLE):
logger.debug('Torrent blackhole')
tor_title = common.removeDisallowedFilenameChars(tor_title)
tor_name = str.replace(str(tor_title), ' ', '_')
if tor_url.startswith('magnet'):
tor_name = tor_name + '.magnet'
else:
tor_name = tor_name + '.torrent'
tor_path = os.path.join(lazylibrarian.TORRENT_DIR, tor_name)
torrent_file = open(tor_path, 'wb')
torrent_file.write(torrent)
torrent_file.close()
logger.debug('Torrent file saved: %s' % tor_title)
download = True
if (lazylibrarian.TOR_DOWNLOADER_UTORRENT):
logger.debug('Utorrent')
hash = CalcTorrentHash(torrent)
download = utorrent.addTorrent(tor_url, hash)
if (lazylibrarian.TOR_DOWNLOADER_TRANSMISSION):
logger.debug('Transmission')
download = transmission.addTorrent(tor_url)
if (lazylibrarian.TOR_DOWNLOADER_DELUGE):
client = DelugeRPCClient(lazylibrarian.DELUGE_HOST,
int(lazylibrarian.DELUGE_PORT),
lazylibrarian.DELUGE_USER,
lazylibrarian.DELUGE_PASS)
if lazylibrarian.DELUGE_USER and lazylibrarian.DELUGE_PASS:
client.connect()
download = client.call('core.add_torrent_url', tor_url, {"name": tor_title})
logger.debug('Deluge return value: %s' % download)
else:
logger.warn('Need user & pass for deluge, check config.')
else:
logger.warn('No torrent download method is enabled, check config.')
return False
if download:
logger.debug(u'Torrent file has been downloaded from %s' % tor_url)
myDB.action('UPDATE books SET status = "Snatched" WHERE BookID="%s"' % bookid)
myDB.action('UPDATE wanted SET status = "Snatched" WHERE NZBurl="%s"' % full_url)
else:
logger.error(u'Failed to download torrent @ <a href="%s">%s</a>' % (full_url, tor_url))
myDB.action('UPDATE wanted SET status = "Failed" WHERE NZBurl="%s"' % full_url)