def get_torrents_status(stalled_for=None, progress={}): ''' Get torrents and calculate status Returns list ''' conf = core.CONFIG['Downloader']['Torrent']['DelugeRPC'] logging.info('Get torrents from DelugeRPC') host = conf['host'] port = conf['port'] user = conf['user'] password = conf['pass'] client = DelugeRPCClient(host, port, user, password) try: client.connect() torrents = [] now = int(datetime.timestamp(datetime.now())) fields = [ 'hash', 'state', 'name', 'last_seen_complete', 'time_since_download', 'total_payload_download' ] for id, torrent in client.call('core.get_torrents_status', { 'id': list(progress.keys()) }, fields).items(): # deluge return empty hash for every requested hash, even when it's missing if not torrent: continue logging.info(torrent) data = { 'hash': torrent[b'hash'].decode(), 'status': torrent[b'state'].lower().decode(), 'name': torrent[b'name'].decode() } if data['status'] == 'downloading' and stalled_for: if b'last_seen_complete' in torrent and b'time_since_download' in torrent: if now > torrent['last_seen_complete'] + stalled_for * 3600 and \ now > torrent['time_since_download'] + stalled_for * 3600: #data['status'] = 'stalled' logging.info('torrent {} detected as stalled'.format( data['name'])) elif data['hash'] in progress: data['progress'] = torrent[b'total_payload_download'] torrent_progress = progress[data['hash']] if data['progress'] == torrent_progress['progress'] and \ now > torrent_progress['time'] + stalled_for * 3600: #data['status'] = 'stalled' logging.info('torrent {} detected as stalled'.format( data['name'])) logging.info(data) torrents.append(data) return torrents except Exception as e: logging.error('Unable to list torrents from DelugeRPC.', exc_info=True) return []
def test_connection(config): ''' Tests connectivity to deluge daemon rpc config: dict of deluge server information Tests if we can open a socket to the rpc and creates DelugeRPC.client if successful Returns Bool True on success or str error message on failure ''' logging.info('Testing connection to DelugeRPC') host = config['host'] port = config['port'] user = config['user'] password = config['pass'] client = DelugeRPCClient(host, port, user, password) try: error = client.connect() if error: return '{}.'.format(error) except Exception as e: logging.error('Unable to connect to Deluge RPC.', exc_info=True) return str(e) else: return True
def test_connection(data): ''' Tests connectivity to deluge daemon rpc data: dict of deluge server information Tests if we can open a socket to the rpc Return True on success or str error message on failure ''' host = data['delugerpchost'] port = int(data['delugerpcport']) user = data['delugerpcuser'] password = data['delugerpcpass'] client = DelugeRPCClient(host, port, user, password) try: error = client.connect() if error: return '{}.'.format(error) except Exception, e: return str(e)
def add_torrent(data): ''' Adds torrent or magnet to Deluge data: dict of torrrent/magnet information Returns dict {'response': True, 'download_id': 'id'} {'response': False, 'error': 'exception'} ''' conf = core.CONFIG['Downloader']['Torrent']['DelugeRPC'] host = conf['host'] port = conf['port'] user = conf['user'] password = conf['pass'] client = DelugeRPCClient(host, port, user, password) try: error = client.connect() if error: return {'response': False, 'error': error} except Exception as e: logging.error('Deluge Add Torrent.', exc_info=True) return {'response': False, 'error': str(e)} try: def_download_path = client.call( 'core.get_config')[b'download_location'].decode('utf-8') except Exception as e: logging.error('Unable to get download path.', exc_info=True) return {'response': False, 'error': 'Unable to get download path.'} download_path = '{}/{}'.format(def_download_path, conf['category']) priority_keys = {'Normal': 0, 'High': 128, 'Max': 255} options = {} options['add_paused'] = conf['addpaused'] options['download_location'] = download_path options['priority'] = priority_keys[conf['priority']] if data['type'] == 'magnet': try: download_id = client.call('core.add_torrent_magnet', data['torrentfile'], options).decode('utf-8') return {'response': True, 'downloadid': download_id} except Exception as e: logging.error('Unable to send magnet.', exc_info=True) return {'response': False, 'error': str(e)} elif data['type'] == 'torrent': try: download_id = client.call('core.add_torrent_url', data['torrentfile'], options).decode('utf-8') return {'response': True, 'downloadid': download_id} except Exception as e: logging.error('Unable to send magnet.', exc_info=True) return {'response': False, 'error': str(e)} return
def cancel_download(downloadid): ''' Cancels download in client downloadid: int download id Returns bool ''' logging.info('Cancelling DelugeRPC download # {}'.format(downloadid)) conf = core.CONFIG['Downloader']['Torrent']['DelugeRPC'] host = conf['host'] port = conf['port'] user = conf['user'] password = conf['pass'] client = DelugeRPCClient(host, port, user, password) try: client.connect() return client.call('core.remove_torrent', downloadid, True) except Exception as e: logging.error('Unable to cancel download.', exc_info=True) return False
def add_torrent(data): ''' Adds torrent or magnet to Deluge data: dict of torrrent/magnet information Returns dict {'response': True, 'download_id': 'id'} {'response': False, 'error': 'exception'} ''' conf = core.CONFIG['Downloader']['Torrent']['DelugeRPC'] host = conf['host'] port = conf['port'] user = conf['user'] password = conf['pass'] client = DelugeRPCClient(host, port, user, password) try: error = client.connect() if error: return {'response': False, 'error': error} except Exception, e: return {'response': False, 'error': str(e)[1:-1]}
def add_torrent(data): ''' Adds torrent or magnet to Deluge data: dict of torrrent/magnet information Returns dict {'response': 'true', 'download_id': 'id'} {'response': 'false', 'error': 'exception'} ''' conf = core.CONFIG['DelugeRPC'] host = conf['delugerpchost'] port = int(conf['delugerpcport']) user = conf['delugerpcuser'] password = conf['delugerpcpass'] client = DelugeRPCClient(host, port, user, password) try: error = client.connect() if error: return {'response': 'false', 'error': error} except Exception, e: return {'response': 'false', 'error': str(e)}
def delete_task(Source, DownloadID, remove_data): try: if Source == "SABNZBD": sabnzbd.SABnzbd(DownloadID, 'delete', remove_data) elif Source == "NZBGET": nzbget.deleteNZB(DownloadID, remove_data) elif Source == "UTORRENT": utorrent.removeTorrent(DownloadID, remove_data) elif Source == "RTORRENT": rtorrent.removeTorrent(DownloadID, remove_data) elif Source == "QBITTORRENT": qbittorrent.removeTorrent(DownloadID, remove_data) elif Source == "TRANSMISSION": transmission.removeTorrent(DownloadID, remove_data) elif Source == "SYNOLOGY_TOR" or Source == "SYNOLOGY_NZB": synology.removeTorrent(DownloadID, remove_data) elif Source == "DELUGEWEBUI": deluge.removeTorrent(DownloadID, remove_data) elif Source == "DELUGERPC": client = DelugeRPCClient(lazylibrarian.DELUGE_HOST, int(lazylibrarian.DELUGE_PORT), lazylibrarian.DELUGE_USER, lazylibrarian.DELUGE_PASS) try: client.connect() client.call('core.remove_torrent', DownloadID, remove_data) except Exception as e: logger.debug('DelugeRPC failed %s' % str(e)) return False else: logger.debug("Unknown source [%s] in delete_task" % Source) return False return True except Exception as e: logger.debug("Failed to delete task %s from %s: %s" % (DownloadID, Source, str(e))) return False
def TORDownloadMethod(bookid=None, tor_prov=None, tor_title=None, tor_url=None): myDB = database.DBConnection() downloadID = False Source = '' full_url = tor_url # keep the url as stored in "wanted" table if tor_url and tor_url.startswith('magnet'): torrent = tor_url # allow magnet link to write to blackhole and hash to utorrent/rtorrent else: if '&file=' in tor_url: # torznab results need to be re-encoded # had a problem with torznab utf-8 encoded strings not matching # our utf-8 strings because of long/short form differences url = tor_url.split('&file=')[0] value = tor_url.split('&file=')[1] if isinstance(value, str): value = value.decode('utf-8') # make unicode value = unicodedata.normalize('NFC', value) # normalize to short form value = value.encode('unicode-escape') # then escape the result value = value.replace(' ', '%20') # and encode any spaces tor_url = url + '&file=' + value # strip url back to the .torrent as some sites add parameters if not tor_url.endswith('.torrent'): if '.torrent' in tor_url: tor_url = tor_url.split('.torrent')[0] + '.torrent' request = urllib2.Request(ur'%s' % tor_url) if lazylibrarian.PROXY_HOST: request.set_proxy(lazylibrarian.PROXY_HOST, lazylibrarian.PROXY_TYPE) request.add_header('Accept-encoding', 'gzip') request.add_header('User-Agent', USER_AGENT) try: response = urllib2.urlopen(request, timeout=90) if response.info().get('Content-Encoding') == 'gzip': buf = StringIO(response.read()) f = gzip.GzipFile(fileobj=buf) torrent = f.read() else: torrent = response.read() except (socket.timeout) as e: logger.warn('Timeout fetching torrent from url: %s' % tor_url) return False except (urllib2.URLError) as e: logger.warn('Error fetching torrent from url: %s, %s' % (tor_url, e.reason)) return False if lazylibrarian.TOR_DOWNLOADER_BLACKHOLE: Source = "BLACKHOLE" logger.debug("Sending %s to blackhole" % tor_title) tor_name = cleanName(tor_title).replace(' ', '_') tor_title = None if tor_url and tor_url.startswith('magnet'): if lazylibrarian.TOR_CONVERT_MAGNET: hashid = CalcTorrentHash(tor_url) tor_name = 'meta-' + hashid + '.torrent' tor_path = os.path.join(lazylibrarian.TORRENT_DIR, tor_name) result = magnet2torrent(tor_url, tor_path) if result is not False: logger.debug('Magnet file saved as: %s' % tor_path) downloadID = Source else: tor_name = tor_name + '.magnet' tor_path = os.path.join(lazylibrarian.TORRENT_DIR, tor_name) with open(tor_path, 'wb') as torrent_file: torrent_file.write(torrent) logger.debug('Magnet file saved: %s' % tor_path) setperm(tor_path) downloadID = Source else: tor_name = tor_name + '.torrent' tor_path = os.path.join(lazylibrarian.TORRENT_DIR, tor_name) with open(tor_path, 'wb') as torrent_file: torrent_file.write(torrent) setperm(tor_path) logger.debug('Torrent file saved: %s' % tor_name) downloadID = Source if (lazylibrarian.TOR_DOWNLOADER_UTORRENT and lazylibrarian.UTORRENT_HOST): logger.debug("Sending %s to Utorrent" % tor_title) Source = "UTORRENT" hashid = CalcTorrentHash(torrent) downloadID = utorrent.addTorrent(tor_url, hashid) # returns hash or False if downloadID: tor_title = utorrent.nameTorrent(downloadID) if (lazylibrarian.TOR_DOWNLOADER_RTORRENT and lazylibrarian.RTORRENT_HOST): logger.debug("Sending %s to rTorrent" % tor_title) Source = "RTORRENT" hashid = CalcTorrentHash(torrent) downloadID = rtorrent.addTorrent(tor_url, hashid) # returns hash or False if downloadID: tor_title = rtorrent.getName(downloadID) if (lazylibrarian.TOR_DOWNLOADER_QBITTORRENT and lazylibrarian.QBITTORRENT_HOST): logger.debug("Sending %s to qbittorrent" % tor_title) Source = "QBITTORRENT" hashid = CalcTorrentHash(torrent) status = qbittorrent.addTorrent(tor_url) # returns hash or False if status: downloadID = hashid tor_title = qbittorrent.getName(hashid) else: logger.debug("qbittorrent returned: %s" % str(response)) if (lazylibrarian.TOR_DOWNLOADER_TRANSMISSION and lazylibrarian.TRANSMISSION_HOST): logger.debug("Sending %s to Transmission" % tor_title) Source = "TRANSMISSION" downloadID = transmission.addTorrent(tor_url) # returns id or False if downloadID: # transmission returns it's own int, but we store hashid instead downloadID = CalcTorrentHash(torrent) tor_title = transmission.getTorrentFolder(downloadID) if (lazylibrarian.TOR_DOWNLOADER_SYNOLOGY and lazylibrarian.USE_SYNOLOGY and lazylibrarian.SYNOLOGY_HOST): logger.debug("Sending %s to Synology" % tor_title) Source = "SYNOLOGY_TOR" downloadID = synology.addTorrent(tor_url) # returns id or False if downloadID: tor_title = synology.getName(downloadID) if (lazylibrarian.TOR_DOWNLOADER_DELUGE and lazylibrarian.DELUGE_HOST): logger.debug("Sending %s to Deluge" % tor_title) if not lazylibrarian.DELUGE_USER: # no username, talk to the webui Source = "DELUGEWEBUI" downloadID = deluge.addTorrent(tor_url) # returns hash or False if downloadID: tor_title = deluge.getTorrentFolder(downloadID) else: # have username, talk to the daemon Source = "DELUGERPC" client = DelugeRPCClient(lazylibrarian.DELUGE_HOST, int(lazylibrarian.DELUGE_PORT), lazylibrarian.DELUGE_USER, lazylibrarian.DELUGE_PASS) try: client.connect() args = {"name": tor_title} if tor_url.startswith('magnet'): downloadID = client.call('core.add_torrent_magnet', tor_url, args) else: downloadID = client.call('core.add_torrent_url', tor_url, args) if downloadID: if lazylibrarian.DELUGE_LABEL: labelled = client.call('label.set_torrent', downloadID, lazylibrarian.DELUGE_LABEL) result = client.call('core.get_torrent_status', downloadID, {}) #for item in result: # logger.debug ('Deluge RPC result %s: %s' % (item, result[item])) if 'name' in result: tor_title = result['name'] except Exception as e: logger.debug('DelugeRPC failed %s' % str(e)) return False if not Source: logger.warn('No torrent download method is enabled, check config.') return False if downloadID: myDB.action('UPDATE books SET status = "Snatched" WHERE BookID="%s"' % bookid) myDB.action('UPDATE wanted SET status = "Snatched", Source = "%s", DownloadID = "%s" WHERE NZBurl="%s"' % (Source, downloadID, full_url)) if tor_title: if downloadID.upper() in tor_title.upper(): logger.warn('%s: name contains hash, probably unresolved magnet' % Source) else: tor_title = unaccented_str(tor_title) logger.debug('%s setting torrent name to [%s]' % (Source, tor_title)) myDB.action('UPDATE wanted SET NZBtitle = "%s" WHERE NZBurl="%s"' % (tor_title, full_url)) return True else: logger.error(u'Failed to download torrent from %s, %s' % (Source, tor_url)) myDB.action('UPDATE wanted SET status = "Failed" WHERE NZBurl="%s"' % full_url) return False
def TORDownloadMethod(bookid=None, tor_title=None, tor_url=None, library='eBook'): myDB = database.DBConnection() downloadID = False Source = '' full_url = tor_url # keep the url as stored in "wanted" table if tor_url and tor_url.startswith('magnet'): torrent = tor_url # allow magnet link to write to blackhole and hash to utorrent/rtorrent else: # h = HTMLParser() # tor_url = h.unescape(tor_url) # HTMLParser is probably overkill, we only seem to get & # tor_url = tor_url.replace('&', '&') if '&file=' in tor_url: # torznab results need to be re-encoded # had a problem with torznab utf-8 encoded strings not matching # our utf-8 strings because of long/short form differences url, value = tor_url.split('&file=', 1) value = makeUnicode(value) # ensure unicode value = unicodedata.normalize('NFC', value) # normalize to short form value = value.encode('unicode-escape') # then escape the result value = value.replace(' ', '%20') # and encode any spaces tor_url = url + '&file=' + value # strip url back to the .torrent as some sites add parameters if not tor_url.endswith('.torrent'): if '.torrent' in tor_url: tor_url = tor_url.split('.torrent')[0] + '.torrent' headers = {'Accept-encoding': 'gzip', 'User-Agent': USER_AGENT} proxies = proxyList() try: r = requests.get(tor_url, headers=headers, timeout=90, proxies=proxies) except requests.exceptions.Timeout: logger.warn('Timeout fetching file from url: %s' % tor_url) return False except Exception as e: if hasattr(e, 'reason'): logger.warn('%s fetching file from url: %s, %s' % (type(e).__name__, tor_url, e.reason)) else: logger.warn('%s fetching file from url: %s, %s' % (type(e).__name__, tor_url, str(e))) return False torrent = r.content if lazylibrarian.CONFIG['TOR_DOWNLOADER_BLACKHOLE']: Source = "BLACKHOLE" logger.debug("Sending %s to blackhole" % tor_title) tor_name = cleanName(tor_title).replace(' ', '_') if tor_url and tor_url.startswith('magnet'): if lazylibrarian.CONFIG['TOR_CONVERT_MAGNET']: hashid = CalcTorrentHash(tor_url) tor_name = 'meta-' + hashid + '.torrent' tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name) result = magnet2torrent(tor_url, tor_path) if result is not False: logger.debug('Magnet file saved as: %s' % tor_path) downloadID = Source else: tor_name += '.magnet' tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name) msg = '' try: msg = 'Opening ' with open(tor_path, 'wb') as torrent_file: msg += 'Writing ' if isinstance(torrent, unicode): torrent = torrent.encode('iso-8859-1') torrent_file.write(torrent) msg += 'SettingPerm' setperm(tor_path) msg += 'Saved' logger.debug('Magnet file saved: %s' % tor_path) downloadID = Source except Exception as e: logger.debug("Failed to write magnet to file: %s %s" % (type(e).__name__, str(e))) logger.debug("Progress: %s" % msg) logger.debug("Filename [%s]" % (repr(tor_path))) return False else: tor_name += '.torrent' tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name) msg = '' try: msg = 'Opening ' with open(tor_path, 'wb') as torrent_file: msg += 'Writing ' if isinstance(torrent, unicode): torrent = torrent.encode('iso-8859-1') torrent_file.write(torrent) msg += 'SettingPerm ' setperm(tor_path) msg += 'Saved' logger.debug('Torrent file saved: %s' % tor_name) downloadID = Source except Exception as e: logger.debug("Failed to write torrent to file: %s %s" % (type(e).__name__, str(e))) logger.debug("Progress: %s" % msg) logger.debug("Filename [%s]" % (repr(tor_path))) return False if lazylibrarian.CONFIG['TOR_DOWNLOADER_UTORRENT'] and lazylibrarian.CONFIG[ 'UTORRENT_HOST']: logger.debug("Sending %s to Utorrent" % tor_title) Source = "UTORRENT" hashid = CalcTorrentHash(torrent) downloadID = utorrent.addTorrent(tor_url, hashid) # returns hash or False if downloadID: tor_title = utorrent.nameTorrent(downloadID) if lazylibrarian.CONFIG['TOR_DOWNLOADER_RTORRENT'] and lazylibrarian.CONFIG[ 'RTORRENT_HOST']: logger.debug("Sending %s to rTorrent" % tor_title) Source = "RTORRENT" hashid = CalcTorrentHash(torrent) downloadID = rtorrent.addTorrent(tor_url, hashid) # returns hash or False if downloadID: tor_title = rtorrent.getName(downloadID) if lazylibrarian.CONFIG[ 'TOR_DOWNLOADER_QBITTORRENT'] and lazylibrarian.CONFIG[ 'QBITTORRENT_HOST']: logger.debug("Sending %s to qbittorrent" % tor_title) Source = "QBITTORRENT" hashid = CalcTorrentHash(torrent) status = qbittorrent.addTorrent(tor_url, hashid) # returns True or False if status: downloadID = hashid tor_title = qbittorrent.getName(hashid) if lazylibrarian.CONFIG[ 'TOR_DOWNLOADER_TRANSMISSION'] and lazylibrarian.CONFIG[ 'TRANSMISSION_HOST']: logger.debug("Sending %s to Transmission" % tor_title) Source = "TRANSMISSION" downloadID = transmission.addTorrent(tor_url) # returns id or False if downloadID: # transmission returns it's own int, but we store hashid instead downloadID = CalcTorrentHash(torrent) tor_title = transmission.getTorrentFolder(downloadID) if lazylibrarian.CONFIG['TOR_DOWNLOADER_SYNOLOGY'] and lazylibrarian.CONFIG['USE_SYNOLOGY'] and \ lazylibrarian.CONFIG['SYNOLOGY_HOST']: logger.debug("Sending %s to Synology" % tor_title) Source = "SYNOLOGY_TOR" downloadID = synology.addTorrent(tor_url) # returns id or False if downloadID: tor_title = synology.getName(downloadID) if lazylibrarian.CONFIG['TOR_DOWNLOADER_DELUGE'] and lazylibrarian.CONFIG[ 'DELUGE_HOST']: logger.debug("Sending %s to Deluge" % tor_title) if not lazylibrarian.CONFIG['DELUGE_USER']: # no username, talk to the webui Source = "DELUGEWEBUI" downloadID = deluge.addTorrent(tor_url) # returns hash or False if downloadID: tor_title = deluge.getTorrentFolder(downloadID) else: # have username, talk to the daemon Source = "DELUGERPC" client = DelugeRPCClient(lazylibrarian.CONFIG['DELUGE_HOST'], lazylibrarian.CONFIG['DELUGE_URL_BASE'], int(lazylibrarian.CONFIG['DELUGE_PORT']), lazylibrarian.CONFIG['DELUGE_USER'], lazylibrarian.CONFIG['DELUGE_PASS']) try: client.connect() args = {"name": tor_title} if tor_url.startswith('magnet'): downloadID = client.call('core.add_torrent_magnet', tor_url, args) else: downloadID = client.call('core.add_torrent_url', tor_url, args) if downloadID: if lazylibrarian.CONFIG['DELUGE_LABEL']: _ = client.call('label.set_torrent', downloadID, lazylibrarian.CONFIG['DELUGE_LABEL']) result = client.call('core.get_torrent_status', downloadID, {}) # for item in result: # logger.debug ('Deluge RPC result %s: %s' % (item, result[item])) if 'name' in result: tor_title = result['name'] except Exception as e: logger.debug('DelugeRPC failed %s %s' % (type(e).__name__, str(e))) return False if not Source: logger.warn('No torrent download method is enabled, check config.') return False if downloadID: if tor_title: if downloadID.upper() in tor_title.upper(): logger.warn( '%s: name contains hash, probably unresolved magnet' % Source) else: tor_title = unaccented_str(tor_title) # need to check against reject words list again as the name may have changed # library = magazine eBook AudioBook to determine which reject list # but we can't easily do the per-magazine rejects if library == 'magazine': reject_list = getList(lazylibrarian.CONFIG['REJECT_MAGS']) elif library == 'eBook': reject_list = getList(lazylibrarian.CONFIG['REJECT_WORDS']) elif library == 'AudioBook': reject_list = getList(lazylibrarian.CONFIG['REJECT_AUDIO']) else: logger.debug("Invalid library [%s] in TORDownloadMethod" % library) reject_list = [] rejected = False lower_title = tor_title.lower() for word in reject_list: if word in lower_title: rejected = True logger.debug("Rejecting torrent name %s, contains %s" % (tor_title, word)) break if rejected: myDB.action( 'UPDATE wanted SET status="Failed" WHERE NZBurl=?', (full_url, )) delete_task(Source, downloadID, True) return False else: logger.debug('%s setting torrent name to [%s]' % (Source, tor_title)) myDB.action('UPDATE wanted SET NZBtitle=? WHERE NZBurl=?', (tor_title, full_url)) if library == 'eBook': myDB.action('UPDATE books SET status="Snatched" WHERE BookID=?', (bookid, )) elif library == 'AudioBook': myDB.action( 'UPDATE books SET audiostatus="Snatched" WHERE BookID=?', (bookid, )) myDB.action( 'UPDATE wanted SET status="Snatched", Source=?, DownloadID=? WHERE NZBurl=?', (Source, downloadID, full_url)) return True logger.error('Failed to download torrent from %s, %s' % (Source, tor_url)) myDB.action('UPDATE wanted SET status="Failed" WHERE NZBurl=?', (full_url, )) return False
def TORDownloadMethod(bookid=None, tor_title=None, tor_url=None, library='eBook'): myDB = database.DBConnection() downloadID = False Source = '' torrent = '' full_url = tor_url # keep the url as stored in "wanted" table if 'magnet:?' in tor_url: # discard any other parameters and just use the magnet link tor_url = 'magnet:?' + tor_url.split('magnet:?')[1] else: # h = HTMLParser() # tor_url = h.unescape(tor_url) # HTMLParser is probably overkill, we only seem to get & # tor_url = tor_url.replace('&', '&') if '&file=' in tor_url: # torznab results need to be re-encoded # had a problem with torznab utf-8 encoded strings not matching # our utf-8 strings because of long/short form differences url, value = tor_url.split('&file=', 1) value = makeUnicode(value) # ensure unicode value = unicodedata.normalize('NFC', value) # normalize to short form value = value.encode('unicode-escape') # then escape the result value = makeUnicode(value) # ensure unicode value = value.replace(' ', '%20') # and encode any spaces tor_url = url + '&file=' + value # strip url back to the .torrent as some sites add extra parameters if not tor_url.endswith('.torrent') and '.torrent' in tor_url: tor_url = tor_url.split('.torrent')[0] + '.torrent' headers = {'Accept-encoding': 'gzip', 'User-Agent': getUserAgent()} proxies = proxyList() try: logger.debug("Fetching %s" % tor_url) r = requests.get(tor_url, headers=headers, timeout=90, proxies=proxies) if str(r.status_code).startswith('2'): torrent = r.content if not len(torrent): res = "Got empty response for %s" % tor_url logger.warn(res) return False, res elif len(torrent) < 100: res = "Only got %s bytes for %s" % (len(torrent), tor_url) logger.warn(res) return False, res else: logger.debug("Got %s bytes for %s" % (len(torrent), tor_url)) else: res = "Got a %s response for %s" % (r.status_code, tor_url) logger.warn(res) return False, res except requests.exceptions.Timeout: res = 'Timeout fetching file from url: %s' % tor_url logger.warn(res) return False, res except Exception as e: # some jackett providers redirect internally using http 301 to a magnet link # which requests can't handle, so throws an exception logger.debug("Requests exception: %s" % str(e)) if "magnet:?" in str(e): tor_url = 'magnet:?' + str(e).split('magnet:?')[1].strip("'") logger.debug("Redirecting to %s" % tor_url) else: if hasattr(e, 'reason'): res = '%s fetching file from url: %s, %s' % ( type(e).__name__, tor_url, e.reason) else: res = '%s fetching file from url: %s, %s' % ( type(e).__name__, tor_url, str(e)) logger.warn(res) return False, res if not torrent and not tor_url.startswith('magnet:?'): res = "No magnet or data, cannot continue" logger.warn(res) return False, res if lazylibrarian.CONFIG['TOR_DOWNLOADER_BLACKHOLE']: Source = "BLACKHOLE" logger.debug("Sending %s to blackhole" % tor_title) tor_name = cleanName(tor_title).replace(' ', '_') if tor_url and tor_url.startswith('magnet'): if lazylibrarian.CONFIG['TOR_CONVERT_MAGNET']: hashid = calculate_torrent_hash(tor_url) if not hashid: hashid = tor_name tor_name = 'meta-' + hashid + '.torrent' tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name) result = magnet2torrent(tor_url, tor_path) if result is not False: logger.debug('Magnet file saved as: %s' % tor_path) downloadID = Source else: tor_name += '.magnet' tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name) msg = '' try: msg = 'Opening ' with open(tor_path, 'wb') as torrent_file: msg += 'Writing ' if isinstance(torrent, text_type): torrent = torrent.encode('iso-8859-1') torrent_file.write(torrent) msg += 'SettingPerm ' setperm(tor_path) msg += 'Saved ' logger.debug('Magnet file saved: %s' % tor_path) downloadID = Source except Exception as e: res = "Failed to write magnet to file: %s %s" % ( type(e).__name__, str(e)) logger.warn(res) logger.debug("Progress: %s Filename [%s]" % (msg, repr(tor_path))) return False, res else: tor_name += '.torrent' tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name) msg = '' try: msg = 'Opening ' with open(tor_path, 'wb') as torrent_file: msg += 'Writing ' if isinstance(torrent, text_type): torrent = torrent.encode('iso-8859-1') torrent_file.write(torrent) msg += 'SettingPerm ' setperm(tor_path) msg += 'Saved ' logger.debug('Torrent file saved: %s' % tor_name) downloadID = Source except Exception as e: res = "Failed to write torrent to file: %s %s" % ( type(e).__name__, str(e)) logger.warn(res) logger.debug("Progress: %s Filename [%s]" % (msg, repr(tor_path))) return False, res hashid = calculate_torrent_hash(tor_url, torrent) if not hashid: res = "Unable to calculate torrent hash from url/data" logger.error(res) logger.debug("url: %s" % tor_url) logger.debug("data: %s" % makeUnicode(str(torrent[:50]))) return False, res if lazylibrarian.CONFIG['TOR_DOWNLOADER_UTORRENT'] and lazylibrarian.CONFIG[ 'UTORRENT_HOST']: logger.debug("Sending %s to Utorrent" % tor_title) Source = "UTORRENT" downloadID, res = utorrent.addTorrent(tor_url, hashid) # returns hash or False if downloadID: tor_title = utorrent.nameTorrent(downloadID) if lazylibrarian.CONFIG['TOR_DOWNLOADER_RTORRENT'] and lazylibrarian.CONFIG[ 'RTORRENT_HOST']: logger.debug("Sending %s to rTorrent" % tor_title) Source = "RTORRENT" if torrent: logger.debug("Sending %s data to rTorrent" % tor_title) downloadID, res = rtorrent.addTorrent(tor_title, hashid, data=torrent) else: logger.debug("Sending %s url to rTorrent" % tor_title) downloadID, res = rtorrent.addTorrent( tor_url, hashid) # returns hash or False if downloadID: tor_title = rtorrent.getName(downloadID) if lazylibrarian.CONFIG[ 'TOR_DOWNLOADER_QBITTORRENT'] and lazylibrarian.CONFIG[ 'QBITTORRENT_HOST']: Source = "QBITTORRENT" if torrent: logger.debug("Sending %s data to qBittorrent" % tor_title) status, res = qbittorrent.addFile(torrent, hashid, tor_title) else: logger.debug("Sending %s url to qBittorrent" % tor_title) status, res = qbittorrent.addTorrent( tor_url, hashid) # returns True or False if status: downloadID = hashid tor_title = qbittorrent.getName(hashid) if lazylibrarian.CONFIG[ 'TOR_DOWNLOADER_TRANSMISSION'] and lazylibrarian.CONFIG[ 'TRANSMISSION_HOST']: Source = "TRANSMISSION" if torrent: logger.debug("Sending %s data to Transmission" % tor_title) # transmission needs b64encoded metainfo to be unicode, not bytes downloadID, res = transmission.addTorrent(None, metainfo=makeUnicode( b64encode(torrent))) else: logger.debug("Sending %s url to Transmission" % tor_title) downloadID, res = transmission.addTorrent( tor_url) # returns id or False if downloadID: # transmission returns it's own int, but we store hashid instead downloadID = hashid tor_title = transmission.getTorrentFolder(downloadID) if lazylibrarian.CONFIG['TOR_DOWNLOADER_SYNOLOGY'] and lazylibrarian.CONFIG['USE_SYNOLOGY'] and \ lazylibrarian.CONFIG['SYNOLOGY_HOST']: logger.debug("Sending %s url to Synology" % tor_title) Source = "SYNOLOGY_TOR" downloadID, res = synology.addTorrent(tor_url) # returns id or False if downloadID: tor_title = synology.getName(downloadID) if lazylibrarian.CONFIG['TOR_DOWNLOADER_DELUGE'] and lazylibrarian.CONFIG[ 'DELUGE_HOST']: if not lazylibrarian.CONFIG['DELUGE_USER']: # no username, talk to the webui Source = "DELUGEWEBUI" if torrent: logger.debug("Sending %s data to Deluge" % tor_title) downloadID, res = deluge.addTorrent(tor_title, data=b64encode(torrent)) else: logger.debug("Sending %s url to Deluge" % tor_title) downloadID, res = deluge.addTorrent( tor_url) # can be link or magnet, returns hash or False if downloadID: tor_title = deluge.getTorrentFolder(downloadID) else: return False, res else: # have username, talk to the daemon Source = "DELUGERPC" client = DelugeRPCClient(lazylibrarian.CONFIG['DELUGE_HOST'], int(lazylibrarian.CONFIG['DELUGE_PORT']), lazylibrarian.CONFIG['DELUGE_USER'], lazylibrarian.CONFIG['DELUGE_PASS']) try: client.connect() args = {"name": tor_title} if tor_url.startswith('magnet'): res = "Sending %s magnet to DelugeRPC" % tor_title logger.debug(res) downloadID = client.call('core.add_torrent_magnet', tor_url, args) elif torrent: res = "Sending %s data to DelugeRPC" % tor_title logger.debug(res) downloadID = client.call('core.add_torrent_file', tor_title, b64encode(torrent), args) else: res = "Sending %s url to DelugeRPC" % tor_title logger.debug(res) downloadID = client.call('core.add_torrent_url', tor_url, args) if downloadID: if lazylibrarian.CONFIG['DELUGE_LABEL']: _ = client.call( 'label.set_torrent', downloadID, lazylibrarian.CONFIG['DELUGE_LABEL'].lower()) result = client.call('core.get_torrent_status', downloadID, {}) if 'name' in result: tor_title = result['name'] else: res += ' failed' logger.error(res) return False, res except Exception as e: res = 'DelugeRPC failed %s %s' % (type(e).__name__, str(e)) logger.error(res) return False, res if not Source: res = 'No torrent download method is enabled, check config.' logger.warn(res) return False, res if downloadID: if tor_title: if downloadID.upper() in tor_title.upper(): logger.warn( '%s: name contains hash, probably unresolved magnet' % Source) else: tor_title = unaccented_str(tor_title) # need to check against reject words list again as the name may have changed # library = magazine eBook AudioBook to determine which reject list # but we can't easily do the per-magazine rejects if library == 'Magazine': reject_list = getList(lazylibrarian.CONFIG['REJECT_MAGS'], ',') elif library == 'eBook': reject_list = getList(lazylibrarian.CONFIG['REJECT_WORDS'], ',') elif library == 'AudioBook': reject_list = getList(lazylibrarian.CONFIG['REJECT_AUDIO'], ',') else: logger.debug("Invalid library [%s] in TORDownloadMethod" % library) reject_list = [] rejected = False lower_title = tor_title.lower() for word in reject_list: if word in lower_title: rejected = "Rejecting torrent name %s, contains %s" % ( tor_title, word) logger.debug(rejected) break if not rejected: rejected = check_contents(Source, downloadID, library, tor_title) if rejected: myDB.action( 'UPDATE wanted SET status="Failed",DLResult=? WHERE NZBurl=?', (rejected, full_url)) delete_task(Source, downloadID, True) return False else: logger.debug('%s setting torrent name to [%s]' % (Source, tor_title)) myDB.action('UPDATE wanted SET NZBtitle=? WHERE NZBurl=?', (tor_title, full_url)) if library == 'eBook': myDB.action('UPDATE books SET status="Snatched" WHERE BookID=?', (bookid, )) elif library == 'AudioBook': myDB.action( 'UPDATE books SET audiostatus="Snatched" WHERE BookID=?', (bookid, )) myDB.action( 'UPDATE wanted SET status="Snatched", Source=?, DownloadID=? WHERE NZBurl=?', (Source, downloadID, full_url)) return True, '' res = 'Failed to send torrent to %s' % Source logger.error(res) return False, res
def TORDownloadMethod(bookid=None, tor_prov=None, tor_title=None, tor_url=None): myDB = database.DBConnection() download = False full_url = tor_url # keep the url as stored in "wanted" table if ( lazylibrarian.TOR_DOWNLOADER_DELUGE or lazylibrarian.TOR_DOWNLOADER_UTORRENT or lazylibrarian.TOR_DOWNLOADER_QBITTORRENT or lazylibrarian.TOR_DOWNLOADER_BLACKHOLE or lazylibrarian.TOR_DOWNLOADER_TRANSMISSION ): if tor_url.startswith("magnet"): torrent = tor_url # allow magnet link to write to blackhole and hash to utorrent else: if "&file=" in tor_url: # torznab results need to be re-encoded # had a problem with torznab utf-8 encoded strings not matching # our utf-8 strings because of long/short form differences url = tor_url.split("&file=")[0] value = tor_url.split("&file=")[1] if isinstance(value, str): value = value.decode("utf-8") # make unicode value = unicodedata.normalize("NFC", value) # normalize to short form value = value.encode("unicode-escape") # then escape the result value = value.replace(" ", "%20") # and encode any spaces tor_url = url + "&file=" + value # strip url back to the .torrent as some sites add parameters if "?" in tor_url: tor_url = tor_url.split("?")[0] request = urllib2.Request(ur"%s" % tor_url) if lazylibrarian.PROXY_HOST: request.set_proxy(lazylibrarian.PROXY_HOST, lazylibrarian.PROXY_TYPE) request.add_header("Accept-encoding", "gzip") request.add_header("User-Agent", common.USER_AGENT) # PAB removed this, KAT serves us html instead of torrent if this header is sent # if tor_prov == 'KAT': # host = lazylibrarian.KAT_HOST # if not str(host)[:4] == "http": # host = 'http://' + host # request.add_header('Referer', host) try: response = urllib2.urlopen(request, timeout=90) if response.info().get("Content-Encoding") == "gzip": buf = StringIO(response.read()) f = gzip.GzipFile(fileobj=buf) torrent = f.read() else: torrent = response.read() except urllib2.URLError as e: logger.warn("Error fetching torrent from url: " + tor_url + " %s" % e.reason) return False if lazylibrarian.TOR_DOWNLOADER_BLACKHOLE: tor_title = common.removeDisallowedFilenameChars(tor_title) logger.debug("Sending %s to blackhole" % tor_title) tor_name = str.replace(str(tor_title), " ", "_") if tor_url.startswith("magnet"): tor_name = tor_name + ".magnet" else: tor_name = tor_name + ".torrent" tor_path = os.path.join(lazylibrarian.TORRENT_DIR, tor_name) with open(tor_path, "wb") as torrent_file: torrent_file.write(torrent) logger.debug("Torrent file saved: %s" % tor_title) download = True if lazylibrarian.TOR_DOWNLOADER_UTORRENT: logger.debug("Sending %s to Utorrent" % tor_title) hash = CalcTorrentHash(torrent) download = utorrent.addTorrent(tor_url, hash) if lazylibrarian.TOR_DOWNLOADER_QBITTORRENT: logger.debug("Sending %s to qbittorrent" % tor_title) download = qbittorrent.addTorrent(tor_url) if lazylibrarian.TOR_DOWNLOADER_TRANSMISSION: logger.debug("Sending %s to Transmission" % tor_title) download = transmission.addTorrent(tor_url) if lazylibrarian.TOR_DOWNLOADER_DELUGE: logger.debug("Sending %s to Deluge" % tor_title) client = DelugeRPCClient( lazylibrarian.DELUGE_HOST, int(lazylibrarian.DELUGE_PORT), lazylibrarian.DELUGE_USER, lazylibrarian.DELUGE_PASS, ) if lazylibrarian.DELUGE_USER and lazylibrarian.DELUGE_PASS: client.connect() download = client.call("core.add_torrent_url", tor_url, {"name": tor_title}) logger.debug("Deluge return value: %s" % download) else: logger.warn("Need user & pass for deluge, check config.") else: logger.warn("No torrent download method is enabled, check config.") return False if download: logger.debug(u"Torrent file has been downloaded from %s" % tor_url) myDB.action('UPDATE books SET status = "Snatched" WHERE BookID="%s"' % bookid) myDB.action('UPDATE wanted SET status = "Snatched" WHERE NZBurl="%s"' % full_url) return True else: logger.error(u'Failed to download torrent @ <a href="%s">%s</a>' % (full_url, tor_url)) myDB.action('UPDATE wanted SET status = "Failed" WHERE NZBurl="%s"' % full_url) return False
def processDir(reset=False): try: threadname = threading.currentThread().name if "Thread-" in threadname: threading.currentThread().name = "POSTPROCESS" processpath = lazylibrarian.DIRECTORY('Download') logger.debug('Checking [%s] for files to post process' % processpath) try: downloads = os.listdir(processpath) except OSError as why: logger.error('Could not access [%s] directory [%s]' % (processpath, why.strerror)) return myDB = database.DBConnection() snatched = myDB.select('SELECT * from wanted WHERE Status="Snatched"') if len(snatched) == 0: logger.info('Nothing marked as snatched.') scheduleJob(action='Stop', target='processDir') return if len(downloads) == 0: logger.info('No downloads are found. Nothing to process yet.') return logger.info("Checking %s download%s for %s snatched file%s" % (len(downloads), plural(len(downloads)), len(snatched), plural(len(snatched)))) ppcount = 0 for book in snatched: # if torrent, see if we can get current status from the downloader as the name # may have been changed once magnet resolved, or download started or completed # depending on torrent downloader. Usenet doesn't change the name. We like usenet. torrentname = '' try: logger.debug("%s was sent to %s" % (book['NZBtitle'], book['Source'])) if book['Source'] == 'TRANSMISSION': torrentname = transmission.getTorrentFolder(book['DownloadID']) elif book['Source'] == 'UTORRENT': torrentname = utorrent.nameTorrent(book['DownloadID']) elif book['Source'] == 'RTORRENT': torrentname = rtorrent.getName(book['DownloadID']) elif book['Source'] == 'QBITTORRENT': torrentname = qbittorrent.getName(book['DownloadID']) elif book['Source'] == 'SYNOLOGY_TOR': torrentname = synology.getName(book['DownloadID']) elif book['Source'] == 'DELUGEWEBUI': torrentname = deluge.getTorrentFolder(book['DownloadID']) elif book['Source'] == 'DELUGERPC': client = DelugeRPCClient(lazylibrarian.DELUGE_HOST, int(lazylibrarian.DELUGE_PORT), lazylibrarian.DELUGE_USER, lazylibrarian.DELUGE_PASS) try: client.connect() result = client.call('core.get_torrent_status', book['DownloadID'], {}) # for item in result: # logger.debug ('Deluge RPC result %s: %s' % (item, result[item])) if 'name' in result: torrentname = unaccented_str(result['name']) except Exception as e: logger.debug('DelugeRPC failed %s' % str(e)) except Exception as e: logger.debug("Failed to get updated torrent name from %s for %s: %s" % (book['Source'], book['DownloadID'], str(e))) matchtitle = unaccented_str(book['NZBtitle']) if torrentname and torrentname != matchtitle: logger.debug("%s Changing [%s] to [%s]" % (book['Source'], matchtitle, torrentname)) myDB.action('UPDATE wanted SET NZBtitle = "%s" WHERE NZBurl = "%s"' % (torrentname, book['NZBurl'])) matchtitle = torrentname # here we could also check percentage downloaded or eta or status? # If downloader says it hasn't completed, no need to look for it. matches = [] logger.info('Looking for %s in %s' % (matchtitle, processpath)) for fname in downloads: # skip if failed before or incomplete torrents, or incomplete btsync extn = os.path.splitext(fname)[1] if extn not in ['.fail', '.part', '.bts', '.!ut']: # This is to get round differences in torrent filenames. # Usenet is ok, but Torrents aren't always returned with the name we searched for # We ask the torrent downloader for the torrent name, but don't always get an answer # so we try to do a "best match" on the name, there might be a better way... if isinstance(fname, str): matchname = fname.decode(lazylibrarian.SYS_ENCODING) else: matchname = fname if ' LL.(' in matchname: matchname = matchname.split(' LL.(')[0] match = 0 if matchtitle: if ' LL.(' in matchtitle: matchtitle = matchtitle.split(' LL.(')[0] match = fuzz.token_set_ratio(matchtitle, matchname) if match and match >= lazylibrarian.DLOAD_RATIO: fname = matchname if os.path.isfile(os.path.join(processpath, fname)): # handle single file downloads here. Book/mag file in download root. # move the file into it's own subdirectory so we don't move/delete things that aren't ours logger.debug('filename [%s] is a file' % os.path.join(processpath, fname)) if is_valid_booktype(fname, booktype="book") \ or is_valid_booktype(fname, booktype="mag"): logger.debug('filename [%s] is a valid book/mag' % os.path.join(processpath, fname)) if bts_file(processpath): logger.debug("Skipping %s, found a .bts file" % processpath) else: fname = os.path.splitext(fname)[0] dirname = os.path.join(processpath, fname) if not os.path.exists(dirname): try: os.makedirs(dirname) setperm(dirname) except OSError as why: logger.debug('Failed to create directory %s, %s' % (dirname, why.strerror)) if os.path.exists(dirname): # move the book and any related files too # ie other book formats, or opf, jpg with same title # can't move metadata.opf or cover.jpg or similar # as can't be sure they are ours # not sure if we need a new listdir here, or whether we can use the old one list_dir = os.listdir(processpath) for ourfile in list_dir: if ourfile.startswith(fname): if is_valid_booktype(ourfile, booktype="book") \ or is_valid_booktype(ourfile, booktype="mag") \ or os.path.splitext(ourfile)[1].lower() in ['.opf', '.jpg']: try: if lazylibrarian.DESTINATION_COPY: shutil.copyfile(os.path.join(processpath, ourfile), os.path.join(dirname, ourfile)) setperm(os.path.join(dirname, ourfile)) else: shutil.move(os.path.join(processpath, ourfile), os.path.join(dirname, ourfile)) setperm(os.path.join(dirname, ourfile)) except Exception as why: logger.debug("Failed to copy/move file %s to %s, %s" % (ourfile, dirname, str(why))) pp_path = os.path.join(processpath, fname) if os.path.isdir(pp_path): logger.debug('Found folder (%s%%) %s for %s' % (match, pp_path, matchtitle)) if not os.listdir(pp_path): logger.debug("Skipping %s, folder is empty" % pp_path) elif bts_file(pp_path): logger.debug("Skipping %s, found a .bts file" % pp_path) else: matches.append([match, pp_path, book]) else: pp_path = os.path.join(processpath, fname) matches.append([match, pp_path, book]) # so we can report closest match else: logger.debug('Skipping %s' % fname) match = 0 if matches: highest = max(matches, key=lambda x: x[0]) match = highest[0] pp_path = highest[1] book = highest[2] if match and match >= lazylibrarian.DLOAD_RATIO: logger.debug(u'Found match (%s%%): %s for %s' % (match, pp_path, book['NZBtitle'])) data = myDB.match('SELECT * from books WHERE BookID="%s"' % book['BookID']) if data: # it's a book logger.debug(u'Processing book %s' % book['BookID']) authorname = data['AuthorName'] bookname = data['BookName'] if 'windows' in platform.system().lower() and '/' in lazylibrarian.EBOOK_DEST_FOLDER: logger.warn('Please check your EBOOK_DEST_FOLDER setting') lazylibrarian.EBOOK_DEST_FOLDER = lazylibrarian.EBOOK_DEST_FOLDER.replace('/', '\\') # Default destination path, should be allowed change per config file. dest_path = lazylibrarian.EBOOK_DEST_FOLDER.replace('$Author', authorname).replace( '$Title', bookname) global_name = lazylibrarian.EBOOK_DEST_FILE.replace('$Author', authorname).replace( '$Title', bookname) global_name = unaccented(global_name) # dest_path = authorname+'/'+bookname # global_name = bookname + ' - ' + authorname # Remove characters we don't want in the filename BEFORE adding to DESTINATION_DIR # as windows drive identifiers have colon, eg c: but no colons allowed elsewhere? dic = {'<': '', '>': '', '...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': '', '*': '', ':': '', ';': '', '\'': ''} dest_path = unaccented_str(replace_all(dest_path, dic)) dest_path = os.path.join(processpath, dest_path).encode(lazylibrarian.SYS_ENCODING) else: data = myDB.match('SELECT * from magazines WHERE Title="%s"' % book['BookID']) if data: # it's a magazine logger.debug(u'Processing magazine %s' % book['BookID']) # AuxInfo was added for magazine release date, normally housed in 'magazines' but if multiple # files are downloading, there will be an error in post-processing, trying to go to the # same directory. mostrecentissue = data['IssueDate'] # keep for processing issues arriving out of order # Remove characters we don't want in the filename before (maybe) adding to DESTINATION_DIR # as windows drive identifiers have colon, eg c: but no colons allowed elsewhere? dic = {'<': '', '>': '', '...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': '', '*': '', ':': '', ';': '', '\'': ''} mag_name = unaccented_str(replace_all(book['BookID'], dic)) # book auxinfo is a cleaned date, eg 2015-01-01 dest_path = lazylibrarian.MAG_DEST_FOLDER.replace( '$IssueDate', book['AuxInfo']).replace('$Title', mag_name) if lazylibrarian.MAG_RELATIVE: if dest_path[0] not in '._': dest_path = '_' + dest_path dest_path = os.path.join(processpath, dest_path).encode( lazylibrarian.SYS_ENCODING) else: dest_path = dest_path.encode(lazylibrarian.SYS_ENCODING) authorname = None bookname = None global_name = lazylibrarian.MAG_DEST_FILE.replace('$IssueDate', book['AuxInfo']).replace( '$Title', mag_name) global_name = unaccented(global_name) else: # not recognised logger.debug('Nothing in database matching "%s"' % book['BookID']) continue else: logger.debug("Snatched %s %s is not in download directory" % (book['NZBmode'], book['NZBtitle'])) if match: logger.debug(u'Closest match (%s%%): %s' % (match, pp_path)) #for match in matches: # logger.info('Match: %s%% %s' % (match[0], match[1])) continue processBook = processDestination(pp_path, dest_path, authorname, bookname, global_name) if processBook: logger.debug("Processing %s, %s" % (global_name, book['NZBurl'])) # update nzbs, only update the snatched ones in case multiple matches for same book / magazine issue controlValueDict = {"BookID": book['BookID'], "NZBurl": book['NZBurl'], "Status": "Snatched"} newValueDict = {"Status": "Processed", "NZBDate": now()} # say when we processed it myDB.upsert("wanted", newValueDict, controlValueDict) if bookname: # it's a book, if None it's a magazine if len(lazylibrarian.IMP_CALIBREDB): logger.debug('Calibre should have created the extras for us') else: processExtras(myDB, dest_path, global_name, data) else: # update mags controlValueDict = {"Title": book['BookID']} if mostrecentissue: if mostrecentissue.isdigit() and str(book['AuxInfo']).isdigit(): older = int(mostrecentissue) > int(book['AuxInfo']) # issuenumber else: older = mostrecentissue > book['AuxInfo'] # YYYY-MM-DD else: older = False if older: # check this in case processing issues arriving out of order newValueDict = {"LastAcquired": today(), "IssueStatus": "Open"} else: newValueDict = {"IssueDate": book['AuxInfo'], "LastAcquired": today(), "IssueStatus": "Open"} myDB.upsert("magazines", newValueDict, controlValueDict) # dest_path is where we put the magazine after processing, but we don't have the full filename # so look for any "book" in that directory dest_file = book_file(dest_path, booktype='mag') controlValueDict = {"Title": book['BookID'], "IssueDate": book['AuxInfo']} newValueDict = {"IssueAcquired": today(), "IssueFile": dest_file, "IssueID": create_id("%s %s" % (book['BookID'], book['AuxInfo'])) } myDB.upsert("issues", newValueDict, controlValueDict) # create a thumbnail cover for the new issue create_cover(dest_file) # calibre or ll copied/moved the files we want, now delete source files to_delete = True if book['NZBmode'] in ['torrent', 'magnet']: # Only delete torrents if we don't want to keep seeding if lazylibrarian.KEEP_SEEDING: logger.warn('%s is seeding %s %s' % (book['Source'], book['NZBmode'], book['NZBtitle'])) to_delete = False else: # ask downloader to delete the torrent, but not the files # we may delete them later, depending on other settings if book['DownloadID'] != "unknown": logger.debug('Removing %s from %s' % (book['NZBtitle'], book['Source'].lower())) delete_task(book['Source'], book['DownloadID'], False) else: logger.warn("Unable to remove %s from %s, no DownloadID" % (book['NZBtitle'], book['Source'].lower())) if to_delete: # only delete the files if not in download root dir and if DESTINATION_COPY not set if not lazylibrarian.DESTINATION_COPY and (pp_path != processpath): if os.path.isdir(pp_path): # calibre might have already deleted it? try: shutil.rmtree(pp_path) except Exception as why: logger.debug("Unable to remove %s, %s" % (pp_path, str(why))) logger.info('Successfully processed: %s' % global_name) ppcount = ppcount + 1 notify_download("%s from %s at %s" % (global_name, book['NZBprov'], now())) else: logger.error('Postprocessing for %s has failed.' % global_name) logger.error('Warning - Residual files remain in %s.fail' % pp_path) controlValueDict = {"NZBurl": book['NZBurl'], "Status": "Snatched"} newValueDict = {"Status": "Failed", "NZBDate": now()} myDB.upsert("wanted", newValueDict, controlValueDict) # if it's a book, reset status so we try for a different version # if it's a magazine, user can select a different one from pastissues table if bookname: myDB.action('UPDATE books SET status = "Wanted" WHERE BookID="%s"' % book['BookID']) # at this point, as it failed we should move it or it will get postprocessed # again (and fail again) try: os.rename(pp_path, pp_path + '.fail') except Exception as e: logger.debug("Unable to rename %s, %s" % (pp_path, str(e))) downloads = os.listdir(processpath) # check in case we processed/deleted some above for directory in downloads: dname, extn = os.path.splitext(directory) if "LL.(" in dname and extn not in ['.fail', '.part', '.bts', '.!ut']: bookID = str(directory).split("LL.(")[1].split(")")[0] logger.debug("Book with id: " + str(bookID) + " found in download directory") pp_path = os.path.join(processpath, directory) if os.path.isfile(pp_path): pp_path = os.path.join(processpath) if (os.path.isdir(pp_path)): if import_book(pp_path, bookID): ppcount = ppcount + 1 if ppcount == 0: logger.info('No snatched books/mags have been found') else: logger.info('%s book%s/mag%s processed.' % (ppcount, plural(ppcount), plural(ppcount))) # Now check for any that are still marked snatched... if lazylibrarian.TASK_AGE: snatched = myDB.select('SELECT * from wanted WHERE Status="Snatched"') if len(snatched) > 0: for snatch in snatched: # FUTURE: we could check percentage downloaded or eta? # if percentage is increasing, it's just slow try: when_snatched = time.strptime(snatch['NZBdate'], '%Y-%m-%d %H:%M:%S') when_snatched = time.mktime(when_snatched) diff = time.time() - when_snatched # time difference in seconds except: diff = 0 hours = int(diff / 3600) if hours >= lazylibrarian.TASK_AGE: logger.warn('%s was sent to %s %s hours ago, deleting failed task' % (snatch['NZBtitle'], snatch['Source'].lower(), hours)) # change status to "Failed", and ask downloader to delete task and files if snatch['BookID'] != 'unknown': myDB.action('UPDATE wanted SET Status="Failed" WHERE BookID="%s"' % snatch['BookID']) myDB.action('UPDATE books SET status = "Wanted" WHERE BookID="%s"' % snatch['BookID']) delete_task(snatch['Source'], snatch['DownloadID'], True) if reset: scheduleJob(action='Restart', target='processDir') except Exception as e: logger.error('Unhandled exception in processDir: %s' % traceback.format_exc())
def TORDownloadMethod(bookid=None, tor_prov=None, tor_title=None, tor_url=None): myDB = database.DBConnection() download = False full_url = tor_url # keep the url as stored in "wanted" table if (lazylibrarian.TOR_DOWNLOADER_DELUGE or lazylibrarian.TOR_DOWNLOADER_UTORRENT or lazylibrarian.TOR_DOWNLOADER_QBITTORRENT or lazylibrarian.TOR_DOWNLOADER_BLACKHOLE or lazylibrarian.TOR_DOWNLOADER_TRANSMISSION): if tor_url and tor_url.startswith('magnet'): torrent = tor_url # allow magnet link to write to blackhole and hash to utorrent else: if '&file=' in tor_url: # torznab results need to be re-encoded # had a problem with torznab utf-8 encoded strings not matching # our utf-8 strings because of long/short form differences url = tor_url.split('&file=')[0] value = tor_url.split('&file=')[1] if isinstance(value, str): value = value.decode('utf-8') # make unicode value = unicodedata.normalize('NFC', value) # normalize to short form value = value.encode( 'unicode-escape') # then escape the result value = value.replace(' ', '%20') # and encode any spaces tor_url = url + '&file=' + value # strip url back to the .torrent as some sites add parameters if not tor_url.endswith('.torrent'): if '.torrent' in tor_url: tor_url = tor_url.split('.torrent')[0] + '.torrent' request = urllib2.Request(ur'%s' % tor_url) if lazylibrarian.PROXY_HOST: request.set_proxy(lazylibrarian.PROXY_HOST, lazylibrarian.PROXY_TYPE) request.add_header('Accept-encoding', 'gzip') request.add_header('User-Agent', USER_AGENT) # PAB removed this, KAT serves us html instead of torrent if this header is sent # if tor_prov == 'KAT': # host = lazylibrarian.KAT_HOST # if not str(host)[:4] == "http": # host = 'http://' + host # request.add_header('Referer', host) try: response = urllib2.urlopen(request, timeout=90) if response.info().get('Content-Encoding') == 'gzip': buf = StringIO(response.read()) f = gzip.GzipFile(fileobj=buf) torrent = f.read() else: torrent = response.read() except (urllib2.URLError, socket.timeout) as e: logger.warn('Error fetching torrent from url: %s, %s' % (tor_url, e)) return False if lazylibrarian.TOR_DOWNLOADER_BLACKHOLE: tor_title = cleanName(tor_title) logger.debug("Sending %s to blackhole" % tor_title) tor_name = str.replace(str(tor_title), ' ', '_') if tor_url and tor_url.startswith('magnet'): tor_name = tor_name + '.magnet' else: tor_name = tor_name + '.torrent' tor_path = os.path.join(lazylibrarian.TORRENT_DIR, tor_name) with open(tor_path, 'wb') as torrent_file: torrent_file.write(torrent) logger.debug('Torrent file saved: %s' % tor_title) download = True if (lazylibrarian.TOR_DOWNLOADER_UTORRENT and lazylibrarian.UTORRENT_HOST): logger.debug("Sending %s to Utorrent" % tor_title) hash = CalcTorrentHash(torrent) download = utorrent.addTorrent(tor_url, hash) if (lazylibrarian.TOR_DOWNLOADER_QBITTORRENT and lazylibrarian.QBITTORRENT_HOST): logger.debug("Sending %s to qbittorrent" % tor_title) download = qbittorrent.addTorrent(tor_url) if (lazylibrarian.TOR_DOWNLOADER_TRANSMISSION and lazylibrarian.TRANSMISSION_HOST): logger.debug("Sending %s to Transmission" % tor_title) download = transmission.addTorrent(tor_url) if (lazylibrarian.TOR_DOWNLOADER_DELUGE and lazylibrarian.DELUGE_HOST): logger.debug("Sending %s to Deluge" % tor_title) if not lazylibrarian.DELUGE_USER: # no username, talk to the webui download = deluge.addTorrent(tor_url) else: # have username, talk to the daemon client = DelugeRPCClient(lazylibrarian.DELUGE_HOST, int(lazylibrarian.DELUGE_PORT), lazylibrarian.DELUGE_USER, lazylibrarian.DELUGE_PASS) client.connect() args = {"name": tor_title} download = client.call('core.add_torrent_url', tor_url, args) logger.debug('Deluge torrent_id: %s' % download) if download and lazylibrarian.DELUGE_LABEL: labelled = client.call('label.set_torrent', download, lazylibrarian.DELUGE_LABEL) logger.debug('Deluge label returned: %s' % labelled) else: logger.warn('No torrent download method is enabled, check config.') return False if download: logger.debug(u'Torrent id %s has been downloaded from %s' % (download, tor_url)) myDB.action('UPDATE books SET status = "Snatched" WHERE BookID="%s"' % bookid) myDB.action('UPDATE wanted SET status = "Snatched" WHERE NZBurl="%s"' % full_url) return True else: logger.error(u'Failed to download torrent @ <a href="%s">%s</a>' % (full_url, tor_url)) myDB.action('UPDATE wanted SET status = "Failed" WHERE NZBurl="%s"' % full_url) return False
def TORDownloadMethod(bookid=None, tor_title=None, tor_url=None): myDB = database.DBConnection() downloadID = False Source = '' full_url = tor_url # keep the url as stored in "wanted" table if tor_url and tor_url.startswith('magnet'): torrent = tor_url # allow magnet link to write to blackhole and hash to utorrent/rtorrent else: # h = HTMLParser() # tor_url = h.unescape(tor_url) # HTMLParser is probably overkill, we only seem to get & # tor_url = tor_url.replace('&', '&') if '&file=' in tor_url: # torznab results need to be re-encoded # had a problem with torznab utf-8 encoded strings not matching # our utf-8 strings because of long/short form differences url, value = tor_url.split('&file=', 1) if isinstance(value, str): value = value.decode('utf-8') # make unicode value = unicodedata.normalize('NFC', value) # normalize to short form value = value.encode('unicode-escape') # then escape the result value = value.replace(' ', '%20') # and encode any spaces tor_url = url + '&file=' + value # strip url back to the .torrent as some sites add parameters if not tor_url.endswith('.torrent'): if '.torrent' in tor_url: tor_url = tor_url.split('.torrent')[0] + '.torrent' request = urllib2.Request(ur'%s' % tor_url) if lazylibrarian.CONFIG['PROXY_HOST']: request.set_proxy(lazylibrarian.CONFIG['PROXY_HOST'], lazylibrarian.CONFIG['PROXY_TYPE']) request.add_header('Accept-encoding', 'gzip') request.add_header('User-Agent', USER_AGENT) try: response = urllib2.urlopen(request, timeout=90) if response.info().get('Content-Encoding') == 'gzip': buf = StringIO(response.read()) f = gzip.GzipFile(fileobj=buf) torrent = f.read() else: torrent = response.read() except socket.timeout: logger.warn('Timeout fetching torrent from url: %s' % tor_url) return False except urllib2.URLError as e: logger.warn('Error fetching torrent from url: %s, %s' % (tor_url, e.reason)) return False except ValueError as e: logger.warn('Error, invalid url: [%s] %s' % (full_url, str(e))) return False if lazylibrarian.CONFIG['TOR_DOWNLOADER_BLACKHOLE']: Source = "BLACKHOLE" logger.debug("Sending %s to blackhole" % tor_title) tor_name = cleanName(tor_title).replace(' ', '_') if tor_url and tor_url.startswith('magnet'): if lazylibrarian.CONFIG['TOR_CONVERT_MAGNET']: hashid = CalcTorrentHash(tor_url) tor_name = 'meta-' + hashid + '.torrent' tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name) result = magnet2torrent(tor_url, tor_path) if result is not False: logger.debug('Magnet file saved as: %s' % tor_path) downloadID = Source else: tor_name += '.magnet' tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name) try: with open(tor_path, 'wb') as torrent_file: torrent_file.write(torrent) logger.debug('Magnet file saved: %s' % tor_path) setperm(tor_path) downloadID = Source except Exception as e: logger.debug("Failed to write magnet to file %s, %s" % (tor_path, str(e))) return False else: tor_name += '.torrent' tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name) try: with open(tor_path, 'wb') as torrent_file: torrent_file.write(torrent) setperm(tor_path) logger.debug('Torrent file saved: %s' % tor_name) downloadID = Source except Exception as e: logger.debug("Failed to write torrent to file %s, %s" % (tor_path, str(e))) return False if lazylibrarian.CONFIG['TOR_DOWNLOADER_UTORRENT'] and lazylibrarian.CONFIG['UTORRENT_HOST']: logger.debug("Sending %s to Utorrent" % tor_title) Source = "UTORRENT" hashid = CalcTorrentHash(torrent) downloadID = utorrent.addTorrent(tor_url, hashid) # returns hash or False if downloadID: tor_title = utorrent.nameTorrent(downloadID) if lazylibrarian.CONFIG['TOR_DOWNLOADER_RTORRENT'] and lazylibrarian.CONFIG['RTORRENT_HOST']: logger.debug("Sending %s to rTorrent" % tor_title) Source = "RTORRENT" hashid = CalcTorrentHash(torrent) downloadID = rtorrent.addTorrent(tor_url, hashid) # returns hash or False if downloadID: tor_title = rtorrent.getName(downloadID) if lazylibrarian.CONFIG['TOR_DOWNLOADER_QBITTORRENT'] and lazylibrarian.CONFIG['QBITTORRENT_HOST']: logger.debug("Sending %s to qbittorrent" % tor_title) Source = "QBITTORRENT" hashid = CalcTorrentHash(torrent) status = qbittorrent.addTorrent(tor_url) # returns hash or False if status: downloadID = hashid tor_title = qbittorrent.getName(hashid) if lazylibrarian.CONFIG['TOR_DOWNLOADER_TRANSMISSION'] and lazylibrarian.CONFIG['TRANSMISSION_HOST']: logger.debug("Sending %s to Transmission" % tor_title) Source = "TRANSMISSION" downloadID = transmission.addTorrent(tor_url) # returns id or False if downloadID: # transmission returns it's own int, but we store hashid instead downloadID = CalcTorrentHash(torrent) tor_title = transmission.getTorrentFolder(downloadID) if lazylibrarian.CONFIG['TOR_DOWNLOADER_SYNOLOGY'] and lazylibrarian.CONFIG['USE_SYNOLOGY'] and lazylibrarian.CONFIG['SYNOLOGY_HOST']: logger.debug("Sending %s to Synology" % tor_title) Source = "SYNOLOGY_TOR" downloadID = synology.addTorrent(tor_url) # returns id or False if downloadID: tor_title = synology.getName(downloadID) if lazylibrarian.CONFIG['TOR_DOWNLOADER_DELUGE'] and lazylibrarian.CONFIG['DELUGE_HOST']: logger.debug("Sending %s to Deluge" % tor_title) if not lazylibrarian.CONFIG['DELUGE_USER']: # no username, talk to the webui Source = "DELUGEWEBUI" downloadID = deluge.addTorrent(tor_url) # returns hash or False if downloadID: tor_title = deluge.getTorrentFolder(downloadID) else: # have username, talk to the daemon Source = "DELUGERPC" client = DelugeRPCClient(lazylibrarian.CONFIG['DELUGE_HOST'], int(lazylibrarian.CONFIG['DELUGE_PORT']), lazylibrarian.CONFIG['DELUGE_USER'], lazylibrarian.CONFIG['DELUGE_PASS']) try: client.connect() args = {"name": tor_title} if tor_url.startswith('magnet'): downloadID = client.call('core.add_torrent_magnet', tor_url, args) else: downloadID = client.call('core.add_torrent_url', tor_url, args) if downloadID: if lazylibrarian.CONFIG['DELUGE_LABEL']: _ = client.call('label.set_torrent', downloadID, lazylibrarian.CONFIG['DELUGE_LABEL']) result = client.call('core.get_torrent_status', downloadID, {}) # for item in result: # logger.debug ('Deluge RPC result %s: %s' % (item, result[item])) if 'name' in result: tor_title = result['name'] except Exception as e: logger.debug('DelugeRPC failed %s' % str(e)) return False if not Source: logger.warn('No torrent download method is enabled, check config.') return False if downloadID: myDB.action('UPDATE books SET status = "Snatched" WHERE BookID="%s"' % bookid) myDB.action('UPDATE wanted SET status = "Snatched", Source = "%s", DownloadID = "%s" WHERE NZBurl="%s"' % (Source, downloadID, full_url)) if tor_title: if downloadID.upper() in tor_title.upper(): logger.warn('%s: name contains hash, probably unresolved magnet' % Source) else: tor_title = unaccented_str(tor_title) logger.debug('%s setting torrent name to [%s]' % (Source, tor_title)) myDB.action('UPDATE wanted SET NZBtitle = "%s" WHERE NZBurl="%s"' % (tor_title, full_url)) return True else: logger.error(u'Failed to download torrent from %s, %s' % (Source, tor_url)) myDB.action('UPDATE wanted SET status = "Failed" WHERE NZBurl="%s"' % full_url) return False
def TORDownloadMethod(bookid=None, tor_title=None, tor_url=None, library='eBook'): myDB = database.DBConnection() downloadID = False Source = '' full_url = tor_url # keep the url as stored in "wanted" table if tor_url and tor_url.startswith('magnet:?'): torrent = tor_url # allow magnet link to write to blackhole and hash to utorrent/rtorrent elif 'magnet:?' in tor_url: # discard any other parameters and just use the magnet link torrent = 'magnet:?' + tor_url.split('magnet:?')[1] else: # h = HTMLParser() # tor_url = h.unescape(tor_url) # HTMLParser is probably overkill, we only seem to get & # tor_url = tor_url.replace('&', '&') if '&file=' in tor_url: # torznab results need to be re-encoded # had a problem with torznab utf-8 encoded strings not matching # our utf-8 strings because of long/short form differences url, value = tor_url.split('&file=', 1) value = makeUnicode(value) # ensure unicode value = unicodedata.normalize('NFC', value) # normalize to short form value = value.encode('unicode-escape') # then escape the result value = makeUnicode(value) # ensure unicode value = value.replace(' ', '%20') # and encode any spaces tor_url = url + '&file=' + value # strip url back to the .torrent as some sites add extra parameters if not tor_url.endswith('.torrent'): if '.torrent' in tor_url: tor_url = tor_url.split('.torrent')[0] + '.torrent' headers = {'Accept-encoding': 'gzip', 'User-Agent': USER_AGENT} proxies = proxyList() try: r = requests.get(tor_url, headers=headers, timeout=90, proxies=proxies) torrent = r.content except requests.exceptions.Timeout: logger.warn('Timeout fetching file from url: %s' % tor_url) return False except Exception as e: # some jackett providers redirect internally using http 301 to a magnet link # which requests can't handle, so throws an exception if "magnet:?" in str(e): torrent = 'magnet:?' + str(e).split('magnet:?')[1]. strip("'") else: if hasattr(e, 'reason'): logger.warn('%s fetching file from url: %s, %s' % (type(e).__name__, tor_url, e.reason)) else: logger.warn('%s fetching file from url: %s, %s' % (type(e).__name__, tor_url, str(e))) return False if lazylibrarian.CONFIG['TOR_DOWNLOADER_BLACKHOLE']: Source = "BLACKHOLE" logger.debug("Sending %s to blackhole" % tor_title) tor_name = cleanName(tor_title).replace(' ', '_') if tor_url and tor_url.startswith('magnet'): if lazylibrarian.CONFIG['TOR_CONVERT_MAGNET']: hashid = CalcTorrentHash(tor_url) tor_name = 'meta-' + hashid + '.torrent' tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name) result = magnet2torrent(tor_url, tor_path) if result is not False: logger.debug('Magnet file saved as: %s' % tor_path) downloadID = Source else: tor_name += '.magnet' tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name) msg = '' try: msg = 'Opening ' with open(tor_path, 'wb') as torrent_file: msg += 'Writing ' if isinstance(torrent, text_type): torrent = torrent.encode('iso-8859-1') torrent_file.write(torrent) msg += 'SettingPerm ' setperm(tor_path) msg += 'Saved ' logger.debug('Magnet file saved: %s' % tor_path) downloadID = Source except Exception as e: logger.warn("Failed to write magnet to file: %s %s" % (type(e).__name__, str(e))) logger.debug("Progress: %s" % msg) logger.debug("Filename [%s]" % (repr(tor_path))) return False else: tor_name += '.torrent' tor_path = os.path.join(lazylibrarian.CONFIG['TORRENT_DIR'], tor_name) msg = '' try: msg = 'Opening ' with open(tor_path, 'wb') as torrent_file: msg += 'Writing ' if isinstance(torrent, text_type): torrent = torrent.encode('iso-8859-1') torrent_file.write(torrent) msg += 'SettingPerm ' setperm(tor_path) msg += 'Saved ' logger.debug('Torrent file saved: %s' % tor_name) downloadID = Source except Exception as e: logger.warn("Failed to write torrent to file: %s %s" % (type(e).__name__, str(e))) logger.debug("Progress: %s" % msg) logger.debug("Filename [%s]" % (repr(tor_path))) return False hashid = CalcTorrentHash(torrent) if lazylibrarian.CONFIG['TOR_DOWNLOADER_UTORRENT'] and lazylibrarian.CONFIG['UTORRENT_HOST']: logger.debug("Sending %s to Utorrent" % tor_title) Source = "UTORRENT" downloadID = utorrent.addTorrent(tor_url, hashid) # returns hash or False if downloadID: tor_title = utorrent.nameTorrent(downloadID) if lazylibrarian.CONFIG['TOR_DOWNLOADER_RTORRENT'] and lazylibrarian.CONFIG['RTORRENT_HOST']: logger.debug("Sending %s to rTorrent" % tor_title) Source = "RTORRENT" downloadID = rtorrent.addTorrent(tor_url, hashid) # returns hash or False if downloadID: tor_title = rtorrent.getName(downloadID) if lazylibrarian.CONFIG['TOR_DOWNLOADER_QBITTORRENT'] and lazylibrarian.CONFIG['QBITTORRENT_HOST']: logger.debug("Sending %s to qbittorrent" % tor_title) Source = "QBITTORRENT" if isinstance(torrent, binary_type) and torrent.startswith(b'magnet'): status = qbittorrent.addTorrent(torrent, hashid) elif isinstance(torrent, text_type) and torrent.startswith('magnet'): status = qbittorrent.addTorrent(torrent, hashid) else: status = qbittorrent.addTorrent(tor_url, hashid) # returns True or False if status: downloadID = hashid tor_title = qbittorrent.getName(hashid) if lazylibrarian.CONFIG['TOR_DOWNLOADER_TRANSMISSION'] and lazylibrarian.CONFIG['TRANSMISSION_HOST']: logger.debug("Sending %s to Transmission" % tor_title) if lazylibrarian.LOGLEVEL & lazylibrarian.log_dlcomms: logger.debug("TORRENT %s [%s] [%s]" % (len(torrent), torrent[:20], torrent[-20:])) Source = "TRANSMISSION" if isinstance(torrent, binary_type) and torrent.startswith(b'magnet'): downloadID = transmission.addTorrent(torrent) # returns id or False elif isinstance(torrent, text_type) and torrent.startswith('magnet'): downloadID = transmission.addTorrent(torrent) elif torrent: downloadID = transmission.addTorrent(None, metainfo=b64encode(torrent)) else: downloadID = transmission.addTorrent(tor_url) # returns id or False if downloadID: # transmission returns it's own int, but we store hashid instead downloadID = hashid tor_title = transmission.getTorrentFolder(downloadID) if lazylibrarian.CONFIG['TOR_DOWNLOADER_SYNOLOGY'] and lazylibrarian.CONFIG['USE_SYNOLOGY'] and \ lazylibrarian.CONFIG['SYNOLOGY_HOST']: logger.debug("Sending %s to Synology" % tor_title) Source = "SYNOLOGY_TOR" downloadID = synology.addTorrent(tor_url) # returns id or False if downloadID: tor_title = synology.getName(downloadID) if lazylibrarian.CONFIG['TOR_DOWNLOADER_DELUGE'] and lazylibrarian.CONFIG['DELUGE_HOST']: logger.debug("Sending %s to Deluge" % tor_title) if not lazylibrarian.CONFIG['DELUGE_USER']: # no username, talk to the webui Source = "DELUGEWEBUI" if isinstance(torrent, binary_type) and torrent.startswith(b'magnet'): downloadID = deluge.addTorrent(torrent) elif isinstance(torrent, text_type) and torrent.startswith('magnet'): downloadID = deluge.addTorrent(torrent) elif torrent: downloadID = deluge.addTorrent(tor_title, data=b64encode(torrent)) else: downloadID = deluge.addTorrent(tor_url) # can be link or magnet, returns hash or False if downloadID: tor_title = deluge.getTorrentFolder(downloadID) else: # have username, talk to the daemon Source = "DELUGERPC" client = DelugeRPCClient(lazylibrarian.CONFIG['DELUGE_HOST'], int(lazylibrarian.CONFIG['DELUGE_PORT']), lazylibrarian.CONFIG['DELUGE_USER'], lazylibrarian.CONFIG['DELUGE_PASS']) try: client.connect() args = {"name": tor_title} if tor_url.startswith('magnet'): downloadID = client.call('core.add_torrent_magnet', tor_url, args) elif isinstance(torrent, binary_type) and torrent.startswith(b'magnet'): downloadID = client.call('core.add_torrent_magnet', torrent, args) elif isinstance(torrent, text_type) and torrent.startswith('magnet'): downloadID = client.call('core.add_torrent_magnet', torrent, args) elif torrent: downloadID = client.call('core.add_torrent_file', tor_title, b64encode(torrent), args) else: downloadID = client.call('core.add_torrent_url', tor_url, args) if downloadID: if lazylibrarian.CONFIG['DELUGE_LABEL']: _ = client.call('label.set_torrent', downloadID, lazylibrarian.CONFIG['DELUGE_LABEL'].lower()) result = client.call('core.get_torrent_status', downloadID, {}) # for item in result: # logger.debug ('Deluge RPC result %s: %s' % (item, result[item])) if 'name' in result: tor_title = result['name'] except Exception as e: logger.error('DelugeRPC failed %s %s' % (type(e).__name__, str(e))) return False if not Source: logger.warn('No torrent download method is enabled, check config.') return False if downloadID: if tor_title: if downloadID.upper() in tor_title.upper(): logger.warn('%s: name contains hash, probably unresolved magnet' % Source) else: tor_title = unaccented_str(tor_title) # need to check against reject words list again as the name may have changed # library = magazine eBook AudioBook to determine which reject list # but we can't easily do the per-magazine rejects if library == 'magazine': reject_list = getList(lazylibrarian.CONFIG['REJECT_MAGS']) elif library == 'eBook': reject_list = getList(lazylibrarian.CONFIG['REJECT_WORDS']) elif library == 'AudioBook': reject_list = getList(lazylibrarian.CONFIG['REJECT_AUDIO']) else: logger.debug("Invalid library [%s] in TORDownloadMethod" % library) reject_list = [] rejected = False lower_title = tor_title.lower() for word in reject_list: if word in lower_title: rejected = True logger.debug("Rejecting torrent name %s, contains %s" % (tor_title, word)) break if rejected: myDB.action('UPDATE wanted SET status="Failed" WHERE NZBurl=?', (full_url,)) delete_task(Source, downloadID, True) return False else: logger.debug('%s setting torrent name to [%s]' % (Source, tor_title)) myDB.action('UPDATE wanted SET NZBtitle=? WHERE NZBurl=?', (tor_title, full_url)) if library == 'eBook': myDB.action('UPDATE books SET status="Snatched" WHERE BookID=?', (bookid,)) elif library == 'AudioBook': myDB.action('UPDATE books SET audiostatus="Snatched" WHERE BookID=?', (bookid,)) myDB.action('UPDATE wanted SET status="Snatched", Source=?, DownloadID=? WHERE NZBurl=?', (Source, downloadID, full_url)) return True logger.error('Failed to download torrent from %s, %s' % (Source, tor_url)) myDB.action('UPDATE wanted SET status="Failed" WHERE NZBurl=?', (full_url,)) return False
def TORDownloadMethod(bookid=None, tor_prov=None, tor_title=None, tor_url=None): myDB = database.DBConnection() download = False full_url = tor_url # keep the url as stored in "wanted" table if (lazylibrarian.TOR_DOWNLOADER_DELUGE or lazylibrarian.TOR_DOWNLOADER_UTORRENT or lazylibrarian.TOR_DOWNLOADER_QBITTORRENT or lazylibrarian.TOR_DOWNLOADER_BLACKHOLE or lazylibrarian.TOR_DOWNLOADER_TRANSMISSION): if tor_url.startswith('magnet'): torrent = tor_url # allow magnet link to write to blackhole and hash to utorrent else: if '&file=' in tor_url: # torznab results need to be re-encoded # had a problem with torznab utf-8 encoded strings not matching # our utf-8 strings because of long/short form differences url = tor_url.split('&file=')[0] value = tor_url.split('&file=')[1] if isinstance(value, str): value = value.decode('utf-8') # make unicode value = unicodedata.normalize('NFC', value) # normalize to short form value = value.encode('unicode-escape') # then escape the result value = value.replace(' ', '%20') # and encode any spaces tor_url = url + '&file=' + value # strip url back to the .torrent as some sites add parameters if '?' in tor_url: tor_url = tor_url.split('?')[0] request = urllib2.Request(ur'%s' % tor_url) if lazylibrarian.PROXY_HOST: request.set_proxy(lazylibrarian.PROXY_HOST, lazylibrarian.PROXY_TYPE) request.add_header('Accept-encoding', 'gzip') request.add_header('User-Agent', common.USER_AGENT) # PAB removed this, KAT serves us html instead of torrent if this header is sent # if tor_prov == 'KAT': # host = lazylibrarian.KAT_HOST # if not str(host)[:4] == "http": # host = 'http://' + host # request.add_header('Referer', host) try: response = urllib2.urlopen(request, timeout=90) if response.info().get('Content-Encoding') == 'gzip': buf = StringIO(response.read()) f = gzip.GzipFile(fileobj=buf) torrent = f.read() else: torrent = response.read() except (urllib2.URLError, socket.timeout) as e: logger.warn('Error fetching torrent from url: %s, %s' % (tor_url, e)) return False if lazylibrarian.TOR_DOWNLOADER_BLACKHOLE: tor_title = common.removeDisallowedFilenameChars(tor_title) logger.debug("Sending %s to blackhole" % tor_title) tor_name = str.replace(str(tor_title), ' ', '_') if tor_url.startswith('magnet'): tor_name = tor_name + '.magnet' else: tor_name = tor_name + '.torrent' tor_path = os.path.join(lazylibrarian.TORRENT_DIR, tor_name) with open(tor_path, 'wb') as torrent_file: torrent_file.write(torrent) logger.debug('Torrent file saved: %s' % tor_title) download = True if (lazylibrarian.TOR_DOWNLOADER_UTORRENT and lazylibrarian.UTORRENT_HOST): logger.debug("Sending %s to Utorrent" % tor_title) hash = CalcTorrentHash(torrent) download = utorrent.addTorrent(tor_url, hash) if (lazylibrarian.TOR_DOWNLOADER_QBITTORRENT and lazylibrarian.QBITTORRENT_HOST): logger.debug("Sending %s to qbittorrent" % tor_title) download = qbittorrent.addTorrent(tor_url) if (lazylibrarian.TOR_DOWNLOADER_TRANSMISSION and lazylibrarian.TRANSMISSION_HOST): logger.debug("Sending %s to Transmission" % tor_title) download = transmission.addTorrent(tor_url) if (lazylibrarian.TOR_DOWNLOADER_DELUGE and lazylibrarian.DELUGE_HOST): logger.debug("Sending %s to Deluge" % tor_title) if not lazylibrarian.DELUGE_USER: # no username, talk to the webui download = deluge.addTorrent(tor_url) else: # have username, talk to the daemon client = DelugeRPCClient(lazylibrarian.DELUGE_HOST, int(lazylibrarian.DELUGE_PORT), lazylibrarian.DELUGE_USER, lazylibrarian.DELUGE_PASS) client.connect() args = {"name": tor_title} download = client.call('core.add_torrent_url', tor_url, args) logger.debug('Deluge torrent_id: %s' % download) if download and lazylibrarian.DELUGE_LABEL: labelled = client.call('label.set_torrent', download, lazylibrarian.DELUGE_LABEL) logger.debug('Deluge label returned: %s' % labelled) else: logger.warn('No torrent download method is enabled, check config.') return False if download: logger.debug(u'Torrent id %s has been downloaded from %s' % (download, tor_url)) myDB.action('UPDATE books SET status = "Snatched" WHERE BookID="%s"' % bookid) myDB.action('UPDATE wanted SET status = "Snatched" WHERE NZBurl="%s"' % full_url) return True else: logger.error(u'Failed to download torrent @ <a href="%s">%s</a>' % (full_url, tor_url)) myDB.action('UPDATE wanted SET status = "Failed" WHERE NZBurl="%s"' % full_url) return False
def DownloadMethod(bookid=None, tor_prov=None, tor_title=None, tor_url=None): myDB = database.DBConnection() download = False if (lazylibrarian.USE_TOR) and ( lazylibrarian.TOR_DOWNLOADER_DELUGE or lazylibrarian.TOR_DOWNLOADER_UTORRENT or lazylibrarian.TOR_DOWNLOADER_BLACKHOLE or lazylibrarian.TOR_DOWNLOADER_TRANSMISSION ): request = urllib2.Request(tor_url) if lazylibrarian.PROXY_HOST: request.set_proxy(lazylibrarian.PROXY_HOST, lazylibrarian.PROXY_TYPE) request.add_header("Accept-encoding", "gzip") request.add_header("User-Agent", common.USER_AGENT) if tor_prov == "KAT": request.add_header("Referer", "http://kat.ph/") response = urllib2.urlopen(request) if response.info().get("Content-Encoding") == "gzip": buf = StringIO(response.read()) f = gzip.GzipFile(fileobj=buf) torrent = f.read() else: torrent = response.read() if lazylibrarian.TOR_DOWNLOADER_BLACKHOLE: logger.info("Torrent blackhole") tor_title = common.removeDisallowedFilenameChars(tor_title) tor_name = str.replace(str(tor_title), " ", "_") + ".torrent" tor_path = os.path.join(lazylibrarian.TORRENT_DIR, tor_name) torrent_file = open(tor_path, "wb") torrent_file.write(torrent) torrent_file.close() logger.info("Torrent file saved: %s" % tor_title) download = True if lazylibrarian.TOR_DOWNLOADER_UTORRENT: logger.info("Utorrent") hash = CalcTorrentHash(torrent) download = utorrent.addTorrent(tor_url, hash) if lazylibrarian.TOR_DOWNLOADER_TRANSMISSION: logger.info("Transmission") download = transmission.addTorrent(tor_url) if lazylibrarian.TOR_DOWNLOADER_DELUGE: client = DelugeRPCClient( lazylibrarian.DELUGE_HOST, int(lazylibrarian.DELUGE_PORT), lazylibrarian.DELUGE_USER, lazylibrarian.DELUGE_PASS, ) client.connect() download = client.call("add_torrent_url", tor_url, {"name": tor_title}) logger.info("Deluge return value: %s" % download) else: logger.error("No torrent download method is enabled, check config.") return False if download: logger.debug("Torrent file has been downloaded from " + str(tor_url)) myDB.action('UPDATE books SET status = "Snatched" WHERE BookID=?', [bookid]) myDB.action('UPDATE wanted SET status = "Snatched" WHERE NZBurl=?', [tor_url]) else: logger.error(u'Failed to download torrent @ <a href="%s">%s</a>' % (tor_url, tor_url)) myDB.action('UPDATE wanted SET status = "Failed" WHERE NZBurl=?', [tor_url])
def add_torrent(torrent): ''' Adds torrent or magnet to Deluge torrent: dict of torrrent/magnet information Returns dict {'response': True, 'downloadid': 'id'} {'response': False, 'error': 'exception'} ''' logging.info('Sending torrent {} to DelugeRPC.'.format(torrent['title'])) conf = core.CONFIG['Downloader']['Torrent']['DelugeRPC'] host = conf['host'] port = conf['port'] user = conf['user'] password = conf['pass'] client = DelugeRPCClient(host, port, user, password) try: error = client.connect() if error: return {'response': False, 'error': error} except Exception as e: logging.error('Deluge Add Torrent.', exc_info=True) return {'response': False, 'error': str(e)} try: def_download_path = client.call( 'core.get_config')[b'download_location'].decode('utf-8') except Exception as e: logging.error('Unable to get download path.', exc_info=True) return {'response': False, 'error': 'Unable to get download path.'} download_path = '{}/{}'.format(def_download_path, conf['category']) priority_keys = { 'Low': 64, 'Normal': 128, 'High': 255, } options = {} options['add_paused'] = conf['addpaused'] options['download_location'] = download_path options['priority'] = priority_keys[conf['priority']] ratio_limit = conf.get('seedratiolimit', '') if ratio_limit != '': options['stop_at_ratio'] = True options['stop_ratio'] = ratio_limit elif ratio_limit == -1: torrent['options']['stop_at_ratio'] = False if conf.get('removetorrents'): options['remove_at_ratio'] = True if torrent['type'] == 'magnet': try: downloadid = client.call('core.add_torrent_magnet', torrent['torrentfile'], options).decode('utf-8') except Exception as e: logging.error('Unable to send magnet.', exc_info=True) return {'response': False, 'error': str(e)} elif torrent['type'] == 'torrent': try: downloadid = (client.call('core.add_torrent_url', torrent['torrentfile'], options) or b'').decode('utf-8') except Exception as e: logging.error('Unable to send torrent.', exc_info=True) return {'response': False, 'error': str(e)} else: return { 'response': False, 'error': 'Invalid torrent type {}'.format(torrent['type']) } _set_label(downloadid, conf['category'], client) return {'response': True, 'downloadid': downloadid}
def TORDownloadMethod(bookid=None, tor_prov=None, tor_title=None, tor_url=None): myDB = database.DBConnection() download = False full_url = tor_url # keep the url as stored in "wanted" table if (lazylibrarian.USE_TOR) and (lazylibrarian.TOR_DOWNLOADER_DELUGE or lazylibrarian.TOR_DOWNLOADER_UTORRENT or lazylibrarian.TOR_DOWNLOADER_BLACKHOLE or lazylibrarian.TOR_DOWNLOADER_TRANSMISSION): if tor_url.startswith('magnet'): torrent = tor_url # allow magnet link to write to blackhole and hash to utorrent else: if '&file=' in tor_url: # torznab results need to be re-encoded url = tor_url.split('&file=')[0] value = tor_url.split('&file=')[1] if isinstance(value, str): value = value.decode('utf-8') # make unicode value = unicodedata.normalize('NFC', value) # normalize to short form value = value.encode('unicode-escape') # then escape the result value = value.replace(' ', '%20') # and encode any spaces tor_url = url + '&file=' + value request = urllib2.Request(ur'%s' % tor_url) if lazylibrarian.PROXY_HOST: request.set_proxy(lazylibrarian.PROXY_HOST, lazylibrarian.PROXY_TYPE) request.add_header('Accept-encoding', 'gzip') request.add_header('User-Agent', common.USER_AGENT) if tor_prov == 'KAT': host = lazylibrarian.KAT_HOST if not str(host)[:4] == "http": host = 'http://' + host request.add_header('Referer', host) try: response = urllib2.urlopen(request) if response.info().get('Content-Encoding') == 'gzip': buf = StringIO(response.read()) f = gzip.GzipFile(fileobj=buf) torrent = f.read() else: torrent = response.read() except urllib2.URLError as e: logger.warn('Error fetching torrent from url: ' + tor_url + ' %s' % e.reason) return # strip url back to the .torrent for passing to downloaders # deluge needs it stripping, transmission doesn't mind # not sure about utorrent if '?' in tor_url: tor_url = tor_url.split('?')[0] if (lazylibrarian.TOR_DOWNLOADER_BLACKHOLE): logger.debug('Torrent blackhole') tor_title = common.removeDisallowedFilenameChars(tor_title) tor_name = str.replace(str(tor_title), ' ', '_') if tor_url.startswith('magnet'): tor_name = tor_name + '.magnet' else: tor_name = tor_name + '.torrent' tor_path = os.path.join(lazylibrarian.TORRENT_DIR, tor_name) torrent_file = open(tor_path, 'wb') torrent_file.write(torrent) torrent_file.close() logger.debug('Torrent file saved: %s' % tor_title) download = True if (lazylibrarian.TOR_DOWNLOADER_UTORRENT): logger.debug('Utorrent') hash = CalcTorrentHash(torrent) download = utorrent.addTorrent(tor_url, hash) if (lazylibrarian.TOR_DOWNLOADER_TRANSMISSION): logger.debug('Transmission') download = transmission.addTorrent(tor_url) if (lazylibrarian.TOR_DOWNLOADER_DELUGE): client = DelugeRPCClient(lazylibrarian.DELUGE_HOST, int(lazylibrarian.DELUGE_PORT), lazylibrarian.DELUGE_USER, lazylibrarian.DELUGE_PASS) if lazylibrarian.DELUGE_USER and lazylibrarian.DELUGE_PASS: client.connect() download = client.call('core.add_torrent_url', tor_url, {"name": tor_title}) logger.debug('Deluge return value: %s' % download) else: logger.warn('Need user & pass for deluge, check config.') else: logger.warn('No torrent download method is enabled, check config.') return False if download: logger.debug(u'Torrent file has been downloaded from %s' % tor_url) myDB.action('UPDATE books SET status = "Snatched" WHERE BookID="%s"' % bookid) myDB.action('UPDATE wanted SET status = "Snatched" WHERE NZBurl="%s"' % full_url) else: logger.error(u'Failed to download torrent @ <a href="%s">%s</a>' % (full_url, tor_url)) myDB.action('UPDATE wanted SET status = "Failed" WHERE NZBurl="%s"' % full_url)