def announce_scrape(user_key): infohashes = request.args.getlist('info_hash') torrents = { 'files': {} } if infohashes: for infohash in infohashes: torrent = r.table('torrents').get(infohash).run() try: complete = len(torrent['seeders']) except TypeError: complete = 0 try: incomplete = len(torrent['leechers']) except TypeError: incomplete = 0 try: downloaded = torrent['completed'] except TypeError: downloaded = 0 t = { 'complete': complete, 'downloaded': downloaded, 'incomplete': incomplete, } torrents['files'][infohash.encode('utf-8')] = t else: allTorrents = r.table('torrents').run() for t in allTorrents: try: complete = len(t['seeders']) except TypeError: complete = 0 try: incomplete = len(t['leechers']) except TypeError: incomplete = 0 try: downloaded = t['completed'] except TypeError: downloaded = 0 torrents['files'][t['id'].encode('utf-8')] = { 'complete': complete, 'downloaded': downloaded, 'incomplete': incomplete } return Response(bencode.encode(torrents))
def get_hash(url, mode='torrent'): if url.startswith('magnet'): return url.split('&')[0].split(':')[-1] else: try: torrent = Url.open(url, stream=True).content metadata = bencode.decode(torrent) hashcontents = bencode.encode(metadata['info']) return hashlib.sha1(hashcontents).hexdigest() except Exception as e: #noqa return None
def CalcTorrentHash(torrent): if torrent and torrent.startswith('magnet'): hashid = re.findall('urn:btih:([\w]{32,40})', torrent)[0] if len(hashid) == 32: hashid = b16encode(b32decode(hashid)).lower() else: # noinspection PyTypeChecker info = dict( decode(torrent))["info"] # python3 decode returns OrderedDict hashid = sha1(encode(info)).hexdigest() logger.debug('Torrent Hash: ' + hashid) return hashid
def CalcTorrentHash(torrent): # torrent could be a unicode magnet link or a bytes object torrent file contents if makeUnicode(torrent[:6]) == 'magnet': # torrent = makeUnicode(torrent) hashid = re.findall('urn:btih:([\w]{32,40})', torrent)[0] if len(hashid) == 32: hashid = b16encode(b32decode(hashid)).lower() else: # noinspection PyTypeChecker info = dict(decode(torrent))["info"] # python3 decode returns OrderedDict hashid = sha1(encode(info)).hexdigest() logger.debug('Torrent Hash: ' + hashid) return hashid
def DirectDownloadMethod(bookid=None, dl_title=None, dl_url=None, library='eBook'): myDB = database.DBConnection() downloadID = False Source = "DIRECT" logger.debug("Starting Direct Download for [%s]" % dl_title) proxies = proxyList() headers = {'Accept-encoding': 'gzip', 'User-Agent': USER_AGENT} try: r = requests.get(dl_url, headers=headers, timeout=90, proxies=proxies) except requests.exceptions.Timeout: logger.warn('Timeout fetching file from url: %s' % dl_url) return False except Exception as e: if hasattr(e, 'reason'): logger.warn('%s fetching file from url: %s, %s' % (type(e).__name__, dl_url, e.reason)) else: logger.warn('%s fetching file from url: %s, %s' % (type(e).__name__, dl_url, str(e))) return False if not str(r.status_code).startswith('2'): logger.debug("Got a %s response for %s" % (r.status_code, dl_url)) elif len(r.content) < 1000: logger.debug("Only got %s bytes for %s, rejecting" % (len(r.content), dl_title)) else: extn = '' basename = '' if ' ' in dl_title: basename, extn = dl_title.rsplit( ' ', 1) # last word is often the extension - but not always... if extn and extn in getList(lazylibrarian.CONFIG['EBOOK_TYPE']): dl_title = '.'.join(dl_title.rsplit(' ', 1)) elif magic: mtype = magic.from_buffer(r.content) if 'EPUB' in mtype: extn = '.epub' elif 'Mobipocket' in mtype: # also true for azw and azw3, does it matter? extn = '.mobi' elif 'PDF' in mtype: extn = '.pdf' else: logger.debug("magic reports %s" % mtype) basename = dl_title else: logger.warn("Don't know the filetype for %s" % dl_title) basename = dl_title logger.debug("File download got %s bytes for %s" % (len(r.content), dl_title)) destdir = os.path.join(lazylibrarian.DIRECTORY('Download'), basename) # destdir = os.path.join(lazylibrarian.DIRECTORY('Download'), '%s LL.(%s)' % (basename, bookid)) if not os.path.isdir(destdir): _ = mymakedirs(destdir) try: hashid = dl_url.split("md5=")[1].split("&")[0] except IndexError: hashid = sha1(encode(dl_url)).hexdigest() destfile = os.path.join(destdir, basename + extn) try: with open(destfile, 'wb') as bookfile: bookfile.write(r.content) setperm(destfile) downloadID = hashid except Exception as e: logger.error("%s writing book to %s, %s" % (type(e).__name__, destfile, e)) if downloadID: logger.debug('File %s has been downloaded from %s' % (dl_title, dl_url)) if library == 'eBook': myDB.action('UPDATE books SET status="Snatched" WHERE BookID=?', (bookid, )) elif library == 'AudioBook': myDB.action( 'UPDATE books SET audiostatus="Snatched" WHERE BookID=?', (bookid, )) myDB.action( 'UPDATE wanted SET status="Snatched", Source=?, DownloadID=? WHERE NZBurl=?', (Source, downloadID, dl_url)) return True else: logger.error('Failed to download file @ <a href="%s">%s</a>' % (dl_url, dl_url)) myDB.action('UPDATE wanted SET status="Failed" WHERE NZBurl=?', (dl_url, )) return False
def DirectDownloadMethod(bookid=None, dl_title=None, dl_url=None, library='eBook'): myDB = database.DBConnection() downloadID = False Source = "DIRECT" logger.debug("Starting Direct Download for [%s]" % dl_title) proxies = proxyList() headers = {'Accept-encoding': 'gzip', 'User-Agent': USER_AGENT} try: r = requests.get(dl_url, headers=headers, timeout=90, proxies=proxies) except requests.exceptions.Timeout: logger.warn('Timeout fetching file from url: %s' % dl_url) return False except Exception as e: if hasattr(e, 'reason'): logger.warn('%s fetching file from url: %s, %s' % (type(e).__name__, dl_url, e.reason)) else: logger.warn('%s fetching file from url: %s, %s' % (type(e).__name__, dl_url, str(e))) return False if not str(r.status_code).startswith('2'): logger.debug("Got a %s response for %s" % (r.status_code, dl_url)) elif len(r.content) < 1000: logger.debug("Only got %s bytes for %s, rejecting" % (len(r.content), dl_title)) else: extn = '' basename = '' if ' ' in dl_title: basename, extn = dl_title.rsplit(' ', 1) # last word is often the extension - but not always... if extn and extn in getList(lazylibrarian.CONFIG['EBOOK_TYPE']): dl_title = '.'.join(dl_title.rsplit(' ', 1)) elif magic: mtype = magic.from_buffer(r.content) if 'EPUB' in mtype: extn = '.epub' elif 'Mobipocket' in mtype: # also true for azw and azw3, does it matter? extn = '.mobi' elif 'PDF' in mtype: extn = '.pdf' else: logger.debug("magic reports %s" % mtype) basename = dl_title else: logger.warn("Don't know the filetype for %s" % dl_title) basename = dl_title logger.debug("File download got %s bytes for %s" % (len(r.content), dl_title)) destdir = os.path.join(lazylibrarian.DIRECTORY('Download'), basename) # destdir = os.path.join(lazylibrarian.DIRECTORY('Download'), '%s LL.(%s)' % (basename, bookid)) if not os.path.isdir(destdir): _ = mymakedirs(destdir) try: hashid = dl_url.split("md5=")[1].split("&")[0] except IndexError: hashid = sha1(encode(dl_url)).hexdigest() destfile = os.path.join(destdir, basename + extn) try: with open(destfile, 'wb') as bookfile: bookfile.write(r.content) setperm(destfile) downloadID = hashid except Exception as e: logger.error("%s writing book to %s, %s" % (type(e).__name__, destfile, e)) if downloadID: logger.debug('File %s has been downloaded from %s' % (dl_title, dl_url)) if library == 'eBook': myDB.action('UPDATE books SET status="Snatched" WHERE BookID=?', (bookid,)) elif library == 'AudioBook': myDB.action('UPDATE books SET audiostatus="Snatched" WHERE BookID=?', (bookid,)) myDB.action('UPDATE wanted SET status="Snatched", Source=?, DownloadID=? WHERE NZBurl=?', (Source, downloadID, dl_url)) return True else: logger.error('Failed to download file @ <a href="%s">%s</a>' % (dl_url, dl_url)) myDB.action('UPDATE wanted SET status="Failed" WHERE NZBurl=?', (dl_url,)) return False
def announce_request(user_key): """TODO: * Check if torrent exists - check via <info_hash> (this will always be unique) - if exists we update said torrent - if not exists create new torrent entry - add peer to completed list if completed - remove if disconnected - add peer to leech list if none of the above * Update peer in user table - get peer using <user_key> - add torrent to peers torrent list - update peers ip - update peers peer_id - update peers port - update peers total download and upload -- each peer has a list of torrents currently active on -- <info_hash>: {upload: <>, download: <>} -- check upload and download for torrent compare to given <uploaded> and <left> -- use difference to upload peers total upload/download """ data = { 'info_hash': request.args.get('info_hash', None), 'peer_id': request.args.get('peer_id', None), 'port': request.args.get('port', None), 'uploaded': request.args.get('uploaded', None), 'downloaded': request.args.get('downloaded', None), 'left': request.args.get('left', None), 'compact': request.args.get('compact', None), # 1 - return compact result; 0 - no compact 'no_peer_id': request.args.get('no_peer_id', None), 'event': request.args.get('event', None), 'ip': request.args.get('ip', request.remote_addr), #optional 'numwant': request.args.get('numwant', 30), #optional 'key': request.args.get('key', None), #optional for public, needed for private 'trackerid': request.args.get('trackerid', 'TRACKER'), #optional, no idea what this is for } for k, v in data.iteritems(): if k not in ['trackerid', 'event', 'no_peer_id']: if not v: return bencode.encode({'failure reason': 'missing ' + k}) else: pass if data.get('compact', None): if data['compact'] == '0': return bencode.encode({'failure reason': 'This tracker only supports compact responses'}) #data['info_hash'] = hashlib.sha1(data['info_hash'].encode('utf-8')).hexdigest() #check if torrent exists if not r.table('torrents').get(data['info_hash']).run(): #torrent doesnt exist; create it #NOTE: do we want to have the frontend create the torrent entries for security purposes? --elgruntox r.table('torrents').insert({ 'id': data['info_hash'], 'peer_list': {}, 'seeders': [], 'leechers': [], 'completed': 0 }).run() #check if user exists if not r.table('users').get(user_key).run(): #user doesnt exist; create it r.table('users').insert({ 'id': user_key, 'last_ip': data['ip'], 'last_port': data['port'], 'seeding': [], 'leeching': [], 'torrents': {}, 'total_upload': 0, 'total_downloaded': 0, }).run() # so I guess torrent clients are mostly shit and dont send a 'completed' event half the time # check if nothing left to download; if nothing left i guess theyre seeding? -- elgruntox if data['left'] == '0': user = r.table('users').get(user_key).run() if data['info_hash'] in user['leeching']: user['leeching'].remove(data['info_hash']) user['seeding'].append(data['info_hash']) user['seeding'] = list(set(user['seeding'])) r.table('users').get(user_key).replace(user).run() torrent = r.table('torrents').get(data['info_hash']).run() if data['peer_id'] in torrent['leechers']: torrent['leechers'].remove(data['peer_id']) torrent['seeders'].append(data['peer_id']) torrent['completed'] = torrent['completed'] + 1 torrent['seeders'] = list(set(torrent['seeders'])) r.table('torrents').get(data['info_hash']).replace(torrent).run() #check if torrent has data specified; if no event just a regular check and we do nothing if data['event']: if data['event'] == 'started': # add torrent to users torrent/leeching list; add peer to torrents peer_list user = r.table('users').get(user_key).run() # remove user from torrents seeding list if in it if data['peer_id'] in user['seeding']: user['seeding'].remove(data['peer_id']) # update users leeching list user['leeching'].append(data['info_hash']) user['leeching'] = list(set(user['leeching'])) # update torrent dict user['torrents'][data['info_hash']] = { 'uploaded': 0, 'downloaded': 0 } # update and save user to db r.table('users').get(user_key).replace(user).run() # update torrents leeching list torrent = r.table('torrents').get(data['info_hash']).run() if data['peer_id'] in torrent['leechers']: pass else: torrent['leechers'].append(data['peer_id']) # update torrents peer_list torrent['peer_list'][data['peer_id']] = {} torrent['peer_list'][data['peer_id']]['ip'] = data['ip'] torrent['peer_list'][data['peer_id']]['port'] = data['port'] # save torrent data to db r.table('torrents').get(data['info_hash']).replace(torrent).run() elif data['event'] == 'stopped': # remove torrent from users seeding or leeching list user = r.table('users').get(user_key).run() # check if in leeching or seeding list; pass if not found if data['info_hash'] in user['seeding']: user['seeding'].remove(data['info_hash']) elif data['info_hash'] in user['leeching']: user['leeching'].remove(data['info_hash']) else: pass # update and save user r.table('users').get(user_key).replace(user).run() # update torrents leeching/seeding/peer list torrent = r.table('torrents').get(data['info_hash']).run() if torrent['peer_list'].get(data['peer_id'], None): del torrent['peer_list'][data['peer_id']] if data['peer_id'] in torrent['seeders']: torrent['seeders'].remove(data['peer_id']) if data['peer_id'] in torrent['leechers']: torrent['leechers'].remove(data['peer_id']) # update and save data to db r.table('torrents').get(data['info_hash']).replace(torrent).run() elif data['event'] == 'completed': user = r.table('users').get(user_key).run() if data['info_hash'] in user['leeching']: user['leeching'].remove(data['peer_id']) user['seeding'].append(data['info_hash']) user['seeding'] = list(set(user['seeding'])) r.table('users').get(user_key).replace(user).run() torrent = r.table('torrents').get(data['info_hash']).run() if data['peer_id'] in torrent['leechers']: torrent['leechers'].remove(data['peer_id']) torrent['seeders'].append(data['peer_id']) torrent['completed'] = torrent['completed'] + 1 torrent['seeders'] = list(set(torrent['seeders'])) r.table('torrents').get(data['info_hash']).replace(torrent).run() else: #malformed request; error out return bencode.encode({'failure reason': 'invalid event specified'}) # update users upload and download stats user = r.table('users').get(user_key).run() if user['torrents'].get(data['info_hash'], None): uploaded = int(data['uploaded']) - user['torrents'][data['info_hash']].get('uploaded', 0) downloaded = int(data['downloaded']) - user['torrents'][data['info_hash']].get('downloaded', 0) user['torrents'][data['info_hash']]['uploaded'] = user['torrents'][data['info_hash']]['uploaded'] + uploaded user['torrents'][data['info_hash']]['downloaded'] = user['torrents'][data['info_hash']]['downloaded'] + downloaded user['total_upload'] = user['total_upload'] + uploaded user['total_downloaded'] = user['total_downloaded'] + downloaded else: user['torrents'][data['info_hash']] = {} user['torrents'][data['info_hash']]['uploaded'] = int(data['uploaded']) user['torrents'][data['info_hash']]['downloaded'] = int(data['downloaded']) user['total_upload'] = user['total_upload'] + int(data['uploaded']) user['total_downloaded'] = user['total_downloaded'] + int(data['downloaded']) r.table('users').get('user_key').update(user).run() # generate response torrent = r.table('torrents').get(data['info_hash']).run() peer_list = [] for peerid, values in torrent['peer_list'].iteritems(): peer = (peerid, torrent['peer_list'][peerid]['ip'], torrent['peer_list'][peerid]['port']) peer_list.append(peer) ares = {} ares['interval'] = 30 ares['min interval'] = 30 ares['trackerid'] = data['trackerid'] ares['complete'] = len(torrent['seeders']) ares['incomplete'] = len(torrent['leechers']) ares['peers'] = bencode.make_compact_peer_list(peer_list) return Response(bencode.encode(ares))