def on_feed_modify(self, feed): # Only scan through accepted entries, as the file must have been downloaded in order to parse anything for entry in feed.accepted: # skip if entry does not have file assigned if not 'file' in entry: log.trace('%s doesn\'t have a file associated' % entry['title']) continue if not os.path.exists(entry['file']): feed.fail(entry, 'File %s does not exists' % entry['file']) continue if os.path.getsize(entry['file']) == 0: feed.fail(entry, 'File %s is 0 bytes in size' % entry['file']) continue if not is_torrent_file(entry['file']): continue log.debug('%s seems to be a torrent' % entry['title']) # create torrent object from torrent try: f = open(entry['file'], 'rb') # NOTE: this reads entire file into memory, but we're pretty sure it's # a small torrent file since it starts with TORRENT_RE data = f.read() f.close() if 'content-length' in entry: if len(data) != entry['content-length']: feed.fail(entry, 'Torrent file length doesn\'t match to the one reported by the server') self.purge(entry) continue # construct torrent object try: torrent = Torrent(data) except SyntaxError, e: feed.fail(entry, '%s - Torrent could not be parsed' % e.message) self.purge(entry) continue entry['torrent'] = torrent entry['torrent_info_hash'] = torrent.get_info_hash() # if we do not have good filename (by download plugin) # for this entry, try to generate one from torrent content if entry.get('filename'): if not entry['filename'].lower().endswith('.torrent'): # filename present but without .torrent extension, add it entry['filename'] = entry['filename'] + '.torrent' else: # generate filename from torrent or fall back to title plus extension entry['filename'] = self.make_filename(torrent, entry) except Exception, e: log.exception(e)
def on_task_modify(self, task): # Only scan through accepted entries, as the file must have been downloaded in order to parse anything for entry in task.accepted: # skip if entry does not have file assigned if not 'file' in entry: log.trace('%s doesn\'t have a file associated' % entry['title']) continue if not os.path.exists(entry['file']): task.fail(entry, 'File %s does not exists' % entry['file']) continue if os.path.getsize(entry['file']) == 0: task.fail(entry, 'File %s is 0 bytes in size' % entry['file']) continue if not is_torrent_file(entry['file']): continue log.debug('%s seems to be a torrent' % entry['title']) # create torrent object from torrent try: f = open(entry['file'], 'rb') # NOTE: this reads entire file into memory, but we're pretty sure it's # a small torrent file since it starts with TORRENT_RE data = f.read() f.close() if 'content-length' in entry: if len(data) != entry['content-length']: task.fail(entry, 'Torrent file length doesn\'t match to the one reported by the server') self.purge(entry) continue # construct torrent object try: torrent = Torrent(data) except SyntaxError, e: task.fail(entry, '%s - broken or invalid torrent file received' % e.message) self.purge(entry) continue entry['torrent'] = torrent entry['torrent_info_hash'] = torrent.get_info_hash() # if we do not have good filename (by download plugin) # for this entry, try to generate one from torrent content if entry.get('filename'): if not entry['filename'].lower().endswith('.torrent'): # filename present but without .torrent extension, add it entry['filename'] += '.torrent' else: # generate filename from torrent or fall back to title plus extension entry['filename'] = self.make_filename(torrent, entry) except Exception, e: log.exception(e)
def test_torrent_scrub(self): # Run task self.execute_task('test_all') for clean, filename in self.test_cases: original = Torrent.from_file(filename) title = os.path.splitext(filename)[0] modified = self.task.find_entry(title=title) assert modified, "%r cannot be found in %r" % (title, self.task) modified = modified.get('torrent') assert modified, "No 'torrent' key in %r" % (title, ) osize = os.path.getsize(filename) msize = os.path.getsize(self.__tmp__ + filename) # Dump small torrents on demand if 0 and not clean: print "original=%r" % original.content print "modified=%r" % modified.content # Make sure essentials survived assert 'announce' in modified.content assert 'info' in modified.content assert 'name' in modified.content['info'] assert 'piece length' in modified.content['info'] assert 'pieces' in modified.content['info'] # Check that hashes have changed accordingly if clean: assert osize == msize, "Filesizes aren't supposed to differ!" assert original.get_info_hash() == modified.get_info_hash( ), 'info dict changed in ' + filename else: assert osize > msize, "Filesizes must be different!" assert original.get_info_hash() != modified.get_info_hash( ), filename + " wasn't scrubbed!" # Check essential keys were scrubbed if filename == 'LICENSE.torrent': assert 'x_cross_seed' in original.content['info'] assert 'x_cross_seed' not in modified.content['info'] if filename == 'LICENSE-resume.torrent': assert 'libtorrent_resume' in original.content assert 'libtorrent_resume' not in modified.content
def test_torrent_scrub(self): # Run task self.execute_task('test_all') for clean, filename in self.test_cases: original = Torrent.from_file(filename) title = os.path.splitext(filename)[0] modified = self.task.find_entry(title=title) assert modified, "%r cannot be found in %r" % (title, self.task) modified = modified.get('torrent') assert modified, "No 'torrent' key in %r" % (title,) osize = os.path.getsize(filename) msize = os.path.getsize(self.__tmp__ + filename) # Dump small torrents on demand if 0 and not clean: print "original=%r" % original.content print "modified=%r" % modified.content # Make sure essentials survived assert 'announce' in modified.content assert 'info' in modified.content assert 'name' in modified.content['info'] assert 'piece length' in modified.content['info'] assert 'pieces' in modified.content['info'] # Check that hashes have changed accordingly if clean: assert osize == msize, "Filesizes aren't supposed to differ!" assert original.info_hash == modified.info_hash, 'info dict changed in ' + filename else: assert osize > msize, "Filesizes must be different!" assert original.info_hash != modified.info_hash, filename + " wasn't scrubbed!" # Check essential keys were scrubbed if filename == 'LICENSE.torrent': assert 'x_cross_seed' in original.content['info'] assert 'x_cross_seed' not in modified.content['info'] if filename == 'LICENSE-resume.torrent': assert 'libtorrent_resume' in original.content assert 'libtorrent_resume' not in modified.content
def load_torrent(self, filename): with open(filename, 'rb') as f: data = f.read() return Torrent(data)
def add_entry(self, client, entry, options, start=True, mkdir=False, fast_resume=False): if 'torrent_info_hash' not in entry: entry.fail('missing torrent_info_hash') return if entry['url'].startswith('magnet:'): torrent_raw = 'd10:magnet-uri%d:%se' % (len( entry['url']), entry['url']) torrent_raw = torrent_raw.encode('ascii') else: # Check that file is downloaded if 'file' not in entry: raise plugin.PluginError( 'Temporary download file is missing from entry') # Verify the temp file exists if not os.path.exists(entry['file']): raise plugin.PluginError( 'Temporary download file is missing from disk') # Verify valid torrent file if not is_torrent_file(entry['file']): entry.fail("Downloaded temp file '%s' is not a torrent file" % entry['file']) return # Modify the torrent with resume data if needed if fast_resume: base = options.get('directory') if not base: base = client.get_directory() piece_size = entry['torrent'].piece_size chunks = int( (entry['torrent'].size + piece_size - 1) / piece_size) files = [] for f in entry['torrent'].get_filelist(): relative_file_path = os.path.join(f['path'], f['name']) if entry['torrent'].is_multi_file: relative_file_path = os.path.join( entry['torrent'].name, relative_file_path) file_path = os.path.join(base, relative_file_path) # TODO should it simply add the torrent anyway? if not os.path.exists(file_path) and not os.path.isfile( file_path): entry.fail( '%s does not exist. Cannot add fast resume data.' % file_path) return # cannot bencode floats, so we need to coerce to int mtime = int(os.path.getmtime(file_path)) # priority 0 should be "don't download" files.append({'priority': 0, 'mtime': mtime}) entry['torrent'].set_libtorrent_resume(chunks, files) # Since we modified the torrent, we need to write it to entry['file'] again with open(entry['file'], 'wb+') as f: f.write(entry['torrent'].encode()) try: with open(entry['file'], 'rb') as f: torrent_raw = f.read() except IOError as e: entry.fail('Failed to add to rTorrent %s' % str(e)) return try: Torrent(torrent_raw) except SyntaxError as e: entry.fail( 'Strange, unable to decode torrent, raise a BUG: %s' % str(e)) return # First check if it already exists try: if client.torrent(entry['torrent_info_hash']): log.warning("Torrent %s already exists, won't add" % entry['title']) return except xmlrpc_client.Error: # No existing found pass try: resp = client.load(torrent_raw, fields=options, start=start, mkdir=mkdir) if resp != 0: entry.fail( 'Failed to add to rTorrent invalid return value %s' % resp) except xmlrpc_client.Error as e: log.exception(e) entry.fail('Failed to add to rTorrent %s' % str(e)) return # Verify the torrent loaded try: self._verify_load(client, entry['torrent_info_hash']) log.info('%s added to rtorrent' % entry['title']) except xmlrpc_client.Error as e: log.warning('Failed to verify torrent %s loaded: %s', entry['title'], str(e))
def add_entry(self, client, entry, options, start=True, mkdir=False): if 'torrent_info_hash' not in entry: entry.fail('missing torrent_info_hash') return if entry['url'].startswith('magnet:'): torrent_raw = 'd10:magnet-uri%d:%se' % (len( entry['url']), entry['url']) else: # Check that file is downloaded if 'file' not in entry: entry.fail('file missing?') return # Verify the temp file exists if not os.path.exists(entry['file']): entry.fail("Downloaded temp file '%s' doesn't exist!?" % entry['file']) return # Verify valid torrent file if not is_torrent_file(entry['file']): entry.fail("Downloaded temp file '%s' is not a torrent file" % entry['file']) return try: with open(entry['file'], 'rb') as f: torrent_raw = f.read() except IOError as e: entry.fail('Failed to add to rTorrent %s' % str(e)) return try: Torrent(torrent_raw) except SyntaxError as e: entry.fail( 'Strange, unable to decode torrent, raise a BUG: %s' % str(e)) return # First check if it already exists try: if client.torrent(entry['torrent_info_hash']): log.warning("Torrent %s already exists, won't add" % entry['title']) return except IOError as e: entry.fail("Error checking if torrent already exists %s" % str(e)) except xmlrpc_client.Error: # No existing found pass try: resp = client.load(torrent_raw, fields=options, start=start, mkdir=mkdir) if resp != 0: entry.fail( 'Failed to add to rTorrent invalid return value %s' % resp) except (IOError, xmlrpc_client.Error) as e: log.exception(e) entry.fail('Failed to add to rTorrent %s' % str(e)) return # Verify the torrent loaded try: self._verify_load(client, entry['torrent_info_hash']) log.info('%s added to rtorrent' % entry['title']) except (IOError, xmlrpc_client.Error) as e: entry.fail('Failed to verify torrent loaded: %s' % str(e))
def load_torrent(self, filename): f = open(filename, 'rb') data = f.read() f.close() return Torrent(data)