def import_torrents(in_dir, out_dir): """Convert deluge and vuze torrent state dirs to json format. Can run incrementally. New torrent data just replaces old torrent data in json dir.""" torrents = load_torrents(out_dir) for f in os.listdir(in_dir): if not f.endswith(".torrent"): continue torrent_path = os.path.join(in_dir, f) data = bencode.bdecode(open(torrent_path).read()) torrent_id = hashlib.sha1(bencode.bencode(data["info"])).hexdigest() torrent_name = f[:-len(".torrent")] if re.match("^[0-9a-f]{40}$", torrent_name): check(torrent_name == torrent_id, torrent_name) t = torrents.setdefault(torrent_id, OrderedDict()) t["data"] = data t.setdefault("download", OrderedDict()).setdefault("mtime", os.path.getmtime(torrent_path)) state_file = os.path.join(in_dir, "torrents.state") if os.path.exists(state_file): state = pickle.load(open(state_file)) check(set(obj_dict(state).keys()) == {"torrents"}) default_torrent = TorrentState() for st in state.torrents: torrent_id = st.torrent_id t = torrents.setdefault(torrent_id, OrderedDict()) t["state"] = tstate = sort_dicts(obj_dict(st)) for k, v in tstate.items(): if v == getattr(default_torrent, k): del tstate[k] resume_file = os.path.join(in_dir, "torrents.fastresume") if os.path.exists(resume_file): resume = bencode.bdecode(open(resume_file).read()) for torrent_id, rt in resume.iteritems(): t = torrents.setdefault(torrent_id, OrderedDict()) t["fastresume"] = bencode.bdecode(rt) # Sort dictionary keys for torrent_id, t in torrents.iteritems(): sort_keys(t, ("data", "state", "fastresume", "download")) save_torrents(torrents, out_dir)
def read_resume_data(self, path): """given the path to resume.dat, decode and return it""" try: with open(path, 'rb') as f: raw = f.read() except (IOError, OSError) as e: log.error('Could not open {0}. Reason{1}'.format(path, e)) raise return bdecode(raw)
def parse(self, filetree=1): if not self.__m_filedata: log.error("No data to process!") return try: self.__m_metadata = bencode.bdecode(self.__m_filedata) except Exception, e: log.warning("Failed to decode torrent data %s: %s", self.filename if self.filename else "", e) raise e
def __init__(self, filename, filetree=1): # Get the torrent data from the torrent file try: log.debug("Attempting to open %s.", filename) self.__m_filedata = open(filename, "rb").read() self.__m_metadata = bencode.bdecode(self.__m_filedata) except Exception, e: log.warning("Unable to open %s: %s", filename, e) raise e
def test_bdecode(self): self.assertEqual(bencode.bdecode(b'3:dEf'), b'dEf') with self.assertRaises(bencode.BTFailure): bencode.bdecode('dEf') with self.assertRaises(bencode.BTFailure): bencode.bdecode(b'dEf') with self.assertRaises(bencode.BTFailure): bencode.bdecode({'dEf': 123})
def test_add_torrent_file(self): options = {} filename = os.path.join(os.path.dirname(__file__), "test.torrent") import base64 torrent_id = self.core.add_torrent_file(filename, base64.encodestring(open(filename).read()), options) # Get the info hash from the test.torrent from deluge.bencode import bdecode, bencode info_hash = sha(bencode(bdecode(open(filename).read())["info"])).hexdigest() self.assertEquals(torrent_id, info_hash)
def test_add_torrent_file(self): options = {} filename = common.get_test_data_file('test.torrent') with open(filename, 'rb') as _file: filedump = base64.encodestring(_file.read()) torrent_id = yield self.core.add_torrent_file(filename, filedump, options) # Get the info hash from the test.torrent from deluge.bencode import bdecode, bencode with open(filename, 'rb') as _file: info_hash = sha(bencode(bdecode(_file.read())[b'info'])).hexdigest() self.assertEquals(torrent_id, info_hash)
def test_add_torrent_file(self): options = {} filename = "../test.torrent" import base64 torrent_id = self.core.add_torrent_file( filename, base64.encodestring(open(filename).read()), options) # Get the info hash from the test.torrent from deluge.bencode import bdecode, bencode info_hash = sha(bencode(bdecode( open(filename).read())["info"])).hexdigest() self.assertEquals(torrent_id, info_hash)
def test_add_torrent_file(self): options = {} filename = common.get_test_data_file('test.torrent') with open(filename, 'rb') as _file: filedump = base64.encodestring(_file.read()) torrent_id = yield self.core.add_torrent_file(filename, filedump, options) # Get the info hash from the test.torrent from deluge.bencode import bdecode, bencode with open(filename, 'rb') as _file: info_hash = sha(bencode(bdecode( _file.read())[b'info'])).hexdigest() self.assertEquals(torrent_id, info_hash)
def read_resume_data(self, path): """given the path to resume.dat, decode and return it""" if not os.path.exists(path): er = ("{0} could not be found. Please check the file exists and " "that you have permission to read it.".format(path)) log.error(er) raise AssertionError(er) if not os.path.isfile(path): er = '{0} is a folder, "Path to resume.dat" must be a file.'.format(path) log.error(er) raise AssertionError(er) try: with open(path, 'rb') as f: raw = f.read() except (IOError, OSError) as e: log.error('Could not open {0}. Reason{1}'.format(path, e)) raise return bdecode(raw)
def __init__(self, filename='', filetree=1, torrent_file=None): self._filedata = None if torrent_file: self._metainfo = torrent_file elif filename: log.debug('Attempting to open %s.', filename) try: with open(filename, 'rb') as _file: self._filedata = _file.read() except IOError as ex: log.warning('Unable to open %s: %s', filename, ex) return try: self._metainfo = bencode.bdecode(self._filedata) except bencode.BTFailure as ex: log.warning('Failed to decode %s: %s', filename, ex) return else: log.warning('Requires valid arguments.') return # info_dict with keys decoded to unicode. info_dict = {k.decode(): v for k, v in self._metainfo[b'info'].items()} self._info_hash = sha(bencode.bencode(info_dict)).hexdigest() # Get encoding from torrent file if available encoding = info_dict.get('encoding', None) codepage = info_dict.get('codepage', None) if not encoding: encoding = codepage if codepage else b'UTF-8' if encoding: encoding = encoding.decode() # Decode 'name' with encoding unless 'name.utf-8' found. if 'name.utf-8' in info_dict: self._name = decode_bytes(info_dict['name.utf-8']) else: self._name = decode_bytes(info_dict['name'], encoding) # Get list of files from torrent info self._files = [] if 'files' in info_dict: paths = {} dirs = {} prefix = self._name if len(info_dict['files']) > 1 else '' for index, f in enumerate(info_dict['files']): f = {k.decode(): v for k, v in f.items()} if 'path.utf-8' in f: path = decode_bytes(os.path.join(*f['path.utf-8'])) del f['path.utf-8'] else: path = decode_bytes(os.path.join(*f['path']), encoding) if prefix: path = os.path.join(prefix, path) self._files.append({ 'path': path, 'size': f['length'], 'download': True }) f['path'] = path f['index'] = index if 'sha1' in f and len(f['sha1']) == 20: f['sha1'] = hexlify(f['sha1']).decode() if 'ed2k' in f and len(f['ed2k']) == 16: f['ed2k'] = hexlify(f['ed2k']).decode() if 'filehash' in f and len(f['filehash']) == 20: f['filehash'] = hexlify(f['filehash']).decode() paths[path] = f dirname = os.path.dirname(path) while dirname: dirinfo = dirs.setdefault(dirname, {}) dirinfo['length'] = dirinfo.get('length', 0) + f['length'] dirname = os.path.dirname(dirname) if filetree == 2: def walk(path, item): if item['type'] == 'dir': item.update(dirs[path]) else: item.update(paths[path]) item['download'] = True file_tree = FileTree2(list(paths)) file_tree.walk(walk) else: def walk(path, item): if isinstance(item, dict): return item return [paths[path]['index'], paths[path]['length'], True] file_tree = FileTree(paths) file_tree.walk(walk) self._files_tree = file_tree.get_tree() else: self._files.append({ 'path': self._name, 'size': info_dict['length'], 'download': True }) if filetree == 2: self._files_tree = { 'contents': { self._name: { 'type': 'file', 'index': 0, 'length': info_dict['length'], 'download': True, } } } else: self._files_tree = {self._name: (0, info_dict['length'], True)}
def __init__(self, filename, filetree=1): # Get the torrent data from the torrent file try: log.debug('Attempting to open %s.', filename) with open(filename, 'rb') as _file: self.__m_filedata = _file.read() except IOError as ex: log.warning('Unable to open %s: %s', filename, ex) raise ex try: self.__m_metadata = bencode.bdecode(self.__m_filedata) except bencode.BTFailure as ex: log.warning('Failed to decode %s: %s', filename, ex) raise ex self.__m_info_hash = sha(bencode.bencode(self.__m_metadata['info'])).hexdigest() # Get encoding from torrent file if available self.encoding = None if 'encoding' in self.__m_metadata: self.encoding = self.__m_metadata['encoding'] elif 'codepage' in self.__m_metadata: self.encoding = str(self.__m_metadata['codepage']) if not self.encoding: self.encoding = 'UTF-8' # Check if 'name.utf-8' is in the torrent and if not try to decode the string # using the encoding found. if 'name.utf-8' in self.__m_metadata['info']: self.__m_name = decode_bytes(self.__m_metadata['info']['name.utf-8']) else: self.__m_name = decode_bytes(self.__m_metadata['info']['name'], self.encoding) # Get list of files from torrent info paths = {} dirs = {} if 'files' in self.__m_metadata['info']: prefix = '' if len(self.__m_metadata['info']['files']) > 1: prefix = self.__m_name for index, f in enumerate(self.__m_metadata['info']['files']): if 'path.utf-8' in f: path = decode_bytes(os.path.join(prefix, *f['path.utf-8'])) del f['path.utf-8'] else: path = os.path.join(prefix, decode_bytes(os.path.join(*f['path']), self.encoding)) f['path'] = path f['index'] = index if 'sha1' in f and len(f['sha1']) == 20: f['sha1'] = f['sha1'].encode('hex') if 'ed2k' in f and len(f['ed2k']) == 16: f['ed2k'] = f['ed2k'].encode('hex') if 'filehash' in f and len(f['filehash']) == 20: f['filehash'] = f['filehash'].encode('hex') paths[path] = f dirname = os.path.dirname(path) while dirname: dirinfo = dirs.setdefault(dirname, {}) dirinfo['length'] = dirinfo.get('length', 0) + f['length'] dirname = os.path.dirname(dirname) if filetree == 2: def walk(path, item): if item['type'] == 'dir': item.update(dirs[path]) else: item.update(paths[path]) item['download'] = True file_tree = FileTree2(list(paths)) file_tree.walk(walk) else: def walk(path, item): if isinstance(item, dict): return item return [paths[path]['index'], paths[path]['length'], True] file_tree = FileTree(paths) file_tree.walk(walk) self.__m_files_tree = file_tree.get_tree() else: if filetree == 2: self.__m_files_tree = { 'contents': { self.__m_name: { 'type': 'file', 'index': 0, 'length': self.__m_metadata['info']['length'], 'download': True } } } else: self.__m_files_tree = { self.__m_name: (0, self.__m_metadata['info']['length'], True) } self.__m_files = [] if 'files' in self.__m_metadata['info']: prefix = '' if len(self.__m_metadata['info']['files']) > 1: prefix = self.__m_name for f in self.__m_metadata['info']['files']: self.__m_files.append({ 'path': f['path'], 'size': f['length'], 'download': True }) else: self.__m_files.append({ 'path': self.__m_name, 'size': self.__m_metadata['info']['length'], 'download': True })
def __init__(self, filename='', filetree=1, metainfo=None, metadata=None): # Get the torrent metainfo from the torrent file if metadata: self._metainfo_dict = {b'info': bencode.bdecode(metadata)} self._metainfo = bencode.bencode(self._metainfo_dict) else: self._metainfo = metainfo if filename and not self._metainfo: log.debug('Attempting to open %s.', filename) try: with open(filename, 'rb') as _file: self._metainfo = _file.read() except IOError as ex: log.warning('Unable to open %s: %s', filename, ex) return try: self._metainfo_dict = bencode.bdecode(self._metainfo) except bencode.BTFailure as ex: log.warning('Failed to decode %s: %s', filename, ex) return info_dict = self._metainfo_dict[b'info'] self._info_hash = sha(bencode.bencode(info_dict)).hexdigest() # Get encoding from torrent file if available encoding = self._metainfo_dict.get(b'encoding', None) codepage = self._metainfo_dict.get(b'codepage', None) if not encoding: encoding = codepage if codepage else b'UTF-8' # Decode 'name' with encoding unless 'name.utf-8' found. if b'name.utf-8' in info_dict: self._name = decode_bytes(info_dict[b'name.utf-8']) else: if encoding: encoding = encoding.decode() self._name = decode_bytes(info_dict[b'name'], encoding) # Get list of files from torrent info if b'files' in info_dict: paths = {} dirs = {} prefix = self._name if len(info_dict[b'files']) > 1 else '' for index, f in enumerate(info_dict[b'files']): if b'path.utf-8' in f: path = decode_bytes(os.path.join(*f[b'path.utf-8'])) del f[b'path.utf-8'] else: path = decode_bytes(os.path.join(*f[b'path']), encoding) if prefix: path = os.path.join(prefix, path) f[b'path'] = path f[b'index'] = index if b'sha1' in f and len(f[b'sha1']) == 20: f[b'sha1'] = f[b'sha1'].encode(b'hex') if b'ed2k' in f and len(f[b'ed2k']) == 16: f[b'ed2k'] = f['ed2k'].encode(b'hex') if b'filehash' in f and len(f[b'filehash']) == 20: f[b'filehash'] = f[b'filehash'].encode(b'hex') paths[path] = f dirname = os.path.dirname(path) while dirname: dirinfo = dirs.setdefault(dirname, {}) dirinfo[b'length'] = dirinfo.get(b'length', 0) + f[b'length'] dirname = os.path.dirname(dirname) if filetree == 2: def walk(path, item): if item['type'] == 'dir': item.update(dirs[path]) else: item.update(paths[path]) item['download'] = True file_tree = FileTree2(list(paths)) file_tree.walk(walk) else: def walk(path, item): if isinstance(item, dict): return item return [ paths[path][b'index'], paths[path][b'length'], True ] file_tree = FileTree(paths) file_tree.walk(walk) self._files_tree = file_tree.get_tree() else: if filetree == 2: self._files_tree = { 'contents': { self._name: { 'type': 'file', 'index': 0, 'length': info_dict[b'length'], 'download': True, } } } else: self._files_tree = { self._name: (0, info_dict[b'length'], True) } self._files = [] if b'files' in info_dict: prefix = '' if len(info_dict[b'files']) > 1: prefix = self._name for f in info_dict[b'files']: self._files.append({ 'path': f[b'path'], 'size': f[b'length'], 'download': True }) else: self._files.append({ 'path': self._name, 'size': info_dict[b'length'], 'download': True })
def parse(self, filetree=1): if not self.__m_filedata: log.error("No data to process!") return try: self.__m_metadata = bencode.bdecode(self.__m_filedata) except Exception as e: log.warning("Failed to decode torrent data %s: %s", self.filename if self.filename else "", e) raise e self.__m_info_hash = sha(bencode.bencode( self.__m_metadata["info"])).hexdigest() # Get encoding from torrent file if available self.encoding = None if "encoding" in self.__m_metadata: self.encoding = self.__m_metadata["encoding"] elif "codepage" in self.__m_metadata: self.encoding = str(self.__m_metadata["codepage"]) if not self.encoding: self.encoding = "UTF-8" # Check if 'name.utf-8' is in the torrent and if not try to decode the string # using the encoding found. if "name.utf-8" in self.__m_metadata["info"]: self.__m_name = decode_string( self.__m_metadata["info"]["name.utf-8"]) else: self.__m_name = decode_string(self.__m_metadata["info"]["name"], self.encoding) # Get list of files from torrent info paths = {} dirs = {} if "files" in self.__m_metadata["info"]: prefix = "" if len(self.__m_metadata["info"]["files"]) > 1: prefix = self.__m_name for index, f in enumerate(self.__m_metadata["info"]["files"]): if "path.utf-8" in f: path = os.path.join(prefix, *f["path.utf-8"]) else: path = decode_string( os.path.join( prefix, decode_string(os.path.join(*f["path"]), self.encoding)), self.encoding) f["index"] = index paths[path] = f dirname = os.path.dirname(path) while dirname: dirinfo = dirs.setdefault(dirname, {}) dirinfo["length"] = dirinfo.get("length", 0) + f["length"] dirname = os.path.dirname(dirname) if filetree == 2: def walk(path, item): if item["type"] == "dir": item.update(dirs[path]) else: item.update(paths[path]) item["download"] = True file_tree = FileTree2(paths.keys()) file_tree.walk(walk) else: def walk(path, item): if type(item) is dict: return item return [paths[path]["index"], paths[path]["length"], True] file_tree = FileTree(paths) file_tree.walk(walk) self.__m_files_tree = file_tree.get_tree() else: if filetree == 2: self.__m_files_tree = { "contents": { self.__m_name: { "type": "file", "index": 0, "length": self.__m_metadata["info"]["length"], "download": True } } } else: self.__m_files_tree = { self.__m_name: (0, self.__m_metadata["info"]["length"], True) } self.__m_files = [] if "files" in self.__m_metadata["info"]: prefix = "" if len(self.__m_metadata["info"]["files"]) > 1: prefix = self.__m_name for f in self.__m_metadata["info"]["files"]: if "path.utf-8" in f: path = os.path.join(prefix, *f["path.utf-8"]) else: path = decode_string( os.path.join( prefix, decode_string(os.path.join(*f["path"]), self.encoding)), self.encoding) self.__m_files.append({ 'path': path, 'size': f["length"], 'download': True }) else: self.__m_files.append({ "path": self.__m_name, "size": self.__m_metadata["info"]["length"], "download": True })
def test_bencode_unicode_metainfo(self): filename = common.get_test_data_file('test.torrent') with open(filename, 'rb') as _file: metainfo = bencode.bdecode(_file.read())[b'info'] bencode.bencode({b'info': metainfo})