def import_torrents(in_dir, out_dir):
"""Convert deluge and vuze torrent state dirs to json format.
Can run incrementally. New torrent data just replaces old torrent data in
json dir."""
torrents = load_torrents(out_dir)
for f in os.listdir(in_dir):
if not f.endswith(".torrent"):
continue
torrent_path = os.path.join(in_dir, f)
data = bencode.bdecode(open(torrent_path).read())
torrent_id = hashlib.sha1(bencode.bencode(data["info"])).hexdigest()
torrent_name = f[:-len(".torrent")]
if re.match("^[0-9a-f]{40}$", torrent_name):
check(torrent_name == torrent_id, torrent_name)
t = torrents.setdefault(torrent_id, OrderedDict())
t["data"] = data
t.setdefault("download",
OrderedDict()).setdefault("mtime",
os.path.getmtime(torrent_path))
state_file = os.path.join(in_dir, "torrents.state")
if os.path.exists(state_file):
state = pickle.load(open(state_file))
check(set(obj_dict(state).keys()) == {"torrents"})
default_torrent = TorrentState()
for st in state.torrents:
torrent_id = st.torrent_id
t = torrents.setdefault(torrent_id, OrderedDict())
t["state"] = tstate = sort_dicts(obj_dict(st))
for k, v in tstate.items():
if v == getattr(default_torrent, k):
del tstate[k]
resume_file = os.path.join(in_dir, "torrents.fastresume")
if os.path.exists(resume_file):
resume = bencode.bdecode(open(resume_file).read())
for torrent_id, rt in resume.iteritems():
t = torrents.setdefault(torrent_id, OrderedDict())
t["fastresume"] = bencode.bdecode(rt)
# Sort dictionary keys
for torrent_id, t in torrents.iteritems():
sort_keys(t, ("data", "state", "fastresume", "download"))
save_torrents(torrents, out_dir)
def read_resume_data(self, path):
"""given the path to resume.dat, decode and return it"""
try:
with open(path, 'rb') as f:
raw = f.read()
except (IOError, OSError) as e:
log.error('Could not open {0}. Reason{1}'.format(path, e))
raise
return bdecode(raw)
def parse(self, filetree=1):
if not self.__m_filedata:
log.error("No data to process!")
return
try:
self.__m_metadata = bencode.bdecode(self.__m_filedata)
except Exception, e:
log.warning("Failed to decode torrent data %s: %s", self.filename if self.filename else "", e)
raise e
def __init__(self, filename, filetree=1):
# Get the torrent data from the torrent file
try:
log.debug("Attempting to open %s.", filename)
self.__m_filedata = open(filename, "rb").read()
self.__m_metadata = bencode.bdecode(self.__m_filedata)
except Exception, e:
log.warning("Unable to open %s: %s", filename, e)
raise e
def __init__(self, filename, filetree=1):
# Get the torrent data from the torrent file
try:
log.debug("Attempting to open %s.", filename)
self.__m_filedata = open(filename, "rb").read()
self.__m_metadata = bencode.bdecode(self.__m_filedata)
except Exception, e:
log.warning("Unable to open %s: %s", filename, e)
raise e
def test_bdecode(self):
self.assertEqual(bencode.bdecode(b'3:dEf'), b'dEf')
with self.assertRaises(bencode.BTFailure):
bencode.bdecode('dEf')
with self.assertRaises(bencode.BTFailure):
bencode.bdecode(b'dEf')
with self.assertRaises(bencode.BTFailure):
bencode.bdecode({'dEf': 123})
def parse(self, filetree=1):
if not self.__m_filedata:
log.error("No data to process!")
return
try:
self.__m_metadata = bencode.bdecode(self.__m_filedata)
except Exception, e:
log.warning("Failed to decode torrent data %s: %s",
self.filename if self.filename else "", e)
raise e
def test_add_torrent_file(self):
options = {}
filename = os.path.join(os.path.dirname(__file__), "test.torrent")
import base64
torrent_id = self.core.add_torrent_file(filename, base64.encodestring(open(filename).read()), options)
# Get the info hash from the test.torrent
from deluge.bencode import bdecode, bencode
info_hash = sha(bencode(bdecode(open(filename).read())["info"])).hexdigest()
self.assertEquals(torrent_id, info_hash)
def test_add_torrent_file(self):
options = {}
filename = common.get_test_data_file('test.torrent')
with open(filename, 'rb') as _file:
filedump = base64.encodestring(_file.read())
torrent_id = yield self.core.add_torrent_file(filename, filedump, options)
# Get the info hash from the test.torrent
from deluge.bencode import bdecode, bencode
with open(filename, 'rb') as _file:
info_hash = sha(bencode(bdecode(_file.read())[b'info'])).hexdigest()
self.assertEquals(torrent_id, info_hash)
def test_add_torrent_file(self):
options = {}
filename = "../test.torrent"
import base64
torrent_id = self.core.add_torrent_file(
filename, base64.encodestring(open(filename).read()), options)
# Get the info hash from the test.torrent
from deluge.bencode import bdecode, bencode
info_hash = sha(bencode(bdecode(
open(filename).read())["info"])).hexdigest()
self.assertEquals(torrent_id, info_hash)
def test_add_torrent_file(self):
options = {}
filename = common.get_test_data_file('test.torrent')
with open(filename, 'rb') as _file:
filedump = base64.encodestring(_file.read())
torrent_id = yield self.core.add_torrent_file(filename, filedump,
options)
# Get the info hash from the test.torrent
from deluge.bencode import bdecode, bencode
with open(filename, 'rb') as _file:
info_hash = sha(bencode(bdecode(
_file.read())[b'info'])).hexdigest()
self.assertEquals(torrent_id, info_hash)
def read_resume_data(self, path):
"""given the path to resume.dat, decode and return it"""
if not os.path.exists(path):
er = ("{0} could not be found. Please check the file exists and "
"that you have permission to read it.".format(path))
log.error(er)
raise AssertionError(er)
if not os.path.isfile(path):
er = '{0} is a folder, "Path to resume.dat" must be a file.'.format(path)
log.error(er)
raise AssertionError(er)
try:
with open(path, 'rb') as f:
raw = f.read()
except (IOError, OSError) as e:
log.error('Could not open {0}. Reason{1}'.format(path, e))
raise
return bdecode(raw)
def read_resume_data(self, path):
"""given the path to resume.dat, decode and return it"""
if not os.path.exists(path):
er = ("{0} could not be found. Please check the file exists and "
"that you have permission to read it.".format(path))
log.error(er)
raise AssertionError(er)
if not os.path.isfile(path):
er = '{0} is a folder, "Path to resume.dat" must be a file.'.format(path)
log.error(er)
raise AssertionError(er)
try:
with open(path, 'rb') as f:
raw = f.read()
except (IOError, OSError) as e:
log.error('Could not open {0}. Reason{1}'.format(path, e))
raise
return bdecode(raw)
def __init__(self, filename='', filetree=1, torrent_file=None):
self._filedata = None
if torrent_file:
self._metainfo = torrent_file
elif filename:
log.debug('Attempting to open %s.', filename)
try:
with open(filename, 'rb') as _file:
self._filedata = _file.read()
except IOError as ex:
log.warning('Unable to open %s: %s', filename, ex)
return
try:
self._metainfo = bencode.bdecode(self._filedata)
except bencode.BTFailure as ex:
log.warning('Failed to decode %s: %s', filename, ex)
return
else:
log.warning('Requires valid arguments.')
return
# info_dict with keys decoded to unicode.
info_dict = {k.decode(): v for k, v in self._metainfo[b'info'].items()}
self._info_hash = sha(bencode.bencode(info_dict)).hexdigest()
# Get encoding from torrent file if available
encoding = info_dict.get('encoding', None)
codepage = info_dict.get('codepage', None)
if not encoding:
encoding = codepage if codepage else b'UTF-8'
if encoding:
encoding = encoding.decode()
# Decode 'name' with encoding unless 'name.utf-8' found.
if 'name.utf-8' in info_dict:
self._name = decode_bytes(info_dict['name.utf-8'])
else:
self._name = decode_bytes(info_dict['name'], encoding)
# Get list of files from torrent info
self._files = []
if 'files' in info_dict:
paths = {}
dirs = {}
prefix = self._name if len(info_dict['files']) > 1 else ''
for index, f in enumerate(info_dict['files']):
f = {k.decode(): v for k, v in f.items()}
if 'path.utf-8' in f:
path = decode_bytes(os.path.join(*f['path.utf-8']))
del f['path.utf-8']
else:
path = decode_bytes(os.path.join(*f['path']), encoding)
if prefix:
path = os.path.join(prefix, path)
self._files.append({
'path': path,
'size': f['length'],
'download': True
})
f['path'] = path
f['index'] = index
if 'sha1' in f and len(f['sha1']) == 20:
f['sha1'] = hexlify(f['sha1']).decode()
if 'ed2k' in f and len(f['ed2k']) == 16:
f['ed2k'] = hexlify(f['ed2k']).decode()
if 'filehash' in f and len(f['filehash']) == 20:
f['filehash'] = hexlify(f['filehash']).decode()
paths[path] = f
dirname = os.path.dirname(path)
while dirname:
dirinfo = dirs.setdefault(dirname, {})
dirinfo['length'] = dirinfo.get('length', 0) + f['length']
dirname = os.path.dirname(dirname)
if filetree == 2:
def walk(path, item):
if item['type'] == 'dir':
item.update(dirs[path])
else:
item.update(paths[path])
item['download'] = True
file_tree = FileTree2(list(paths))
file_tree.walk(walk)
else:
def walk(path, item):
if isinstance(item, dict):
return item
return [paths[path]['index'], paths[path]['length'], True]
file_tree = FileTree(paths)
file_tree.walk(walk)
self._files_tree = file_tree.get_tree()
else:
self._files.append({
'path': self._name,
'size': info_dict['length'],
'download': True
})
if filetree == 2:
self._files_tree = {
'contents': {
self._name: {
'type': 'file',
'index': 0,
'length': info_dict['length'],
'download': True,
}
}
}
else:
self._files_tree = {self._name: (0, info_dict['length'], True)}
def __init__(self, filename, filetree=1):
# Get the torrent data from the torrent file
try:
log.debug('Attempting to open %s.', filename)
with open(filename, 'rb') as _file:
self.__m_filedata = _file.read()
except IOError as ex:
log.warning('Unable to open %s: %s', filename, ex)
raise ex
try:
self.__m_metadata = bencode.bdecode(self.__m_filedata)
except bencode.BTFailure as ex:
log.warning('Failed to decode %s: %s', filename, ex)
raise ex
self.__m_info_hash = sha(bencode.bencode(self.__m_metadata['info'])).hexdigest()
# Get encoding from torrent file if available
self.encoding = None
if 'encoding' in self.__m_metadata:
self.encoding = self.__m_metadata['encoding']
elif 'codepage' in self.__m_metadata:
self.encoding = str(self.__m_metadata['codepage'])
if not self.encoding:
self.encoding = 'UTF-8'
# Check if 'name.utf-8' is in the torrent and if not try to decode the string
# using the encoding found.
if 'name.utf-8' in self.__m_metadata['info']:
self.__m_name = decode_bytes(self.__m_metadata['info']['name.utf-8'])
else:
self.__m_name = decode_bytes(self.__m_metadata['info']['name'], self.encoding)
# Get list of files from torrent info
paths = {}
dirs = {}
if 'files' in self.__m_metadata['info']:
prefix = ''
if len(self.__m_metadata['info']['files']) > 1:
prefix = self.__m_name
for index, f in enumerate(self.__m_metadata['info']['files']):
if 'path.utf-8' in f:
path = decode_bytes(os.path.join(prefix, *f['path.utf-8']))
del f['path.utf-8']
else:
path = os.path.join(prefix, decode_bytes(os.path.join(*f['path']), self.encoding))
f['path'] = path
f['index'] = index
if 'sha1' in f and len(f['sha1']) == 20:
f['sha1'] = f['sha1'].encode('hex')
if 'ed2k' in f and len(f['ed2k']) == 16:
f['ed2k'] = f['ed2k'].encode('hex')
if 'filehash' in f and len(f['filehash']) == 20:
f['filehash'] = f['filehash'].encode('hex')
paths[path] = f
dirname = os.path.dirname(path)
while dirname:
dirinfo = dirs.setdefault(dirname, {})
dirinfo['length'] = dirinfo.get('length', 0) + f['length']
dirname = os.path.dirname(dirname)
if filetree == 2:
def walk(path, item):
if item['type'] == 'dir':
item.update(dirs[path])
else:
item.update(paths[path])
item['download'] = True
file_tree = FileTree2(list(paths))
file_tree.walk(walk)
else:
def walk(path, item):
if isinstance(item, dict):
return item
return [paths[path]['index'], paths[path]['length'], True]
file_tree = FileTree(paths)
file_tree.walk(walk)
self.__m_files_tree = file_tree.get_tree()
else:
if filetree == 2:
self.__m_files_tree = {
'contents': {
self.__m_name: {
'type': 'file',
'index': 0,
'length': self.__m_metadata['info']['length'],
'download': True
}
}
}
else:
self.__m_files_tree = {
self.__m_name: (0, self.__m_metadata['info']['length'], True)
}
self.__m_files = []
if 'files' in self.__m_metadata['info']:
prefix = ''
if len(self.__m_metadata['info']['files']) > 1:
prefix = self.__m_name
for f in self.__m_metadata['info']['files']:
self.__m_files.append({
'path': f['path'],
'size': f['length'],
'download': True
})
else:
self.__m_files.append({
'path': self.__m_name,
'size': self.__m_metadata['info']['length'],
'download': True
})
def __init__(self, filename='', filetree=1, metainfo=None, metadata=None):
# Get the torrent metainfo from the torrent file
if metadata:
self._metainfo_dict = {b'info': bencode.bdecode(metadata)}
self._metainfo = bencode.bencode(self._metainfo_dict)
else:
self._metainfo = metainfo
if filename and not self._metainfo:
log.debug('Attempting to open %s.', filename)
try:
with open(filename, 'rb') as _file:
self._metainfo = _file.read()
except IOError as ex:
log.warning('Unable to open %s: %s', filename, ex)
return
try:
self._metainfo_dict = bencode.bdecode(self._metainfo)
except bencode.BTFailure as ex:
log.warning('Failed to decode %s: %s', filename, ex)
return
info_dict = self._metainfo_dict[b'info']
self._info_hash = sha(bencode.bencode(info_dict)).hexdigest()
# Get encoding from torrent file if available
encoding = self._metainfo_dict.get(b'encoding', None)
codepage = self._metainfo_dict.get(b'codepage', None)
if not encoding:
encoding = codepage if codepage else b'UTF-8'
# Decode 'name' with encoding unless 'name.utf-8' found.
if b'name.utf-8' in info_dict:
self._name = decode_bytes(info_dict[b'name.utf-8'])
else:
if encoding:
encoding = encoding.decode()
self._name = decode_bytes(info_dict[b'name'], encoding)
# Get list of files from torrent info
if b'files' in info_dict:
paths = {}
dirs = {}
prefix = self._name if len(info_dict[b'files']) > 1 else ''
for index, f in enumerate(info_dict[b'files']):
if b'path.utf-8' in f:
path = decode_bytes(os.path.join(*f[b'path.utf-8']))
del f[b'path.utf-8']
else:
path = decode_bytes(os.path.join(*f[b'path']), encoding)
if prefix:
path = os.path.join(prefix, path)
f[b'path'] = path
f[b'index'] = index
if b'sha1' in f and len(f[b'sha1']) == 20:
f[b'sha1'] = f[b'sha1'].encode(b'hex')
if b'ed2k' in f and len(f[b'ed2k']) == 16:
f[b'ed2k'] = f['ed2k'].encode(b'hex')
if b'filehash' in f and len(f[b'filehash']) == 20:
f[b'filehash'] = f[b'filehash'].encode(b'hex')
paths[path] = f
dirname = os.path.dirname(path)
while dirname:
dirinfo = dirs.setdefault(dirname, {})
dirinfo[b'length'] = dirinfo.get(b'length',
0) + f[b'length']
dirname = os.path.dirname(dirname)
if filetree == 2:
def walk(path, item):
if item['type'] == 'dir':
item.update(dirs[path])
else:
item.update(paths[path])
item['download'] = True
file_tree = FileTree2(list(paths))
file_tree.walk(walk)
else:
def walk(path, item):
if isinstance(item, dict):
return item
return [
paths[path][b'index'], paths[path][b'length'], True
]
file_tree = FileTree(paths)
file_tree.walk(walk)
self._files_tree = file_tree.get_tree()
else:
if filetree == 2:
self._files_tree = {
'contents': {
self._name: {
'type': 'file',
'index': 0,
'length': info_dict[b'length'],
'download': True,
}
}
}
else:
self._files_tree = {
self._name: (0, info_dict[b'length'], True)
}
self._files = []
if b'files' in info_dict:
prefix = ''
if len(info_dict[b'files']) > 1:
prefix = self._name
for f in info_dict[b'files']:
self._files.append({
'path': f[b'path'],
'size': f[b'length'],
'download': True
})
else:
self._files.append({
'path': self._name,
'size': info_dict[b'length'],
'download': True
})
def parse(self, filetree=1):
if not self.__m_filedata:
log.error("No data to process!")
return
try:
self.__m_metadata = bencode.bdecode(self.__m_filedata)
except Exception as e:
log.warning("Failed to decode torrent data %s: %s",
self.filename if self.filename else "", e)
raise e
self.__m_info_hash = sha(bencode.bencode(
self.__m_metadata["info"])).hexdigest()
# Get encoding from torrent file if available
self.encoding = None
if "encoding" in self.__m_metadata:
self.encoding = self.__m_metadata["encoding"]
elif "codepage" in self.__m_metadata:
self.encoding = str(self.__m_metadata["codepage"])
if not self.encoding:
self.encoding = "UTF-8"
# Check if 'name.utf-8' is in the torrent and if not try to decode the string
# using the encoding found.
if "name.utf-8" in self.__m_metadata["info"]:
self.__m_name = decode_string(
self.__m_metadata["info"]["name.utf-8"])
else:
self.__m_name = decode_string(self.__m_metadata["info"]["name"],
self.encoding)
# Get list of files from torrent info
paths = {}
dirs = {}
if "files" in self.__m_metadata["info"]:
prefix = ""
if len(self.__m_metadata["info"]["files"]) > 1:
prefix = self.__m_name
for index, f in enumerate(self.__m_metadata["info"]["files"]):
if "path.utf-8" in f:
path = os.path.join(prefix, *f["path.utf-8"])
else:
path = decode_string(
os.path.join(
prefix,
decode_string(os.path.join(*f["path"]),
self.encoding)), self.encoding)
f["index"] = index
paths[path] = f
dirname = os.path.dirname(path)
while dirname:
dirinfo = dirs.setdefault(dirname, {})
dirinfo["length"] = dirinfo.get("length", 0) + f["length"]
dirname = os.path.dirname(dirname)
if filetree == 2:
def walk(path, item):
if item["type"] == "dir":
item.update(dirs[path])
else:
item.update(paths[path])
item["download"] = True
file_tree = FileTree2(paths.keys())
file_tree.walk(walk)
else:
def walk(path, item):
if type(item) is dict:
return item
return [paths[path]["index"], paths[path]["length"], True]
file_tree = FileTree(paths)
file_tree.walk(walk)
self.__m_files_tree = file_tree.get_tree()
else:
if filetree == 2:
self.__m_files_tree = {
"contents": {
self.__m_name: {
"type": "file",
"index": 0,
"length": self.__m_metadata["info"]["length"],
"download": True
}
}
}
else:
self.__m_files_tree = {
self.__m_name:
(0, self.__m_metadata["info"]["length"], True)
}
self.__m_files = []
if "files" in self.__m_metadata["info"]:
prefix = ""
if len(self.__m_metadata["info"]["files"]) > 1:
prefix = self.__m_name
for f in self.__m_metadata["info"]["files"]:
if "path.utf-8" in f:
path = os.path.join(prefix, *f["path.utf-8"])
else:
path = decode_string(
os.path.join(
prefix,
decode_string(os.path.join(*f["path"]),
self.encoding)), self.encoding)
self.__m_files.append({
'path': path,
'size': f["length"],
'download': True
})
else:
self.__m_files.append({
"path": self.__m_name,
"size": self.__m_metadata["info"]["length"],
"download": True
})
def test_bencode_unicode_metainfo(self):
filename = common.get_test_data_file('test.torrent')
with open(filename, 'rb') as _file:
metainfo = bencode.bdecode(_file.read())[b'info']
bencode.bencode({b'info': metainfo})
