def parse_extended_message(self):
extended_message_type = self.field[0]
message = self.field[1:]
if extended_message_type == 0:
try:
message = bdecode(message)
except DecodingError:
self.error.set()
return
if b'm' not in message:
logger.debug('"m" not in extended handshake.')
self.error.set()
return
self.extended_message_types = message[b'm']
if b'ut_metadata' not in self.extended_message_types:
logger.debug('Peer does not support metadata protocol.')
self.error.set()
return
if b'metadata_size' not in message:
logger.debug('Peer did not send "metadata_size" in extended handshake.')
self.error.set()
return
self.metadata_size = message[b'metadata_size']
logger.info('metadata size: {}'.format(self.metadata_size))
self.extended_handshake_complete.set()
self.write_message(15, b'') # have none
logger.debug('Sent HAVE NONE.')
self.write_message(0, b'') # choke
logger.debug('Sent CHOKE.')
self.write_message(3, b'') # not interesete
logger.debug('Sent NOT INTERESTED.')
elif extended_message_type == self.extended_message_types[b'ut_metadata']:
original_message = message
try:
message = bdecode(message)
except DecodingError:
self.error.set()
return
if message[b'msg_type'] == 0:
reply = {
'msg_type': 2,
'piece': message[b'piece']
}
elif message[b'msg_type'] == 2:
logger.debug('Request for metadata rejected.')
return
elif message[b'msg_type'] == 1:
size = len(original_message) - len(bencode(message))
logger.debug('Got a metadata block of size: {}'.format(size))
self.metadata_block = original_message[-size:]
self.metadata_block_received.set()
def parse_extended_message(self):
extended_message_type = self.field[0]
message = self.field[1:]
if extended_message_type == 0:
try:
message = bdecode(message)
except DecodingError:
self.error.set()
return
if b'm' not in message:
logger.debug('"m" not in extended handshake.')
self.error.set()
return
self.extended_message_types = message[b'm']
if b'ut_metadata' not in self.extended_message_types:
logger.debug('Peer does not support metadata protocol.')
self.error.set()
return
if b'metadata_size' not in message:
logger.debug('Peer did not send "metadata_size" in extended handshake.')
self.error.set()
return
self.metadata_size = message[b'metadata_size']
logger.info('metadata size: {}'.format(self.metadata_size))
self.extended_handshake_complete.set()
self.write_message(15, b'') # have none
logger.debug('Sent HAVE NONE.')
self.write_message(0, b'') # choke
logger.debug('Sent CHOKE.')
self.write_message(3, b'') # not interesete
logger.debug('Sent NOT INTERESTED.')
elif extended_message_type == self.extended_message_types[b'ut_metadata']:
original_message = message
try:
message = bdecode(message)
except DecodingError:
self.error.set()
return
if message[b'msg_type'] == 0:
reply = {
'msg_type': 2,
'piece': message[b'piece']
}
elif message[b'msg_type'] == 2:
logger.debug('Request for metadata rejected.')
return
elif message[b'msg_type'] == 1:
size = len(original_message) - len(bencode(message))
logger.debug('Got a metadata block of size: {}'.format(size))
self.metadata_block = original_message[-size:]
self.metadata_block_received.set()
def test_decode_parameter():
"""Ensure non-strings raise an exception."""
# TODO: BTL implementation currently chokes on this type of input
# self.assertRaises(BTFailure, bdecode, 0)
# self.assertRaises(BTFailure, bdecode, None)
# self.assertRaises(BTFailure, bdecode, 1.0)
with pytest.raises(BencodeDecodeError):
bdecode([1, 2])
with pytest.raises(BencodeDecodeError):
bdecode({'foo': 'bar'})
def verify_torrent_contents(torrent_file, path):
orig_path = path
if hasattr(torrent_file, 'seek') and hasattr(torrent_file, 'read'):
torrent_file.seek(0)
torrent = bdecode(torrent_file.read())
else:
try:
with open(torrent_file, 'rb') as f:
torrent = bdecode(f.read())
except (IOError, TypeError):
torrent = bdecode(torrent_file)
path = path_join(path, torrent[b'info'][b'name'].decode('utf-8'))
is_a_file = False
try:
with open(path, 'rb'):
is_a_file = True
except IOError:
pass
if not isdir(path) and not is_a_file:
raise IOError('Path specified for torrent data is invalid')
piece_length = torrent[b'info'][b'piece length']
piece_hashes = torrent[b'info'][b'pieces']
piece_hashes = struct.unpack('<{}B'.format(len(piece_hashes)),
piece_hashes)
piece_hashes = _chunks(piece_hashes, 20)
try:
filenames = ['/'.join([y.decode('utf-8') for y in x[b'path']])
for x in torrent[b'info'][b'files']]
except KeyError as e:
if e.args[0] != b'files':
raise e
# Single file torrent, never has a directory
filenames = [path]
path = orig_path
pieces = _get_torrent_pieces(filenames, path, piece_length)
for known_hash, piece in zip(piece_hashes, pieces):
try:
file_hash = sha1(piece).digest()
except TypeError:
raise VerificationError('Unable to get hash for piece')
if not compare_digest(bytes(known_hash), file_hash):
raise VerificationError('Computed hash does not match torrent file\'s hash')
def scrape_http(parsed_tracker, hashes):
qs = []
for hash in hashes:
url_param = binascii.a2b_hex(hash)
qs.append(("info_hash", url_param))
qs = urlencode(qs)
pt = parsed_tracker
url = urlunsplit((pt.scheme, pt.netloc, pt.path, qs, pt.fragment))
try:
handle = urlopen(url, timeout=5)
except Exception:
return {}
if handle.getcode() is not 200:
raise RuntimeError("%s status code returned" % handle.getcode())
decoded = bdecode(handle.read())
decoded = {k.decode('utf8'): v for k, v in decoded.items()}
if 'files' not in decoded:
return {}
ret = {}
for hash, stats in decoded['files'].items():
nice_hash = binascii.b2a_hex(hash).decode('utf8')
stats = {k.decode('utf8'): v for k, v in stats.items()}
s = None if 'complete' not in stats else stats["complete"]
p = None if 'incomplete' not in stats else stats["incomplete"]
c = None if 'downloaded' not in stats else stats["downloaded"]
ret[nice_hash] = {"seeds": s, "peers": p, "complete": c}
return ret
def datagram_received(self, data, addr):
try:
message = bdecode(data)
except DecodingError:
logger.debug('Received invalid bencoding in reply. Discarded.')
return
if b't' not in message:
logger.debug('Received invalid reply. Discarded')
return
if message[b't'] != self.tid:
logger.debug(
'Received reply with invalid transaction ID. Discarded.')
return
if b'r' not in message or b'id' not in message[b'r']:
logger.debug('Received invalid reply. Discarded.')
return
logger.debug('Received DHT reply from {}:{} with node ID {}.'.format(
addr[0], addr[1],
hexlify(message[b'r'][b'id']).decode()))
self.reply = message[b'r']
self.reply_received.set()
def tracker_info(response):
info = bdecode(response)
failure = info.get(b'failure reason')
if not failure:
return info
raise Tracker.TrackerError(
"Tracker return with error: {}".format(failure)
)
def _decode(self, torrent_file):
with open(torrent_file, 'rb') as f:
self.torrent = f.read()
odict = bdecode(self.torrent)
self.announce = odict[b'announce'].decode('utf-8')
self.info = odict[b'info']
self.length = self._length(self.info)
self.hash = sha1(bencode(self.info)).digest()
def test_decode_dict():
"""Ensure bytes can be decoded."""
value = bdecode('d5:title7:Examplee')
# Ensure a dict is returned
assert isinstance(value, dict)
# Validate items
assert value == {b'title': b'Example'}
def test_decode_errors():
"""Illegally formatted strings should raise an exception when decoded."""
with pytest.raises(BencodeDecodeError):
bdecode("foo")
with pytest.raises(BencodeDecodeError):
bdecode("x:foo")
with pytest.raises(BencodeDecodeError):
bdecode("x42e")
def scrape_http(parsed_tracker, hashes):
print("Scraping HTTP: %s for %s hashes" %
(parsed_tracker.geturl(), len(hashes)))
qs = []
for hash in hashes:
url_param = binascii.a2b_hex(hash)
qs.append(("info_hash", url_param))
qs = urlencode(qs)
pt = parsed_tracker
url = urlunsplit((pt.scheme, pt.netloc, pt.path, qs, pt.fragment))
handle = requests.get(url)
if handle.status_code != 200:
raise RuntimeError("%s status code returned" % handle.status_code)
decoded_dict = bencodepy.bdecode(handle.content)
decoded = get_decoded_dict(decoded_dict)
ret = {}
for hash, stats in decoded['files'].items():
nice_hash = binascii.b2a_hex(hash).decode('utf-8')
s = stats["complete"]
p = stats["incomplete"]
c = stats["downloaded"]
ret[nice_hash] = {"seeds": s, "peers": p, "complete": c}
return ret
def datagram_received(self, data, addr):
try:
message = bdecode(data)
except DecodingError:
logger.debug('Received invalid bencoding in reply. Discarded.')
return
if b't' not in message:
logger.debug('Received invalid reply. Discarded')
return
if message[b't'] != self.tid:
logger.debug('Received reply with invalid transaction ID. Discarded.')
return
if b'r' not in message or b'id' not in message[b'r']:
logger.debug('Received invalid reply. Discarded.')
return
logger.debug('Received DHT reply from {}:{} with node ID {}.'.format(
addr[0], addr[1], hexlify(message[b'r'][b'id']).decode()))
self.reply = message[b'r']
self.reply_received.set()
async def get_metadata(host, port, infohash):
global metadata, metadata_size, keep_running, full_metadata, get_metadatas_in_progress
if not keep_running:
return True
get_metadatas_in_progress += 1
try:
logger.info('Getting metadata from: {}:{}'.format(host, port))
loop = asyncio.get_running_loop()
try:
transport, protocol = await loop.create_connection(
lambda: BitTorrentProtocol(infohash, nodeid), host, port)
except OSError as e:
logger.debug('Connection error: {}'.format(e))
return False
logger.debug('Connected to peer: {}:{}'.format(host, port))
done, pending = await asyncio.wait(
[asyncio.create_task(protocol.handshake_complete.wait()),
asyncio.create_task(protocol.error.wait())],
return_when=FIRST_COMPLETED,
timeout=TIMEOUT)
for task in pending:
task.cancel()
if not done or protocol.error.is_set():
logger.debug('Error communicating with the peer while waiting for the handshake.')
transport.close()
return False
done, pending = await asyncio.wait(
[asyncio.create_task(protocol.extended_handshake_complete.wait()),
asyncio.create_task(protocol.error.wait())],
return_when=FIRST_COMPLETED,
timeout=TIMEOUT)
for task in pending:
task.cancel()
if not done or protocol.error.is_set():
logger.debug('Error communicating with the peer while waiting for the extended handshake.')
transport.close()
return False
if metadata_size > 0 and metadata_size != protocol.metadata_size:
logger.warning('Inconsistent metadata size received.')
metadata_size = protocol.metadata_size
metadata_nblocks = int(metadata_size / (16 * 1024))
metadata_nblocks += 0 if metadata_size % (16 * 1024) == 0 else 1
while keep_running:
protocol.metadata_block_received.clear()
try:
i = next(i for i in range(metadata_nblocks)
if i not in [m[0] for m in metadata])
except StopIteration as e:
transport.close()
return True
protocol.get_metadata_block(i)
done, pending = await asyncio.wait(
[asyncio.create_task(protocol.metadata_block_received.wait()),
asyncio.create_task(protocol.error.wait())],
return_when=FIRST_COMPLETED,
timeout=TIMEOUT)
for task in pending:
task.cancel()
if not done or protocol.error.is_set():
logger.debug('Error communicating with the peer while waiting for metadata block.')
transport.close()
return False
metadata.add((i, protocol.metadata_block))
if {m[0] for m in metadata} == set(range(metadata_nblocks)):
# metadata complete. hash check.
m = hashlib.sha1()
full_metadata = b''
for i, b in sorted(metadata, key=lambda m: m[0]):
full_metadata += b
m.update(b)
if m.digest() != infohash:
logger.debug('Invalid metadata received. Hash does not checkout. Discarding.')
metadata_size = 0
metadata = set()
return False
logger.info('Metadata received.')
full_metadata = bdecode(full_metadata)
keep_running = False
finally:
get_metadatas_in_progress -= 1
return True
def test_decode_bytes():
"""Ensure bytes can be decoded."""
assert bdecode(b'1:\x9c') == b'\x9c'
def test_encode_roundtrip():
"""Consecutive calls to decode and encode should deliver the original data again."""
for plain, encoded in ENCODE:
assert encoded == bencode(bdecode(encoded))
def test_decode_roundtrip():
"""Consecutive calls to encode and decode should deliver the original data again."""
for plain, encoded in VALUES:
assert plain == bdecode(bencode(plain))
def test_decode():
"""Decode should give known result with known input."""
for plain, encoded in DECODE:
assert plain == bdecode(encoded)
def get_metadata(loop, host, port, infohash):
global metadata, metadata_size, keep_running, full_metadata, get_metadatas_in_progress
if not keep_running:
return True
get_metadatas_in_progress += 1
try:
logger.info('Getting metadata from: {}:{}'.format(host, port))
try:
transport, protocol = yield from loop.create_connection(
lambda: BitTorrentProtocol(infohash, nodeid), host, port)
except OSError as e:
logger.debug('Connection error: {}'.format(e))
return False
logger.debug('Connected to peer: {}:{}'.format(host, port))
done, pending = yield from asyncio.wait(
[protocol.handshake_complete.wait(),
protocol.error.wait()],
return_when=FIRST_COMPLETED,
timeout=TIMEOUT)
for task in pending:
task.cancel()
if not done or protocol.error.is_set():
logger.debug('Error communicating with the peer while waiting for the handshake.')
transport.close()
return False
done, pending = yield from asyncio.wait(
[protocol.extended_handshake_complete.wait(),
protocol.error.wait()],
return_when=FIRST_COMPLETED,
timeout=TIMEOUT)
for task in pending:
task.cancel()
if not done or protocol.error.is_set():
logger.debug('Error communicating with the peer while waiting for the extended handshake.')
transport.close()
return False
if metadata_size > 0 and metadata_size != protocol.metadata_size:
logger.warning('Inconsistent metadata size received.')
metadata_size = protocol.metadata_size
metadata_nblocks = int(metadata_size / (16 * 1024))
metadata_nblocks += 0 if metadata_size % (16 * 1024) == 0 else 1
while keep_running:
protocol.metadata_block_received.clear()
try:
i = next(i for i in range(metadata_nblocks)
if i not in [m[0] for m in metadata])
except StopIteration as e:
transport.close()
return True
protocol.get_metadata_block(i)
done, pending = yield from asyncio.wait(
[protocol.metadata_block_received.wait(),
protocol.error.wait()],
return_when=FIRST_COMPLETED,
timeout=TIMEOUT)
for task in pending:
task.cancel()
if not done or protocol.error.is_set():
logger.debug('Error communicating with the peer while waiting for metadata block.')
transport.close()
return False
metadata.add((i, protocol.metadata_block))
if {m[0] for m in metadata} == set(range(metadata_nblocks)):
# metadata complete. hash check.
m = hashlib.sha1()
full_metadata = b''
for i, b in sorted(metadata, key=lambda m: m[0]):
full_metadata += b
m.update(b)
if m.digest() != infohash:
logger.debug('Invalid metadata received. Hash does not checkout. Discarding.')
metadata_size = 0
metadata = set()
return False
logger.info('Metadata received.')
full_metadata = bdecode(full_metadata)
keep_running = False
finally:
get_metadatas_in_progress -= 1
return True