def test_decode_large_int(benchmark):
assert bdecode(b'i1455189890e') == 1455189890
assert bdecode(b'i25735241490e') == 25735241490
MAX_SIZE = sys.maxsize + 1
BENCODED_MAXSIZE = ('i%de' % MAX_SIZE).encode()
assert benchmark(bdecode, BENCODED_MAXSIZE) == MAX_SIZE
def test_ordered_dict(benchmark):
from bencoder import OrderedDict
rv = benchmark(bdecode, b'd2:ka2:va2:kbi2ee')
assert isinstance(rv, OrderedDict)
assert list(rv.keys()) == [b'ka', b'kb']
assert list(bdecode(b'd2:kc2:va2:kei2ee').keys()) == [b'kc', b'ke']
assert list(bdecode(b'd2:ke2:va2:kci2ee').keys()) == [b'ke', b'kc']
def test_decode_large_int(benchmark):
assert bdecode(b'i1455189890e') == 1455189890
assert bdecode(b'i25735241490e') == 25735241490
MAX_SIZE = sys.maxsize + 1
BENCODED_MAXSIZE = ('i%de' % MAX_SIZE).encode()
assert benchmark(bdecode, BENCODED_MAXSIZE) == MAX_SIZE
def test_ordered_dict(benchmark):
from bencoder import OrderedDict
rv = benchmark(bdecode, b'd2:ka2:va2:kbi2ee')
assert isinstance(rv, OrderedDict)
assert list(rv.keys()) == [b'ka', b'kb']
assert list(bdecode(b'd2:kc2:va2:kei2ee').keys()) == [b'kc', b'ke']
assert list(bdecode(b'd2:ke2:va2:kci2ee').keys()) == [b'ke', b'kc']
def parse_torrent():
with open('../test.torrent', 'rb') as f:
info = bencoder.bdecode(f.read())
print(info.keys())
# for key, value in info.get(b'info').items():
# print(key)
# print(value)
with open('../test2.torrent', 'rb') as f:
info = bencoder.bdecode(f.read())
print(info.keys())
print(info[b'nodes'])
def __init__(self, reactor, peer_id, port, raw_data = None, torrent_dict = None):
super(Torrent, self).__init__()
if(raw_data == None and torrent_dict == None):
Logger.error("Not Enough Information to get Torrent Data.\nCannot Ignore Error. Program will now Terminate")
elif (torrent_dict == None and raw_data != None):
torrent_dict = Bencoder.bdecode(raw_data)
self.started = False
self.reactor = reactor
self.start_time = time.time()
self.comment = torrent_dict['comment']
self.info = torrent_dict['info']
self.announce_list = torrent_dict['announce-list']
self.announce = torrent_dict['announce']
self.peer_id = peer_id
self.port = port
self.payload = self.generateBasicPayload()
self.protocol = Core.BitTorrentFactory(self)
self.fileManager = FileManager.FileManager(self.info)
self.requester = RequestManager.RequestManager(self)
self.payload = self.updatePayload()
# The handhshake message to be sent is constant for a given torrent
# str() has been overloaded
self.handshake_message = str(Messages.Handshake(self.payload['info_hash'], self.payload['peer_id']))
print "Total number of pieces :", len(self.info['pieces'])
def __init__(self, sock):
print("Peer started ...")
# Start RPC daemon for listening to his messages
RPCThread = threading.Thread(target=self.RPCHandler, args=())
RPCThread.daemon = True
RPCThread.start()
helloThread = threading.Thread(target=self.sendHello, args=())
helloThread.daemon = True
helloThread.start()
# Get some time variable out of while loop
helloTime = time.time()
while True:
# print('\nwaiting to receive message')
data, address = sock.recvfrom(4096)
# print('received {} bytes from {}'.format(
# len(data), address))
data = data.decode("utf-8")
dataDecoded = bdecode(data)
message = self.parseMessage(dataDecoded)
for line in message:
if line == 'type':
if message[line] == 'message':
print("[SERVER] I got message from someone")
print(message)
bencoded = bencode(b'ACK')
sent = sock.sendto(bencoded, address)
def sendGetList(self):
# Create a UDP socket
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
server_address = (REG_IPV4, int(REG_PORT))
data = {
"type":"getlist",
"txid": TXID
}
while True:
bencoded = bencode(data)
try:
# Send data
# print('sending {!r}'.format(bencoded))
sent = sock.sendto(bencoded, server_address)
# Receive response
# print('waiting to receive')
data, server = sock.recvfrom(4096)
decodedData = bdecode(data)
# print('received {!r}'.format(decodedData))
self.sendACK()
finally:
# print('closing socket')
sock.close()
return decodedData
def test_encode_complex():
od = dict()
od[b'KeyA'] = [b'listitemA', {b'k': b'v'}, 3]
od[b'KeyB'] = {b'k': b'v'}
od[b'KeyC'] = 3
od[b'KeyD'] = b'AString'
expected_result = b'd4:KeyAl9:listitemAd1:k1:vei3ee4:KeyBd1:k1:ve4:KeyCi3e4:KeyD7:AStringe'
assert bdecode(expected_result) == od
def decode_krpc(message):
'''
decode with bencoding. When exception is thrown, return None.
'''
try:
return bdecode(message)
except BTFailure:
return None
def test_encode_complex():
od = dict()
od[b'KeyA'] = [b'listitemA', {b'k': b'v'}, 3]
od[b'KeyB'] = {b'k': b'v'}
od[b'KeyC'] = 3
od[b'KeyD'] = b'AString'
expected_result = b'd4:KeyAl9:listitemAd1:k1:vei3ee4:KeyBd1:k1:ve4:KeyCi3e4:KeyD7:AStringe'
assert bdecode(expected_result) == od
def test_write_with_creation_date(generated_singlefile_torrent, tmpdir):
f = tmpdir.join('a.torrent')
now = int(time.time())
generated_singlefile_torrent.creation_date = now
generated_singlefile_torrent.write(str(f))
metainfo = bdecode(open(str(f), 'rb').read())
assert metainfo[b'creation date'] == now
def read_torrent_from_filepath(path):
f = open(path, 'rb')
content = None
try:
f_cont = f.read()
content = bdecode(f_cont)
finally:
f.close()
return content
def receive_forever(self):
"""一直接收外部发来的信息"""
self.logger.info('start receive forever...')
while True:
try:
data, addr = self.udp_socket.recvfrom(BUFSIZE)
self.handle_receive_things(bdecode(data), addr)
except Exception:
pass
def receive_response_forever(self):
self.bootstrap()
while True:
try:
d, a = self.s.recvfrom(UDP_RECV_BUFFSIZE)
msg = bencoder.bdecode(d)
self.on_message(a, msg)
sleep(SLEEP_TIME)
except Exception as e:
pass
def parse_torrent(file):
dot_torrent = file
with open(file, "rb") as f:
content = f.read()
# data = decoder.decode(content)
torrent = bdecode(content)
# hash = gutils.sha1("bencode(torrent['info'])")
hash = gutils.sha1(file)
torrent_name = torrent['info']['name']
try:
files = torrent['info']['files']
except KeyError:
# return {
# 'name': torrent_name
# }
files = [{'path': [torrent_name]}]
# print files
counter = populate_counter()
report_files = []
for file in files:
fname = '/'.join(file['path'])
report_files.append(fname)
if not is_ignored(fname):
cat = file_cat(fname)
counter['globl'] += 1
counter[cat] += 1
for case in switch(cat):
if case(MUSIC_CAT):
break
if case():
break
without_unk = removekey(counter, UNKNOWN_CAT, 'globl')
most_matched_cat = max(without_unk, key=without_unk.get)
if counter[most_matched_cat] == 0:
most_matched_cat = 'misc'
return {
'matched': counter[most_matched_cat],
'total': counter['globl'],
'assumed_category': most_matched_cat,
'name': torrent_name,
'files': report_files,
'source': dot_torrent,
'info_hash': hash
}
def receive_response(self):
self.join_DHT()
while True:
try:
# 接收返回报文
data, address = self.udp.recvfrom(UDP_RECV_BUFFSIZE)
# 解码返回的数据
msg = bdecode(data)
# 处理返回的信息
self.on_message(msg, address)
time.sleep(SLEEP_TIME)
except Exception as e:
print(e)
def pieces_complete(self):
metainfo = b''.join(self.pieces)
if len(metainfo) != self.metadata_size:
# Wrong size
return self.close()
infohash = hashlib.sha1(metainfo).hexdigest()
if binascii.unhexlify(infohash.upper()) != self.infohash:
# Wrong infohash
return self.close()
return bdecode(metainfo)
def datagram_received(self, data, addr):
try:
msg = bencoder.bdecode(data)
except:
return
try:
self.handle_message(msg, addr)
except Exception as e:
self.send_message(data={
"t": msg["t"],
"y": "e",
"e": [202, "Server Error"]
}, addr=addr)
raise e
def datagram_received(self, data, addr):
try:
msg = bencoder.bdecode(data)
except:
return
try:
self.handle_message(msg, addr)
except Exception as e:
self.send_message(data={
't': msg[b't'],
'y': 'e',
'e': [202, 'Server Error']
}, addr=addr)
raise e
def parse_dtorrent(self, path):
with open(path, "rb") as f:
torbytes = f.read()
self.dtor_dict = bdecode(torbytes)
announce = self.dtor_dict[b'announce'].decode()
tr_domain = re.search(r"https?://(.+?)/.+", announce).group(1)
for t, data in tr_data.items():
if tr_domain in data['tracker']:
self.src_tr = t
break
info = self.dtor_dict[b'info']
self.info_hash = sha1(bencode(info)).hexdigest()
def datagram_received(self, data, addr):
try:
msg = bencoder.bdecode(data)
except:
return
try:
self.handle_message(msg, addr)
except Exception as e:
self.send_message(data={
"t": msg["t"],
"y": "e",
"e": [202, "Server Error"]
}, addr=addr)
raise e
async def work(self):
self.writer.write(BT_HEADER + self.infohash + self.peer_id)
while True:
if not self.handshaked:
if self.check_handshake(await self.reader.readexactly(68)):
self.handshaked = True
# Send EXT Handshake
self.write_message(EXT_HANDSHAKE_MESSAGE)
else:
return self.close()
total_message_length, msg_id = struct.unpack(
"!IB", await self.reader.readexactly(5))
# Total message length contains message id length, remove it
payload_length = total_message_length - 1
payload = await self.reader.readexactly(payload_length)
if msg_id != EXT_ID:
continue
extended_id, extend_payload = payload[0], payload[1:]
if extended_id == 0 and not self.ut_metadata:
# Extend handshake, receive ut_metadata and metadata_size
try:
self.ut_metadata = get_ut_metadata(extend_payload)
self.metadata_size = get_metadata_size(extend_payload)
except:
return self.close()
self.pieces_num = math.ceil(self.metadata_size / BLOCK)
self.pieces = [False] * self.pieces_num
self.request_piece(0)
continue
try:
split_index = extend_payload.index(b"ee") + 2
info = bdecode(extend_payload[:split_index])
if info[b'msg_type'] != MessageType.DATA:
return self.close()
if info[b'piece'] != self.pieces_received_num:
return self.close()
self.pieces[info[b'piece']] = extend_payload[split_index:]
except:
return self.close()
self.pieces_received_num += 1
if self.pieces_received_num == self.pieces_num:
return self.pieces_complete()
else:
self.request_piece(self.pieces_received_num)
def receive_response_forever(self):
"""
循环接受 udp 数据
"""
self.logger.info("receive response forever {}:{}".format(
self.bind_ip, self.bind_port))
# 首先加入到 DHT 网络
self.bootstrap()
while True:
try:
# 接受返回报文
data, address = self.udp.recvfrom(UDP_RECV_BUFFSIZE)
# 使用 bdecode 解码返回数据
msg = bencoder.bdecode(data)
# 处理返回信息
self.on_message(msg, address)
time.sleep(SLEEP_TIME)
except Exception as e:
self.logger.warning(e)
def receive_response_forever(self):
"""
循环接受 udp 数据
"""
self.logger.info(
"receive response forever {}:{}".format(
self.bind_ip, self.bind_port
)
)
# 首先加入到 DHT 网络
self.bootstrap()
while True:
try:
# 接受返回报文
data, address = self.udp.recvfrom(UDP_RECV_BUFFSIZE)
# 使用 bdecode 解码返回数据
msg = bencoder.bdecode(data)
# 处理返回信息
self.on_message(msg, address)
time.sleep(SLEEP_TIME)
except Exception as e:
self.logger.warning(e)
def __init__(self):
print("[INFO] init")
print("Server is starting ...")
# Start RPC daemon for listening to his messages
RPCThread = threading.Thread(target=self.RPCHandler, args=())
RPCThread.daemon = True
RPCThread.start()
# Create a UDP socket
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
# Bind the socket to the port
server_address = (REG_IPV4, int(REG_PORT))
print('Serverstarting up socket on {} port {}'.format(*server_address))
sock.bind(server_address)
while True:
# print('\nwaiting to receive message')
data, address = sock.recvfrom(4096)
# print('received {} bytes from {}'.format(
# len(data), address))
data = data.decode("utf-8")
dataDecoded = bdecode(data)
message = self.parseMessage(dataDecoded)
# print(message)
for line in message:
self.checkPeerHelloInterval()
if line == 'type':
if message[line] == 'getlist':
print("[SERVER] I got GETLIST")
data = self.preparePeersData()
bencoded = bencode(data)
sent = sock.sendto(bencoded, address)
elif message[line] == 'hello':
print("[SERVER] I got HELO")
self.checkPeer(message)
elif message[line] == 'ack':
print("ACK")
def run(self):
self.join_dht_network()
count = 0
while True:
try:
(data, address) = self.transport.recvfrom(65536)
except:
if count > 8:
count = 0
self.join_dht_network()
else:
count += 1
continue
try:
msg = bdecode(data)
except Exception:
msg = None
print('编码解析错误:', data.decode('latin1'))
if msg:
self.handle_message(msg, address)
def test_decode_debian_torrent():
with open(TORRENT_PATH, "rb") as f:
torrent = bdecode(f.read())
assert torrent[b'announce'] == b'http://bttracker.debian.org:6969/announce'
assert torrent[b'comment'] == b'"Debian CD from cdimage.debian.org"'
def meta_info(self):
encode_info = bencoder.bdecode(self.handler.get_data())
return self.handler.utf_8_to_unicode(encode_info)
def info_hash(self):
raw_info = bencoder.bdecode(self.handler.get_data()).get(b'info')
return hashlib.sha1(bencoder.bencode(raw_info)).hexdigest()
def test_infohash():
import hashlib
with open(TORRENT_PATH, "rb") as f:
torrent = bdecode(f.read())
infohash = hashlib.sha1(bencode(torrent[b'info'])).hexdigest()
assert infohash == "4194e473d6c49630e1c6247d6716076809bb96ae"
def test_infohash():
import hashlib
with open(TORRENT_PATH, "rb") as f:
torrent = bdecode(f.read())
infohash = hashlib.sha1(bencode(torrent[b'info'])).hexdigest()
assert infohash == "4194e473d6c49630e1c6247d6716076809bb96ae"
def get_meta_info(torrent):
"""
返回解码后的 meta info 字典
"""
with open(torrent, "rb") as f:
return bdecode(f.read())
def decode_torrent(self, torrent):
with open(torrent, 'rb') as f:
content = bencoder.bdecode(f.read())
return content
def test_decode_debian_torrent():
with open(TORRENT_PATH, "rb") as f:
torrent = bdecode(f.read())
assert torrent[b'announce'] == b'http://bttracker.debian.org:6969/announce'
assert torrent[b'comment'] == b'"Debian CD from cdimage.debian.org"'
async def get_metadata(self):
if not self.writer or not self.reader:
return
try:
# handshake
self.send_handshake()
try:
data = await self.reader.readexactly(68)
except asyncio.IncompleteReadError:
logger.debug('Handshake failed')
return
if not self.check_handshake(data):
logger.error('Handshake response error: %s', data)
return
# ext handshake
self.send_ext_handshake()
pieces_list = None
pieces_count = 0
metadata_size = 0
while True:
data = await self.read_message()
if data.read(1) != BT_MSG_ID:
continue
if data.read(1) == EXT_HANDSHAKE_ID:
if pieces_list is not None:
logger.error("Metadata pended before handshake!")
return
packet = data.read()
meta_dict = bencoder.bdecode(packet)
ut_metadata, metadata_size = meta_dict[b'm'][b'ut_metadata'], meta_dict.get(b'metadata_size', 0)
if not metadata_size:
logger.error('Empty metadata size: %s', dict(meta_dict))
return
pieces_count = int(math.ceil(metadata_size / BLOCK_SIZE))
pieces_list = [b''] * pieces_count
logger.debug("request metadata pieces count [%d]", pieces_count)
for piece_index in range(pieces_count):
self.request_metadata(ut_metadata, piece_index)
else:
if pieces_list is None:
logger.error("metadata_size not defined")
return
# piece是个控制块，根据控制块下载数据
packet = data.read()
piece_dict, index = bencoder.decode_dict(packet, 0)
if piece_dict[b'msg_type'] != 1:
logger.debug("Message type [%d]", piece_dict[b'msg_type'])
continue
piece_index = piece_dict[b'piece']
piece_data = packet[index:]
if not self.check_piece(piece_data, piece_index, pieces_count, metadata_size):
logger.error("Piece check failed [%s]", [piece_data, piece_index, pieces_count, metadata_size])
return
pieces_list[piece_index] = piece_data
if self.pieces_done(pieces_list):
break
continue
logger.debug("Merge pieces together")
pieces = b''.join(pieces_list)
sha1_encoder = hashlib.sha1()
sha1_encoder.update(pieces)
if self.infohash != sha1_encoder.digest():
logger.error("Infohash %s not equal to info sha1 %s", self.infohash.hex(), sha1_encoder.hexdigest())
return
result = bencoder.bdecode(pieces)
del pieces
gc.collect()
logger.debug("Fetched metadata")
return result
except Exception as e:
logger.exception(e)
finally:
self.close()
def _deserialize(data):
return bencoder.bdecode(data)
def get_meta_info(torrent):
"""
返回解码后的 meta info 字典
"""
with open(torrent, "rb") as f:
return bdecode(f.read())
def _parse_torrent(filename):
with open(filename, 'rb') as f:
torrent = bdecode(f.read())
info = torrent[INFO]
return torrent, info
