def _download_torrent(self):
res = self.session.get(self.dl_link)
torrent = bytes()
for chunk in res.iter_content(100000):
torrent += chunk
ggn_dir = os.path.join(TORRENT_DIR, 'ggn/')
if not os.path.exists(ggn_dir):
os.makedirs(ggn_dir)
with open(
os.path.join(
ggn_dir,
os.path.basename('[GGn]{}.torrent'.format(
self.release_title))), 'wb') as t:
t.write(torrent)
torrent = bencodepy.decode(torrent)
torrent[
b'announce'] = b'https://tracker.pterclub.com/announce?passkey=' + bytes(
PTER_KEY, encoding='utf-8')
torrent[b'info'][b'source'] = bytes('[pterclub.com] ＰＴ之友俱乐部',
encoding='utf-8')
del torrent[b'comment']
torrent = bencodepy.encode(torrent)
with open(
os.path.join(
'torrents',
os.path.basename('[PTer]{}.torrent'.format(
self.release_title))), 'wb') as t:
t.write(torrent)
def peers_b2l(response):
"""
Parse the response fron tracker, dig out the peers addresses
:param response: binary data, response from the tracker
"""
if response == b'':
raise ValueError('No data returned')
data = b.decode(response)
lst = list()
if not isinstance(data[b'peers'], bytes):
raise TypeError('Peer list is {0}'.format(data[b'peers']))
# parse the response
for offset in range(0, len(data[b'peers']), 6):
# all data are stored as big endian
# dig out the IP address (integer)
raw_ip = s.unpack_from(">i", data[b'peers'], offset)[0]
ip = socket.inet_ntoa(s.pack(">i", raw_ip))
# do the same for the port number (insigned short)
port = s.unpack_from(">H", data[b'peers'], offset + 4)[0]
lst.append('{0}:{1}'.format(ip, port))
return lst
async def announce(self, server_port: int, event: EventType):
params = {
'info_hash': self._download_info.info_hash,
'peer_id': self._our_peer_id,
'port': server_port,
'uploaded': self._statistics.total_uploaded,
'downloaded': self._statistics.total_downloaded,
'left': self._download_info.bytes_left,
'compact': 1,
}
if event != EventType.none:
params['event'] = event.name
if self._tracker_id is not None:
params['trackerid'] = self._tracker_id
params = {name: urllib.parse.quote(value) if isinstance(value, bytes) else value
for name, value in params.items()}
url = yarl.URL(self._announce_url).update_query(params)
with async_timeout.timeout(HTTPTrackerClient.REQUEST_TIMEOUT):
async with aiohttp.ClientSession() as session:
async with session.get(url) as conn:
response = await conn.read()
response = bencodepy.decode(response)
if not response:
if event == EventType.started:
raise ValueError('Tracker returned an empty answer on start announcement')
return
response = cast(OrderedDict, response)
self._handle_primary_response_fields(response)
self._handle_optional_response_fields(response)
def get_torrent_file(torrent_id, return_file=False):
rows = db2.query(Torrent_File).filter_by(torrent_id=torrent_id).all()
if len(rows) != 0:
if return_file is False:
return
else:
return {"file": rows[0].file, "source": "Cached"}
login()
r = gazelle.get("{0}/torrents.php?action=download&id={1}".format(
gazelle_url, torrent_id))
try:
data = bencodepy.decode(r.content)
except:
print("Error decoding torrent %d" % torrent_id)
exit()
# data[b"announce"] = ""
db2.merge(Torrent_File(torrent_id=torrent_id, file=bencodepy.encode(data)))
db2.commit()
#return {"file": None, "source": ""}
if return_file is True:
return {"file": bencodepy.encode(data), "source": "Fetched"}
def __init__(self, bencontent):
"""
Args:
bencontent (bytes): bencoded torrent file contents
"""
if not bencontent:
raise TorrentDecodeError('Empty torrent file')
try:
content = bencodepy.decode(bencontent)
except bencodepy.DecodingError as e:
# TODO: better msg
raise TorrentDecodeError from e
# TODO: validate shape using voluptuous or schema libraries
# 'encoding' field defines character encoding for 'pieces' field.
encoding = content.get(b'encoding')
if encoding and encoding.decode('utf-8').lower() != 'utf-8':
raise(TorrentDecodeError('Unsupported encoding: %s' % encoding))
self.announce = content[b'announce'].decode('utf-8')
try:
vol.Url()(self.announce)
except vol.UrlInvalid as e:
msg = 'Invalid announce URL: %s' % self.announce
raise TorrentDecodeError(msg) from e
# Ignore 'creation date', 'comment', 'created by', 'announce-list'
info_dict = content[b'info']
self.info_hash = hashlib.sha1(bencodepy.encode(info_dict)).digest()
self.info = self._decode_info_dict(info_dict)
def _decode(self, content):
try:
response = bencodepy.decode(content)
self._decode_peers(response[b'peers'])
except bencodepy.exceptions.DecodingError:
raise TrackerError('Failed to decode tracker response {}'
.format(self._metadata.announce()))
def get_torrent_file(torrent_id, return_file=False):
rows = db2.query(Torrent_File).filter_by(torrent_id=torrent_id).all()
if len(rows) != 0:
if return_file is False:
return
else:
return {"file": rows[0].file, "source": "Cached"}
login()
r = gazelle.get("{0}/torrents.php?action=download&id={1}".format(gazelle_url, torrent_id))
try:
data = bencodepy.decode(r.content)
except:
print("Error decoding torrent %d" % torrent_id)
exit()
# data[b"announce"] = ""
db2.merge(Torrent_File(torrent_id=torrent_id, file=bencodepy.encode(data)))
db2.commit()
#return {"file": None, "source": ""}
if return_file is True:
return {"file": bencodepy.encode(data), "source": "Fetched"}
async def get_file_structure(self, location: str) -> List[str]:
"""
Given a URL to a .torrent file, it will then return a list
of the file names inside.
Parameters
----------
location: str
A URL to a .torrent file.
Returns
-------
List[str]
List of file names.
"""
async with self.session.get(location) as resp:
torrent_bytes = await resp.read()
metadata = bencodepy.decode(torrent_bytes)
is_folder = b"files" in metadata[b"info"]
file_names = []
if is_folder:
for item in metadata[b"info"][b"files"]:
try:
file_names.append(item[b"path"][0].decode("utf-8"))
except IndexError:
pass
else:
file_names.append(metadata[b"info"][b"name"].decode("utf-8"))
return file_names
def handle(request):
logging.warning('Tracker connection received!')
def create_sock(_peer, ip):
server = yield from loop.create_server(lambda: TorrentProxy(_peer), ip, 0)
port = server.sockets[0].getsockname()[1]
return port
ip = request.GET['ip'] if ip in request.GET else '127.0.0.1'
port = yield from create_sock({
'ip': ip,
'port': request.GET['port']
}, '0.0.0.0')
path = request.path_qs.replace('port={}'.format(request.GET['port']),
'port={}'.format(port))
r = yield from aiohttp.get(sys.argv[1] + path)
data = yield from r.read()
yield from r.release()
torrent = bencodepy.decode(data)
proxied_peers = []
base_ip = ip2int('127.13.37.0')
for index, peer in enumerate(decode_peers(torrent[b'peers'])):
ip = int2ip(base_ip + index)
port = yield from create_sock(peer, ip)
proxied_peers.append({ 'ip': ip, 'port': port })
logging.info('Opened listener on {}'.format(port))
torrent[b'peers'] = encode_peers(proxied_peers)
return web.Response(body=bencodepy.encode(torrent))
def peers_b2l(response):
"""
Parse the response fron tracker, dig out the peers addresses
:param response: binary data, response from the tracker
"""
if response == b'':
raise ValueError('No data returned')
data = b.decode(response)
lst = list()
if not isinstance(data[b'peers'], bytes):
raise TypeError('Peer list is {0}'.format(data[b'peers']))
# parse the response
for offset in range(0,len(data[b'peers']), 6):
# all data are stored as big endian
# dig out the IP address (integer)
raw_ip = s.unpack_from(">i", data[b'peers'], offset)[0]
ip = socket.inet_ntoa(s.pack(">i", raw_ip))
# do the same for the port number (insigned short)
port = s.unpack_from(">H", data[b'peers'], offset+4)[0]
lst.append('{0}:{1}'.format(ip, port))
return lst
def get_torrent_stats(url):
response = requests.get(url)
data = bencodepy.decode(response.content)
files = data[b'info'][b'files']
size = 0
for file in files:
size += file[b'length']
size = size * 1e-9
size = round(size, 2)
info_hash = hashlib.sha1(bencodepy.bencode(data[b"info"])).hexdigest()
trackers_list = data[b'announce-list']
stats = {'seeds': 0, 'peers': 0}
for tracker_url in trackers_list:
tracker_url = tracker_url[0].decode('utf-8')
result = scrape(tracker_url, [info_hash])
if not result:
continue
if result[info_hash]['seeds'] is None or result[info_hash][
'peers'] is None:
continue
stats['seeds'] = max(stats['seeds'], result[info_hash]['seeds'])
stats['peers'] = max(stats['peers'], result[info_hash]['peers'])
stats['size_gb'] = size
return stats
def spawn(url):
torrent_url_components = urlparse(url)
torrent_url_query = torrent_url_components.query
match = re.search('ref=(.*)', torrent_url_query)
if match is None:
pass
else:
torrent_file = str(match.groups(0)[0]) + '.torrent'
blob = download_torrent(url)
dbx = dropbox.Dropbox(
'JUEnSrL_pnAAAAAAAAAADGNPxjjbk3nYLnatbTN8vvJ01JM8yQIhn-MI5DqW41nR')
path = '/torrents/%s' % torrent_file
dbx.files_upload(blob, path)
link = dbx.sharing_create_shared_link_with_settings(path).url
link = re.sub('dl=0', 'dl=1', link)
try:
metainfo = decode(blob)
info = metainfo[b'info']
btih = hashlib.sha1(encode(info)).hexdigest()
dn = metainfo[b'info'][b'name']
magnet = 'magnet:?xt=urn:btih:{btih}&dn={dn}'.format(btih=btih,
dn=dn)
torrent = {'status': 'OK', 'magnet': magnet, 'torrent': link}
r.hmset(url, torrent)
record_event({'type': 'aisex.newtorrent'})
except:
torrent = {'status': 'ERROR', 'error': 'not a valid torrent file'}
return torrent
def get_peers_from_tracker(announce, torrent_data, http=True) -> list:
if http:
response = connect_to_http_tracker(announce, torrent_data)
else:
transaction_id = get_udp_transaction_id() # randomized transaction ID for UDP
response = connect_to_udp_tracker(announce, torrent_data, transaction_id)
if response == b'':
log('Response of tracker was empty', 2)
return [] # if error has occurred empty list of peers will be returned
if http:
# response from tracker is bencoded binary representation of peers addresses will be parsed to readable form
decoded = bencodepy.decode(response)
if not decoded:
log("Decoded response is empty", 1)
return []
elif b'peers' not in decoded:
log("Decoded response has no peers data in it", 1)
return []
bin_peers = decoded[b'peers']
else:
bin_peers = parse_udp_announce_response(response, transaction_id)
log("There should be %u peers" % (len(bin_peers) / 6), 1)
peers = []
# bin peers data is field of bytes, where each 6 bytes represent one peer
# for each peer will be appended to peers list
for i in range(0, len(bin_peers), 6):
peers.append(parse_bin_peer(bin_peers[i:i + 6]))
return peers
def _receiver(self):
logger = logging.getLogger("btdht")
logger.info("Receiver thread started.")
while self.running:
try:
data = self.sock.recv(4096)
except OSError:
time.sleep(SLEEP_QUANTUM)
continue
try:
recvd = bencodepy.decode(data)
if recvd[b'y'] == b'r':
tid = recvd[b't']
response = recvd[b'r']
with self.lock:
try:
req = self.requests[tid]
except KeyError:
logger.info("Received response to unknown request.")
else:
del self.requests[tid]
if req.type == REQUEST_TYPE_FIND_NODE:
if self.ipv == 'ip4':
nodes = parse_compact_node_info(response[b'nodes'])
else:
nodes = parse_compact_node6_info(response[b'nodes6'])
for node in nodes:
addr = node['addr']
nodeid = node['id']
# if the unique option is enabled, ignore duplicates
if self.unique is not None:
if addr[0] in self.unique:
continue
self.unique.add(addr[0])
self.addr_cache.put((addr, nodeid))
self.requests_success += 1
if len(self.addr_pool) > 100:
self.addr_pool.popleft()
self.addr_pool.append(addr)
else:
logger.info("Unknown request type: {}. Seems like I've created a Request record with a unexpected request type.".format(req.type))
except bencodepy.DecodingError:
logger.info("bencode decoding of incoming packet failed.")
except KeyError:
logger.info("Malformed? packet received.")
except Exception:
logger.exception("Unknown error happened during response handling.")
logger.info("Shutting down receiver.")
self.sock.close()
def read_metadata_from_file(self):
"""
Read a .torrent file and decode metadata.
:return: None
"""
with open(self.file_path, 'rb') as f:
self.metadata = bencodepy.decode(f.read())
self.metadata = utils.dict_to_utf8(self.metadata)
def loadb(self, b: bytes, options: dict) -> Any:
kwargs = {}
kwargs.update(Options.pop_origin_kwargs(options))
self.check_options(options)
try:
return bencodepy.decode(b, **kwargs)
except bencodepy.BencodeDecodeError as e:
raise SerializeError(e)
def parseTorrent(fd):
"""
parse a torrent from a file descriptor
return dict that was bencoded
will raise of parse error
"""
data = fd.read()
return bencodepy.decode(data)
def __init__(self, filepath):
self.file = filepath
with open(self.file, 'rb') as f:
self.meta_info = f.read()
self.meta_info = bencodepy.decode(self.meta_info)
info = bencodepy.encode(self.meta_info[b'info'])
self.info_hash = sha1(info).digest()
self.count_files()
def run(self):
self.re_join_DHT()
while True:
try:
(data, address) = self.ufd.recvfrom(65536)
msg = decode(data)
self.on_message(msg, address)
except Exception:
pass
def cleanup_torrentsdir(self, torrents_dir, dryrun: bool):
try:
tor_filenames = os.listdir(torrents_dir)
except FileNotFoundError:
echo(f'unable list file from {torrents_dir!r}.')
return
class _FileEntry:
def __init__(self, name: str) -> None:
self.name = name
self.info_hash: str = None
self.torrents = []
@property
def path(self):
return os.path.join(torrents_dir, self.name)
# get files from disk before get torrents from transmission
# ensure no new torrents will be delete.
file_entries: Dict[str, _FileEntry] = {}
file_entries_by_infohash: Dict[str, List[_FileEntry]] = {}
for name in tor_filenames:
assert name not in file_entries
file_entries[name] = fe = _FileEntry(name)
path = os.path.join(torrents_dir, name)
with open(path, 'rb') as f:
tor_body = bencodepy.decode(f.read())
magnet_info = tor_body.get(b'magnet-info')
if magnet_info:
# this is a magnet
info_hash: str = magnet_info[b'info_hash'].hex()
else:
info_hash: str = compute_info_hash(tor_body)
info_hash = info_hash.lower()
fe.info_hash = info_hash
file_entries_by_infohash.setdefault(info_hash, []).append(fe)
echo(f'read {len(file_entries)} torrents from torrents dir.')
drift_torrents = []
torrents = self.tc.get_torrents(
arguments=['id', 'hashString', 'torrentFile'])
for tor in torrents:
assert isinstance(tor.torrentFile, str)
info_hash = tor.hashString.lower()
tfn = os.path.basename(tor.torrentFile)
fe = file_entries.get(tfn)
if fe:
fe.torrents.append(tor)
else:
if len(fels := file_entries_by_infohash.get(info_hash, (
))) == 1:
fels[0].torrents.append(tor)
else:
drift_torrents.append(tor)
def run(self):
self.re_join_dht()
while True:
try:
(data, address) = self.socket.recvfrom(PACKET_SIZE)
msg = bencodepy.decode(data)
self.process_message(msg, address)
except Exception:
pass
def peer_tracker():
with open("./file.torrent", "rb") as f:
raw_data = f.read()
data = bc.decode(raw_data)
print(data.keys())
info_hash = hashlib.sha1(bc.bencode(data[b"info"])).hexdigest()
print(data[b"announce-list"])
return info_hash
def run(self):
self.re_join_dht()
while True:
try:
(data, address) = self.socket.recvfrom(PACKET_SIZE)
msg = bencodepy.decode(data)
self.process_message(msg, address)
except Exception:
pass
def run(self): self.re_join_DHT() while True: try: data, address = self.sock.recvfrom(65536) message = decode(data) self.on_message(message, address) except Exception as error: pass
def handle_announce_response(self, http_resp):
resp = bencodepy.decode(http_resp.text.encode('latin-1'))
d = self.decode_announce_response(resp)
# TODO: use 'interval', 'tracker id', 'complete', 'incomplete'
for peer_dict in d['peers']:
# TODO: raise error or warning on port = 0?
if peer_dict['ip'] and peer_dict['port'] > 0:
self.torrent.add_peer(peer_dict)
def parse_tracker_response(self, content):
resp = bencodepy.decode(content)
split_peers = [
resp[b'peers'][i:i + 6] for i in range(0, len(resp[b'peers']), 6)
]
peers = [(socket.inet_ntoa(p[:4]), _decode_port(p[4:]))
for p in split_peers]
return TrackerResponse(resp[b'complete'], resp.get(b'crypto_flags'),
resp.get(b'incomplete'), resp.get(b'interval'),
peers)
def make_request_to_tracker(self):
"""
Makes the GET request to the tracker
:return: Decoded response content
"""
raw_response = requests.get(self.torrent.metadata['announce'], params=self.params)
print('status code: ', raw_response.status_code)
if raw_response.status_code != 200:
raise ConnectionError(f'Couldn\'t connect to tracker: {raw_response.status_code}')
self.response = bencodepy.decode(raw_response.content)
return self.response
def torrentToMagnet(cls, torrentByte):
try:
metadata = bencodepy.decode(torrentByte)
subj = metadata[b'info']
hashcontents = bencodepy.encode(subj)
digest = hashlib.sha1(hashcontents).digest()
b32hash = base64.b32encode(digest).decode()
return 'magnet:?' \
+ 'xt=urn:btih:' + b32hash
except:
return ""
def handle_announce_responce(self, http_resp):
resp = bencodepy.decode(http_resp.text.encode('latin-1'))
d = self.decode_announce_response(resp)
peer_list = d['peers']
return peer_list
for peer_dict in d['peers']:
# TODO: raise error or warning on port = 0?
if peer_dict['ip'] and peer_dict['port'] > 0:
# print(peer_dict)
self.torrent.add_peer(peer_dict)
def run(self):
"""
Wait for DHT responses and query the parser if it seems valid.
"""
while True:
try:
(data, address) = self.ufd.recvfrom(65536)
msg = bencodepy.decode(data)
if msg[b'y'] == b'r':
self.parse_response(msg, address)
except Exception:
pass
async def work(self):
self.writer.write(BT_HEADER + self.infohash + self.peer_id)
print(172)
while True:
if not self.handshaked:
print(175)
if self.check_handshake(await self.reader.readexactly(68)):
self.handshaked = True
# Send EXT Handshake
self.write_message(EXT_HANDSHAKE_MESSAGE)
else:
return self.close()
total_message_length, msg_id = struct.unpack(
"!IB", await self.reader.readexactly(5))
# Total message length contains message id length, remove it
payload_length = total_message_length - 1
payload = await self.reader.readexactly(payload_length)
if msg_id != EXT_ID:
continue
extended_id, extend_payload = payload[0], payload[1:]
if extended_id == 0 and not self.ut_metadata:
# Extend handshake, receive ut_metadata and metadata_size
try:
self.ut_metadata = get_ut_metadata(extend_payload)
self.metadata_size = get_metadata_size(extend_payload)
except:
return self.close()
self.pieces_num = math.ceil(self.metadata_size / BLOCK)
self.pieces = [False] * self.pieces_num
self.request_piece(0)
continue
try:
split_index = extend_payload.index(b"ee") + 2
info = decode(extend_payload[:split_index])
if info[b'msg_type'] != MessageType.DATA:
return self.close()
if info[b'piece'] != self.pieces_received_num:
return self.close()
self.pieces[info[b'piece']] = extend_payload[split_index:]
except:
return self.close()
self.pieces_received_num += 1
if self.pieces_received_num == self.pieces_num:
# return self.pieces_complete()
loop = asyncio.get_event_loop()
loop.create_task(self.pieces_complete())
else:
self.request_piece(self.pieces_received_num)
def validateRSS(self):
try:
if self.cookies:
success, status = self.add_cookies_from_ui()
if not success:
return False, status
# Access to a protected member of a client class
data = self.cache._get_rss_data()['entries']
if not data:
return False, 'No items found in the RSS feed {0}'.format(
self.url)
title, url = self._get_title_and_url(data[0])
if not title:
return False, 'Unable to get title from first item'
if not url:
return False, 'Unable to get torrent url from first item'
if url.startswith('magnet:') and re.search(
r'urn:btih:([\w]{32,40})', url):
return True, 'RSS feed Parsed correctly'
else:
torrent_file = self.get_url(url, returns='content')
try:
bencodepy.decode(torrent_file)
except (bencodepy.exceptions.BencodeDecodeError,
Exception) as error:
self.dumpHTML(torrent_file)
return False, 'Torrent link is not a valid torrent file: {0}'.format(
error)
return True, 'RSS feed Parsed correctly'
except Exception as error:
return False, 'Error when trying to load RSS: {0}'.format(
str(error))
def handle_tracker_response(self):
print("Raw Response: ", self.response)
#decoded_response = self.response.decode('utf-8')
#print("Decoded Response: ", decoded_response)
split_decoded = self.response.split(b'\r\n\r\n')
response_dict = decode(split_decoded[1])
print("Tracker response: ", response_dict)
#check for failure reason
if b'failure reason' in response_dict:
print(response_dict[b'failure reason'].decode('utf-8'))
else:
self.interval = response_dict[b'interval']
#self.tracker_id = response_dict[b'tracker id']
self.complete = response_dict[b'complete']
self.incomplete = response_dict[b'incomplete']
#parse list of peers
peerlist = list()
unparsed_peers = response_dict[b'peers']
print(unparsed_peers)
#add peers to list of tuples (IP, port)
for x in range(len(unparsed_peers) // 6):
ip = socket.inet_ntoa(unparsed_peers[x * 6:x * 6 + 4])
port = int.from_bytes(unparsed_peers[x * 6 + 4:x * 6 + 6],
byteorder='big')
#print("Reading peer ", ip, port)
peerlist.append((ip, port))
print(peerlist)
self.peer_list = peerlist
#for each peer in peer list, check if connected
for (IP, port) in self.peer_list:
#print(self.ip_addr, IP, self.ip_addr != IP)
#print(self.port, port, self.port != port)
if (IP != self.ip_addr) or (port != self.port):
for connection in self.connection_list:
if (IP == connection.peer_ip_addr) and (
port == connection.peer_port):
break
else:
#if not connected, initiate handshake with peer
print("Connect to peer ", IP, port)
handshake_thread = threading.Thread(
target=self.initiate_handshaking, args=(IP, port))
handshake_thread.daemon = True
handshake_thread.start()
handshake_thread.join()
def from_data(cls, data: bytes, **kwargs):
# TODO: refactor from_{data,filename}
dictionary = cast(OrderedDict, bencodepy.decode(data))
download_info = DownloadInfo.from_dict(dictionary[b'info'])
if b'announce-list' in dictionary:
announce_list = [[url.decode() for url in tier]
for tier in dictionary[b'announce-list']]
else:
announce_list = [[dictionary[b'announce'].decode()]]
return cls(download_info, announce_list, **kwargs)
def main():
arg = ArgumentParser.ArgumentParser()
arg.do_parsing()
torrent_dict = decode_from_file(os.getcwd() + "\\" + arg.torrent_file_url)
info_dict = torrent_dict[b"info"]
piece_length = info_dict[b"piece length"]
piece_count = int(len(info_dict[b"pieces"])/20)
file_name = info_dict[b"name"]
file_name = file_name.decode("utf-8")
if b"files" in info_dict:
print(info_dict[b"files"])
global chunk_number
chunk_number = int(piece_length/chunk_size)
if piece_length % chunk_size != 0:
chunk_number += 1
global pieces_bitfield
pieces_bitfield = [None] * piece_count
global currently_used
currently_used = [False] * piece_count
dictionary_sha1 = helper.get_sha1(encode(info_dict))
bencoded_info_dict = encode(info_dict)
info_hash = helper.get_sha1(bencoded_info_dict)
peer_id = os.urandom(20)
listen_port = 2710
file_length = info_dict[b"length"]
payload = {"info_hash": info_hash, "peer_id": peer_id,
"port": listen_port, "uploaded": 0, "downloaded": 0, "left": file_length}
r = requests.get(torrent_dict[b"announce"], params=payload)
response = decode(r.content)
if b"failure reason" in response:
return
interval = response[b"interval"]
peers = response[b"peers"]
peers_list = []
peer_num = len(peers)/6
for elem in range(int(peer_num)):
start_ind = 6 * elem
peer_ip = socket.inet_ntoa(peers[start_ind:start_ind+4])
peer_port = struct.unpack("!H", peers[start_ind+4:start_ind+6])[0]
peers_list.append((peer_ip, peer_port))
print(peers_list)
for elem in peers_list:
test_ip = elem[0]
test_port = elem[1]
cur_thread = Thread(target=do_for_each_peer,
args=(dictionary_sha1, test_ip, test_port, piece_length, info_dict, peer_id, file_length,
file_name))
# do_for_each_peer(dictionary_sha1, test_ip, test_port, piece_length, info_dict,peer_id)
# print(cur_thread.name)
cur_thread.start()
def request_torrent_information(self):
# try establishing a connection to the tracker
try:
# the reponse from HTTP tracker is an bencoded dictionary
bencoded_response = requests.get(self.tracker_url, self.request_parameters, timeout=5)
# decode the bencoded dictionary to python ordered dictionary
raw_response_dict = bencodepy.decode(bencoded_response.content)
# parse the dictionary containing raw data
self.parse_http_tracker_response(raw_response_dict)
return True
except Exception as error_msg:
# cannont establish a connection with the tracker
self.tracker_logger.log(self.tracker_url + ' connection failed !' + FAILURE)
return False
def datagram_received(self, data, addr):
try:
msg = bencodepy.decode(data)
except:
return
try:
self.handle_message(msg, addr)
except Exception as e:
self.send_message(data={
"t": msg["t"],
"y": "e",
"e": [202, "Server Error"]
},
addr=addr)
raise e
async def get_magnet(self, location: str) -> str:
"""
Will take an internet location for a torrent file.
The magnet URL for that torrent is then resolved and returned.
If the location parameter is already detected to be a magnet URL,
it will instantly return it.
Parameters
----------
location: str
A file location or web address.
Returns
-------
str
The magnet URL for the torrent at the given location.
"""
pattern = re.compile(r"\burn:btih:([A-z\d]+)\b")
def b32_to_sha1(match: re.Match) -> str:
hash_ = match.group(1)
if len(hash_) == 40:
return match.group(0)
elif len(hash_) == 32:
return "urn:btih:" + base64.b32decode(hash_.upper()).hex()
return match.group(0)
if location.startswith("magnet:?"):
return re.sub(pattern, b32_to_sha1, location)
else:
async with self.session.get(location) as resp:
torrent_bytes = await resp.read()
metadata = bencodepy.decode(torrent_bytes)
subject = metadata[b'info']
hash_data = bencodepy.encode(subject)
digest = hashlib.sha1(hash_data).hexdigest()
magnet_url = ("magnet:?"
f"xt=urn:btih:{digest}"
f"&dn={metadata[b'info'][b'name'].decode()}"
f"&tr={metadata[b'announce'].decode()}")
return re.sub(pattern, b32_to_sha1, magnet_url)
def create_magnet_uri(data: bytes):
# noinspection PyTypeChecker
metadata: dict = bencodepy.decode(data)
subj = metadata[b'info']
hashcontents = bencodepy.encode(subj)
digest = hashlib.sha1(hashcontents).digest()
b32hash = base64.b32encode(digest).decode()
magnet_uri = 'magnet:?' + 'xt=urn:btih:' + b32hash
if b"announce" in metadata:
magnet_uri += ('&tr=' + metadata[b'announce'].decode())
if b"info" in metadata:
metadata_info = metadata[b'info']
if b"name" in metadata_info:
magnet_uri += ('&dn=' + metadata[b'info'][b'name'].decode())
if b"length" in metadata_info:
magnet_uri += ('&xl=' + str(metadata[b'info'][b'length']))
return magnet_uri
def __init__(self, file_name):
"""参数定义"""
self.announce = None # Tracker服务器地址
self.info = None # torrent文件中的info0字节内容
self.info_hash = None # info字节内容的sha1 hash值
self.name = None # 资源若为单个资源文件，则为文件名称，若为资源文件夹，则为最顶层目录名称
"""读取torrent文件内容"""
with open(file_name, 'rb') as f:
content = f.read()
content = bencodepy.decode(content)
self.announce = content[b'announce'].decode('utf-8')
self.announce_list = [
t.decode('utf-8')
for t in self.parse_list(content[b'announce-list'])
]
self.info = self._decode_info_dict(content[b'info'])
self.info_hash = hashlib.sha1(bencodepy.encode(
content[b'info'])).digest()
def get_peers_ips(self):
root_url = self.torrent_dict[b'announce'].decode()
if validators.url(root_url):
request_params = self.create_tracker_params()
response = make_get_request(root_url, request_params)
if response.code == 200:
response_body = response.read()
if b'failure reason' not in response_body:
response_dict = bencodepy.decode(response_body)
return calculate_peers(response_dict[b'peers'])
else:
raise Exception("Failure connecting to Tracker")
else:
raise Exception("Can't connect to Tracker")
else:
raise Exception("Not valid tracker url in torrent file.\n"
"Can't connect to Tracker!")
def initialise_variables(torrent_file_name):
modules.multi_torrent_flag = False
modules.temp_name = ""
torr = open(torrent_file_name, "rb")
_dic = decode(torr.read())
modules.hash_string = _dic[b"info"][b"pieces"]
modules.file_name = _dic[b"info"][b"name"].decode()
modules.total_size = 0
if b"files" in _dic[b"info"].keys():
modules.multi_torrent_flag = True
modules.temp_name = "TEMP"
ls = _dic[b"info"][b"files"]
# input(ls)
files_details = []
for e in ls:
modules.total_size += e[b"length"]
files_details.append((e[b"path"][0].decode(), e[b"length"]))
modules.files_details = files_details
# input(modules.files_details)
else:
modules.total_size = _dic[b"info"][b"length"]
modules.total_pieces = len(modules.hash_string) // 20
modules.piece_len = _dic[b"info"][b"piece length"]
tr = encode(_dic[b"info"])
modules.hashes = [hashlib.sha1(tr).hexdigest()]
ty = _dic[b"announce-list"]
modules.trackers = []
for i in range(len(ty)):
a = []
url = ty[i][0].decode()
a.append(url)
modules.trackers.append(a)
modules.recieved_data = [0 for _ in range(modules.total_pieces)]
modules.bitfield = [0 for _ in range(modules.total_pieces)]
while len(modules.bitfield) % 8 != 0:
modules.bitfield.append(0)
modules.connected_peers = []
modules.download_rates = defaultdict(lambda: 0)
modules.upload_rates = defaultdict(lambda: 0)
modules.end_all_threads = False
def datagramReceived(self, data, addressPort):
(address, port) = addressPort
print("Datagram %s received from %s" % (repr(data), repr((address, port))))
data = bencodepy.decode(data)
if data[b'y'] == b'r' and data[b't'] == b'42':
token = data[b'r'][b'token']
newData = {
'y': 'q',
'q': 'announce_peer',
'a': {
'id': hashlib.sha1(b'bla').digest(),
'info_hash': hashlib.sha1(b'someTorrent').digest(),
'port': 8046,
'token': token
},
't': b'43'
}
self.transport.write(bencodepy.encode(newData), ('127.0.0.1', 8043))
print("Sent announce")
def fetch_metadata(nid, infohash, address, timeout=5):
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(timeout)
s.connect(address)
# handshake
handshake(s, nid, infohash)
msg = s.recv(4096)
# verification
if not check_handshake_response(msg, infohash):
return
# advertize to support ut_metadata(BEP-09)
ext_handshake(s)
msg = s.recv(4096)
ut_metadata, metadata_size = decode_ext_handshake_msg(msg)
#print("ut_metadata", ut_metadata, "metadata_size", metadata_size)
metadata = []
piece_tot = int(ceil(metadata_size / (1024 * 16)))
for i in range(piece_tot):
send_request_metadata(s, ut_metadata, i)
piece = recv_piece(s, timeout)
metadata.append(piece)
metadata = bencodepy.decode(b"".join(metadata))
print(metadata[b"name"].decode(), "size", get_length(metadata))
except socket.timeout:
pass
except Exception as e:
#print(e)
pass
finally:
if s:
s.close()
def get_hash(binary): decoded_data = bencodepy.decode(binary) decoded_info = decoded_data[b'info'] encoded_info = bencodepy.encode(decoded_info) torrent_hash = hashlib.sha1(encoded_info).hexdigest() return torrent_hash
def bdecode(data):
return bencodepy.decode(data)
def find_new(self):
# find our shows
builtins = ('daemon', 'transmission', 'defaults', 'tvdb')
shows = [s for s in self.config.sections() if s not in builtins]
defaults = self.config.items('defaults', SHOW_DEFAULTS)
for cfg_name in shows:
show = self.config.items(cfg_name, defaults)
show['name'] = show.get('name', cfg_name)
show['feed_search'] = show.get('feed_search', show['name'])
log.debug('Looking up %s' % show['feed_search'])
# if we're at downloading max_concurrent episodes, then stop
# processing this show
max_concurrent = int(show.get('max_concurrent'))
c = self.db.cursor()
c.execute(
'select count(*) from shows where cfg_name=? and status=?',
(cfg_name, STATUS_INCOMPLETE)
)
count = c.fetchone()[0]
if count >= max_concurrent:
log.debug(
'Reached maximum concurrent torrents (%d) for "%s".' % (
max_concurrent, cfg_name)
)
continue
# Get the show data from TVDB
if show.get('tvdb_id'):
try:
tvdb_show = self.tvdb.get(show['tvdb_id'], self.tvdb_lang)
except tvdb_error.TVDBIdError:
result = []
else:
result = self.tvdb.search(show['name'], self.tvdb_lang)
if not len(result):
log.error('Show not found on tvdb: %s' % show['name'])
continue
if len(result) > 1:
log.warning('Multiple matches found for "{r.search}"'
.format(r=result))
tvdb_show = result[0]
try:
# if show has a season 0 (extras), don't count it in the total
# number of seasons.
tvdb_show[0]
except tvdb_error.TVDBIndexError:
num_seasons = len(tvdb_show)
else:
num_seasons = len(tvdb_show) - 1
if num_seasons <= 0:
log.error('No seasons found for "{r.search}"'.format(r=result))
continue
# Get last downloaded season from the database to see which season
# to start with. If no downloads, use start_season
c = self.db.cursor()
c.execute(
'select max(season) from shows where cfg_name=?', (cfg_name,))
max = c.fetchone()[0]
start_season = max or show['start_season']
# load torrent feeds one season at a time, since the feed only
# returns a max of 30 shows.
entries = []
for season in range(int(start_season), num_seasons + 1):
# load rss feed:
feed_params = {
'mode': 'rss',
'show_name': show['feed_search'],
'quality': show.get('quality'),
'season': season
}
if show.get('feed_search_exact', 'false').lower() != 'false':
feed_params['show_name_exact'] = 'true'
show_feed_url = feed_url + '?' + urlencode(feed_params)
log.debug('checking feed url: %s' % show_feed_url)
feed = feedparser.parse(show_feed_url)
log.debug('found %d entries for %s, season %s' %
(len(feed['entries']), show['name'], season))
# assume that feed has given episodes sorted by seed quality,
# and maintain that order.
for i, e in enumerate(feed['entries']):
feed['entries'][i]['ord'] = i
# sort feed entries by episode
def ordkey(ep):
summary = self._parse_summary(ep['summary'])
return summary['episode'] * 100 + ep['ord']
entries += sorted(feed['entries'], key=ordkey)
eps = [self._parse_summary(e['summary'])['episode']
for e in entries]
log.debug(' Found episodes: {}'
.format(str([int(s) for s in set(eps)])))
added = 0
for entry in entries:
if count >= max_concurrent:
log.info(
'Reached maximum concurrent torrents (%d) for this '
'show "%s".' % (max_concurrent, cfg_name))
break
link = entry['link']
summary = entry['summary']
# parse summary details (assuming ezrss keeps this consistent)
# ex: 'Show Name: Dexter; Episode Title: My Bad; Season: 5;
# Episode: 1'
info = self._parse_summary(summary)
log.debug(
'Found: %(show_name)s: Season: %(season)s; '
'Episode: %(episode)s; Title: %(title)s' % info
)
season = int(info['season'])
episode = int(info['episode'])
# skip if less than start_episode. eg, s04e06 would be 406
e2n = lambda s, e: int(s) * 100 + int(e)
start_ssn = show['start_season']
start_ep = show['start_episode']
if (e2n(season, episode) < e2n(start_ssn, start_ep)):
log.debug(
'Skipping, s%02de%02d is earlier than start_episode'
% (season, episode))
continue
# Check and see if we need this episode
c = self.db.cursor()
c.execute(
'SELECT COUNT() FROM shows WHERE cfg_name=? '
'AND season=? AND episode=?', (cfg_name, season, episode)
)
if c.fetchone()[0] > 0:
# already have this one, or are already downloading it.
log.debug(
'"%(show_name)s-%(season)s-%(episode)s" has already '
'been downloaded or is currently downloading' % info
)
continue
# Get torrent file so that we can parse info out of it
log.debug('Decoding torrent...')
try:
request = Request(link)
request.add_header('Accept-encoding', 'gzip')
response = urlopen(request)
except HTTPError as e:
log.debug('Could not download torrent: %s, %s' % (link, e))
continue
if response.info().get('Content-Encoding') == 'gzip':
buf = io.BytesIO(response.read())
f = gzip.GzipFile(fileobj=buf)
data = f.read()
else:
data = response.read()
try:
torrent = bencodepy.decode(data)
except DecodingError as e:
log.debug(str(e))
log.error('Could not parse torrent: %s' % link)
continue
filename = torrent[b'info'].get(b'name').decode()
if not filename:
files = torrent[b'info'][b'files']
# get largest file
files = sorted(
files, key=lambda f: f['length'], reverse=True)
filename = files[0]['path']
ext = os.path.splitext(filename)[1][1:]
if ext in show['exclude_extensions'].split(','):
log.debug(
'Skipping %s, file extension blacklisted' % filename)
continue
# Add the show
added += 1
log.info(
'Adding %(show_name)s-%(season)s-%(episode)s to '
'transmission queue' % info)
log.debug(link)
b64_data = base64.b64encode(data).decode()
try:
trans_info = self.transmission.add_torrent(b64_data)
except transmissionrpc.error.TransmissionError as e:
if '"duplicate torrent"' in str(e):
log.info('Torrent already exists. Resuming.')
# TODO: Find the duplicate torrent
binfo = bencodepy.encode(torrent[b'info'])
hash = hashlib.sha1(binfo)
trans_info = self.transmission.inf(hash.hexdigest())
self.transmission.start(trans_info.id)
else:
raise
# Record in db
c = self.db.cursor()
show_name = tvdb_show.SeriesName
try:
title = tvdb_show[season][episode].EpisodeName
except (tvdb_error.TVDBIndexError, KeyError):
title = info.get('title', '(no title)')
c.execute(
'INSERT INTO shows (name, season, episode, title, status, '
'url, transid, cfg_name) VALUES (?, ?, ? ,? ,? ,?, ?, ?)',
(show_name, season, episode, title, STATUS_INCOMPLETE,
link, trans_info.id, cfg_name)
)
self.db.commit()
count += 1
if added == 0:
log.info('No new episodes found for %s' % cfg_name)
def get_torrent(self, url):
"""Downloads a torrent file, if it hasn't already been d/led
Note that this function does NOT check in the DB for existence!
Verification of that should be done prior, by calling
start_process(). This function assumes you know you want to
try to download the torrent file, and only checks for existence
on disk of the path specified in the torrent. The reason for this
is to allow for logic between checking if the URL was processed
and actually downloading the torrent file, on a per-caller
basis. If these two steps were combined, every torrent would
be downloaded from every tracker RSS feed every time this is
called, which generates far too many useless, duplicate requests
url - `string` with URL of the torrent file to get
Returns `bool` indicating whether torrent was downloaded (true)
or skipped due to pre-existence (false)
"""
headers = {'User-Agent':'Mozilla/4.0 (compatible;MSIE 5.5;Windows NT)'}
req = urllib.request.Request(url, headers=headers)
resp = urllib.request.urlopen(req)
torrent = resp.read()
torrent_data = decode(torrent)
# Extract the base directory from the torrent file to check for
# duplicates.
#
# Per the torrent spec, the 'name' field in multi-file
# torrents is advised to contain the name of the directory
# containing all of the data files.
#
# https://wiki.theory.org/BitTorrentSpecification
torrent_path = torrent_data[b'info'][b'name'].decode(encoding='UTF-8')
# Check incomplete and complete dirs for path
# Slight TOCTOU race condition here... But I don't care
found = False
if self.check_dupey:
for d in self.dupey_dirs:
if os.path.exists('%s%s%s' % (d, os.sep, torrent_path)):
found = True
# Now check the DB for this path.... This can happen if two
# torrents exist on the same server with the same path...
cursor = self.db.cursor()
cursor.execute("SELECT path FROM dls WHERE path=?", (torrent_path,))
res = cursor.fetchone()
if res:
found = True
if not found:
# Drop the torrent file into the torrent_dir
fn = '%s%s%s.torrent' % (os.path.abspath(self.torrent_dir),
os.sep,
re.sub(r'[^\w\.\-]', '_', torrent_path))
try:
#pylint:disable=locally-disabled, bad-open-mode
torfile = open(fn, 'xb')
torfile.write(torrent)
torfile.close()
except FileExistsError:
found = True
if found:
# We found either the torrent or the path, so mark it as
# processed for the future
self.skip_torrent(url)
return False
else:
# It was new, so update the DB to reflect success
cursor = self.db.cursor()
cursor.execute("""
UPDATE dls SET path=?, downloaded=1, finished=1
WHERE url=?
""", (torrent_path, url))
self.db.commit()
return True
with open(rss, "w") as xml:
xml.write("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n")
xml.write("<rss version=\"2.0\">\n")
xml.write("\t<channel>\n")
xml.write("\t\t<title>Plailect Guide Feed</title>\n")
xml.write("\t\t<lastBuildDate>{0}</lastBuildDate>\n".format(datetime.datetime.utcnow().strftime("%a, %d %b %Y %X +0000")))
xml.write("\t\t<link>https://github.com/Plailect/Guide_3DS/</link>\n")
for filename in os.listdir(dir):
if filename.endswith(".torrent"):
filepath = os.path.join(dir, filename)
with open(filepath, "rb") as a:
raw = a.read()
tor = bencodepy.decode(raw)
trackers = []
infohash = hashlib.sha1(bencodepy.encode(tor[b"info"])).hexdigest().upper()
magp = {"xt": "urn:btih:{0}".format(infohash), "dn": tor[b"info"][b"name"], "xl": tor[b"info"][b"length"]}
magstr = urllib.parse.urlencode(magp)
for anncl in tor[b"announce-list"]:
if isinstance(anncl, list):
for annc in anncl:
trackers.append(annc.decode("utf-8"))
else:
trackers.append(anncl.decode("utf-8"))
length = tor[b"info"][b"length"]
name = tor[b"info"][b"name"].decode("utf-8")
ts = tor[b"creation date"]
pubdate = datetime.datetime.utcfromtimestamp(int(ts))
def datagramReceived(self, data, addressPort):
(address, port) = addressPort
print("Datagram %s received from %s" % (repr(data), repr((address, port))))
print(bencodepy.decode(data))
def message_extended(self, payload):
extension, payload = unpack('>B', payload)
payload = bencodepy.decode(payload)
logging.info('Extension {}: {}'.format(extension, payload))
return extension, payload
