def download_file(url, fname=""):
if not fname:
fname = os.getcwd() + '/' + url.split('title=')[-1] + '.torrent'
if not fname:
fname = 'torrent.torrent'
log('Downloading >> ' + fname, 1)
try:
get_request = request.Request(url, method='GET', headers={'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64)',
'Accept-Charset': 'utf-8', 'Accept-Language': '*',
'Connection': '*'})
response = request.urlopen(get_request)
if response.getcode() is not 200:
error(response.getcode(), 2, 'http')
with open(fname, 'wb') as f:
shutil.copyfileobj(response, f)
# server response can be gzipped, this will decode the response so torrent will can be parsed
return open_file(fname, response.getheader('Content-Encoding') == 'gzip')
except Exception as e:
error('Exception happened\n\n' + str(e), 3)
def torrent_from_xml(rss_xml):
url = rss_xml.find("./channel/item/enclosure").attrib['url']
fname = get_fname(rss_xml.find("./channel/item"))
get_peers_for_torrent(download_file(url, fname))
with open('movies_announce_list.txt', 'w') as f:
f.write(generate_txt(rss_xml))
log('movies_announce_list.txt has been created', 3)
def check_response(response):
if re.search(r'HTTP/1.[01].*\r\n\r\n', response, flags=re.DOTALL | re.MULTILINE):
if re.search(r'HTTP/1.[01] 200.*\r\n\r\n', response, flags=re.DOTALL | re.MULTILINE):
return 200
elif re.search(r'HTTP/1.[01] 404.*\r\n\r\n', response, flags=re.DOTALL | re.MULTILINE):
return 404
else:
return -1
else:
log("response is weird -> %s" % response, 3)
return None
def open_file(name, gzipped=False) -> str:
"""Function will open and read the data of the torrent file, if needed it will also gzip decode it
:param str name: file name of the torrent to open and read
:param bool gzipped: signalize if file is gzipped or not (default is False)
:return: data of the opened torrent
"""
if gzipped:
return open_gzipped_file(name)
else:
log("File should not be compressed according to response header", 2)
return open_standard_file(name)
def open_gzipped_file(name) -> bytes:
"""Function to read gzipped standard file
:param str name: file name to open and read
:return: data of the opened and decoded file
"""
log("Decompressing response", 2)
with open(name, 'rb') as f:
torrent = gzip.decompress(f.read())
with open(name, 'wb') as f:
f.write(torrent)
return torrent
def write_peers_to_file(name, peers):
"""Function to read standard file
Args:
name (str): file name to write to
peers (list): list of peers to write
"""
with open(name + ".peerlist", 'w') as f:
for peer in peers:
f.write(peer + "\n")
log("Peers have been written to file %s.peerlist" % name, 1)
def receiving(sock, content_length):
received = b''
received_length = 0
while received_length < content_length:
received += sock.recv(1)
received_length += 1
if len(received) != content_length:
sock.close()
log("Corruption while receiving, length is not right", 0)
# log('Received from server: %s' % received, 3)
return received
def get_peers_from_tracker(announce, torrent_data, http=True) -> list:
if http:
response = connect_to_http_tracker(announce, torrent_data)
else:
transaction_id = get_udp_transaction_id() # randomized transaction ID for UDP
response = connect_to_udp_tracker(announce, torrent_data, transaction_id)
if response == b'':
log('Response of tracker was empty', 2)
return [] # if error has occurred empty list of peers will be returned
if http:
# response from tracker is bencoded binary representation of peers addresses will be parsed to readable form
decoded = bencodepy.decode(response)
if not decoded:
log("Decoded response is empty", 1)
return []
elif b'peers' not in decoded:
log("Decoded response has no peers data in it", 1)
return []
bin_peers = decoded[b'peers']
else:
bin_peers = parse_udp_announce_response(response, transaction_id)
log("There should be %u peers" % (len(bin_peers) / 6), 1)
peers = []
# bin peers data is field of bytes, where each 6 bytes represent one peer
# for each peer will be appended to peers list
for i in range(0, len(bin_peers), 6):
peers.append(parse_bin_peer(bin_peers[i:i + 6]))
return peers
def split_trackers(tracker, trackers):
# Due to extend of standard list of backups can be passed as single tracker, this will check them all one by one
# to see if there is any of http ones
if isinstance(tracker, list):
# Iterate through the all backups of the tracker or through the all trackers in the list, if more than one
# tracker is passed (announce-list was present in torrent file metadata)
for backup in tracker:
split_trackers(backup, trackers)
else:
if tracker[:4] == b'http':
trackers['http'].append(tracker.decode("utf-8"))
elif tracker[:3] == b'udp':
trackers['udp'].append(tracker.decode("utf-8"))
else:
log("tracker is not http or upd ( " + tracker + " )", 3)
def receive_header(sock):
request = b''
while True:
try:
request += sock.recv(1)
except Exception as e:
log("Exception during receiving: %s" % e, 0)
return None
if len(request) > 4 and request[len(request) - 4:] == b'\r\n\r\n':
break
if len(request) > 4 and request[len(request) - 4:] == b'\r\n<!':
return None
return request
def generate_txt(rss_xml):
log('Generating movies_announce_list.txt file', 3)
text = txt_append(rss_xml, './channel/title')
text += txt_append(rss_xml, './channel/link')
text += txt_append(rss_xml, './channel/description') + '\n'
for item_xml in rss_xml.findall('./channel/item'):
text += '\n'
text += txt_append(item_xml, './title')
text += txt_append(item_xml, './category')
text += txt_append(item_xml, './author')
text += txt_append(item_xml, './link')
text += txt_append(item_xml, './pubDate')
# prints elements that are at namespace
for ns_item in item_xml.findall('./'):
if re.search(r'\{.*\}infoHash', ns_item.tag):
text += 'torrent:infoHash: ' + ns_item.text + '\n'
elif re.search(r'\{.*\}fileName', ns_item.tag):
text += 'torrent:fileName: ' + ns_item.text + '\n'
return text
def parse_torrent(torrent) -> dict:
# Decoded torrent_metadata object contains b'value' as binary data, that is why we need use it explicitly sometimes,
# otherwise it wont match the values
# Also this contains additional items that will be used for connection to announce
torrent_data = {'metadata': bencodepy.decode(torrent), 'trackers': {'http': [], 'udp': []}, 'info_hash': '',
'hex_info_hash': '', 'peer_id': "PY3-ISA-project-2015",
'uploaded': 0, 'downloaded': 0, 'left': 1000, 'compact': 1, 'port': 6886, 'numwant': -1,
}
# if 'announce-list' keyword is missing in list, there is only one tracker for this torrent available
# and therefor 'announce' will be used to connect to the torrent tracker
if b'announce-list' in torrent_data['metadata']:
log("announce-list keyword is present, there are more possible trackers", 2)
split_trackers(torrent_data['metadata'][b'announce-list'], torrent_data['trackers'])
else:
log("announce-list keyword is NOT present in dictionary, only one choice for tracker", 2)
# here will be value under keyword announce used as only tracker
split_trackers(torrent_data['metadata'][b'announce'], torrent_data['trackers'])
torrent_data['info_hash'] = get_info_hash(torrent_data['metadata'][b'info'])
torrent_data['hex_info_hash'] = get_info_hash(torrent_data['metadata'][b'info'], True)
return torrent_data
def send_get_message(sock, path, headers):
"""Function will send http GET message to tracer ip opened in passed socket
Args:
sock (object): socket that will be used for connection
"""
message = "GET " + path + " HTTP/1.0\r\n" + headers + "\r\n"
message = bytes(message, 'utf-8')
log("Message is %s" % message, 3)
try:
log('Sending GET message to tracker', 2)
sock.send(message)
except Exception as e:
log('Sending data to server has failed. Exception %s' % e, 0)
sock.close()
def create_socket(hostname, port, http=True) -> socket:
if http:
log('Creating socket', 2)
# TODO: there probably could be IPv6 socket condition
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
else:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.settimeout(isaCommon.TIMEOUT) # TODO: set and test timeout
try:
log("Connecting to tracker socket", 2)
s.connect((hostname, port))
except socket.gaierror:
log('Can not translate %s to IP address' % hostname, 0)
s.close()
return None
except Exception as e:
log('Connection to server has failed %s' % str(e), 0)
s.close()
return None
return s # return opened and connected socket, if error happens closed socked is returned
def parse_udp_connection_response(response, sent_transaction_id):
# Handle the errors of udp announce response
if len(response) < 16: # constant 16 represents min byte length that is required only for metadata
log("Too short response length %s, returning empty response" % len(response), 0)
return b''
action, res_transaction_id = struct.unpack_from("!II", response)
if res_transaction_id != sent_transaction_id:
log("Transaction ID is not same as Connection ID in response, Expected %s and got %s, returning empty response"
% (sent_transaction_id, res_transaction_id), 0)
return b''
if action == 0x0: # Connection established, getting Connection ID
return struct.unpack_from("!Q", response, 8)[0] # unpack 8 bytes, should be the connection_id
elif action == 0x3: # Error message in response
log("Error message in response: { %s }, returning empty response" % struct.unpack_from("!s", response, 8)[0], 0)
return b''
def chunked_receive(sock):
all_received = b''
while True:
chunk_size, bin_chunk_size = receive_size(sock)
log('chunk_size: %s' % chunk_size, 3)
if chunk_size is None:
sock.close()
log("Received chunk size was None, returning empty data", 2)
return b''
if chunk_size == 0:
break
received = receiving(sock, chunk_size)
all_received += received
if len(received) != chunk_size:
sock.close()
log("Received size is not same as chunk size should be, returning empty data", 2)
return b''
return all_received
def connect_to_http_tracker(announce, torrent_data):
hostname = urlparse(announce).hostname
port = urlparse(announce).port
s = create_socket(hostname, port, http=True)
if not isinstance(s, socket.socket):
log("Returned socket is empty", 2)
return b''
headers = "User-Agent: Mozilla/5.0 (X11; Linux x86_64)\r\n" + \
"Host: " + hostname + ":" + str(port) + "\r\n" + \
"Accept-Charset: utf-8\r\n" + \
"Connection: *\r\n" + \
"Accept: application/x-bittorrent\r\n"
# Send the GET message
# GET request is created from announce path part, torrent data part and headers
send_get_message(s, create_tracker_request(urlparse(announce).path, torrent_data), headers)
# Receive a response
response, content_length, is_chunked, encoding = receive_response(s)
log('Response received', 2)
if response is None:
log("No response form server", 0)
return b''
if is_chunked:
log("Starting chunked communication", 3)
received = chunked_receive(s).decode(encoding)
elif content_length is not None:
log("Length of content being received is %s Bytes" % content_length, 3)
received = receiving(s, content_length)
else:
log("communication is NOT chunked and length of content is not specified", 3)
s = s.makefile(newline='', mode='rb')
received = s.read()
log("Length of content is %i Bytes" % len(received), 3)
log('Closing socket', 2)
s.close()
return received
def get_peers_for_torrent(torrent):
torrent_data = parse_torrent(torrent)
# announce = torrent_data['trackers']['http'][2]
peers = []
if not torrent_data['trackers']['http']:
log('List of HTTP trackers is empty', 0) # TODO: return or end script
# Get peers for trackers
if isaCommon.params.tracker_annonce_url:
announce = isaCommon.params.tracker_annonce_url
log("Used only announce %s" % announce, 1)
pprint(announce)
if announce[:4] == 'http':
log("Getting peers for announce %s" % announce, 1)
peers.extend(get_peers_from_tracker(announce, torrent_data))
elif announce[:3] == 'udp':
log("Getting peers for announce %s" % announce, 1)
peers.extend(get_peers_from_tracker(announce, torrent_data, http=False))
else:
for announce in torrent_data['trackers']['http']:
log("Getting peers for announce %s" % announce, 1)
peers.extend(get_peers_from_tracker(announce, torrent_data))
for announce in torrent_data['trackers']['udp']:
log("Getting peers for announce %s" % announce, 1)
peers.extend(get_peers_from_tracker(announce, torrent_data, http=False))
write_peers_to_file(torrent_data['hex_info_hash'], peers)
def parse_udp_announce_response(response, transaction_id) -> bytes:
# Handle the errors of udp announce response
if len(response) < 20: # constant 20 represents min byte length that is required only for metadata without peers
log("Too short response length %s, returning empty response" % len(response), 0)
return b''
log("Response length is %s" % len(response), 1)
action = struct.unpack_from("!I", response)[0]
if action != 0x1:
log("Received wrong action number %d, returning empty response" % action, 0)
return b''
received_transaction_id = struct.unpack_from("!I", response, 4)[0] # next 4 bytes is transaction id
if received_transaction_id != transaction_id:
log("Transaction ID is wrong. Expected %s, received %s, returning empty response"
% (transaction_id, received_transaction_id), 0)
return b''
log("Reading Response", 3)
meta = dict()
offset = 8 # Action and Transaction ID has been at first 8 bytes, that is why offset 8 is needed
meta['interval'], meta['leeches'], meta['seeds'] = struct.unpack_from("!iii", response, offset)
offset += 12 # three integers by 4 bytes
log("Interval = %d, Leeches = %d, Seeds = %d" % (meta['interval'], meta['leeches'], meta['seeds']), 3)
log("Actual peer data in response are %s" % response[offset:], 3)
return response[offset:] # all the rest data in response are peers coded as 6 byte entities
def receive_response(sock):
log('Waiting for response', 3)
response = receive_header(sock)
log('Response from server: %s' % response)
if response is None:
return None, None, None, None
response = response.decode('utf-8')
code = check_response(response)
if code is None:
log("Tracker response HTTP error code [ %d ]" % code, 2)
elif code == 404:
log("Response from server is Error 404", 0)
elif code == 200:
log("Response from server is OK", 2)
else:
log("Unknown response from server -> code %s" % code, 2)
content_length = get_conntent_length(response)
is_chunked = is_set_chunk(response)
if is_chunked:
log("Communication will be chunked", 2)
encoding = get_encoding(response)
log("Encoding of content is: %s" % encoding, 2)
return response, content_length, is_chunked, encoding
def connect_to_udp_tracker(announce, torrent_data, transaction_id):
hostname = urlparse(announce).hostname
port = urlparse(announce).port
s = create_socket(hostname, port, http=False)
if s is None:
log("Returned socket is empty", 2)
return b''
connection_id = 0x41727101980 # default connection id
s.sendto(create_udp_connection_request(connection_id, transaction_id), (socket.gethostbyname(hostname), port))
try:
con_response = s.recvfrom(2048)[0]
except socket.timeout:
log("Socket connection has timed out", 0)
return b''
except socket.error as e:
log("Cannot connect %s" % e, 0)
s.close()
return b''
except Exception as e:
log('Connection to server has failed %s' % str(e), 0)
s.close()
return b''
connection_id = parse_udp_connection_response(con_response, transaction_id)
log("New connection ID is %u" % connection_id, 3)
socket_port = s.getsockname()[1] # port number on which socket is communicating
s.sendto(create_udp_announce_request(connection_id, transaction_id, torrent_data, socket_port),
(socket.gethostbyname(hostname), port))
try:
response = s.recvfrom(2048)[0]
except socket.timeout:
log("Socket connection has timed out", 0)
return b''
return response
