def download_file(url, fname=""): if not fname: fname = os.getcwd() + '/' + url.split('title=')[-1] + '.torrent' if not fname: fname = 'torrent.torrent' log('Downloading >> ' + fname, 1) try: get_request = request.Request(url, method='GET', headers={'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64)', 'Accept-Charset': 'utf-8', 'Accept-Language': '*', 'Connection': '*'}) response = request.urlopen(get_request) if response.getcode() is not 200: error(response.getcode(), 2, 'http') with open(fname, 'wb') as f: shutil.copyfileobj(response, f) # server response can be gzipped, this will decode the response so torrent will can be parsed return open_file(fname, response.getheader('Content-Encoding') == 'gzip') except Exception as e: error('Exception happened\n\n' + str(e), 3)
def torrent_from_xml(rss_xml): url = rss_xml.find("./channel/item/enclosure").attrib['url'] fname = get_fname(rss_xml.find("./channel/item")) get_peers_for_torrent(download_file(url, fname)) with open('movies_announce_list.txt', 'w') as f: f.write(generate_txt(rss_xml)) log('movies_announce_list.txt has been created', 3)
def check_response(response): if re.search(r'HTTP/1.[01].*\r\n\r\n', response, flags=re.DOTALL | re.MULTILINE): if re.search(r'HTTP/1.[01] 200.*\r\n\r\n', response, flags=re.DOTALL | re.MULTILINE): return 200 elif re.search(r'HTTP/1.[01] 404.*\r\n\r\n', response, flags=re.DOTALL | re.MULTILINE): return 404 else: return -1 else: log("response is weird -> %s" % response, 3) return None
def open_file(name, gzipped=False) -> str: """Function will open and read the data of the torrent file, if needed it will also gzip decode it :param str name: file name of the torrent to open and read :param bool gzipped: signalize if file is gzipped or not (default is False) :return: data of the opened torrent """ if gzipped: return open_gzipped_file(name) else: log("File should not be compressed according to response header", 2) return open_standard_file(name)
def open_gzipped_file(name) -> bytes: """Function to read gzipped standard file :param str name: file name to open and read :return: data of the opened and decoded file """ log("Decompressing response", 2) with open(name, 'rb') as f: torrent = gzip.decompress(f.read()) with open(name, 'wb') as f: f.write(torrent) return torrent
def write_peers_to_file(name, peers): """Function to read standard file Args: name (str): file name to write to peers (list): list of peers to write """ with open(name + ".peerlist", 'w') as f: for peer in peers: f.write(peer + "\n") log("Peers have been written to file %s.peerlist" % name, 1)
def receiving(sock, content_length): received = b'' received_length = 0 while received_length < content_length: received += sock.recv(1) received_length += 1 if len(received) != content_length: sock.close() log("Corruption while receiving, length is not right", 0) # log('Received from server: %s' % received, 3) return received
def get_peers_from_tracker(announce, torrent_data, http=True) -> list: if http: response = connect_to_http_tracker(announce, torrent_data) else: transaction_id = get_udp_transaction_id() # randomized transaction ID for UDP response = connect_to_udp_tracker(announce, torrent_data, transaction_id) if response == b'': log('Response of tracker was empty', 2) return [] # if error has occurred empty list of peers will be returned if http: # response from tracker is bencoded binary representation of peers addresses will be parsed to readable form decoded = bencodepy.decode(response) if not decoded: log("Decoded response is empty", 1) return [] elif b'peers' not in decoded: log("Decoded response has no peers data in it", 1) return [] bin_peers = decoded[b'peers'] else: bin_peers = parse_udp_announce_response(response, transaction_id) log("There should be %u peers" % (len(bin_peers) / 6), 1) peers = [] # bin peers data is field of bytes, where each 6 bytes represent one peer # for each peer will be appended to peers list for i in range(0, len(bin_peers), 6): peers.append(parse_bin_peer(bin_peers[i:i + 6])) return peers
def split_trackers(tracker, trackers): # Due to extend of standard list of backups can be passed as single tracker, this will check them all one by one # to see if there is any of http ones if isinstance(tracker, list): # Iterate through the all backups of the tracker or through the all trackers in the list, if more than one # tracker is passed (announce-list was present in torrent file metadata) for backup in tracker: split_trackers(backup, trackers) else: if tracker[:4] == b'http': trackers['http'].append(tracker.decode("utf-8")) elif tracker[:3] == b'udp': trackers['udp'].append(tracker.decode("utf-8")) else: log("tracker is not http or upd ( " + tracker + " )", 3)
def receive_header(sock): request = b'' while True: try: request += sock.recv(1) except Exception as e: log("Exception during receiving: %s" % e, 0) return None if len(request) > 4 and request[len(request) - 4:] == b'\r\n\r\n': break if len(request) > 4 and request[len(request) - 4:] == b'\r\n<!': return None return request
def generate_txt(rss_xml): log('Generating movies_announce_list.txt file', 3) text = txt_append(rss_xml, './channel/title') text += txt_append(rss_xml, './channel/link') text += txt_append(rss_xml, './channel/description') + '\n' for item_xml in rss_xml.findall('./channel/item'): text += '\n' text += txt_append(item_xml, './title') text += txt_append(item_xml, './category') text += txt_append(item_xml, './author') text += txt_append(item_xml, './link') text += txt_append(item_xml, './pubDate') # prints elements that are at namespace for ns_item in item_xml.findall('./'): if re.search(r'\{.*\}infoHash', ns_item.tag): text += 'torrent:infoHash: ' + ns_item.text + '\n' elif re.search(r'\{.*\}fileName', ns_item.tag): text += 'torrent:fileName: ' + ns_item.text + '\n' return text
def parse_torrent(torrent) -> dict: # Decoded torrent_metadata object contains b'value' as binary data, that is why we need use it explicitly sometimes, # otherwise it wont match the values # Also this contains additional items that will be used for connection to announce torrent_data = {'metadata': bencodepy.decode(torrent), 'trackers': {'http': [], 'udp': []}, 'info_hash': '', 'hex_info_hash': '', 'peer_id': "PY3-ISA-project-2015", 'uploaded': 0, 'downloaded': 0, 'left': 1000, 'compact': 1, 'port': 6886, 'numwant': -1, } # if 'announce-list' keyword is missing in list, there is only one tracker for this torrent available # and therefor 'announce' will be used to connect to the torrent tracker if b'announce-list' in torrent_data['metadata']: log("announce-list keyword is present, there are more possible trackers", 2) split_trackers(torrent_data['metadata'][b'announce-list'], torrent_data['trackers']) else: log("announce-list keyword is NOT present in dictionary, only one choice for tracker", 2) # here will be value under keyword announce used as only tracker split_trackers(torrent_data['metadata'][b'announce'], torrent_data['trackers']) torrent_data['info_hash'] = get_info_hash(torrent_data['metadata'][b'info']) torrent_data['hex_info_hash'] = get_info_hash(torrent_data['metadata'][b'info'], True) return torrent_data
def send_get_message(sock, path, headers): """Function will send http GET message to tracer ip opened in passed socket Args: sock (object): socket that will be used for connection """ message = "GET " + path + " HTTP/1.0\r\n" + headers + "\r\n" message = bytes(message, 'utf-8') log("Message is %s" % message, 3) try: log('Sending GET message to tracker', 2) sock.send(message) except Exception as e: log('Sending data to server has failed. Exception %s' % e, 0) sock.close()
def create_socket(hostname, port, http=True) -> socket: if http: log('Creating socket', 2) # TODO: there probably could be IPv6 socket condition s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) else: s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.settimeout(isaCommon.TIMEOUT) # TODO: set and test timeout try: log("Connecting to tracker socket", 2) s.connect((hostname, port)) except socket.gaierror: log('Can not translate %s to IP address' % hostname, 0) s.close() return None except Exception as e: log('Connection to server has failed %s' % str(e), 0) s.close() return None return s # return opened and connected socket, if error happens closed socked is returned
def parse_udp_connection_response(response, sent_transaction_id): # Handle the errors of udp announce response if len(response) < 16: # constant 16 represents min byte length that is required only for metadata log("Too short response length %s, returning empty response" % len(response), 0) return b'' action, res_transaction_id = struct.unpack_from("!II", response) if res_transaction_id != sent_transaction_id: log("Transaction ID is not same as Connection ID in response, Expected %s and got %s, returning empty response" % (sent_transaction_id, res_transaction_id), 0) return b'' if action == 0x0: # Connection established, getting Connection ID return struct.unpack_from("!Q", response, 8)[0] # unpack 8 bytes, should be the connection_id elif action == 0x3: # Error message in response log("Error message in response: { %s }, returning empty response" % struct.unpack_from("!s", response, 8)[0], 0) return b''
def chunked_receive(sock): all_received = b'' while True: chunk_size, bin_chunk_size = receive_size(sock) log('chunk_size: %s' % chunk_size, 3) if chunk_size is None: sock.close() log("Received chunk size was None, returning empty data", 2) return b'' if chunk_size == 0: break received = receiving(sock, chunk_size) all_received += received if len(received) != chunk_size: sock.close() log("Received size is not same as chunk size should be, returning empty data", 2) return b'' return all_received
def connect_to_http_tracker(announce, torrent_data): hostname = urlparse(announce).hostname port = urlparse(announce).port s = create_socket(hostname, port, http=True) if not isinstance(s, socket.socket): log("Returned socket is empty", 2) return b'' headers = "User-Agent: Mozilla/5.0 (X11; Linux x86_64)\r\n" + \ "Host: " + hostname + ":" + str(port) + "\r\n" + \ "Accept-Charset: utf-8\r\n" + \ "Connection: *\r\n" + \ "Accept: application/x-bittorrent\r\n" # Send the GET message # GET request is created from announce path part, torrent data part and headers send_get_message(s, create_tracker_request(urlparse(announce).path, torrent_data), headers) # Receive a response response, content_length, is_chunked, encoding = receive_response(s) log('Response received', 2) if response is None: log("No response form server", 0) return b'' if is_chunked: log("Starting chunked communication", 3) received = chunked_receive(s).decode(encoding) elif content_length is not None: log("Length of content being received is %s Bytes" % content_length, 3) received = receiving(s, content_length) else: log("communication is NOT chunked and length of content is not specified", 3) s = s.makefile(newline='', mode='rb') received = s.read() log("Length of content is %i Bytes" % len(received), 3) log('Closing socket', 2) s.close() return received
def get_peers_for_torrent(torrent): torrent_data = parse_torrent(torrent) # announce = torrent_data['trackers']['http'][2] peers = [] if not torrent_data['trackers']['http']: log('List of HTTP trackers is empty', 0) # TODO: return or end script # Get peers for trackers if isaCommon.params.tracker_annonce_url: announce = isaCommon.params.tracker_annonce_url log("Used only announce %s" % announce, 1) pprint(announce) if announce[:4] == 'http': log("Getting peers for announce %s" % announce, 1) peers.extend(get_peers_from_tracker(announce, torrent_data)) elif announce[:3] == 'udp': log("Getting peers for announce %s" % announce, 1) peers.extend(get_peers_from_tracker(announce, torrent_data, http=False)) else: for announce in torrent_data['trackers']['http']: log("Getting peers for announce %s" % announce, 1) peers.extend(get_peers_from_tracker(announce, torrent_data)) for announce in torrent_data['trackers']['udp']: log("Getting peers for announce %s" % announce, 1) peers.extend(get_peers_from_tracker(announce, torrent_data, http=False)) write_peers_to_file(torrent_data['hex_info_hash'], peers)
def parse_udp_announce_response(response, transaction_id) -> bytes: # Handle the errors of udp announce response if len(response) < 20: # constant 20 represents min byte length that is required only for metadata without peers log("Too short response length %s, returning empty response" % len(response), 0) return b'' log("Response length is %s" % len(response), 1) action = struct.unpack_from("!I", response)[0] if action != 0x1: log("Received wrong action number %d, returning empty response" % action, 0) return b'' received_transaction_id = struct.unpack_from("!I", response, 4)[0] # next 4 bytes is transaction id if received_transaction_id != transaction_id: log("Transaction ID is wrong. Expected %s, received %s, returning empty response" % (transaction_id, received_transaction_id), 0) return b'' log("Reading Response", 3) meta = dict() offset = 8 # Action and Transaction ID has been at first 8 bytes, that is why offset 8 is needed meta['interval'], meta['leeches'], meta['seeds'] = struct.unpack_from("!iii", response, offset) offset += 12 # three integers by 4 bytes log("Interval = %d, Leeches = %d, Seeds = %d" % (meta['interval'], meta['leeches'], meta['seeds']), 3) log("Actual peer data in response are %s" % response[offset:], 3) return response[offset:] # all the rest data in response are peers coded as 6 byte entities
def receive_response(sock): log('Waiting for response', 3) response = receive_header(sock) log('Response from server: %s' % response) if response is None: return None, None, None, None response = response.decode('utf-8') code = check_response(response) if code is None: log("Tracker response HTTP error code [ %d ]" % code, 2) elif code == 404: log("Response from server is Error 404", 0) elif code == 200: log("Response from server is OK", 2) else: log("Unknown response from server -> code %s" % code, 2) content_length = get_conntent_length(response) is_chunked = is_set_chunk(response) if is_chunked: log("Communication will be chunked", 2) encoding = get_encoding(response) log("Encoding of content is: %s" % encoding, 2) return response, content_length, is_chunked, encoding
def connect_to_udp_tracker(announce, torrent_data, transaction_id): hostname = urlparse(announce).hostname port = urlparse(announce).port s = create_socket(hostname, port, http=False) if s is None: log("Returned socket is empty", 2) return b'' connection_id = 0x41727101980 # default connection id s.sendto(create_udp_connection_request(connection_id, transaction_id), (socket.gethostbyname(hostname), port)) try: con_response = s.recvfrom(2048)[0] except socket.timeout: log("Socket connection has timed out", 0) return b'' except socket.error as e: log("Cannot connect %s" % e, 0) s.close() return b'' except Exception as e: log('Connection to server has failed %s' % str(e), 0) s.close() return b'' connection_id = parse_udp_connection_response(con_response, transaction_id) log("New connection ID is %u" % connection_id, 3) socket_port = s.getsockname()[1] # port number on which socket is communicating s.sendto(create_udp_announce_request(connection_id, transaction_id, torrent_data, socket_port), (socket.gethostbyname(hostname), port)) try: response = s.recvfrom(2048)[0] except socket.timeout: log("Socket connection has timed out", 0) return b'' return response