def create_torrent(file, port, try_vod = True): ''' Creates a torrent for the given file. Returns a string representing the path to the torrent file. If try_vod is True try to add the playtime to the torrent making it "streamable". ''' # generate torrent tdef = TorrentDef() url = "http://localhost:%s/announce/" % port tdef.set_tracker(url) #tdef.set_create_merkle_torrent(True) #tdef.set_piece_length(self._pieceSize) if try_vod: torrent_name = FileUtils.get_relative_filename(file) + constants.TORRENT_VOD_EXT # set playtime playtime = videostats_via_ffmpeg(file)['playtime'] print "playtime", playtime else: torrent_name = FileUtils.get_relative_filename(file) + constants.TORRENT_DOWNLOAD_EXT playtime = None print "Create a non-streamable torrent" tdef.add_content(file, playtime=playtime) tdef.finalize() torrent_dir = os.getcwd() torrent = os.path.join(torrent_dir, torrent_name) tdef.save(torrent) print "created torrent file: %s" % torrent print "Tracker uses the announce URL: %s" % tdef.get_tracker() return tdef
def isValidRemoteVal(d, selversion): if not isinstance(d, dict): if DEBUG: print >> sys.stderr, "rqmh: reply: a: value not dict" return False if selversion >= OLPROTO_VER_TWELFTH: if not ('content_name' in d and 'length' in d and 'leecher' in d and 'seeder' in d and 'category' in d and 'torrent_size' in d and 'channel_permid' in d and 'channel_name' in d): if DEBUG: print >> sys.stderr, "rqmh: reply: torrentrec12: key missing, got", d.keys( ) return False if 'metatype' in d and 'metadata' in d: try: metatype = d['metatype'] metadata = d['metadata'] if metatype == URL_MIME_TYPE: tdef = TorrentDef.load_from_url(metadata) else: metainfo = bdecode(metadata) tdef = TorrentDef.load_from_dict(metainfo) except: if DEBUG: print >> sys.stderr, "rqmh: reply: torrentrec12: metadata invalid" print_exc() return False elif selversion >= OLPROTO_VER_ELEVENTH: if not ('content_name' in d and 'length' in d and 'leecher' in d and 'seeder' in d and 'category' in d and 'torrent_size' in d and 'channel_permid' in d and 'channel_name' in d): if DEBUG: print >> sys.stderr, "rqmh: reply: torrentrec11: key missing, got", d.keys( ) return False elif selversion >= OLPROTO_VER_NINETH: if not ('content_name' in d and 'length' in d and 'leecher' in d and 'seeder' in d and 'category' in d and 'torrent_size' in d): if DEBUG: print >> sys.stderr, "rqmh: reply: torrentrec9: key missing, got", d.keys( ) return False else: if not ('content_name' in d and 'length' in d and 'leecher' in d and 'seeder' in d and 'category' in d): if DEBUG: print >> sys.stderr, "rqmh: reply: torrentrec6: key missing, got", d.keys( ) return False # if not (isinstance(d['content_name'],str) and isinstance(d['length'],int) and isinstance(d['leecher'],int) and isinstance(d['seeder'],int)): # return False # if len(d) > 4: # no other keys # return False return True
def setUpPostSession(self): """ override TestAsServer """ TestAsServer.setUpPostSession(self) self.hispermid = str(self.his_keypair.pub().get_der()) self.my_permid = str(self.my_keypair.pub().get_der()) self.content_name = 'Hallo S22E44' self.tdef = TorrentDef() self.tdef.set_tracker('http://localhost:0/announce') self.tdef.set_piece_length(2**15) self.tdef.create_live(self.content_name, 2**16) self.tdef.finalize()
def addTorrentToDB(self, filename, torrent_hash, metadata, source='BC', extra_info={}, hack=False): """ Arno: no need to delegate to olbridge, this is already run by OverlayThread """ # 03/02/10 Boudewijn: addExternalTorrent now requires a # torrentdef, consequently we provide the filename through the # extra_info dictionary torrentdef = TorrentDef.load(filename) if not 'filename' in extra_info: extra_info['filename'] = filename torrent = self.torrent_db.addExternalTorrent(torrentdef, source, extra_info) if torrent is None: return # Arno, 2008-10-20: XXX torrents are filtered out in the final display stage self.launchmany.set_activity(NTFY_ACT_GOT_METADATA,unicode('"'+torrent['name']+'"'),torrent['category']) if self.initialized: self.num_torrents += 1 # for free disk limitation if not extra_info: self.refreshTrackerStatus(torrent) if len(self.recently_collected_torrents) < 50: # Queue of 50 self.recently_collected_torrents.append(torrent_hash) else: self.recently_collected_torrents.pop(0) self.recently_collected_torrents.append(torrent_hash)
def testRetrieveLocalPeersForActiveTorrents(self): tdef = TorrentDef.load("torrents/ubuntu.torrent") infohash_hex = common_utils.get_id(tdef) torrent_dict = self.wsclient.retrieve_local_peers_for_active_torrents(list_of_torrent_ids=[infohash_hex], maximum_number_of_peers=10, include_seeds=False) assert len(torrent_dict) >= 0, torrent_dict
def start_torrent(self, torrent): tdef = TorrentDef.load(torrent) if not os.access(self._directory, os.F_OK): os.makedirs(self._directory) dscfg = DownloadStartupConfig() dscfg.set_dest_dir(self._directory) dscfg.set_video_events([ simpledefs.VODEVENT_START, simpledefs.VODEVENT_PAUSE, simpledefs.VODEVENT_RESUME ]) dscfg.set_max_speed(simpledefs.DOWNLOAD, self._max_dl_rate) dscfg.set_max_speed(simpledefs.UPLOAD, self._max_ul_rate) dscfg.set_peer_type("S") #dscfg.set_video_event_callback(self.video_callback) # supporter should not play the files ! d = self._session.start_download(tdef, dscfg) d.set_state_callback(self.state_callback) time.sleep(1) # give the download some time to fully initialize d.sd.dow.choker.set_supporter_server(True) self._tracker_url = tdef.get_tracker()[:tdef.get_tracker(). find("announce")] self._id = d.sd.peerid self._choke_objects.append(d.sd.dow.choker)
def start_torrent(self, torrent): tdef = TorrentDef.load(torrent) if not os.access(self._directory, os.F_OK): os.makedirs(self._directory) dscfg = DownloadStartupConfig() dscfg.set_dest_dir(self._directory) dscfg.set_video_events([simpledefs.VODEVENT_START, simpledefs.VODEVENT_PAUSE, simpledefs.VODEVENT_RESUME]) dscfg.set_max_speed(simpledefs.DOWNLOAD, self._max_dl_rate) dscfg.set_max_speed(simpledefs.UPLOAD, self._max_ul_rate) dscfg.set_peer_type("S") #dscfg.set_video_event_callback(self.video_callback) # supporter should not play the files ! d = self._session.start_download(tdef, dscfg) d.set_state_callback(self.state_callback) time.sleep(1) # give the download some time to fully initialize d.sd.dow.choker.set_supporter_server(True) self._tracker_url = tdef.get_tracker()[:tdef.get_tracker().find("announce")] self._id = d.sd.peerid self._choke_objects.append(d.sd.dow.choker)
def isValidRemoteVal(d,selversion): if not isinstance(d,dict): if DEBUG: print >>sys.stderr,"rqmh: reply: a: value not dict" return False if selversion >= OLPROTO_VER_TWELFTH: if not ('content_name' in d and 'length' in d and 'leecher' in d and 'seeder' in d and 'category' in d and 'torrent_size' in d and 'channel_permid' in d and 'channel_name' in d): if DEBUG: print >>sys.stderr,"rqmh: reply: torrentrec12: key missing, got",d.keys() return False if 'metatype' in d and 'metadata' in d: try: metatype = d['metatype'] metadata = d['metadata'] if metatype == URL_MIME_TYPE: tdef = TorrentDef.load_from_url(metadata) else: metainfo = bdecode(metadata) tdef = TorrentDef.load_from_dict(metainfo) except: if DEBUG: print >>sys.stderr,"rqmh: reply: torrentrec12: metadata invalid" print_exc() return False elif selversion >= OLPROTO_VER_ELEVENTH: if not ('content_name' in d and 'length' in d and 'leecher' in d and 'seeder' in d and 'category' in d and 'torrent_size' in d and 'channel_permid' in d and 'channel_name' in d): if DEBUG: print >>sys.stderr,"rqmh: reply: torrentrec11: key missing, got",d.keys() return False elif selversion >= OLPROTO_VER_NINETH: if not ('content_name' in d and 'length' in d and 'leecher' in d and 'seeder' in d and 'category' in d and 'torrent_size' in d): if DEBUG: print >>sys.stderr,"rqmh: reply: torrentrec9: key missing, got",d.keys() return False else: if not ('content_name' in d and 'length' in d and 'leecher' in d and 'seeder' in d and 'category' in d): if DEBUG: print >>sys.stderr,"rqmh: reply: torrentrec6: key missing, got",d.keys() return False # if not (isinstance(d['content_name'],str) and isinstance(d['length'],int) and isinstance(d['leecher'],int) and isinstance(d['seeder'],int)): # return False # if len(d) > 4: # no other keys # return False return True
def read_and_send_metadata(self, permid, infohash, torrent_path, selversion): torrent_data = self.read_torrent(torrent_path) if torrent_data: # Arno: Don't send private torrents try: metainfo = bdecode(torrent_data) if 'info' in metainfo and 'private' in metainfo[ 'info'] and metainfo['info']['private']: if DEBUG: print >> sys.stderr, "metadata: Not sending torrent", ` torrent_path `, "because it is private" return 0 except: print_exc() return 0 if DEBUG: print >> sys.stderr, "metadata: sending torrent", ` torrent_path `, len( torrent_data) torrent = {} torrent['torrent_hash'] = infohash # P2PURLs: If URL compat then send URL tdef = TorrentDef.load_from_dict(metainfo) if selversion >= OLPROTO_VER_ELEVENTH and tdef.get_url_compat(): torrent['metatype'] = URL_MIME_TYPE torrent['metadata'] = tdef.get_url() else: torrent['metatype'] = TSTREAM_MIME_TYPE torrent['metadata'] = torrent_data if selversion >= OLPROTO_VER_FOURTH: data = self.torrent_db.getTorrent(infohash) if data is None: # DB inconsistency return 0 nleechers = data.get('leecher', -1) nseeders = data.get('seeder', -1) last_check_ago = int(time()) - data.get('last_check_time', 0) # relative time if last_check_ago < 0: last_check_ago = 0 status = data.get('status', 'unknown') torrent.update({ 'leecher': nleechers, 'seeder': nseeders, 'last_check_time': last_check_ago, 'status': status }) return self.do_send_metadata(permid, torrent, selversion) else: # deleted before sending it self.torrent_db.deleteTorrent(infohash, delete_file=True, updateFlag=True) if DEBUG: print >> sys.stderr, "metadata: GET_METADATA: no torrent data to send" return 0
def setUpPostSession(self): """ override TestAsServer """ TestAsServer.setUpPostSession(self) self.mypermid = str(self.my_keypair.pub().get_der()) self.hispermid = str(self.his_keypair.pub().get_der()) # Create URL compat torrents and save in Torrent database. self.tdef1 = TorrentDef.load_from_url(P2PURL_SCHEME+'://127.2.3.42:7764/announce?SjaakCam.mpegts&k=MHowDQYJKoZIhvcNAQEBBQADaQAwZgJhAN0Khlp5ZhWC7VfLynCkKts71b8h8tZXH87PkDtJUTJaX_SS1Cddxkv63PRmKOvtAHhkTLSsWOZbSeHkOlPIq_FGg2aDLDJ05g3lQ-8mSmo05ff4SLqNUTShWO2CR2TPhQIBAw&l=HCAAAA&s=15&a=RSA&b=AAIAAA') self.torrentfn1 = os.path.join(self.session.get_torrent_collecting_dir(),"live.torrent") self.tdef1.save(self.torrentfn1) self.tdef2 = TorrentDef.load_from_url(P2PURL_SCHEME+'://127.1.0.10:6969/announce?trailer.mkv&r=TTgcifG0Ot7STCY2JL8SUOxROFo&l=AKK35A&s=15&b=AAFnGg') self.torrentfn2 = os.path.join(self.session.get_torrent_collecting_dir(),"vod.torrent") self.tdef2.save(self.torrentfn2) self.torrent_db = self.session.open_dbhandler(NTFY_TORRENTS) extra_info = {'status':'good'} self.torrent_db.addExternalTorrent(self.torrentfn1, source='',extra_info=extra_info) self.torrent_db.addExternalTorrent(self.torrentfn2, source='',extra_info=extra_info)
def setUpPostSession(self): """ override TestAsServer """ TestAsServer.setUpPostSession(self) self.hispermid = str(self.his_keypair.pub().get_der()) self.my_permid = str(self.my_keypair.pub().get_der()) self.content_name = 'Hallo S22E44' self.tdef = TorrentDef() self.tdef.set_tracker('http://localhost:0/announce') self.tdef.set_piece_length(2 ** 15) self.tdef.create_live(self.content_name,2 ** 16) self.tdef.finalize()
def read_and_send_metadata(self, permid, infohash, torrent_path, selversion): torrent_data = self.read_torrent(torrent_path) if torrent_data: # Arno: Don't send private torrents try: metainfo = bdecode(torrent_data) if 'info' in metainfo and 'private' in metainfo['info'] and metainfo['info']['private']: if DEBUG: print >> sys.stderr,"metadata: Not sending torrent", `torrent_path`,"because it is private" return 0 except: print_exc() return 0 if DEBUG: print >> sys.stderr,"metadata: sending torrent", `torrent_path`, len(torrent_data) torrent = {} torrent['torrent_hash'] = infohash # P2PURLs: If URL compat then send URL tdef = TorrentDef.load_from_dict(metainfo) if selversion >= OLPROTO_VER_ELEVENTH and tdef.get_url_compat(): torrent['metatype'] = URL_MIME_TYPE torrent['metadata'] = tdef.get_url() else: torrent['metatype'] = TSTREAM_MIME_TYPE torrent['metadata'] = torrent_data if selversion >= OLPROTO_VER_FOURTH: data = self.torrent_db.getTorrent(infohash) if data is None: # DB inconsistency return 0 nleechers = data.get('leecher', -1) nseeders = data.get('seeder', -1) last_check_ago = int(time()) - data.get('last_check_time', 0) # relative time if last_check_ago < 0: last_check_ago = 0 status = data.get('status', 'unknown') torrent.update({'leecher':nleechers, 'seeder':nseeders, 'last_check_time':last_check_ago, 'status':status}) return self.do_send_metadata(permid, torrent, selversion) else: # deleted before sending it self.torrent_db.deleteTorrent(infohash, delete_file=True, updateFlag=True) if DEBUG: print >> sys.stderr,"metadata: GET_METADATA: no torrent data to send" return 0
def valid_metadata(self, infohash, metadata): try: metainfo = bdecode(metadata) tdef = TorrentDef.load_from_dict(metainfo) got_infohash = tdef.get_infohash() if infohash != got_infohash: print >> sys.stderr, "metadata: infohash doesn't match the torrent " + \ "hash. Required: " + `infohash` + ", but got: " + `got_infohash` return False return True except: print_exc() #print >> sys.stderr, "problem metadata:", repr(metadata) return False
def main(): global __TORRENT__, __TORRENT_NEW__, __DOWNLOAD__, active_procs if len(sys.argv[1:]) != 1: print >>sys.stderr, 'Usage: %s <full download>' % sys.argv[0] sys.exit(1) __DOWNLOAD__ = sys.argv[1] __TORRENT__ = FileUtils.get_relative_filename(__DOWNLOAD__) + constants.TORRENT_DOWNLOAD_EXT __TORRENT_NEW__ = __TORRENT__ + '-new' check_dependencies() prepare_scenario() start_ts = time.time() last_vodclient_spawned = time.time()-30.0 print >>sys.stderr, 'Spawning tracker process...' spawn_tracker() time.sleep(10) print >>sys.stderr, 'Fetching torrent from tracker...' shutil.copyfile(os.path.join('basic_scenario_tracker', __TORRENT__), __TORRENT__) print >>sys.stderr, 'Adding playtime information to %s' % __TORRENT__ add_playtime_to_torrent(__TORRENT__) tdef = TorrentDef.load(__TORRENT_NEW__) bitrate = tdef.get_bitrate() print >>sys.stderr, 'Bitrate is: %i' % bitrate print >>sys.stderr, 'Spawning seeder process...' spawn_headless(bitrate*4) FINISH_TIME = 90 # experiment duration in seconds count = 0 while (time.time() - start_ts <= FINISH_TIME): if count == 10: count = 0 print >>sys.stderr, 'Scenario progress '+str(time.time() - start_ts) count+=1 # remove finished vodclient processes active_procs = [p for p in active_procs if p.poll() == None] # check if we have to spawn a new vodclient ts = time.time() if ts - last_vodclient_spawned >= 30.0 and len(active_procs) < 2: print >>sys.stderr, '%s: Spawning new vod client... experiment progress is %f' % (str(ts), ts/start_ts*100) spawn_vodclient(bitrate/4, bitrate) last_vodclient_spawned = ts time.sleep(1.0) print >>sys.stderr, 'Emulation finished, finish and exit...' cleanup() os._exit(0)
def setUp(self): """ override TestAsServer """ TestAsServer.setUp(self) print >>sys.stderr,time.asctime(),'-', "test: Giving MyLaunchMany time to startup" time.sleep(5) print >>sys.stderr,time.asctime(),'-', "test: MyLaunchMany should have started up" # the metadata that we want to transfer self.tdef = TorrentDef() self.tdef.add_content(os.path.join(os.getcwd(), "API", "file.wmv")) self.tdef.set_tracker(self.session.get_internal_tracker_url()) # we use a small piece length to obtain multiple pieces self.tdef.set_piece_length(1) self.tdef.finalize() # self.tdef.save(os.path.join(self.session.get_state_dir(), "gen.torrent")) MagnetHelpers.__init__(self, self.tdef)
def setUpPostSession(self): """ override TestAsServer """ TestAsServer.setUpPostSession(self) # Let Tribler start downloading an non-functioning torrent, so # we can talk to a normal download engine. self.torrentfn = os.path.join('extend_hs_dir','dummydata.merkle.torrent') tdef = TorrentDef.load(self.torrentfn) dscfg = self.setUpDownloadConfig() self.session.start_download(tdef,dscfg) # This is the infohash of the torrent in test/extend_hs_dir self.infohash = '\xccg\x07\xe2\x9e!]\x16\xae{\xb8\x10?\xf9\xa5\xf9\x07\xfdBk' self.mylistenport = 4810
def pretest_q(self,queryprefix,keyword): query = queryprefix+' '+keyword self.content_name = keyword.upper()+' S22E44' self.tdef = TorrentDef() self.tdef.set_tracker('http://localhost:0/announce') self.tdef.set_piece_length(2 ** 15) self.tdef.create_live(self.content_name,2 ** 16) self.tdef.finalize() # 1. First connect to Tribler self.openconn = OLConnection(self.my_keypair,'localhost',self.hisport) sleep(3) # 2. Make Tribler send query self.query = query self.session.query_connected_peers(query,self.query_usercallback,max_peers_to_query=10)
def setUpPostSession(self): """ override TestAsServer """ TestAsServer.setUpPostSession(self) # Let Tribler start downloading an non-functioning torrent, so # we can talk to a normal download engine. self.torrentfn = os.path.join('extend_hs_dir', 'dummydata.merkle.torrent') tdef = TorrentDef.load(self.torrentfn) dscfg = self.setUpDownloadConfig() self.session.start_download(tdef, dscfg) # This is the infohash of the torrent in test/extend_hs_dir self.infohash = '\xccg\x07\xe2\x9e!]\x16\xae{\xb8\x10?\xf9\xa5\xf9\x07\xfdBk' self.mylistenport = 4810
def get_torrents(folder): ''' Returns the dictionary {infohash : (torrent_definition, file_name)} for all torrent files in the given directory. ''' fileObjects = os.listdir(folder) torrent_files = [] for file in fileObjects: if str(file).endswith(constants.TORRENT_DOWNLOAD_EXT) or str(file).endswith(constants.TORRENT_VOD_EXT): torrent_files.append(os.path.join(folder, str(file))) files = sorted(torrent_files) torrents = dict() for file in files: tdef = TorrentDef.load(file) id = get_id(tdef) torrents[id] = (tdef, file) return torrents
def search_torrents(self, kws, maxhits=None, sendtorrents=False): if DEBUG: print >> sys.stderr, "rquery: search for torrents matching", ` kws ` allhits = self.torrent_db.searchNames(kws, local=False) print >> sys.stderr, "rquery: got matches", ` allhits ` if maxhits is None: hits = allhits else: hits = allhits[:maxhits] colltorrdir = self.session.get_torrent_collecting_dir() if sendtorrents: print >> sys.stderr, "rqmh: search_torrents: adding torrents" for hit in hits: filename = os.path.join(colltorrdir, hit['torrent_file_name']) try: tdef = TorrentDef.load(filename) if tdef.get_url_compat(): metatype = URL_MIME_TYPE metadata = tdef.get_url() else: metatype = TSTREAM_MIME_TYPE metadata = bencode(tdef.get_metainfo()) except: print_exc() metadata = None hit['metatype'] = metatype hit['metadata'] = metadata # Filter out hits for which we could not read torrent file (rare) newhits = [] for hit in hits: if hit['metadata'] is not None: newhits.append(hit) hits = newhits return hits
def search_torrents(self,kws,maxhits=None,sendtorrents=False): if DEBUG: print >>sys.stderr,"rquery: search for torrents matching",`kws` allhits = self.torrent_db.searchNames(kws,local=False) print >>sys.stderr,"rquery: got matches",`allhits` if maxhits is None: hits = allhits else: hits = allhits[:maxhits] colltorrdir = self.session.get_torrent_collecting_dir() if sendtorrents: print >>sys.stderr,"rqmh: search_torrents: adding torrents" for hit in hits: filename = os.path.join(colltorrdir,hit['torrent_file_name']) try: tdef = TorrentDef.load(filename) if tdef.get_url_compat(): metatype = URL_MIME_TYPE metadata = tdef.get_url() else: metatype = TSTREAM_MIME_TYPE metadata = bencode(tdef.get_metainfo()) except: print_exc() metadata = None hit['metatype'] = metatype hit['metadata'] = metadata # Filter out hits for which we could not read torrent file (rare) newhits = [] for hit in hits: if hit['metadata'] is not None: newhits.append(hit) hits = newhits return hits
def setUp(self): """ override TestAsServer """ # listener for incoming connections from MiniBitTorrent self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.server.bind(("localhost", LISTEN_PORT)) self.server.listen(1) # the metadata that we want to transfer self.tdef = TorrentDef() self.tdef.add_content(os.path.join(os.getcwd(), "API", "file.wmv")) self.tdef.set_tracker("http://fake.net/announce") # we use a small piece length to obtain multiple pieces self.tdef.set_piece_length(1) self.tdef.finalize() MagnetHelpers.__init__(self, self.tdef) # startup the client TestAsServer.setUp(self) print >>sys.stderr,time.asctime(),'-', "test: Giving MyLaunchMany time to startup" time.sleep(5) print >>sys.stderr,time.asctime(),'-', "test: MyLaunchMany should have started up"
def create_torrent(self): [srchandle, self.sourcefn] = mkstemp() self.content = Rand.rand_bytes(self.contentlen) os.write(srchandle, self.content) os.close(srchandle) self.tdef = TorrentDef() self.tdef.add_content(self.sourcefn) self.tdef.set_piece_length(self.piecelen) self.tdef.set_tracker("http://127.0.0.1:12/announce") self.tdef.finalize() self.torrentfn = os.path.join(self.session.get_state_dir(), "gen.torrent") self.tdef.save(self.torrentfn) dscfg = DownloadStartupConfig() destdir = os.path.dirname(self.sourcefn) dscfg.set_dest_dir(destdir) dscfg.set_video_event_callback(self.sesscb_vod_event_callback) self.session.set_download_states_callback(self.states_callback) self.session.start_download(self.tdef, dscfg)
def test_good_transfer(self): def torrentdef_retrieved(tdef): tags["retrieved"] = True tags["metainfo"] = tdef.get_metainfo() tags = {"retrieved":False} assert TorrentDef.retrieve_from_magnet(self.create_good_url(), torrentdef_retrieved) # supply fake addresses (regular dht obviously wont work here) for magnetlink in MagnetHandler.get_instance().get_magnets(): magnetlink._swarm.add_potential_peers([("localhost", LISTEN_PORT)]) # accept incoming connection self.server.settimeout(10.0) sock, address = self.server.accept() assert sock, "No incoming connection" # handshakes conn = BTConnection(address[0], address[1], opensock=sock, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) # serve pieces for counter in xrange(len(self.metadata_list)): piece = self.read_extend_metadata_request(conn) assert 0 <= piece < len(self.metadata_list) conn.send(self.create_good_extend_metadata_reply(metadata_id, piece)) # no more metadata request may be send and the connection must # be closed self.read_extend_metadata_close(conn) time.sleep(5) assert tags["retrieved"] assert tags["metainfo"]["info"] == self.tdef.get_metainfo()["info"]
def got_metadata(self, permid, message, selversion): """ receive torrent file from others """ # Arno, 2007-06-20: Disabled the following code. What's this? Somebody sends # us something and we refuse? Also doesn't take into account download help #and remote-query extension. #if self.upload_rate <= 0: # if no upload, no download, that's the game # return True # don't close connection try: message = bdecode(message[1:]) except: print_exc() return False if not isinstance(message, dict): return False try: infohash = message['torrent_hash'] if not isValidInfohash(infohash): return False if not infohash in self.requested_torrents: # got a torrent which was not requested return True if self.torrent_db.hasMetaData(infohash): return True # P2PURL goturl = False if selversion >= OLPROTO_VER_ELEVENTH: if 'metatype' in message and message['metatype'] == URL_MIME_TYPE: try: tdef = TorrentDef.load_from_url(metadata['metadata']) # Internal storage format is still .torrent file metainfo = tdef.get_metainfo() metadata = bencode(metainfo) goturl = True except: print_exc() return False else: metadata = message['metadata'] else: metadata = message['metadata'] if not self.valid_metadata(infohash, metadata): return False if DEBUG: torrent_size = len(metadata) if goturl: mdt = "URL" else: mdt = "torrent" print >> sys.stderr,"metadata: Recvd",mdt,`infohash`,sha(infohash).hexdigest(), torrent_size extra_info = {} if selversion >= OLPROTO_VER_FOURTH: try: extra_info = {'leecher': message.get('leecher', -1), 'seeder': message.get('seeder', -1), 'last_check_time': message.get('last_check_time', -1), 'status':message.get('status', 'unknown')} except Exception, msg: print_exc() print >> sys.stderr, "metadata: wrong extra info in msg - ", message extra_info = {} filename = self.save_torrent(infohash, metadata, extra_info=extra_info) self.requested_torrents.remove(infohash) #if DEBUG: # print >>sys.stderr,"metadata: Was I asked to dlhelp someone",self.dlhelper if filename is not None: self.notify_torrent_is_in(infohash,metadata,filename) # BarterCast: add bytes of torrent to BarterCastDB # Save exchanged KBs in BarterCastDB if permid is not None and BARTERCAST_TORRENTS: self.overlay_bridge.add_task(lambda:self.olthread_bartercast_torrentexchange(permid, 'downloaded'), 0)
scfg = SessionStartupConfig() scfg.set_state_dir(tempfile.mkdtemp()) scfg.set_listen_port(options.port) scfg.set_overlay(False) scfg.set_megacache(False) scfg.set_upnp_mode(simpledefs.UPNPMODE_DISABLED) scfg.set_dialback(False) scfg.set_social_networking(False) scfg.set_buddycast(False) scfg.set_crawler(False) scfg.set_internal_tracker(False) s = Session(scfg) tdef = TorrentDef.load(__TORRENT_FILE__) # tdef.get_tracker() returns the announce-url; we must omit the "announce" part tracker_url = tdef.get_tracker()[:tdef.get_tracker().find("announce")] if tdef.get_bitrate() == None: print >> sys.stderr, "Provided torrent file has no bitrate information. Exiting." sys.exit(1) BITRATE = tdef.get_bitrate() print >> sys.stderr, "Calculated bitrate is %d" % BITRATE client_stats['video_duration'] = int(tdef.get_length() / tdef.get_bitrate()) if not os.access(options.directory, os.F_OK): os.makedirs(options.directory)
torrent = fileargs[1] if not os.path.exists(torrent): print "Error: Could not find torrent file '%s'"%torrent raise SystemExit(1) if not config['key_file']: config['key_file'] = torrent + ".tkey" if not os.path.exists(config['key_file']): print "Error: Could not find key file '%s'"%config['key_file'] raise SystemExit(1) # Load the torrent file try: t = TorrentDef.load(torrent) except Exception,e: print "Bad torrent file:",e raise SystemExit(1) if not t.get_cs_keys(): print "Not a closed swarm torrent" raise SystemExit(1) try: torrent_keypair = ClosedSwarm.read_cs_keypair(config['key_file']) except Exception,e: print "Bad torrent key file",e raise SystemExit(1) # Need permid of the receiving node if not config['node_id']:
def recv_query_reply(self, permid, message, selversion): #print "****** recv query reply", len(message) if selversion < OLPROTO_VER_SIXTH: return False #if len(message) > MAX_QUERY_REPLY_LEN: # return True # don't close # Unpack try: d = bdecode(message[1:]) except: if DEBUG: print >> sys.stderr, "rquery: Cannot bdecode QUERY_REPLY message", selversion return False if not isValidQueryReply(d, selversion): if DEBUG: print >> sys.stderr, "rquery: not valid QUERY_REPLY message", selversion return False # Check auth queryrec = self.is_registered_query_id(d['id']) if not queryrec: if DEBUG: print >> sys.stderr, "rquery: QUERY_REPLY has unknown query ID", selversion return False if selversion >= OLPROTO_VER_TWELFTH: if queryrec['query'].startswith('SIMPLE+METADATA'): for infohash, torrentrec in d['a'].iteritems(): if not 'metatype' in torrentrec: if DEBUG: print >> sys.stderr, "rquery: QUERY_REPLY has no metatype field", selversion return False if not 'metadata' in torrentrec: if DEBUG: print >> sys.stderr, "rquery: QUERY_REPLY has no metadata field", selversion return False if torrentrec['torrent_size'] != len( torrentrec['metadata']): if DEBUG: print >> sys.stderr, "rquery: QUERY_REPLY torrent_size != len metadata", selversion return False try: # Validity test if torrentrec['metatype'] == URL_MIME_TYPE: tdef = TorrentDef.load_from_url( torrentrec['metadata']) else: metainfo = bdecode(torrentrec['metadata']) tdef = TorrentDef.load_from_dict(metainfo) except: if DEBUG: print_exc() return False # Process self.process_query_reply(permid, queryrec['query'], queryrec['usercallback'], d) return True
class TestVideoOnDemand(TestAsServer): """ Testing Merkle hashpiece messages for both: * Merkle BEP style * old Tribler <= 4.5.2 that did not use the Extention protocol (BEP 10). See BitTornado/BT1/Connecter.py """ def setUp(self): """ override TestAsServer """ TestAsServer.setUp(self) self.vodstarted = False def setUpPreSession(self): """ override TestAsServer """ TestAsServer.setUpPreSession(self) self.config.set_overlay(False) self.config.set_megacache(False) def setUpPostSession(self): """ override TestAsServer """ TestAsServer.setUpPostSession(self) def tearDown(self): print >> sys.stderr, time.asctime(), "-", "Test: Sleep before tear down" time.sleep(10) TestAsServer.tearDown(self) def create_torrent(self): [srchandle, self.sourcefn] = mkstemp() self.content = Rand.rand_bytes(self.contentlen) os.write(srchandle, self.content) os.close(srchandle) self.tdef = TorrentDef() self.tdef.add_content(self.sourcefn) self.tdef.set_piece_length(self.piecelen) self.tdef.set_tracker("http://127.0.0.1:12/announce") self.tdef.finalize() self.torrentfn = os.path.join(self.session.get_state_dir(), "gen.torrent") self.tdef.save(self.torrentfn) dscfg = DownloadStartupConfig() destdir = os.path.dirname(self.sourcefn) dscfg.set_dest_dir(destdir) dscfg.set_video_event_callback(self.sesscb_vod_event_callback) self.session.set_download_states_callback(self.states_callback) self.session.start_download(self.tdef, dscfg) def states_callback(self, dslist): ds = dslist[0] d = ds.get_download() # print >>sys.stderr,time.asctime(),'-', `d.get_def().get_name()`,dlstatus_strings[ds.get_status()],ds.get_progress(),"%",ds.get_error(),"up",ds.get_current_speed(UPLOAD),"down",ds.get_current_speed(DOWNLOAD) print >> sys.stderr, time.asctime(), "-", "%s %s %5.2f%% %s up %8.2fKB/s down %8.2fKB/s" % ( d.get_def().get_name(), dlstatus_strings[ds.get_status()], ds.get_progress() * 100, ds.get_error(), ds.get_current_speed(UPLOAD), ds.get_current_speed(DOWNLOAD), ) return (1.0, False) def sesscb_vod_event_callback(self, d, event, params): if self.vodstarted: return self.vodstarted = True print >> sys.stderr, time.asctime(), "-", "Test: vod_event_callback", event, params if event == VODEVENT_START: stream = params["stream"] # Read last piece lastpieceoff = ((self.contentlen - 1) / self.piecelen) * self.piecelen lastpiecesize = self.contentlen - lastpieceoff print >> sys.stderr, time.asctime(), "-", "Test: stream: lastpieceoff", lastpieceoff, lastpiecesize self.stream_read(stream, lastpieceoff, lastpiecesize, self.piecelen) # Read second,3rd,4th byte, only secoff = 1 secsize = 3 blocksize = 3 self.stream_read(stream, secoff, secsize, blocksize) # Read last byte lastoff = self.contentlen - 1 lastsize = 1 self.stream_read(stream, lastoff, lastsize, self.piecelen) print >> sys.stderr, time.asctime(), "-", "Test: stream: Passed?" def stream_read(self, stream, off, size, blocksize): stream.seek(off) data = stream.read(blocksize) print >> sys.stderr, time.asctime(), "-", "Test: stream: Got data", len(data) self.assertEquals(len(data), size) self.assertEquals(data, self.content[off : off + size]) def singtest_99(self): self.contentlen = 99 self.piecelen = 10 self.create_torrent() print >> sys.stderr, time.asctime(), "-", "Test: Letting network thread create Download, sleeping" time.sleep(5) dlist = self.session.get_downloads() d = dlist[0] vs = d.sd.videostatus goodrange = ((0, 0), (9, 8)) self.assertEqual(vs.movie_range, goodrange) self.assertEqual(vs.first_piecelen, 10) self.assertEqual(vs.last_piecelen, 9) self.assertEqual(vs.first_piece, 0) self.assertEqual(vs.last_piece, 9) self.assertEqual(vs.movie_numpieces, 10) print >> sys.stderr, time.asctime(), "-", "Test: status: Passed? ****************************************************************" def singtest_100(self): self.contentlen = 100 self.piecelen = 10 self.create_torrent() print >> sys.stderr, time.asctime(), "-", "Test: Letting network thread create Download, sleeping" time.sleep(5) dlist = self.session.get_downloads() d = dlist[0] vs = d.sd.videostatus goodrange = ((0, 0), (9, 9)) self.assertEqual(vs.movie_range, goodrange) self.assertEqual(vs.first_piecelen, 10) self.assertEqual(vs.last_piecelen, 10) self.assertEqual(vs.first_piece, 0) self.assertEqual(vs.last_piece, 9) self.assertEqual(vs.movie_numpieces, 10) print >> sys.stderr, time.asctime(), "-", "Test: status: Passed? ****************************************************************" def singtest_101(self): self.contentlen = 101 self.piecelen = 10 self.create_torrent() print >> sys.stderr, time.asctime(), "-", "Test: Letting network thread create Download, sleeping" time.sleep(5) dlist = self.session.get_downloads() d = dlist[0] vs = d.sd.videostatus goodrange = ((0, 0), (10, 0)) self.assertEqual(vs.movie_range, goodrange) self.assertEqual(vs.first_piecelen, 10) self.assertEqual(vs.last_piecelen, 1) self.assertEqual(vs.first_piece, 0) self.assertEqual(vs.last_piece, 10) self.assertEqual(vs.movie_numpieces, 11) print >> sys.stderr, time.asctime(), "-", "Test: status: Passed? ****************************************************************"
def testECreateRichTorrents(self): _log.info("Create rich metadata torrents ------------") if not TestMetaGenerator.run_test: _log.debug("Test avoided.") return data = None self.assertTrue(TestMetaGenerator.data != None, "Test E depends on data gathered in test D, set testD attribute to True") if TestMetaGenerator.data != None: data = TestMetaGenerator.data else: return rmg = RichMetadataGenerator.getInstance() torrents = [] for k, v in data.items(): if v.content_base != None: # Choose only a small subset of content to download if v.content_base == "http://www.rtvslo.si/podcasts/gori_doli_naokoli.xml": mm_file = tempfile.NamedTemporaryFile() try: urllib.urlretrieve(v.id, mm_file.name) torrent_def = TorrentDef() torrent_def.add_content(mm_file.name) torrent_def.set_tracker("http://wanabe.a.tracker.com:8080") torrent_def.set_metadata(k) torrent_def.finalize() torrent_def.save(mm_file.name + ".torrent") torrents.append(mm_file.name + ".torrent") except Exception, e: _log.error("Failed to generate torrent file, reason: " + str(e))
class TestMagnetMiniBitTorrent(TestAsServer, MagnetHelpers): """ A MiniBitTorrent instance is used to connect to BitTorrent clients and download the info part from the metadata. """ def setUp(self): """ override TestAsServer """ # listener for incoming connections from MiniBitTorrent self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.server.bind(("localhost", LISTEN_PORT)) self.server.listen(1) # the metadata that we want to transfer self.tdef = TorrentDef() self.tdef.add_content(os.path.join(os.getcwd(), "API", "file.wmv")) self.tdef.set_tracker("http://fake.net/announce") # we use a small piece length to obtain multiple pieces self.tdef.set_piece_length(1) self.tdef.finalize() MagnetHelpers.__init__(self, self.tdef) # startup the client TestAsServer.setUp(self) print >>sys.stderr,time.asctime(),'-', "test: Giving MyLaunchMany time to startup" time.sleep(5) print >>sys.stderr,time.asctime(),'-', "test: MyLaunchMany should have started up" def create_good_url(self, infohash=None, title=None, tracker=None): url = "magnet:?xt=urn:btih:" if infohash: assert isinstance(infohash, str) url += hexlify(infohash) else: url += hexlify(self.tdef.get_infohash()) if title: assert isinstance(title, str) url += "&dn=" + title if tracker: assert isinstance(tracker, str) url += "&tr=" + tracker return url def test_good_transfer(self): def torrentdef_retrieved(tdef): tags["retrieved"] = True tags["metainfo"] = tdef.get_metainfo() tags = {"retrieved":False} assert TorrentDef.retrieve_from_magnet(self.create_good_url(), torrentdef_retrieved) # supply fake addresses (regular dht obviously wont work here) for magnetlink in MagnetHandler.get_instance().get_magnets(): magnetlink._swarm.add_potential_peers([("localhost", LISTEN_PORT)]) # accept incoming connection self.server.settimeout(10.0) sock, address = self.server.accept() assert sock, "No incoming connection" # handshakes conn = BTConnection(address[0], address[1], opensock=sock, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) # serve pieces for counter in xrange(len(self.metadata_list)): piece = self.read_extend_metadata_request(conn) assert 0 <= piece < len(self.metadata_list) conn.send(self.create_good_extend_metadata_reply(metadata_id, piece)) # no more metadata request may be send and the connection must # be closed self.read_extend_metadata_close(conn) time.sleep(5) assert tags["retrieved"] assert tags["metainfo"]["info"] == self.tdef.get_metainfo()["info"]
def setUpPostSession(self): """ override TestAsServer """ TestAsServer.setUpPostSession(self) # Let Tribler start downloading an non-functioning torrent, so # we can talk to a normal download engine. self.tdef = TorrentDef() self.sourcefn = os.path.join(os.getcwd(),"API","file2.wmv") self.tdef.add_content(self.sourcefn) self.tdef.set_create_merkle_torrent(True) self.tdef.set_tracker("http://127.0.0.1:12/announce") self.tdef.finalize() self.torrentfn = os.path.join(self.session.get_state_dir(),"gen.torrent") self.tdef.save(self.torrentfn) dscfg = self.setUpDownloadConfig() self.session.start_download(self.tdef,dscfg) self.infohash = self.tdef.get_infohash() self.mylistenport = 4810 self.numpieces = (self.tdef.get_length()+self.tdef.get_piece_length()-1) / self.tdef.get_piece_length() b = Bitfield(self.numpieces) for i in range(self.numpieces): b[i] = True self.assert_(b.complete()) self.seederbitfieldstr = b.tostring() #piece_hashes = ['\x01\x02\x03\x04\x05\x06\x07\x08\x07\x06\x05\x04\x03\x02\x01\x00\x01\x02\x03\x04' ] * npieces # Construct Merkle tree tdef2 = TorrentDef() tdef2.add_content(self.sourcefn) tdef2.set_create_merkle_torrent(False) tdef2.set_tracker("http://127.0.0.1:12/announce") tdef2.set_piece_length(self.tdef.get_piece_length()) tdef2.finalize() metainfo = tdef2.get_metainfo() piecesstr = metainfo['info']['pieces'] print >>sys.stderr,"test: pieces has len",len(piecesstr) piece_hashes = [] for i in range(0,len(piecesstr),20): hash = piecesstr[i:i+20] print >>sys.stderr,"test: piece",i/20,"hash",`hash` piece_hashes.append(hash) print >>sys.stderr,"test: Putting",len(piece_hashes),"into MerkleTree, size",self.tdef.get_piece_length(),tdef2.get_piece_length() self.tree = MerkleTree(self.tdef.get_piece_length(),self.tdef.get_length(),None,piece_hashes) f = open(self.sourcefn,"rb") piece1 = f.read(2 ** 18) piece2 = f.read(2 ** 18) print >>sys.stderr,"read piece1",len(piece1) print >>sys.stderr,"read piece2",len(piece2) f.close() hash1 = sha(piece1).digest() hash2 = sha(piece2).digest() print >>sys.stderr,"hash piece1",`hash1` print >>sys.stderr,"hash piece2",`hash2` f2 = open("piece1.bin","wb") f2.write(piece2) f2.close()
def test_tdef_has_video_files_expected_false(self): try: tdef = TorrentDef.load("torrents/ubuntu.torrent") self.assertFalse(has_torrent_video_files(tdef)) except: self.fail("There was an error while loading the torrent files.")
def got_metadata(self, permid, message, selversion): """ receive torrent file from others """ # Arno, 2007-06-20: Disabled the following code. What's this? Somebody sends # us something and we refuse? Also doesn't take into account download help #and remote-query extension. #if self.upload_rate <= 0: # if no upload, no download, that's the game # return True # don't close connection try: message = bdecode(message[1:]) except: print_exc() return False if not isinstance(message, dict): return False try: infohash = message['torrent_hash'] if not isValidInfohash(infohash): return False if not infohash in self.requested_torrents: # got a torrent which was not requested return True if self.torrent_db.hasMetaData(infohash): return True # P2PURL goturl = False if selversion >= OLPROTO_VER_ELEVENTH: if 'metatype' in message and message[ 'metatype'] == URL_MIME_TYPE: try: tdef = TorrentDef.load_from_url(metadata['metadata']) # Internal storage format is still .torrent file metainfo = tdef.get_metainfo() metadata = bencode(metainfo) goturl = True except: print_exc() return False else: metadata = message['metadata'] else: metadata = message['metadata'] if not self.valid_metadata(infohash, metadata): return False if DEBUG: torrent_size = len(metadata) if goturl: mdt = "URL" else: mdt = "torrent" print >> sys.stderr, "metadata: Recvd", mdt, ` infohash `, sha( infohash).hexdigest(), torrent_size extra_info = {} if selversion >= OLPROTO_VER_FOURTH: try: extra_info = { 'leecher': message.get('leecher', -1), 'seeder': message.get('seeder', -1), 'last_check_time': message.get('last_check_time', -1), 'status': message.get('status', 'unknown') } except Exception, msg: print_exc() print >> sys.stderr, "metadata: wrong extra info in msg - ", message extra_info = {} filename = self.save_torrent(infohash, metadata, extra_info=extra_info) self.requested_torrents.remove(infohash) #if DEBUG: # print >>sys.stderr,"metadata: Was I asked to dlhelp someone",self.dlhelper if filename is not None: self.notify_torrent_is_in(infohash, metadata, filename) # BarterCast: add bytes of torrent to BarterCastDB # Save exchanged KBs in BarterCastDB if permid is not None and BARTERCAST_TORRENTS: self.overlay_bridge.add_task( lambda: self.olthread_bartercast_torrentexchange( permid, 'downloaded'), 0)
class TestMerkleMessage(TestAsServer): """ Testing Merkle hashpiece messages for both: * Merkle BEP style * old Tribler <= 4.5.2 that did not use the Extention protocol (BEP 10). See BitTornado/BT1/Connecter.py """ def setUp(self): """ override TestAsServer """ TestAsServer.setUp(self) print >>sys.stderr,"test: Giving Session time to startup" time.sleep(5) print >>sys.stderr,"test: Session should have started up" def setUpPreSession(self): """ override TestAsServer """ TestAsServer.setUpPreSession(self) self.config.set_overlay(False) self.config.set_megacache(False) def setUpPostSession(self): """ override TestAsServer """ TestAsServer.setUpPostSession(self) # Let Tribler start downloading an non-functioning torrent, so # we can talk to a normal download engine. self.tdef = TorrentDef() self.sourcefn = os.path.join(os.getcwd(),"API","file2.wmv") self.tdef.add_content(self.sourcefn) self.tdef.set_create_merkle_torrent(True) self.tdef.set_tracker("http://127.0.0.1:12/announce") self.tdef.finalize() self.torrentfn = os.path.join(self.session.get_state_dir(),"gen.torrent") self.tdef.save(self.torrentfn) dscfg = self.setUpDownloadConfig() self.session.start_download(self.tdef,dscfg) self.infohash = self.tdef.get_infohash() self.mylistenport = 4810 self.numpieces = (self.tdef.get_length()+self.tdef.get_piece_length()-1) / self.tdef.get_piece_length() b = Bitfield(self.numpieces) for i in range(self.numpieces): b[i] = True self.assert_(b.complete()) self.seederbitfieldstr = b.tostring() #piece_hashes = ['\x01\x02\x03\x04\x05\x06\x07\x08\x07\x06\x05\x04\x03\x02\x01\x00\x01\x02\x03\x04' ] * npieces # Construct Merkle tree tdef2 = TorrentDef() tdef2.add_content(self.sourcefn) tdef2.set_create_merkle_torrent(False) tdef2.set_tracker("http://127.0.0.1:12/announce") tdef2.set_piece_length(self.tdef.get_piece_length()) tdef2.finalize() metainfo = tdef2.get_metainfo() piecesstr = metainfo['info']['pieces'] print >>sys.stderr,"test: pieces has len",len(piecesstr) piece_hashes = [] for i in range(0,len(piecesstr),20): hash = piecesstr[i:i+20] print >>sys.stderr,"test: piece",i/20,"hash",`hash` piece_hashes.append(hash) print >>sys.stderr,"test: Putting",len(piece_hashes),"into MerkleTree, size",self.tdef.get_piece_length(),tdef2.get_piece_length() self.tree = MerkleTree(self.tdef.get_piece_length(),self.tdef.get_length(),None,piece_hashes) f = open(self.sourcefn,"rb") piece1 = f.read(2 ** 18) piece2 = f.read(2 ** 18) print >>sys.stderr,"read piece1",len(piece1) print >>sys.stderr,"read piece2",len(piece2) f.close() hash1 = sha(piece1).digest() hash2 = sha(piece2).digest() print >>sys.stderr,"hash piece1",`hash1` print >>sys.stderr,"hash piece2",`hash2` f2 = open("piece1.bin","wb") f2.write(piece2) f2.close() def setUpDownloadConfig(self): dscfg = DownloadStartupConfig() print >>sys.stderr,"test: Downloading to",self.config_path dscfg.set_dest_dir(self.config_path) dscfg.set_breakup_seed_bitfield(False) return dscfg def tearDown(self): TestAsServer.tearDown(self) try: os.remove('piece1.bin') except: pass def singtest_good_hashpiece_bepstyle(self): self.subtest_good_hashpiece(False) def singtest_good_hashpiece_oldstyle(self): self.subtest_good_hashpiece(True) def singtest_good_request_bepstyle(self): # Let Session download file first self.subtest_good_hashpiece(False) # Now connect as different peer and download print >>sys.stderr,"\n\ntest: test_good_request: STARTING" self._test_good_request() def singtest_bad_hashpiece_bepstyle(self): self.subtest_bad_hashpiece(False) def singtest_bad_hashpiece_oldstyle(self): self.subtest_bad_hashpiece(True) # # Good hashpiece message # def subtest_good_hashpiece(self,oldstyle): print >>sys.stderr,"test: Testing good hashpiece, oldstyle",oldstyle if oldstyle: self._test_good(self.create_good_hashpiece,oldstyle,self.create_good_tribler_extend_hs,infohash=self.infohash) else: options = '\x00\x00\x00\x00\x00\x10\x00\x00' self._test_good(self.create_good_hashpiece,oldstyle,self.create_good_nontribler_extend_hs,options=options,infohash=self.infohash) def _test_good(self,msg_gen_func,oldstyle,extend_hs_gen_func,options=None,infohash=None): if options is None and infohash is None: s = BTConnection('localhost',self.hisport) elif options is None: s = BTConnection('localhost',self.hisport,user_infohash=infohash) elif infohash is None: s = BTConnection('localhost',self.hisport,user_option_pattern=options) else: s = BTConnection('localhost',self.hisport,user_option_pattern=options,user_infohash=infohash) print >>sys.stderr,"test: test_good: Create EXTEND HS" msg = extend_hs_gen_func() print >>sys.stderr,"test: test_good: Sending EXTEND HS",`msg` s.send(msg) print >>sys.stderr,"test: test_good: Waiting for BT HS" s.read_handshake_medium_rare() # Tribler should send an EXTEND message back try: print >>sys.stderr,"test: Waiting for reply" s.s.settimeout(10.0) resp = s.recv() self.assert_(len(resp) > 0) print >>sys.stderr,"test: Got reply",getMessageName(resp[0]) self.assert_(resp[0] == EXTEND) self.check_tribler_extend_hs(resp[1:]) # 1. Pretend we're seeder: send BITFIELD and UNCHOKE msg = BITFIELD+self.seederbitfieldstr s.send(msg) msg = UNCHOKE s.send(msg) print >>sys.stderr,"test: Pretend we are seeder" while True: resp = s.recv() self.assert_(len(resp) > 0) print >>sys.stderr,"test: Got reply2",getMessageName(resp[0]) self.assert_(resp[0] == REQUEST or resp[0] == INTERESTED or resp[0] == UNCHOKE or resp[0] == HAVE or resp[0] == NOT_INTERESTED) if resp[0] == REQUEST: chunkid = self.check_request(resp) # 2. Reply to REQUEST with HASHPIECE (oldstyle) or Tr_hashpiece msg = msg_gen_func(oldstyle,chunkid) s.send(msg) elif resp[0] == NOT_INTERESTED: break #s.close() except socket.timeout: print >> sys.stderr,"test: Timeout, bad, peer didn't reply in time" self.assert_(False) destfn = os.path.join(self.config_path,"file2.wmv") sf = open(self.sourcefn,"rb") df = open(destfn,"rb") n = self.tdef.get_piece_length() while True: sdata = sf.read(n) if len(sdata) == 0: break ddata = df.read(n) self.assert_(sdata == ddata) time.sleep(3) s.close() def create_good_nontribler_extend_hs(self): """ Merkle BEP style """ d = {} d['m'] = {'Tr_hashpiece':250} d['p'] = self.mylistenport d['v'] = 'TestSweet 1.2.3.4' bd = bencode(d) return EXTEND+chr(0)+bd def create_good_tribler_extend_hs(self): """ old Tribler style """ d = {} d['m'] = {'Tr_OVERLAYSWARM':253} d['p'] = self.mylistenport d['v'] = 'Tribler 3.5.1' bd = bencode(d) return EXTEND+chr(0)+bd def check_tribler_extend_hs(self,data): self.assert_(data[0] == chr(0)) d = bdecode(data[1:]) self.assert_(type(d) == DictType) self.assert_('m' in d.keys()) m = d['m'] self.assert_(type(m) == DictType) self.assert_('Tr_hashpiece' in m.keys()) val = m['Tr_hashpiece'] self.assert_(type(val) == IntType) self.assert_(val == 250) def check_request(self,data): index = toint(data[1:5]) begin = toint(data[5:9]) length = toint(data[9:]) return (index,begin,length) def create_good_hashpiece(self,oldstyle,chunkid): index, begin, length = chunkid if begin == 0: ohlist = self.tree.get_hashes_for_piece(index) else: ohlist = [] chunk = self.read_chunk(index,begin,length) bohlist = bencode(ohlist) print >>sys.stderr,"test: create_good_hashpiece:",index,begin,length,"==len",len(chunk) payload = tobinary(index)+tobinary(begin)+tobinary(len(bohlist))+bohlist+chunk if oldstyle: msg = HASHPIECE+payload else: # Offical: use the msg ID he defined in his handshake msg = EXTEND+HASHPIECE+payload return msg def read_chunk(self,index,begin,length): offset = index*self.tdef.get_piece_length() + begin f = open(self.sourcefn,"rb") f.seek(offset) chunk = f.read(length) f.close() return chunk # # Test whether Tribler sends good Tr_hashpiece on our requests # def _test_good_request(self): options = '\x00\x00\x00\x00\x00\x10\x00\x00' myid = Rand.rand_bytes(20) s = BTConnection('localhost',self.hisport,user_option_pattern=options,user_infohash=self.infohash,myid=myid) msg = self.create_good_nontribler_extend_hs() s.send(msg) s.read_handshake_medium_rare() # Tribler should send an EXTEND message back try: print >>sys.stderr,"test: Waiting for reply" s.s.settimeout(10.0) resp = s.recv() self.assert_(len(resp) > 0) print >>sys.stderr,"test: Got reply",getMessageName(resp[0]) self.assert_(resp[0] == EXTEND) self.check_tribler_extend_hs(resp[1:]) # 1. Pretend we're leecher: send INTERESTED msg = INTERESTED s.send(msg) print >>sys.stderr,"test: Pretend we are leecher" while True: resp = s.recv() self.assert_(len(resp) > 0) print >>sys.stderr,"test: Got reply2",getMessageName(resp[0]) if resp[0] == EXTEND: print >>sys.stderr,"test: Got EXTEND type",getMessageName(resp[1]) self.assert_(resp[0] == UNCHOKE or resp[0] == BITFIELD or resp[0] == EXTEND or resp[0] == HAVE) if resp[0] == UNCHOKE: # 2. Reply with REQUESTs for index in range(0,self.numpieces): plen = self.get_piece_length(index) for begin in range(0,plen,2 ** 14): length = self.get_chunk_length(index,begin) print >>sys.stderr,"RETRIEVE",index,begin,length chunkid = (index,begin,length) msg = self.create_request(chunkid) s.send(msg) #s.send(NOT_INTERESTED) elif resp[0] == EXTEND and resp[1] == HASHPIECE: done = self.check_hashpiece(resp) if done: break elif resp[0] == BITFIELD: self.check_bitfield(resp) #s.close() except socket.timeout: print >> sys.stderr,"test: Timeout, bad, peer didn't reply in time" self.assert_(False) time.sleep(3) s.close() def get_piece_length(self,index): if index == (self.numpieces-1): plen = self.tdef.get_length() % self.tdef.get_piece_length() else: plen = self.tdef.get_piece_length() return plen def get_chunk_length(self,index,begin): plen = self.get_piece_length(index) length = 2 ** 14 if index == (self.numpieces-1): if (begin+2 ** 14) > plen: length = plen - begin return length def create_request(self,chunkid): index,begin,length = chunkid return REQUEST+tobinary(index)+tobinary(begin)+tobinary(length) def check_hashpiece(self,resp): """ Merkle BEP style """ print >>sys.stderr,"test: good_request: check_hashpiece" self.assert_(resp[0] == EXTEND) self.assert_(resp[1] == HASHPIECE) index = toint(resp[2:2+4]) begin = toint(resp[6:6+4]) ohlen = toint(resp[10:10+4]) print >>sys.stderr,"test: good_request: check_hashpiece",index,begin,ohlen bohlist = resp[14:14+ohlen] hisohlist = bdecode(bohlist) hischunk = resp[14+ohlen:] if begin == 0: self.assert_(type(hisohlist) == ListType) for oh in hisohlist: self.assert_(type(oh) == ListType) self.assert_(len(oh) == 2) self.assert_(type(oh[0]) == IntType) self.assert_(type(oh[1]) == StringType) hisohlist.sort() print >>sys.stderr,"test: good_request: check_hashpiece",`hisohlist` myohlist = self.tree.get_hashes_for_piece(index) myohlist.sort() self.assert_(len(hisohlist) == len(myohlist)) for i in range(0,len(hisohlist)): hisoh = hisohlist[i] myoh = myohlist[i] self.assert_(hisoh == myoh) else: self.assert_(len(hisohlist) == 0) mylength = self.get_chunk_length(index,begin) mychunk = self.read_chunk(index,begin,mylength) self.assert_(hischunk == mychunk) return index == self.numpieces-1 and mylength != 2 ** 14 def check_bitfield(self,data): self.assert_(data[0] == BITFIELD) bitmap = data[1:] self.assert_(len(bitmap) == 1) # Must have set_breakup_seed_bitfield() set to False self.assert_(bitmap == '\xc0') # # Bad EXTEND handshake message # def subtest_bad_hashpiece(self,oldstyle): if not oldstyle: # Test becomes equivalent to BT keep alive message (len 0, payload '') self._test_bad(self.create_empty,oldstyle) self._test_bad(self.create_ext_id_not_byte,oldstyle) self._test_bad(self.create_not_hashpiece,oldstyle) self._test_bad(self.create_not_index,oldstyle) self._test_bad(self.create_not_begin,oldstyle) self._test_bad(self.create_not_len_bohlist,oldstyle) self._test_bad(self.create_ohlist_not_bdecodable,oldstyle) self._test_bad(self.create_ohlist_wrong_no_hashes,oldstyle) self._test_bad(self.create_ohlist_wrong_no_root_hash,oldstyle) self._test_bad(self.create_ohlist_wrong_bad_offset,oldstyle) self._test_bad(self.create_ohlist_wrong_bad_hash,oldstyle) # TODO: need working peer kicking for that ##self._test_bad(self.create_bad_chunk,oldstyle) # # Main test code for bad EXTEND handshake messages # def _test_bad(self,msg_gen_func,oldstyle): print >>sys.stderr,"test: test_BAD: Create EXTEND HS",`msg_gen_func`,oldstyle if oldstyle: options = None exthsmsg = self.create_good_tribler_extend_hs() else: options = '\x00\x00\x00\x00\x00\x10\x00\x00' exthsmsg = self.create_good_nontribler_extend_hs() s = BTConnection('localhost',self.hisport,user_option_pattern=options,user_infohash=self.infohash) s.send(exthsmsg) s.read_handshake_medium_rare() # Tribler should send an EXTEND message back try: print >>sys.stderr,"test: Waiting for reply" s.s.settimeout(10.0) resp = s.recv() self.assert_(len(resp) > 0) print >>sys.stderr,"test: Got reply",getMessageName(resp[0]) self.assert_(resp[0] == EXTEND) self.check_tribler_extend_hs(resp[1:]) # 1. Pretend we're seeder: send BITFIELD and UNCHOKE msg = BITFIELD+self.seederbitfieldstr s.send(msg) msg = UNCHOKE s.send(msg) print >>sys.stderr,"test: Pretend we are seeder" while True: resp = s.recv() self.assert_(len(resp) > 0) print >>sys.stderr,"test: Got reply 2",getMessageName(resp[0]) self.assert_(resp[0] == REQUEST or resp[0] == INTERESTED or resp[0] == UNCHOKE or resp[0] == HAVE or resp[0] == NOT_INTERESTED) if resp[0] == REQUEST: chunkid = self.check_request(resp) # 2. Reply to REQUEST with *bad* HASHPIECE msg = msg_gen_func(chunkid) if oldstyle: if len(msg) == 1: msg = '' else: msg = msg[1:] # Strip EXTEND byte s.send(msg) break #s.close() except socket.timeout: print >> sys.stderr,"test: Timeout, bad, peer didn't reply in time" self.assert_(False) time.sleep(3) # Should have closed the connection try: s.send(UNCHOKE) self.assert_(False) except: print_exc() s.close() # # Bad message creators (all create Merkle BEP style, I strip first byte # later for oldstyle # def create_empty(self,chunkid): return EXTEND def create_ext_id_not_byte(self,chunkid): return EXTEND+'Hallo kijkbuiskinderen' def create_not_hashpiece(self,chunkid): index,begin,length = chunkid ohlist = [] bohlist = bencode(ohlist) chunk = self.read_chunk(index,begin,length) payload = tobinary(index)+tobinary(begin)+tobinary(len(bohlist))+bohlist+chunk return EXTEND+chr(231)+payload def create_not_index(self,chunkid): payload = 'bla' return EXTEND+HASHPIECE+payload def create_not_begin(self,chunkid): index,begin,length = chunkid payload = tobinary(index)+'bla' return EXTEND+HASHPIECE+payload def create_not_len_bohlist(self,chunkid): index,begin,length = chunkid payload = tobinary(index)+tobinary(begin)+'bla' return EXTEND+HASHPIECE+payload def create_ohlist_not_bdecodable(self,chunkid): index,begin,length = chunkid bohlist = 'bla' chunk = '*' * (2 ** 14) payload = tobinary(index)+tobinary(begin)+tobinary(len(bohlist))+bohlist+chunk return EXTEND+HASHPIECE+payload def create_ohlist_wrong_no_hashes(self,chunkid): index,begin,length = chunkid ohlist = [ (0,'#' * 20),(1,'$' * 20)] # should contain 3 for file2.wmv: own, sibling and root bohlist = bencode(ohlist) chunk = '*' * (2 ** 14) payload = tobinary(index)+tobinary(begin)+tobinary(len(bohlist))+bohlist+chunk return EXTEND+HASHPIECE+payload def create_ohlist_wrong_no_root_hash(self,chunkid): index,begin,length = chunkid ohlist = self.tree.get_hashes_for_piece(index) newohlist = [] # Remove root hash for oh in ohlist: if oh[0] != 0: newohlist.append(oh) ohlist = newohlist bohlist = bencode(ohlist) chunk = self.read_chunk(index,begin,length) payload = tobinary(index)+tobinary(begin)+tobinary(len(bohlist))+bohlist+chunk return EXTEND+HASHPIECE+payload def create_ohlist_wrong_bad_offset(self,chunkid): index,begin,length = chunkid ohlist = self.tree.get_hashes_for_piece(index) ohlist[1][0] = 481 bohlist = bencode(ohlist) chunk = self.read_chunk(index,begin,length) payload = tobinary(index)+tobinary(begin)+tobinary(len(bohlist))+bohlist+chunk return EXTEND+HASHPIECE+payload def create_ohlist_wrong_bad_hash(self,chunkid): index,begin,length = chunkid ohlist = self.tree.get_hashes_for_piece(index) ohlist[1][1] = '$' * 20 bohlist = bencode(ohlist) chunk = self.read_chunk(index,begin,length) payload = tobinary(index)+tobinary(begin)+tobinary(len(bohlist))+bohlist+chunk return EXTEND+HASHPIECE+payload def create_bad_chunk(self,chunkid): index,begin,length = chunkid ohlist = self.tree.get_hashes_for_piece(index) bohlist = bencode(ohlist) chunk = '*' * length payload = tobinary(index)+tobinary(begin)+tobinary(len(bohlist))+bohlist+chunk return EXTEND+HASHPIECE+payload
class TestQueryReplyActive(TestAsServer): """ Testing QUERY_REPLY message of Query extension V1 This test checks how the Tribler code responds to good and bad QUERY_REPLY messages. I.e. the Tribler client initiates the dialback by connecting to us and sending a QUERY and we reply with good and bad messages. This test allows authoritative answers from superpeers. WARNING: Each of the test_ methods should be tested by running the TestCase in a separate Python interpreter to prevent problems with our singleton classes, e.g. SuperPeerDB, etc. """ def setUpPreSession(self): """ override TestAsServer """ print >> sys.stderr, "test: Pre Tribler Init" TestAsServer.setUpPreSession(self) print >> sys.stderr, "test: Pre Tribler Init: config_path", self.config_path # Enable remote querying self.config.set_remote_query(True) def setUpPostSession(self): """ override TestAsServer """ TestAsServer.setUpPostSession(self) self.hispermid = str(self.his_keypair.pub().get_der()) self.my_permid = str(self.my_keypair.pub().get_der()) self.content_name = 'Hallo S22E44' self.tdef = TorrentDef() self.tdef.set_tracker('http://localhost:0/announce') self.tdef.set_piece_length(2**15) self.tdef.create_live(self.content_name, 2**16) self.tdef.finalize() def pretest_simple(self): self.pretest_q('SIMPLE hallo') def pretest_simpleplustorrents(self): self.pretest_q('SIMPLE+METADATA hallo') def pretest_q(self, query): # 1. First connect to Tribler self.openconn = OLConnection(self.my_keypair, 'localhost', self.hisport) sleep(3) # 2. Make Tribler send query self.query = query self.session.query_connected_peers(query, self.query_usercallback, max_peers_to_query=10) def query_usercallback(self, permid, query, hits): print >> sys.stderr, "test: query_usercallback:", ` permid `, ` query `, ` hits ` self.assert_(query == self.query) self.assert_(permid == self.my_permid) self.check_good_qreply(hits) # TODO: if SIMPLE+METADATA: check torrent now in db. # # Good SIMPLE QUERY, builds on TestQueryReply code # def singtest_good_simple_reply(self): self.pretest_simple() self._test_qreply(self.create_good_simple_reply, True) # # Good SIMPLE+METADATA QUERY, builds on TestQueryReply code # def singtest_good_simpleplustorrents_reply(self): self.pretest_simpleplustorrents() self._test_qreply(self.create_good_simpleplustorrents_reply, True) # # Bad QUERY, builds on TestQueryReply code # def singtest_bad_not_bdecodable(self): self.pretest_simple() self._test_qreply(self.create_not_bdecodable, False) # # Bad SIMPLE+METADATA QUERY, builds on TestQueryReply code # def singtest_bad_not_bdecodable_torrentfile(self): self.pretest_simpleplustorrents() self._test_qreply(self.create_not_bdecodable_torrentfile, False) ### TODO: send different valid answers so consensus not reached # # Main test code # def _test_qreply(self, gen_qreply, good): print >> sys.stderr, "test: waiting for reply" s = self.openconn msg = s.recv() self.assert_(len(msg) > 0) print >> sys.stderr, "test: Received overlay message", getMessageName( msg[0]) self.assert_(msg[0] == QUERY) id = self.check_rquery(msg[1:]) resp = gen_qreply(id) print >> sys.stderr, "test: sending QUERY_REPLY" s.send(resp) if good: time.sleep(10) # the other side should not have closed the connection, as # this is all valid, so this should not throw an exception: s.send('bla') s.close() else: # the other side should not like this and close the connection self.assert_(len(s.recv()) == 0) s.close() def create_good_simple_reply_dict(self, id): r = {} r['content_name'] = self.content_name r['length'] = LENGTH r['leecher'] = LEECHERS r['seeder'] = SEEDERS r['category'] = CATEGORY # OLPROTO_PROTO_ELEVENTH # set later r['torrent_size'] = 42 r['channel_permid'] = '$' * 83 r['channel_name'] = 'Nitin Channel' d2 = {} d2[self.tdef.get_infohash()] = r d = {} d['id'] = id d['a'] = d2 return d def create_good_simple_reply(self, id): d = self.create_good_simple_reply_dict(id) bmetainfo = bencode(self.tdef.get_metainfo()) d['a'][self.tdef.get_infohash()]['torrent_size'] = len(bmetainfo) b = bencode(d) return QUERY_REPLY + b def create_good_simpleplustorrents_reply(self, id): d = self.create_good_simple_reply_dict(id) bmetainfo = bencode(self.tdef.get_metainfo()) d['a'][self.tdef.get_infohash()]['torrent_size'] = len(bmetainfo) d['a'][self.tdef.get_infohash( )]['metatype'] = 'application/x-tribler-stream' d['a'][self.tdef.get_infohash()]['metadata'] = bmetainfo b = bencode(d) return QUERY_REPLY + b def check_good_qreply(self, hits): self.assert_(len(hits) == 1) self.assert_(hits.keys()[0] == self.tdef.get_infohash()) hit = hits[self.tdef.get_infohash()] self.assert_(hit['content_name'] == self.content_name) self.assert_(hit['length'] == LENGTH) self.assert_(hit['leecher'] == LEECHERS) self.assert_(hit['seeder'] == SEEDERS) self.assert_(hit['category'] == CATEGORY) # OLPROTO_VERSION_ELEVENTH bmetainfo = bencode(self.tdef.get_metainfo()) self.assert_(hit['torrent_size'] == len(bmetainfo)) if self.query.startswith('SIMPLE+METADATA'): self.assert_(hit['metadata'] == bmetainfo) def create_not_bdecodable(self, id): return QUERY_REPLY + "bla" def create_not_bdecodable_torrentfile(self, id): d = self.create_good_simple_reply_dict(id) d['a'][self.tdef.get_infohash( )]['torrent_size'] = 3 # consistent with metadata. Should be named "metasize" d['a'][self.tdef.get_infohash()]['metadata'] = 'bla' b = bencode(d) return QUERY_REPLY + b def check_rquery(self, data): d = bdecode(data) self.assert_(type(d) == DictType) self.assert_(d.has_key('q')) q = d['q'] self.assert_(type(q) == StringType) self.assert_(d.has_key('id')) id = d['id'] self.assert_(type(id) == StringType) self.assert_(q == self.query) return d['id']
def test_tdef_has_video_files_expected_true(self): try: tdef = TorrentDef.load("torrents/Locality-Demo.mp4.tstream") self.assertTrue(has_torrent_video_files(tdef)) except: self.fail("There was some error while loading the torrent file.")
class TestMetadata(TestAsServer, MagnetHelpers): """ Once we are downloading a torrent, our client should respond to the ut_metadata extention message. This allows other clients to obtain the info part of the metadata from us. """ def setUp(self): """ override TestAsServer """ TestAsServer.setUp(self) print >>sys.stderr,time.asctime(),'-', "test: Giving MyLaunchMany time to startup" time.sleep(5) print >>sys.stderr,time.asctime(),'-', "test: MyLaunchMany should have started up" # the metadata that we want to transfer self.tdef = TorrentDef() self.tdef.add_content(os.path.join(os.getcwd(), "API", "file.wmv")) self.tdef.set_tracker(self.session.get_internal_tracker_url()) # we use a small piece length to obtain multiple pieces self.tdef.set_piece_length(1) self.tdef.finalize() # self.tdef.save(os.path.join(self.session.get_state_dir(), "gen.torrent")) MagnetHelpers.__init__(self, self.tdef) def setup_seeder(self): self.seeder_setup_complete = False self.seeder_teardown_complete = False self.seeder_teardown = False self.dscfg = DownloadStartupConfig() self.dscfg.set_dest_dir(os.getcwd()) self.download = self.session.start_download(self.tdef, self.dscfg) self.download.set_state_callback(self.seeder_state_callback) counter = 0 while not self.seeder_setup_complete: counter += 1 time.sleep(1) assert counter < 30, "timeout" print >> sys.stderr, time.asctime(),'-', "test: setup_seeder() complete" def teardown_seeder(self): self.seeder_teardown_complete = False self.session.remove_download(self.download) counter = 0 while not self.seeder_setup_complete: counter += 1 time.sleep(1) assert counter < 30, "timeout" print >> sys.stderr, time.asctime(),'-', "test: teardown_seeder() complete" def seeder_state_callback(self,ds): assert not self.seeder_teardown_complete self.seeder_setup_complete = (ds.get_status() == DLSTATUS_DOWNLOADING) d = ds.get_download() print >> sys.stderr, time.asctime(),'-', "test: seeder:", `d.get_def().get_name()`, dlstatus_strings[ds.get_status()], ds.get_progress() if self.seeder_teardown: self.seeder_teardown_complete = True else: return (1.0, False) def test_all(self): self.setup_seeder() try: self.subtest_good_flood() finally: self.teardown_seeder() self.setup_seeder() try: self.subtest_good_request() self.subtest_bad_request() finally: self.teardown_seeder() def subtest_good_request(self): conn = BTConnection("localhost", self.hisport, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) # request metadata block 0, 2, 3, and the last conn.send(self.create_good_extend_metadata_request(metadata_id, 0)) conn.send(self.create_good_extend_metadata_request(metadata_id, 2)) conn.send(self.create_good_extend_metadata_request(metadata_id, 3)) conn.send(self.create_good_extend_metadata_request(metadata_id, len(self.metadata_list) - 1)) self.read_extend_metadata_reply(conn, 0) self.read_extend_metadata_reply(conn, 2) self.read_extend_metadata_reply(conn, 3) self.read_extend_metadata_reply(conn, len(self.metadata_list) - 1) def subtest_good_flood(self): conn = BTConnection("localhost", self.hisport, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) for counter in xrange(len(self.metadata_list) * 2): piece = counter % len(self.metadata_list) conn.send(self.create_good_extend_metadata_request(metadata_id, piece)) if counter > len(self.metadata_list): self.read_extend_metadata_reject(conn, piece) else: self.read_extend_metadata_reply(conn, piece) def subtest_bad_request(self): self.bad_request_and_disconnect({"msg_type":0, "piece":len(self.metadata_list)}) self.bad_request_and_disconnect({"msg_type":0, "piece":-1}) self.bad_request_and_disconnect({"msg_type":0, "piece":"1"}) self.bad_request_and_disconnect({"msg_type":0, "piece":[1,2]}) self.bad_request_and_disconnect({"msg_type":0, "PIECE":1}) def bad_request_and_disconnect(self, payload): conn = BTConnection("localhost", self.hisport, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) conn.send(EXTEND + chr(metadata_id) + bencode(payload)) self.read_extend_metadata_close(conn)
def parsedir(directory, parsed, files, blocked, exts = ['.torrent', TRIBLER_TORRENT_EXT], return_metainfo = False, errfunc = _errfunc): if DEBUG: errfunc('checking dir') dirs_to_check = [directory] new_files = {} new_blocked = {} torrent_type = {} while dirs_to_check: # first, recurse directories and gather torrents directory = dirs_to_check.pop() newtorrents = False for f in os.listdir(directory): newtorrent = None for ext in exts: if f.endswith(ext): newtorrent = ext[1:] break if newtorrent: newtorrents = True p = os.path.join(directory, f) new_files[p] = [(int(os.path.getmtime(p)), os.path.getsize(p)), 0] torrent_type[p] = newtorrent if not newtorrents: for f in os.listdir(directory): p = os.path.join(directory, f) if os.path.isdir(p): dirs_to_check.append(p) new_parsed = {} to_add = [] added = {} removed = {} # files[path] = [(modification_time, size), hash], hash is 0 if the file # has not been successfully parsed for p, v in new_files.items(): # re-add old items and check for changes oldval = files.get(p) if not oldval: # new file to_add.append(p) continue h = oldval[1] if oldval[0] == v[0]: # file is unchanged from last parse if h: if blocked.has_key(p): # parseable + blocked means duplicate to_add.append(p) # other duplicate may have gone away else: new_parsed[h] = parsed[h] new_files[p] = oldval else: new_blocked[p] = 1 # same broken unparseable file continue if parsed.has_key(h) and not blocked.has_key(p): if DEBUG: errfunc('removing '+p+' (will re-add)') removed[h] = parsed[h] to_add.append(p) to_add.sort() for p in to_add: # then, parse new and changed torrents new_file = new_files[p] v, h = new_file if new_parsed.has_key(h): # duplicate if not blocked.has_key(p) or files[p][0] != v: errfunc('**warning** '+ p +' is a duplicate torrent for '+new_parsed[h]['path']) new_blocked[p] = 1 continue if DEBUG: errfunc('adding '+p) try: # Arno: P2PURL tdef = TorrentDef.load(p) h = tdef.get_infohash() d = tdef.get_metainfo() new_file[1] = h if new_parsed.has_key(h): errfunc('**warning** '+ p +' is a duplicate torrent for '+new_parsed[h]['path']) new_blocked[p] = 1 continue a = {} a['path'] = p f = os.path.basename(p) a['file'] = f a['type'] = torrent_type[p] if tdef.get_url_compat(): a['url'] = tdef.get_url() i = d['info'] l = 0 nf = 0 if i.has_key('length'): l = i.get('length', 0) nf = 1 elif i.has_key('files'): for li in i['files']: nf += 1 if li.has_key('length'): l += li['length'] a['numfiles'] = nf a['length'] = l a['name'] = i.get('name', f) def setkey(k, d = d, a = a): if d.has_key(k): a[k] = d[k] setkey('failure reason') setkey('warning message') setkey('announce-list') # Arno, LOOKUP SERVICE if tdef.get_urllist() is not None: httpseedhashes = [] for url in tdef.get_urllist(): # TODO: normalize? urlhash = sha(url).digest() httpseedhashes.append(urlhash) a['url-hash-list'] = httpseedhashes if return_metainfo: a['metainfo'] = d except: print_exc() errfunc('**warning** '+p+' has errors') new_blocked[p] = 1 continue if DEBUG: errfunc('... successful') new_parsed[h] = a added[h] = a for p, v in files.items(): # and finally, mark removed torrents if not new_files.has_key(p) and not blocked.has_key(p): if DEBUG: errfunc('removing '+p) removed[v[1]] = parsed[v[1]] if DEBUG: errfunc('done checking') return (new_parsed, new_files, new_blocked, added, removed)
scfg = SessionStartupConfig() scfg.set_state_dir(tempfile.mkdtemp()) scfg.set_listen_port(options.port) scfg.set_overlay(False) scfg.set_megacache(False) scfg.set_upnp_mode(simpledefs.UPNPMODE_DISABLED) scfg.set_dialback(False) scfg.set_social_networking(False) scfg.set_buddycast(False) scfg.set_crawler(False) scfg.set_internal_tracker(False) s = Session(scfg) tdef = TorrentDef.load(__TORRENT_FILE__) # tdef.get_tracker() returns the announce-url; we must omit the "announce" part tracker_url = tdef.get_tracker()[:tdef.get_tracker().find("announce")] if tdef.get_bitrate() == None: print >>sys.stderr, "Provided torrent file has no bitrate information. Exiting." sys.exit(1) BITRATE = tdef.get_bitrate() print >>sys.stderr, "Calculated bitrate is %d" % BITRATE client_stats['video_duration'] = int(tdef.get_length() / tdef.get_bitrate()) if not os.access(options.directory, os.F_OK): os.makedirs(options.directory) dscfg = DownloadStartupConfig()
class TestQueryReplyActive(TestAsServer): """ Testing QUERY_REPLY message of Query extension V1 This test checks how the Tribler code responds to good and bad QUERY_REPLY messages. I.e. the Tribler client initiates the dialback by connecting to us and sending a QUERY and we reply with good and bad messages. This test allows authoritative answers from superpeers. WARNING: Each of the test_ methods should be tested by running the TestCase in a separate Python interpreter to prevent problems with our singleton classes, e.g. SuperPeerDB, etc. """ def setUpPreSession(self): """ override TestAsServer """ print >> sys.stderr,"test: Pre Tribler Init" TestAsServer.setUpPreSession(self) print >> sys.stderr,"test: Pre Tribler Init: config_path",self.config_path # Enable remote querying self.config.set_remote_query(True) def setUpPostSession(self): """ override TestAsServer """ TestAsServer.setUpPostSession(self) self.hispermid = str(self.his_keypair.pub().get_der()) self.my_permid = str(self.my_keypair.pub().get_der()) self.content_name = 'Hallo S22E44' self.tdef = TorrentDef() self.tdef.set_tracker('http://localhost:0/announce') self.tdef.set_piece_length(2 ** 15) self.tdef.create_live(self.content_name,2 ** 16) self.tdef.finalize() def pretest_simple(self): self.pretest_q('SIMPLE hallo') def pretest_simpleplustorrents(self): self.pretest_q('SIMPLE+METADATA hallo') def pretest_q(self,query): # 1. First connect to Tribler self.openconn = OLConnection(self.my_keypair,'localhost',self.hisport) sleep(3) # 2. Make Tribler send query self.query = query self.session.query_connected_peers(query,self.query_usercallback,max_peers_to_query=10) def query_usercallback(self,permid,query,hits): print >>sys.stderr,"test: query_usercallback:",`permid`,`query`,`hits` self.assert_(query == self.query) self.assert_(permid == self.my_permid) self.check_good_qreply(hits) # TODO: if SIMPLE+METADATA: check torrent now in db. # # Good SIMPLE QUERY, builds on TestQueryReply code # def singtest_good_simple_reply(self): self.pretest_simple() self._test_qreply(self.create_good_simple_reply,True) # # Good SIMPLE+METADATA QUERY, builds on TestQueryReply code # def singtest_good_simpleplustorrents_reply(self): self.pretest_simpleplustorrents() self._test_qreply(self.create_good_simpleplustorrents_reply,True) # # Bad QUERY, builds on TestQueryReply code # def singtest_bad_not_bdecodable(self): self.pretest_simple() self._test_qreply(self.create_not_bdecodable,False) # # Bad SIMPLE+METADATA QUERY, builds on TestQueryReply code # def singtest_bad_not_bdecodable_torrentfile(self): self.pretest_simpleplustorrents() self._test_qreply(self.create_not_bdecodable_torrentfile,False) ### TODO: send different valid answers so consensus not reached # # Main test code # def _test_qreply(self,gen_qreply,good): print >> sys.stderr,"test: waiting for reply" s = self.openconn msg = s.recv() self.assert_(len(msg) > 0) print >> sys.stderr,"test: Received overlay message",getMessageName(msg[0]) self.assert_(msg[0] == QUERY) id = self.check_rquery(msg[1:]) resp = gen_qreply(id) print >> sys.stderr,"test: sending QUERY_REPLY" s.send(resp) if good: time.sleep(10) # the other side should not have closed the connection, as # this is all valid, so this should not throw an exception: s.send('bla') s.close() else: # the other side should not like this and close the connection self.assert_(len(s.recv())==0) s.close() def create_good_simple_reply_dict(self,id): r = {} r['content_name'] = self.content_name r['length'] = LENGTH r['leecher'] = LEECHERS r['seeder'] = SEEDERS r['category'] = CATEGORY # OLPROTO_PROTO_ELEVENTH # set later r['torrent_size'] = 42 r['channel_permid'] = '$' * 83 r['channel_name'] = 'Nitin Channel' d2 = {} d2[self.tdef.get_infohash()] = r d = {} d['id'] = id d['a'] = d2 return d def create_good_simple_reply(self,id): d = self.create_good_simple_reply_dict(id) bmetainfo = bencode(self.tdef.get_metainfo()) d['a'][self.tdef.get_infohash()]['torrent_size'] = len(bmetainfo) b = bencode(d) return QUERY_REPLY+b def create_good_simpleplustorrents_reply(self,id): d = self.create_good_simple_reply_dict(id) bmetainfo = bencode(self.tdef.get_metainfo()) d['a'][self.tdef.get_infohash()]['torrent_size'] = len(bmetainfo) d['a'][self.tdef.get_infohash()]['metatype'] = 'application/x-tribler-stream' d['a'][self.tdef.get_infohash()]['metadata'] = bmetainfo b = bencode(d) return QUERY_REPLY+b def check_good_qreply(self,hits): self.assert_(len(hits) == 1) self.assert_(hits.keys()[0] == self.tdef.get_infohash()) hit = hits[self.tdef.get_infohash()] self.assert_(hit['content_name'] == self.content_name) self.assert_(hit['length'] == LENGTH) self.assert_(hit['leecher'] == LEECHERS) self.assert_(hit['seeder'] == SEEDERS) self.assert_(hit['category'] == CATEGORY) # OLPROTO_VERSION_ELEVENTH bmetainfo = bencode(self.tdef.get_metainfo()) self.assert_(hit['torrent_size'] == len(bmetainfo)) if self.query.startswith('SIMPLE+METADATA'): self.assert_(hit['metadata'] == bmetainfo) def create_not_bdecodable(self,id): return QUERY_REPLY+"bla" def create_not_bdecodable_torrentfile(self,id): d = self.create_good_simple_reply_dict(id) d['a'][self.tdef.get_infohash()]['torrent_size'] = 3 # consistent with metadata. Should be named "metasize" d['a'][self.tdef.get_infohash()]['metadata'] = 'bla' b = bencode(d) return QUERY_REPLY+b def check_rquery(self,data): d = bdecode(data) self.assert_(type(d) == DictType) self.assert_(d.has_key('q')) q = d['q'] self.assert_(type(q) == StringType) self.assert_(d.has_key('id')) id = d['id'] self.assert_(type(id) == StringType) self.assert_(q == self.query) return d['id']
def recv_query_reply(self,permid,message,selversion): #print "****** recv query reply", len(message) if selversion < OLPROTO_VER_SIXTH: return False #if len(message) > MAX_QUERY_REPLY_LEN: # return True # don't close # Unpack try: d = bdecode(message[1:]) except: if DEBUG: print >>sys.stderr,"rquery: Cannot bdecode QUERY_REPLY message", selversion return False if not isValidQueryReply(d,selversion): if DEBUG: print >>sys.stderr,"rquery: not valid QUERY_REPLY message", selversion return False # Check auth queryrec = self.is_registered_query_id(d['id']) if not queryrec: if DEBUG: print >>sys.stderr,"rquery: QUERY_REPLY has unknown query ID", selversion return False if selversion >= OLPROTO_VER_TWELFTH: if queryrec['query'].startswith('SIMPLE+METADATA'): for infohash,torrentrec in d['a'].iteritems(): if not 'metatype' in torrentrec: if DEBUG: print >>sys.stderr,"rquery: QUERY_REPLY has no metatype field", selversion return False if not 'metadata' in torrentrec: if DEBUG: print >>sys.stderr,"rquery: QUERY_REPLY has no metadata field", selversion return False if torrentrec['torrent_size'] != len(torrentrec['metadata']): if DEBUG: print >>sys.stderr,"rquery: QUERY_REPLY torrent_size != len metadata", selversion return False try: # Validity test if torrentrec['metatype'] == URL_MIME_TYPE: tdef = TorrentDef.load_from_url(torrentrec['metadata']) else: metainfo = bdecode(torrentrec['metadata']) tdef = TorrentDef.load_from_dict(metainfo) except: if DEBUG: print_exc() return False # Process self.process_query_reply(permid,queryrec['query'],queryrec['usercallback'],d) return True
def parsedir(directory, parsed, files, blocked, exts=['.torrent', TRIBLER_TORRENT_EXT], return_metainfo=False, errfunc=_errfunc): if DEBUG: errfunc('checking dir') dirs_to_check = [directory] new_files = {} new_blocked = {} torrent_type = {} while dirs_to_check: # first, recurse directories and gather torrents directory = dirs_to_check.pop() newtorrents = False for f in os.listdir(directory): newtorrent = None for ext in exts: if f.endswith(ext): newtorrent = ext[1:] break if newtorrent: newtorrents = True p = os.path.join(directory, f) new_files[p] = [(int(os.path.getmtime(p)), os.path.getsize(p)), 0] torrent_type[p] = newtorrent if not newtorrents: for f in os.listdir(directory): p = os.path.join(directory, f) if os.path.isdir(p): dirs_to_check.append(p) new_parsed = {} to_add = [] added = {} removed = {} # files[path] = [(modification_time, size), hash], hash is 0 if the file # has not been successfully parsed for p, v in new_files.items(): # re-add old items and check for changes oldval = files.get(p) if not oldval: # new file to_add.append(p) continue h = oldval[1] if oldval[0] == v[0]: # file is unchanged from last parse if h: if blocked.has_key(p): # parseable + blocked means duplicate to_add.append(p) # other duplicate may have gone away else: new_parsed[h] = parsed[h] new_files[p] = oldval else: new_blocked[p] = 1 # same broken unparseable file continue if parsed.has_key(h) and not blocked.has_key(p): if DEBUG: errfunc('removing ' + p + ' (will re-add)') removed[h] = parsed[h] to_add.append(p) to_add.sort() for p in to_add: # then, parse new and changed torrents new_file = new_files[p] v, h = new_file if new_parsed.has_key(h): # duplicate if not blocked.has_key(p) or files[p][0] != v: errfunc('**warning** ' + p + ' is a duplicate torrent for ' + new_parsed[h]['path']) new_blocked[p] = 1 continue if DEBUG: errfunc('adding ' + p) try: # Arno: P2PURL tdef = TorrentDef.load(p) h = tdef.get_infohash() d = tdef.get_metainfo() new_file[1] = h if new_parsed.has_key(h): errfunc('**warning** ' + p + ' is a duplicate torrent for ' + new_parsed[h]['path']) new_blocked[p] = 1 continue a = {} a['path'] = p f = os.path.basename(p) a['file'] = f a['type'] = torrent_type[p] if tdef.get_url_compat(): a['url'] = tdef.get_url() i = d['info'] l = 0 nf = 0 if i.has_key('length'): l = i.get('length', 0) nf = 1 elif i.has_key('files'): for li in i['files']: nf += 1 if li.has_key('length'): l += li['length'] a['numfiles'] = nf a['length'] = l a['name'] = i.get('name', f) def setkey(k, d=d, a=a): if d.has_key(k): a[k] = d[k] setkey('failure reason') setkey('warning message') setkey('announce-list') if return_metainfo: a['metainfo'] = d except: errfunc('**warning** ' + p + ' has errors') new_blocked[p] = 1 continue if DEBUG: errfunc('... successful') new_parsed[h] = a added[h] = a for p, v in files.items(): # and finally, mark removed torrents if not new_files.has_key(p) and not blocked.has_key(p): if DEBUG: errfunc('removing ' + p) removed[v[1]] = parsed[v[1]] if DEBUG: errfunc('done checking') return (new_parsed, new_files, new_blocked, added, removed)
def create_torrent(file, port, try_vod=True): ''' Creates a torrent for the given file. Returns a string representing the path to the torrent file. If try_vod is True try to add the playtime to the torrent making it "streamable". ''' # generate torrent tdef = TorrentDef() url = "http://localhost:%s/announce/" % port tdef.set_tracker(url) #tdef.set_create_merkle_torrent(True) #tdef.set_piece_length(self._pieceSize) if try_vod: torrent_name = FileUtils.get_relative_filename( file) + constants.TORRENT_VOD_EXT # set playtime playtime = videostats_via_ffmpeg(file)['playtime'] print "playtime", playtime else: torrent_name = FileUtils.get_relative_filename( file) + constants.TORRENT_DOWNLOAD_EXT playtime = None print "Create a non-streamable torrent" tdef.add_content(file, playtime=playtime) tdef.finalize() torrent_dir = os.getcwd() torrent = os.path.join(torrent_dir, torrent_name) tdef.save(torrent) print "created torrent file: %s" % torrent print "Tracker uses the announce URL: %s" % tdef.get_tracker() return tdef