def test_add_content_file_and_copy(self): """ Add a single file to a TorrentDef """ t = TorrentDef() fn = os.path.join(TESTS_DATA_DIR, self.VIDEO_FILE_NAME) t.add_content(fn) t.set_tracker(TRACKER) t.finalize() s = os.path.getsize(fn) metainfo = t.get_metainfo() self.general_check(metainfo) self.assertEqual(metainfo['info']['name'], self.VIDEO_FILE_NAME) self.assertEqual(metainfo['info']['length'], s) self.assertTrue(t.get_pieces()) self.assertEqual(len(t.get_infohash()), INFOHASH_LENGTH) self.assertTrue(t.get_name()) # test copy constructor nt = TorrentDef(t.input, t.metainfo, t.infohash) self.assertEqual(nt.input, t.input) self.assertEqual(nt.metainfo, t.metainfo) self.assertEqual(nt.infohash, t.infohash) # test removing content nt.remove_content("/test123") self.assertEqual(len(nt.input['files']), 1) nt.remove_content(unicode(fn)) self.assertEqual(len(nt.input['files']), 0) nt.remove_content(unicode(fn))
def test_add_content_dir(self): """ Add a single dir to a TorrentDef """ t = TorrentDef() dn = os.path.join(TESTS_DATA_DIR, "contentdir") t.add_content(dn, "dirintorrent") t.set_tracker(TRACKER) t.finalize() exps = 0 for f in os.listdir(dn): if f.startswith('.'): continue p = os.path.join(dn, f) s = os.path.getsize(p) exps += s self._logger.debug("Expected size %s %d", f, s) self._logger.debug("Expected total size of files in torrent %d", exps) metainfo = t.get_metainfo() self.general_check(metainfo) self.assertEqual(metainfo['info']['name'], 'dirintorrent') reals = 0 for file in metainfo['info']['files']: s = file['length'] self._logger.debug("real size %s %d", file['path'], s) reals += s self._logger.debug("Real size of files in torrent %d", reals) self.assertEqual(exps, reals)
def test_add_content_dir_and_file(self): """ Add a single dir and single file to a TorrentDef """ t = TorrentDef() dn = os.path.join(TESTS_DATA_DIR, "contentdir") t.add_content(dn, "dirintorrent") fn = os.path.join(TESTS_DATA_DIR, self.VIDEO_FILE_NAME) t.add_content(fn, os.path.join("dirintorrent", self.VIDEO_FILE_NAME)) t.set_tracker(TRACKER) t.finalize() # Check exps = os.path.getsize(fn) for f in os.listdir(dn): if f.startswith('.'): continue p = os.path.join(dn, f) exps += os.path.getsize(p) metainfo = t.get_metainfo() self.general_check(metainfo) self.assertEqual(metainfo['info']['name'], 'dirintorrent') reals = 0 for file in metainfo['info']['files']: reals += file['length'] self.assertEqual(exps, reals)
def setupSeeder(self): from Tribler.Core.Session import Session from Tribler.Core.TorrentDef import TorrentDef from Tribler.Core.DownloadConfig import DownloadStartupConfig self.setUpPreSession() self.config.set_libtorrent(True) self.config2 = self.config.copy() self.session2 = Session(self.config2, ignore_singleton=True) upgrader = self.session2.prestart() while not upgrader.is_done: time.sleep(0.1) assert not upgrader.failed, upgrader.current_status self.session2.start() tdef = TorrentDef() tdef.add_content(os.path.join(TESTS_DATA_DIR, "video.avi")) tdef.set_tracker("http://fake.net/announce") tdef.finalize() torrentfn = os.path.join(self.session.get_state_dir(), "gen.torrent") tdef.save(torrentfn) dscfg = DownloadStartupConfig() dscfg.set_dest_dir(TESTS_DATA_DIR) # basedir of the file we are seeding d = self.session2.start_download(tdef, dscfg) d.set_state_callback(self.seeder_state_callback) return torrentfn
def setup_tunnel_seeder(self, hops): """ Setup the seeder. """ from Tribler.Core.Session import Session self.seed_config = self.config.copy() self.seed_config.set_state_dir(self.getStateDir(2)) self.seed_config.set_megacache_enabled(True) self.seed_config.set_tunnel_community_socks5_listen_ports(self.get_socks5_ports()) if self.session2 is None: self.session2 = Session(self.seed_config) self.session2.start() tdef = TorrentDef() tdef.add_content(os.path.join(TESTS_DATA_DIR, "video.avi")) tdef.set_tracker("http://localhost/announce") tdef.finalize() torrentfn = os.path.join(self.session2.config.get_state_dir(), "gen.torrent") tdef.save(torrentfn) self.seed_tdef = tdef if hops > 0: # Safe seeding enabled self.tunnel_community_seeder = self.load_tunnel_community_in_session(self.session2) self.tunnel_community_seeder.build_tunnels(hops) else: self.sanitize_network(self.session2) dscfg = DownloadStartupConfig() dscfg.set_dest_dir(TESTS_DATA_DIR) # basedir of the file we are seeding dscfg.set_hops(hops) d = self.session2.start_download_from_tdef(tdef, dscfg) d.set_state_callback(self.seeder_state_callback)
def subtest_add_content_dir_and_file(self): """ Add a single dir and single file to a TorrentDef """ t = TorrentDef() dn = os.path.join(TESTS_API_DIR, "contentdir") t.add_content(dn, "dirintorrent") fn = os.path.join(TESTS_API_DIR, "video.avi") t.add_content(fn, os.path.join("dirintorrent", "video.avi")) t.set_tracker(TRACKER) t.finalize() # Check exps = os.path.getsize(fn) for f in os.listdir(dn): if f.startswith('.'): continue p = os.path.join(dn, f) exps += os.path.getsize(p) metainfo = t.get_metainfo() self.general_check(metainfo) self.assert_(metainfo['info']['name'] == 'dirintorrent') reals = 0 for file in metainfo['info']['files']: reals += file['length'] self.assert_(exps == reals)
def setupSeeder(self): from Tribler.Core.Session import Session from Tribler.Core.TorrentDef import TorrentDef from Tribler.Core.DownloadConfig import DownloadStartupConfig self.setUpPreSession() self.config.set_libtorrent(True) self.config2 = self.config.copy() self.session2 = Session(self.config2, ignore_singleton=True) upgrader = self.session2.prestart() while not upgrader.is_done: time.sleep(0.1) assert not upgrader.failed, upgrader.current_status self.session2.start() tdef = TorrentDef() tdef.add_content(os.path.join(TESTS_DATA_DIR, "video.avi")) tdef.set_tracker("http://fake.net/announce") tdef.finalize() torrentfn = os.path.join(self.session.get_state_dir(), "gen.torrent") tdef.save(torrentfn) dscfg = DownloadStartupConfig() dscfg.set_dest_dir( TESTS_DATA_DIR) # basedir of the file we are seeding d = self.session2.start_download(tdef, dscfg) d.set_state_callback(self.seeder_state_callback) return torrentfn
def test_multifile_torrent(self): tdef = TorrentDef() dn = os.path.join(TESTS_DATA_DIR, "contentdir") tdef.add_content(dn, "dirintorrent") fn = os.path.join(TESTS_DATA_DIR, "video.avi") tdef.add_content(fn, os.path.join("dirintorrent", "video.avi")) tdef.set_tracker("http://tribler.org/announce") tdef.finalize() impl = LibtorrentDownloadImpl(self.session, tdef) # Override the add_torrent because it will be called impl.ltmgr = MockObject() impl.ltmgr.add_torrent = lambda _, _dummy2: fake_handler impl.set_selected_files = lambda: None fake_handler = MockObject() fake_handler.is_valid = lambda: True fake_handler.status = lambda: fake_status fake_handler.set_share_mode = lambda _: None fake_handler.resume = lambda: None fake_handler.resolve_countries = lambda _: None fake_status = MockObject() fake_status.share_mode = False # Create a dummy download config impl.dlconfig = DownloadStartupConfig().dlconfig.copy() # Create a dummy pstate pstate = CallbackConfigParser() pstate.add_section("state") test_dict = dict() test_dict["a"] = "b" pstate.set("state", "engineresumedata", test_dict) return impl.network_create_engine_wrapper(pstate)
def setupSeeder(self, hops=0, session=None): from Tribler.Core.Session import Session from Tribler.Core.TorrentDef import TorrentDef from Tribler.Core.DownloadConfig import DownloadStartupConfig self.setUpPreSession() self.config.set_libtorrent(True) self.config2 = self.config.copy() self.config2.set_state_dir(self.getStateDir(2)) if session is None: self.session2 = Session(self.config2, ignore_singleton=True, autoload_discovery=False) upgrader = self.session2.prestart() while not upgrader.is_done: time.sleep(0.1) self.session2.start() session = self.session2 tdef = TorrentDef() tdef.add_content(os.path.join(TESTS_DATA_DIR, "video.avi")) tdef.set_tracker("http://fake.net/announce") tdef.set_private() # disable dht tdef.finalize() torrentfn = os.path.join(session.get_state_dir(), "gen.torrent") tdef.save(torrentfn) dscfg = DownloadStartupConfig() dscfg.set_dest_dir(TESTS_DATA_DIR) # basedir of the file we are seeding dscfg.set_hops(hops) d = session.start_download(tdef, dscfg) d.set_state_callback(self.seeder_state_callback) return torrentfn
def subtest_add_content_dir(self): """ Add a single dir to a TorrentDef """ t = TorrentDef() dn = os.path.join(TESTS_API_DIR, "contentdir") t.add_content(dn, "dirintorrent") t.set_tracker(TRACKER) t.finalize() exps = 0 for f in os.listdir(dn): if f.startswith('.'): continue p = os.path.join(dn, f) s = os.path.getsize(p) exps += s print "test: expected size", f, s print "test: expected total size of files in torrent", exps metainfo = t.get_metainfo() self.general_check(metainfo) self.assert_(metainfo['info']['name'] == 'dirintorrent') reals = 0 for file in metainfo['info']['files']: s = file['length'] print "test: real size", file['path'], s reals += s print "test: real size of files in torrent", reals self.assert_(exps == reals)
def setup_tunnel_seeder(self, hops): """ Setup the seeder. """ from Tribler.Core.Session import Session self.seed_config = self.config.copy() self.seed_config.set_state_dir(self.getStateDir(2)) self.seed_config.set_megacache_enabled(True) self.seed_config.set_tunnel_community_socks5_listen_ports( self.get_socks5_ports()) if self.session2 is None: self.session2 = Session(self.seed_config, autoload_discovery=False) self.session2.start() tdef = TorrentDef() tdef.add_content(os.path.join(TESTS_DATA_DIR, "video.avi")) tdef.set_tracker("http://localhost/announce") tdef.finalize() torrentfn = os.path.join(self.session2.config.get_state_dir(), "gen.torrent") tdef.save(torrentfn) self.seed_tdef = tdef if hops > 0: # Safe seeding enabled self.tunnel_community_seeder = self.load_tunnel_community_in_session( self.session2) self.tunnel_community_seeder.build_tunnels(hops) dscfg = DownloadStartupConfig() dscfg.set_dest_dir( TESTS_DATA_DIR) # basedir of the file we are seeding dscfg.set_hops(hops) d = self.session2.start_download_from_tdef(tdef, dscfg) d.set_state_callback(self.seeder_state_callback)
def test_get_name_utf8_unknown(self): """ Add a TorrentDef with non-utf8 encoding""" t = TorrentDef() t.set_name('\xA1\xC0') t.set_tracker(TRACKER) t.finalize() self.assertEqual(t.get_name_utf8(), u'\xa1\xc0')
def createTorrent(self): tdef = TorrentDef() tdef.add_content(os.path.join(TESTS_DATA_DIR, "video.avi")) tdef.set_tracker("http://fake.net/announce") tdef.finalize() torrentfn = os.path.join(self.session.get_state_dir(), "gen.torrent") tdef.save(torrentfn) return torrentfn
def subtest_add_content_piece_length(self): """ Add a single file with piece length to a TorrentDef """ t = TorrentDef() fn = os.path.join(TESTS_API_DIR, "video.avi") t.add_content(fn) t.set_piece_length(2**16) t.set_tracker(TRACKER) t.finalize() metainfo = t.get_metainfo() self.general_check(metainfo) self.assert_(metainfo['info']['piece length'] == 2**16)
def test_add_content_piece_length(self): """ Add a single file with piece length to a TorrentDef """ t = TorrentDef() fn = os.path.join(TESTS_DATA_DIR, self.VIDEO_FILE_NAME) t.add_content(fn) t.set_piece_length(2 ** 16) t.set_tracker(TRACKER) t.finalize() metainfo = t.get_metainfo() self.general_check(metainfo) self.assertEqual(metainfo['info']['piece length'], 2 ** 16)
def subtest_add_content_piece_length(self): """ Add a single file with piece length to a TorrentDef """ t = TorrentDef() fn = os.path.join(TESTS_API_DIR, "video.avi") t.add_content(fn) t.set_piece_length(2 ** 16) t.set_tracker(TRACKER) t.finalize() metainfo = t.get_metainfo() self.general_check(metainfo) self.assert_(metainfo['info']['piece length'] == 2 ** 16)
def test_add_content_piece_length(self): """ Add a single file with piece length to a TorrentDef """ t = TorrentDef() fn = os.path.join(TESTS_DATA_DIR, self.VIDEO_FILE_NAME) t.add_content(fn) t.set_piece_length(2**16) t.set_tracker(TRACKER) t.finalize() metainfo = t.get_metainfo() self.general_check(metainfo) self.assertEqual(metainfo['info']['piece length'], 2**16)
def create_tdef(self): """ create and save torrent definition used in this test file """ tdef = TorrentDef() sourcefn = os.path.join(TESTS_DATA_DIR, 'video.avi') tdef.add_content(sourcefn) tdef.set_tracker("http://localhost/announce") tdef.finalize() torrentfn = os.path.join(self.session.get_state_dir(), "gen.torrent") tdef.save(torrentfn) return tdef
def create_tdef(self): """ create and save torrent definition used in this test file """ tdef = TorrentDef() sourcefn = os.path.join(TESTS_DATA_DIR, 'video.avi') tdef.add_content(sourcefn) tdef.set_tracker("http://localhost/announce") tdef.finalize() torrentfn = os.path.join(self.session.config.get_state_dir(), "gen.torrent") tdef.save(torrentfn) return tdef
def subtest_add_content_httpseeds(self): """ Add a single file with BitTornado httpseeds to a TorrentDef """ t = TorrentDef() fn = os.path.join(TESTS_API_DIR, "video.avi") t.add_content(fn) t.set_tracker(TRACKER) expseeds = ['http://www.cs.vu.nl/index.html', 'http://www.st.ewi.tudelft.nl/index.html'] t.set_httpseeds(expseeds) t.finalize() metainfo = t.get_metainfo() self.general_check(metainfo) realseeds = metainfo['httpseeds'] self.assert_(realseeds == expseeds)
def subtest_add_content_file(self): """ Add a single file to a TorrentDef """ t = TorrentDef() fn = os.path.join(TESTS_API_DIR, "video.avi") t.add_content(fn) t.set_tracker(TRACKER) t.finalize() s = os.path.getsize(fn) metainfo = t.get_metainfo() self.general_check(metainfo) self.assert_(metainfo['info']['name'] == "video.avi") self.assert_(metainfo['info']['length'] == s)
def create_local_torrent(self, source_file): ''' This method creates a torrent from a local file and saves the torrent in the session state dir. Note that the source file needs to exist. ''' self.assertTrue(os.path.exists(source_file)) tdef = TorrentDef() tdef.add_content(source_file) tdef.set_tracker("http://localhost/announce") tdef.finalize() torrent_path = os.path.join(self.session.config.get_state_dir(), "seed.torrent") tdef.save(torrent_path) return tdef, torrent_path
def subtest_add_content_announce_list(self): """ Add a single file with announce-list to a TorrentDef """ t = TorrentDef() fn = os.path.join(TESTS_API_DIR, "video.avi") t.add_content(fn) t.set_tracker(TRACKER) exphier = [[TRACKER], ['http://tracker1.tribler.org:6969/announce', 'http://tracker2.tribler.org:7070/ann'], ['http://www.cs.vu.nl', 'http://www.st.ewi.tudelft.nl', 'http://www.vuze.com']] t.set_tracker_hierarchy(exphier) t.finalize() metainfo = t.get_metainfo() self.general_check(metainfo) realhier = metainfo['announce-list'] self.assert_(realhier == exphier)
def _create_and_save_torrent(self, session, filename, createTdef=True): if createTdef: tdef = TorrentDef() sourcefn = os.path.join(BASE_DIR, "API", filename) tdef.add_content(sourcefn) tdef.set_tracker("http://fake.net/announce") tdef.finalize() torrentfn = os.path.join(session.get_state_dir(), "gen.torrent") tdef.save(torrentfn) else: tdef = None torrentfn = os.path.join(BASE_DIR, "API", filename) sdef, swiftpath = session.lm.rtorrent_handler._write_to_collected(torrentfn) return tdef.get_id() if tdef else None, sdef.get_id()
def _create_and_save_torrent(self, session, filename, createTdef=True): if createTdef: tdef = TorrentDef() sourcefn = os.path.join(BASE_DIR, "API", filename) tdef.add_content(sourcefn) tdef.set_tracker("http://fake.net/announce") tdef.finalize() torrentfn = os.path.join(session.get_state_dir(), "gen.torrent") tdef.save(torrentfn) else: tdef = None torrentfn = os.path.join(BASE_DIR, "API", filename) sdef, swiftpath = session.lm.rtorrent_handler._write_to_collected( torrentfn) return tdef.get_id() if tdef else None, sdef.get_id()
def test_add_content_httpseeds(self): """ Add a single file with BitTornado httpseeds to a TorrentDef """ t = TorrentDef() fn = os.path.join(TESTS_DATA_DIR, self.VIDEO_FILE_NAME) t.add_content(fn) t.set_tracker(TRACKER) expseeds = [ 'http://www.cs.vu.nl/index.html', 'http://www.st.ewi.tudelft.nl/index.html' ] t.set_httpseeds(expseeds) t.finalize() metainfo = t.get_metainfo() self.general_check(metainfo) realseeds = metainfo['httpseeds'] self.assertEqual(realseeds, expseeds)
def test_add_content_file_save(self): """ Add a single file to a TorrentDef and save the latter""" t = TorrentDef() fn = os.path.join(TESTS_DATA_DIR, self.VIDEO_FILE_NAME) t.add_content(fn) t.set_tracker(TRACKER) t.finalize() tfn = os.path.join(os.getcwd(), "gen.torrent") t.save(tfn) f = open(tfn, "rb") bdata = f.read() f.close() os.remove(tfn) data = bdecode(bdata) metainfo = t.get_metainfo() self.general_check(metainfo) self.assertEqual(metainfo, data)
def subtest_add_content_file_save(self): """ Add a single file to a TorrentDef and save the latter""" t = TorrentDef() fn = os.path.join(TESTS_API_DIR, "video.avi") t.add_content(fn) t.set_tracker(TRACKER) t.finalize() tfn = os.path.join(os.getcwd(), "gen.torrent") t.save(tfn) f = open(tfn, "rb") bdata = f.read() f.close() os.remove(tfn) data = bdecode(bdata) metainfo = t.get_metainfo() self.general_check(metainfo) self.assert_(metainfo == data)
def setupSeeder(self): from Tribler.Core.Session import Session from Tribler.Core.TorrentDef import TorrentDef from Tribler.Core.DownloadConfig import DownloadStartupConfig self.setUpPreSession() self.config.set_libtorrent(True) self.session2 = Session(self.config, ignore_singleton=True) self.session2.start() tdef = TorrentDef() tdef.add_content(os.path.join(BASE_DIR, "data", "video.avi")) tdef.set_tracker("http://fake.net/announce") tdef.finalize() torrentfn = os.path.join(self.session.get_state_dir(), "gen.torrent") tdef.save(torrentfn) dscfg = DownloadStartupConfig() dscfg.set_dest_dir(os.path.join(BASE_DIR, "data")) # basedir of the file we are seeding self.session2.start_download(tdef, dscfg) return torrentfn
def test_add_content_announce_list(self): """ Add a single file with announce-list to a TorrentDef """ t = TorrentDef() fn = os.path.join(TESTS_DATA_DIR, self.VIDEO_FILE_NAME) t.add_content(fn) t.set_tracker(TRACKER) exphier = [[TRACKER], [ 'http://tracker1.tribler.org:6969/announce', 'http://tracker2.tribler.org:7070/ann' ], [ 'http://www.cs.vu.nl', 'http://www.st.ewi.tudelft.nl', 'http://www.vuze.com' ]] t.set_tracker_hierarchy(exphier) t.finalize() metainfo = t.get_metainfo() self.general_check(metainfo) realhier = metainfo['announce-list'] self.assertEqual(realhier, exphier)
def test_multifile_torrent(self): tdef = TorrentDef() dn = os.path.join(TESTS_DATA_DIR, "contentdir") tdef.add_content(dn, "dirintorrent") fn = os.path.join(TESTS_DATA_DIR, "video.avi") tdef.add_content(fn, os.path.join("dirintorrent", "video.avi")) tdef.set_tracker("http://tribler.org/announce") tdef.finalize() impl = LibtorrentDownloadImpl(self.session, tdef) # Override the add_torrent because it will be called impl.ltmgr = MockObject() impl.ltmgr.add_torrent = lambda _, _dummy2: succeed(fake_handler) impl.set_selected_files = lambda: None fake_handler = MockObject() fake_handler.is_valid = lambda: True fake_handler.status = lambda: fake_status fake_handler.set_share_mode = lambda _: None fake_handler.set_priority = lambda _: None fake_handler.set_sequential_download = lambda _: None fake_handler.resume = lambda: None fake_handler.set_max_connections = lambda _: None fake_status = MockObject() fake_status.share_mode = False # Create a dummy download config impl.dlconfig = DownloadStartupConfig().dlconfig.copy() # Create a dummy pstate pstate = CallbackConfigParser() pstate.add_section("state") test_dict = dict() test_dict["a"] = "b" pstate.set("state", "engineresumedata", test_dict) return impl.network_create_engine_wrapper(pstate)
def setupSeeder(self, hops=0, session=None): from Tribler.Core.Session import Session from Tribler.Core.TorrentDef import TorrentDef from Tribler.Core.DownloadConfig import DownloadStartupConfig self.setUpPreSession() self.config.set_libtorrent(True) self.config2 = self.config.copy() self.config2.set_state_dir(self.getStateDir(2)) if session is None: self.session2 = Session(self.config2, ignore_singleton=True, autoload_discovery=False) upgrader = self.session2.prestart() while not upgrader.is_done: time.sleep(0.1) self.session2.start() session = self.session2 tdef = TorrentDef() tdef.add_content(os.path.join(TESTS_DATA_DIR, "video.avi")) tdef.set_tracker("http://fake.net/announce") tdef.set_private() # disable dht tdef.finalize() torrentfn = os.path.join(session.get_state_dir(), "gen.torrent") tdef.save(torrentfn) dscfg = DownloadStartupConfig() dscfg.set_dest_dir( TESTS_DATA_DIR) # basedir of the file we are seeding dscfg.set_hops(hops) d = session.start_download(tdef, dscfg) d.set_state_callback(self.seeder_state_callback) return torrentfn
def setupSeeder(self): from Tribler.Core.Session import Session from Tribler.Core.TorrentDef import TorrentDef from Tribler.Core.DownloadConfig import DownloadStartupConfig self.setUpPreSession() self.config.set_libtorrent(True) self.session2 = Session(self.config, ignore_singleton=True) self.session2.start() tdef = TorrentDef() tdef.add_content(os.path.join(BASE_DIR, "data", "video.avi")) tdef.set_tracker("http://fake.net/announce") tdef.finalize() torrentfn = os.path.join(self.session.get_state_dir(), "gen.torrent") tdef.save(torrentfn) dscfg = DownloadStartupConfig() dscfg.set_dest_dir(os.path.join( BASE_DIR, "data")) # basedir of the file we are seeding self.session2.start_download(tdef, dscfg) return torrentfn
def make_meta_file(srcpaths, params, userabortflag, progressCallback, torrentfilenameCallback): tdef = TorrentDef() if len(srcpaths) > 1: basepath = [] for srcpath in srcpaths: path, filename = os.path.split(srcpath) basepath.append(path) basepath, _ = os.path.split(os.path.commonprefix(basepath)) for srcpath in srcpaths: outpath = os.path.relpath(srcpath, basepath) # h4x0r playtime if 'playtime' in params: tdef.add_content(srcpath, outpath, playtime=params['playtime']) else: tdef.add_content(srcpath, outpath) else: srcpath = srcpaths[0] basepath, _ = os.path.split(srcpath) if 'playtime' in params: tdef.add_content(srcpath,playtime=params['playtime']) else: tdef.add_content(srcpath) if params['comment']: tdef.set_comment(params['comment']) if params['created by']: tdef.set_created_by(params['created by']) if params['announce']: tdef.set_tracker(params['announce']) if params['announce-list']: tdef.set_tracker_hierarchy(params['announce-list']) if params['nodes']: # mainline DHT tdef.set_dht_nodesmax(params['nodes']) if params['httpseeds']: tdef.set_httpseeds(params['httpseeds']) if params['encoding']: tdef.set_encoding(params['encoding']) if params['piece length']: tdef.set_piece_length(params['piece length']) if params['makehash_md5']: print >>sys.stderr,"TorrentMaker: make MD5" tdef.set_add_md5hash(params['makehash_md5']) if params['makehash_crc32']: print >>sys.stderr,"TorrentMaker: make CRC32" tdef.set_add_crc32(params['makehash_crc32']) if params['makehash_sha1']: print >>sys.stderr,"TorrentMaker: make SHA1" tdef.set_add_sha1hash(params['makehash_sha1']) if params['createmerkletorrent']: tdef.set_create_merkle_torrent(params['createmerkletorrent']) if params['torrentsigkeypairfilename']: tdef.set_signature_keypair_filename(params['torrentsigkeypairfilename']) if params['thumb']: tdef.set_thumbnail(params['thumb']) tdef.finalize(userabortflag=userabortflag,userprogresscallback=progressCallback) if params['createmerkletorrent']: postfix = TRIBLER_TORRENT_EXT else: postfix = '.torrent' if 'target' in params and params['target']: torrentfilename = os.path.join(params['target'], os.path.split(os.path.normpath(srcpath))[1] + postfix) else: a, b = os.path.split(srcpaths[0]) if b == '': torrentfilename = a + postfix else: torrentfilename = os.path.join(a, b + postfix) tdef.save(torrentfilename) # Inform higher layer we created torrent torrentfilenameCallback(basepath, torrentfilename)
class TestVideoOnDemand(TestAsServer): """ Testing Merkle hashpiece messages for both: * Merkle BEP style * old Tribler <= 4.5.2 that did not use the Extention protocol (BEP 10). See BitTornado/BT1/Connecter.py """ @blocking_call_on_reactor_thread @inlineCallbacks def setUp(self, autoload_discovery=True): yield TestAsServer.setUp(self, autoload_discovery=autoload_discovery) self.content = None self.tdef = None self.test_deferred = Deferred() self.contentlen = None self.piecelen = 0 def setUpPreSession(self): TestAsServer.setUpPreSession(self) self.config.set_libtorrent_enabled(True) def create_torrent(self): [srchandle, sourcefn] = mkstemp() self.content = Rand.rand_bytes(self.contentlen) os.write(srchandle, self.content) os.close(srchandle) self.tdef = TorrentDef() self.tdef.add_content(sourcefn) self.tdef.set_piece_length(self.piecelen) self.tdef.set_tracker("http://127.0.0.1:12/announce") self.tdef.finalize() torrentfn = os.path.join(self.session.config.get_state_dir(), "gen.torrent") self.tdef.save(torrentfn) dscfg = DownloadStartupConfig() destdir = os.path.dirname(sourcefn) dscfg.set_dest_dir(destdir) dscfg.set_mode(DLMODE_VOD) download = self.session.start_download_from_tdef(self.tdef, dscfg) download.set_state_callback(self.state_callback) self.session.set_download_states_callback(self.states_callback) def states_callback(self, dslist): ds = dslist[0] d = ds.get_download() self._logger.debug('%s %s %5.2f%% %s up %8.2fKB/s down %8.2fKB/s', d.get_def().get_name(), dlstatus_strings[ds.get_status()], ds.get_progress() * 100, ds.get_error(), ds.get_current_speed(UPLOAD), ds.get_current_speed(DOWNLOAD)) return [] def state_callback(self, ds): download = ds.get_download() if ds.get_vod_prebuffering_progress() == 1.0: self._logger.debug("Test: state_callback") stream = VODFile(open(download.get_content_dest(), 'rb'), download) # Read last piece lastpieceoff = ( (self.contentlen - 1) / self.piecelen) * self.piecelen lastpiecesize = self.contentlen - lastpieceoff self._logger.debug("stream: lastpieceoff %s %s", lastpieceoff, lastpiecesize) self.stream_read(stream, lastpieceoff, lastpiecesize, self.piecelen) # Read second,3rd,4th byte, only secoff = 1 secsize = 3 blocksize = 3 self.stream_read(stream, secoff, secsize, blocksize) # Read last byte lastoff = self.contentlen - 1 lastsize = 1 self.stream_read(stream, lastoff, lastsize, self.piecelen) self.test_deferred.callback(None) return 0 return 1.0 def stream_read(self, stream, off, size, blocksize): stream.seek(off) data = stream.read(blocksize) self._logger.debug("stream: Got data %s", len(data)) self.assertEquals(len(data), size) self.assertEquals(data, self.content[off:off + size]) @deferred(timeout=10) def test_99(self): self.contentlen = 99 self.piecelen = 16 self.create_torrent() self._logger.debug("Letting network thread create Download, sleeping") return self.test_deferred @deferred(timeout=10) def test_100(self): self.contentlen = 100 self.piecelen = 16 self.create_torrent() self._logger.debug("Letting network thread create Download, sleeping") return self.test_deferred @deferred(timeout=10) def test_101(self): self.contentlen = 101 self.piecelen = 16 self.create_torrent() self._logger.debug("Letting network thread create Download, sleeping") return self.test_deferred
class TestSeeding(TestAsServer): """ Test whether the seeding works correctly. """ @inlineCallbacks def setUp(self): yield super(TestSeeding, self).setUp() self._logger = logging.getLogger(self.__class__.__name__) self.test_deferred = Deferred() self.tdef = None self.sourcefn = None def setUpPreSession(self): super(TestSeeding, self).setUpPreSession() self.config.set_libtorrent_enabled(True) def generate_torrent(self): self.tdef = TorrentDef() self.sourcefn = os.path.join(TESTS_DATA_DIR, 'video.avi') self.tdef.add_content(self.sourcefn) self.tdef.set_tracker("http://localhost/announce") self.tdef.finalize() self.tdef.save(os.path.join(self.session.config.get_state_dir(), "gen.torrent")) def start_download(self, dscfg): download = self.session.start_download_from_tdef(self.tdef, dscfg) download.set_state_callback(self.downloader_state_callback) download.add_peer(("127.0.0.1", self.seeder_session.config.get_libtorrent_port())) @trial_timeout(60) def test_seeding(self): """ Test whether a torrent is correctly seeded """ self.generate_torrent() def start_download(_): dscfg = self.dscfg_seed.copy() dscfg.set_dest_dir(self.getDestDir()) self.start_download(dscfg) self.setup_seeder(self.tdef, TESTS_DATA_DIR).addCallback(start_download) return self.test_deferred def downloader_state_callback(self, ds): d = ds.get_download() self._logger.debug("download status: %s %s %s", repr(d.get_def().get_name()), dlstatus_strings[ds.get_status()], ds.get_progress()) if ds.get_status() == DLSTATUS_SEEDING: # File is in destfn = os.path.join(self.getDestDir(), "video.avi") f = open(destfn, "rb") realdata = f.read() f.close() f = open(self.sourcefn, "rb") expdata = f.read() f.close() self.assertEqual(realdata, expdata) self.test_deferred.callback(None) return 0.0 return 1.0
def lineReceived(self, line): anon_tunnel = self.anon_tunnel profile = self.profile if line == 'threads': for thread in threading.enumerate(): print "%s \t %d" % (thread.name, thread.ident) elif line == 'p': if profile: for func_stats in yappi.get_func_stats().sort("subtime")[:50]: print "YAPPI: %10dx %10.3fs" % (func_stats.ncall, func_stats.tsub), func_stats.name else: logger.error("Profiling disabled!") elif line == 'P': if profile: filename = 'callgrindc_%d.yappi' % anon_tunnel.dispersy.lan_address[1] yappi.get_func_stats().save(filename, type='callgrind') else: logger.error("Profiling disabled!") elif line == 't': if profile: yappi.get_thread_stats().sort("totaltime").print_all() else: logger.error("Profiling disabled!") elif line == 'c': print "========\nCircuits\n========\nid\taddress\t\t\t\t\tgoal\thops\tIN (MB)\tOUT (MB)\tinfohash\ttype" for circuit_id, circuit in anon_tunnel.community.circuits.items(): info_hash = circuit.info_hash.encode('hex')[:10] if circuit.info_hash else '?' print "%d\t%s:%d\t%d\t%d\t\t%.2f\t\t%.2f\t\t%s\t%s" % (circuit_id, circuit.first_hop[0], circuit.first_hop[1], circuit.goal_hops, len(circuit.hops), circuit.bytes_down / 1024.0 / 1024.0, circuit.bytes_up / 1024.0 / 1024.0, info_hash, circuit.ctype) elif line.startswith('s'): cur_path = os.getcwd() line_split = line.split(' ') filename = 'test_file' if len(line_split) == 1 else line_split[1] if not os.path.exists(filename): logger.info("Creating torrent..") with open(filename, 'wb') as fp: fp.write(os.urandom(50 * 1024 * 1024)) tdef = TorrentDef() tdef.add_content(os.path.join(cur_path, filename)) tdef.set_tracker("udp://fake.net/announce") tdef.set_private() tdef.finalize() tdef.save(os.path.join(cur_path, filename + '.torrent')) else: logger.info("Loading existing torrent..") tdef = TorrentDef.load(filename + '.torrent') logger.info("loading torrent done, infohash of torrent: %s" % (tdef.get_infohash().encode('hex')[:10])) defaultDLConfig = DefaultDownloadStartupConfig.getInstance() dscfg = defaultDLConfig.copy() dscfg.set_hops(1) dscfg.set_dest_dir(cur_path) anon_tunnel.session.lm.threadpool.call(0, anon_tunnel.session.start_download, tdef, dscfg) elif line.startswith('i'): # Introduce dispersy port from other main peer to this peer line_split = line.split(' ') to_introduce_ip = line_split[1] to_introduce_port = int(line_split[2]) self.anon_tunnel.community.add_discovered_candidate(Candidate((to_introduce_ip, to_introduce_port), tunnel=False)) elif line.startswith('d'): line_split = line.split(' ') filename = 'test_file' if len(line_split) == 1 else line_split[1] logger.info("Loading torrent..") tdef = TorrentDef.load(filename + '.torrent') logger.info("Loading torrent done") defaultDLConfig = DefaultDownloadStartupConfig.getInstance() dscfg = defaultDLConfig.copy() dscfg.set_hops(1) dscfg.set_dest_dir(os.path.join(os.getcwd(), 'downloader%s' % anon_tunnel.session.get_dispersy_port())) def start_download(): def cb(ds): logger.info('Download infohash=%s, down=%s, progress=%s, status=%s, seedpeers=%s, candidates=%d' % (tdef.get_infohash().encode('hex')[:10], ds.get_current_speed('down'), ds.get_progress(), dlstatus_strings[ds.get_status()], sum(ds.get_num_seeds_peers()), sum(1 for _ in anon_tunnel.community.dispersy_yield_verified_candidates()))) return 1.0, False download = anon_tunnel.session.start_download(tdef, dscfg) download.set_state_callback(cb, delay=1) anon_tunnel.session.lm.threadpool.call(0, start_download) elif line == 'q': anon_tunnel.stop() return elif line == 'r': print "circuit\t\t\tdirection\tcircuit\t\t\tTraffic (MB)" from_to = anon_tunnel.community.relay_from_to for key in from_to.keys(): relay = from_to[key] logger.info("%s-->\t%s\t\t%.2f" % ((key[0], key[1]), (relay.sock_addr, relay.circuit_id), relay.bytes[1] / 1024.0 / 1024.0,))
def setUpPostSession(self): """ override TestAsServer """ TestAsServer.setUpPostSession(self) # Let Tribler start downloading an non-functioning torrent, so # we can talk to a normal download engine. self.tdef = TorrentDef() self.sourcefn = os.path.join(os.getcwd(), "API", "file2.wmv") self.tdef.add_content(self.sourcefn) self.tdef.set_create_merkle_torrent(True) self.tdef.set_tracker("http://127.0.0.1:12/announce") self.tdef.finalize() self.torrentfn = os.path.join(self.session.get_state_dir(), "gen.torrent") self.tdef.save(self.torrentfn) dscfg = self.setUpDownloadConfig() self.session.start_download(self.tdef, dscfg) self.infohash = self.tdef.get_infohash() self.mylistenport = 4810 self.numpieces = (self.tdef.get_length() + self.tdef.get_piece_length() - 1) / self.tdef.get_piece_length() b = Bitfield(self.numpieces) for i in range(self.numpieces): b[i] = True self.assert_(b.complete()) self.seederbitfieldstr = b.tostring() # piece_hashes = ['\x01\x02\x03\x04\x05\x06\x07\x08\x07\x06\x05\x04\x03\x02\x01\x00\x01\x02\x03\x04' ] * npieces # Construct Merkle tree tdef2 = TorrentDef() tdef2.add_content(self.sourcefn) tdef2.set_create_merkle_torrent(False) tdef2.set_tracker("http://127.0.0.1:12/announce") tdef2.set_piece_length(self.tdef.get_piece_length()) tdef2.finalize() metainfo = tdef2.get_metainfo() piecesstr = metainfo['info']['pieces'] print >> sys.stderr, "test: pieces has len", len(piecesstr) piece_hashes = [] for i in range(0, len(piecesstr), 20): hash = piecesstr[i:i + 20] print >> sys.stderr, "test: piece", i / 20, "hash", repr(hash) piece_hashes.append(hash) print >> sys.stderr, "test: Putting", len(piece_hashes), "into MerkleTree, size", self.tdef.get_piece_length(), tdef2.get_piece_length() self.tree = MerkleTree(self.tdef.get_piece_length(), self.tdef.get_length(), None, piece_hashes) f = open(self.sourcefn, "rb") piece1 = f.read(2 ** 18) piece2 = f.read(2 ** 18) print >> sys.stderr, "read piece1", len(piece1) print >> sys.stderr, "read piece2", len(piece2) f.close() hash1 = sha(piece1).digest() hash2 = sha(piece2).digest() print >> sys.stderr, "hash piece1", repr(hash1) print >> sys.stderr, "hash piece2", repr(hash2) f2 = open("piece1.bin", "wb") f2.write(piece2) f2.close()
class TestVideoServerSession(TestAsServer): """ Class for testing HTTP-based video server in a session. Mainly HTTP range queries. """ @blocking_call_on_reactor_thread @inlineCallbacks def setUp(self, autoload_discovery=True): """ unittest test setup code """ yield TestAsServer.setUp(self, autoload_discovery=autoload_discovery) self.port = self.session.config.get_video_server_port() self.sourcefn = os.path.join(TESTS_DATA_DIR, "video.avi") self.sourcesize = os.path.getsize(self.sourcefn) self.tdef = None self.expsize = 0 yield self.start_vod_download() def setUpPreSession(self): TestAsServer.setUpPreSession(self) self.config.set_libtorrent_enabled(True) self.config.set_video_server_enabled(True) # # Tests # @deferred(timeout=10) def test_specific_range(self): return self.range_check(115, 214) @deferred(timeout=10) def test_last_100(self): return self.range_check(self.sourcesize - 100, None) @deferred(timeout=10) def test_first_100(self): return self.range_check(None, 100) @deferred(timeout=10) def test_combined(self): return self.range_check(115, 214, setset=True) def start_vod_download(self): self.tdef = TorrentDef() self.tdef.add_content(self.sourcefn) self.tdef.set_tracker("http://127.0.0.1:12/announce") self.tdef.finalize() dscfg = DownloadStartupConfig() dscfg.set_dest_dir(os.path.dirname(self.sourcefn)) download = self.session.start_download_from_tdef(self.tdef, dscfg) return download.get_handle() def get_std_header(self): msg = "GET /%s/0 HTTP/1.1\r\n" % binascii.hexlify( self.tdef.get_infohash()) msg += "Host: 127.0.0.1:" + str(self.port) + "\r\n" return msg @staticmethod def create_range_str(firstbyte, lastbyte): head = "" if firstbyte is not None: head += str(firstbyte) head += "-" if lastbyte is not None: head += str(lastbyte) return head def get_header(self, firstbyte, lastbyte, setset=False): head = self.get_std_header() head += "Range: bytes=" head += self.create_range_str(firstbyte, lastbyte) if setset: # Make into set of byte ranges, VideoHTTPServer should refuse. head += ",0-99" head += "\r\n" head += "Connection: close\r\n" return head + "\r\n" def range_check(self, firstbyte, lastbyte, setset=False): test_deferred = Deferred() self._logger.debug("range_test: %s %s %s setset %s", firstbyte, lastbyte, self.sourcesize, setset) if firstbyte is not None and lastbyte is None: exp_byte_range = (firstbyte, self.sourcesize - 1) elif firstbyte is None and lastbyte is not None: exp_byte_range = (self.sourcesize - lastbyte, self.sourcesize - 1) else: exp_byte_range = (firstbyte, lastbyte) # the amount of bytes actually requested. (Content-length) self.expsize = exp_byte_range[1] - exp_byte_range[0] + 1 f = open(self.sourcefn, "rb") f.seek(exp_byte_range[0]) expdata = f.read(self.expsize) f.close() def on_connected(p): p.sendMessage(self.get_header(firstbyte, lastbyte, setset)) endpoint = TCP4ClientEndpoint(reactor, "localhost", self.port) connectProtocol(endpoint, VideoServerProtocol(test_deferred, self.sourcesize, expdata, setset, exp_byte_range))\ .addCallback(on_connected) return test_deferred
class TestVideoOnDemand(TestAsServer): """ Testing Merkle hashpiece messages for both: * Merkle BEP style * old Tribler <= 4.5.2 that did not use the Extention protocol (BEP 10). See BitTornado/BT1/Connecter.py """ @inlineCallbacks def setUp(self): yield TestAsServer.setUp(self) self.content = None self.tdef = None self.test_deferred = Deferred() self.contentlen = None self.piecelen = 0 def setUpPreSession(self): TestAsServer.setUpPreSession(self) self.config.set_libtorrent_enabled(True) def create_torrent(self): [srchandle, sourcefn] = mkstemp() self.content = Rand.rand_bytes(self.contentlen) os.write(srchandle, self.content) os.close(srchandle) self.tdef = TorrentDef() self.tdef.add_content(sourcefn) self.tdef.set_piece_length(self.piecelen) self.tdef.set_tracker("http://127.0.0.1:12/announce") self.tdef.finalize() torrentfn = os.path.join(self.session.config.get_state_dir(), "gen.torrent") self.tdef.save(torrentfn) dscfg = DownloadStartupConfig() destdir = os.path.dirname(sourcefn) dscfg.set_dest_dir(destdir) dscfg.set_mode(DLMODE_VOD) download = self.session.start_download_from_tdef(self.tdef, dscfg) download.set_state_callback(self.state_callback) self.session.set_download_states_callback(self.states_callback) def states_callback(self, dslist): ds = dslist[0] d = ds.get_download() self._logger.debug('%s %s %5.2f%% %s up %8.2fKB/s down %8.2fKB/s', d.get_def().get_name(), dlstatus_strings[ds.get_status()], ds.get_progress() * 100, ds.get_error(), ds.get_current_speed(UPLOAD), ds.get_current_speed(DOWNLOAD)) return [] def state_callback(self, ds): download = ds.get_download() if ds.get_vod_prebuffering_progress() == 1.0: self._logger.debug("Test: state_callback") stream = VODFile(open(download.get_content_dest(), 'rb'), download) # Read last piece lastpieceoff = ((self.contentlen - 1) / self.piecelen) * self.piecelen lastpiecesize = self.contentlen - lastpieceoff self._logger.debug("stream: lastpieceoff %s %s", lastpieceoff, lastpiecesize) self.stream_read(stream, lastpieceoff, lastpiecesize, self.piecelen) # Read second,3rd,4th byte, only secoff = 1 secsize = 3 blocksize = 3 self.stream_read(stream, secoff, secsize, blocksize) # Read last byte lastoff = self.contentlen - 1 lastsize = 1 self.stream_read(stream, lastoff, lastsize, self.piecelen) self.test_deferred.callback(None) return 0 return 1.0 def stream_read(self, stream, off, size, blocksize): stream.seek(off) data = stream.read(blocksize) self._logger.debug("stream: Got data %s", len(data)) self.assertEquals(len(data), size) self.assertEquals(data, self.content[off:off + size]) @trial_timeout(10) def test_99(self): self.contentlen = 99 self.piecelen = 16 self.create_torrent() self._logger.debug("Letting network thread create Download, sleeping") return self.test_deferred @trial_timeout(10) def test_100(self): self.contentlen = 100 self.piecelen = 16 self.create_torrent() self._logger.debug("Letting network thread create Download, sleeping") return self.test_deferred @trial_timeout(10) def test_101(self): self.contentlen = 101 self.piecelen = 16 self.create_torrent() self._logger.debug("Letting network thread create Download, sleeping") return self.test_deferred
class TestSeeding(TestAsServer): """ Test whether the seeding works correctly. """ @blocking_call_on_reactor_thread @inlineCallbacks def setUp(self, autoload_discovery=True): yield super(TestSeeding, self).setUp(autoload_discovery=autoload_discovery) self._logger = logging.getLogger(self.__class__.__name__) self.test_deferred = Deferred() self.tdef = None self.sourcefn = None def setUpPreSession(self): super(TestSeeding, self).setUpPreSession() self.config.set_libtorrent_enabled(True) def generate_torrent(self): self.tdef = TorrentDef() self.sourcefn = os.path.join(TESTS_DATA_DIR, 'video.avi') self.tdef.add_content(self.sourcefn) self.tdef.set_tracker("http://localhost/announce") self.tdef.finalize() self.tdef.save(os.path.join(self.session.config.get_state_dir(), "gen.torrent")) def start_download(self, dscfg): download = self.session.start_download_from_tdef(self.tdef, dscfg) download.set_state_callback(self.downloader_state_callback) download.add_peer(("127.0.0.1", self.seeder_session.config.get_libtorrent_port())) @deferred(timeout=60) def test_seeding(self): """ Test whether a torrent is correctly seeded """ self.generate_torrent() def start_download(_): dscfg = self.dscfg_seed.copy() dscfg.set_dest_dir(self.getDestDir()) self.start_download(dscfg) self.setup_seeder(self.tdef, TESTS_DATA_DIR).addCallback(start_download) return self.test_deferred def downloader_state_callback(self, ds): d = ds.get_download() self._logger.debug("download status: %s %s %s", repr(d.get_def().get_name()), dlstatus_strings[ds.get_status()], ds.get_progress()) if ds.get_status() == DLSTATUS_SEEDING: # File is in destfn = os.path.join(self.getDestDir(), "video.avi") f = open(destfn, "rb") realdata = f.read() f.close() f = open(self.sourcefn, "rb") expdata = f.read() f.close() self.assertEqual(realdata, expdata) self.test_deferred.callback(None) return 0.0 return 1.0
def setUpPostSession(self): """ override TestAsServer """ TestAsServer.setUpPostSession(self) # Let Tribler start downloading an non-functioning torrent, so # we can talk to a normal download engine. self.tdef = TorrentDef() self.sourcefn = os.path.join(os.getcwd(), "API", "file2.wmv") self.tdef.add_content(self.sourcefn) self.tdef.set_create_merkle_torrent(True) self.tdef.set_tracker("http://127.0.0.1:12/announce") self.tdef.finalize() self.torrentfn = os.path.join(self.session.get_state_dir(), "gen.torrent") self.tdef.save(self.torrentfn) dscfg = self.setUpDownloadConfig() self.session.start_download(self.tdef, dscfg) self.infohash = self.tdef.get_infohash() self.mylistenport = 4810 self.numpieces = (self.tdef.get_length() + self.tdef.get_piece_length() - 1) / self.tdef.get_piece_length() b = Bitfield(self.numpieces) for i in range(self.numpieces): b[i] = True self.assert_(b.complete()) self.seederbitfieldstr = b.tostring() # piece_hashes = ['\x01\x02\x03\x04\x05\x06\x07\x08\x07\x06\x05\x04\x03\x02\x01\x00\x01\x02\x03\x04' ] * npieces # Construct Merkle tree tdef2 = TorrentDef() tdef2.add_content(self.sourcefn) tdef2.set_create_merkle_torrent(False) tdef2.set_tracker("http://127.0.0.1:12/announce") tdef2.set_piece_length(self.tdef.get_piece_length()) tdef2.finalize() metainfo = tdef2.get_metainfo() piecesstr = metainfo["info"]["pieces"] print >> sys.stderr, "test: pieces has len", len(piecesstr) piece_hashes = [] for i in range(0, len(piecesstr), 20): hash = piecesstr[i : i + 20] print >> sys.stderr, "test: piece", i / 20, "hash", repr(hash) piece_hashes.append(hash) print >> sys.stderr, "test: Putting", len( piece_hashes ), "into MerkleTree, size", self.tdef.get_piece_length(), tdef2.get_piece_length() self.tree = MerkleTree(self.tdef.get_piece_length(), self.tdef.get_length(), None, piece_hashes) f = open(self.sourcefn, "rb") piece1 = f.read(2 ** 18) piece2 = f.read(2 ** 18) print >> sys.stderr, "read piece1", len(piece1) print >> sys.stderr, "read piece2", len(piece2) f.close() hash1 = sha(piece1).digest() hash2 = sha(piece2).digest() print >> sys.stderr, "hash piece1", repr(hash1) print >> sys.stderr, "hash piece2", repr(hash2) f2 = open("piece1.bin", "wb") f2.write(piece2) f2.close()
class TestMetadata(TestAsServer, MagnetHelpers): """ Once we are downloading a torrent, our client should respond to the ut_metadata extention message. This allows other clients to obtain the info part of the metadata from us. """ def setUp(self): """ override TestAsServer """ TestAsServer.setUp(self) print >>sys.stderr,"test: Giving MyLaunchMany time to startup" time.sleep(5) print >>sys.stderr,"test: MyLaunchMany should have started up" # the metadata that we want to transfer self.tdef = TorrentDef() self.tdef.add_content(os.path.join(os.getcwd(), "API", "file.wmv")) self.tdef.set_tracker(self.session.get_internal_tracker_url()) # we use a small piece length to obtain multiple pieces self.tdef.set_piece_length(1) self.tdef.finalize() # self.tdef.save(os.path.join(self.session.get_state_dir(), "gen.torrent")) MagnetHelpers.__init__(self, self.tdef) def setup_seeder(self): self.seeder_setup_complete = False self.seeder_teardown_complete = False self.seeder_teardown = False self.dscfg = DownloadStartupConfig() self.dscfg.set_dest_dir(os.getcwd()) self.download = self.session.start_download(self.tdef, self.dscfg) self.download.set_state_callback(self.seeder_state_callback) counter = 0 while not self.seeder_setup_complete: counter += 1 time.sleep(1) assert counter < 30, "timeout" print >> sys.stderr, "test: setup_seeder() complete" def teardown_seeder(self): self.seeder_teardown_complete = False self.session.remove_download(self.download) counter = 0 while not self.seeder_setup_complete: counter += 1 time.sleep(1) assert counter < 30, "timeout" print >> sys.stderr, "test: teardown_seeder() complete" def seeder_state_callback(self,ds): assert not self.seeder_teardown_complete self.seeder_setup_complete = (ds.get_status() == DLSTATUS_DOWNLOADING) d = ds.get_download() print >> sys.stderr, "test: seeder:", `d.get_def().get_name()`, dlstatus_strings[ds.get_status()], ds.get_progress() if self.seeder_teardown: self.seeder_teardown_complete = True else: return (1.0, False) def test_all(self): self.setup_seeder() try: self.subtest_good_flood() finally: self.teardown_seeder() self.setup_seeder() try: self.subtest_good_request() self.subtest_bad_request() finally: self.teardown_seeder() def subtest_good_request(self): conn = BTConnection("localhost", self.hisport, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) # request metadata block 0, 2, 3, and the last conn.send(self.create_good_extend_metadata_request(metadata_id, 0)) conn.send(self.create_good_extend_metadata_request(metadata_id, 2)) conn.send(self.create_good_extend_metadata_request(metadata_id, 3)) conn.send(self.create_good_extend_metadata_request(metadata_id, len(self.metadata_list) - 1)) self.read_extend_metadata_reply(conn, 0) self.read_extend_metadata_reply(conn, 2) self.read_extend_metadata_reply(conn, 3) self.read_extend_metadata_reply(conn, len(self.metadata_list) - 1) def subtest_good_flood(self): conn = BTConnection("localhost", self.hisport, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) for counter in xrange(len(self.metadata_list) * 2): piece = counter % len(self.metadata_list) conn.send(self.create_good_extend_metadata_request(metadata_id, piece)) if counter > len(self.metadata_list): self.read_extend_metadata_reject(conn, piece) else: self.read_extend_metadata_reply(conn, piece) def subtest_bad_request(self): self.bad_request_and_disconnect({"msg_type":0, "piece":len(self.metadata_list)}) self.bad_request_and_disconnect({"msg_type":0, "piece":-1}) self.bad_request_and_disconnect({"msg_type":0, "piece":"1"}) self.bad_request_and_disconnect({"msg_type":0, "piece":[1,2]}) self.bad_request_and_disconnect({"msg_type":0, "PIECE":1}) def bad_request_and_disconnect(self, payload): conn = BTConnection("localhost", self.hisport, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) conn.send(EXTEND + chr(metadata_id) + bencode(payload)) self.read_extend_metadata_close(conn)
class TestMetadataFakePeer(TestAsServer, MagnetHelpers): """ Once we are downloading a torrent, our client should respond to the ut_metadata extention message. This allows other clients to obtain the info part of the metadata from us. """ def setUp(self): TestAsServer.setUp(self) # the metadata that we want to transfer self.tdef = TorrentDef() self.tdef.add_content(os.path.join(TESTS_API_DIR, "file.wmv")) self.tdef.set_tracker("http://fake.net/announce") # we use a small piece length to obtain multiple pieces self.tdef.set_piece_length(1) self.tdef.finalize() self.setup_seeder() MagnetHelpers.__init__(self, self.tdef) def setUpPreSession(self): TestAsServer.setUpPreSession(self) self.config.set_libtorrent(True) self.config2 = self.config.copy() self.config2.set_state_dir(self.getStateDir(2)) def tearDown(self): self.teardown_seeder() TestAsServer.tearDown(self) def setup_seeder(self): self.seeder_setup_complete = threading.Event() self.dscfg = DownloadStartupConfig() self.dscfg.set_dest_dir(TESTS_API_DIR) self.download = self.session.start_download(self.tdef, self.dscfg) self.download.set_state_callback(self.seeder_state_callback) assert self.seeder_setup_complete.wait(30) def teardown_seeder(self): self.session.remove_download(self.download) def seeder_state_callback(self, ds): if ds.get_status() == DLSTATUS_SEEDING: self.seeder_setup_complete.set() d = ds.get_download() self._logger.debug("seeder: %s %s %s", repr(d.get_def().get_name()), dlstatus_strings[ds.get_status()], ds.get_progress()) return 1.0, False def test_good_request(self): conn = BTConnection("localhost", self.session.get_listen_port(), user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) # request metadata block 0, 2, 3, and the last conn.send(self.create_good_extend_metadata_request(metadata_id, 0)) conn.send(self.create_good_extend_metadata_request(metadata_id, 2)) conn.send(self.create_good_extend_metadata_request(metadata_id, 3)) conn.send( self.create_good_extend_metadata_request( metadata_id, len(self.metadata_list) - 1)) self.read_extend_metadata_reply(conn, 0) self.read_extend_metadata_reply(conn, 2) self.read_extend_metadata_reply(conn, 3) self.read_extend_metadata_reply(conn, len(self.metadata_list) - 1) def test_good_flood(self): conn = BTConnection("localhost", self.session.get_listen_port(), user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) for counter in xrange(len(self.metadata_list) * 2): piece = counter % len(self.metadata_list) conn.send( self.create_good_extend_metadata_request(metadata_id, piece)) if counter > len(self.metadata_list): self.read_extend_metadata_reject(conn, piece) else: self.read_extend_metadata_reply(conn, piece) def test_bad_request(self): self.bad_request_and_disconnect({ "msg_type": 0, "piece": len(self.metadata_list) }) self.bad_request_and_disconnect({"msg_type": 0, "piece": -1}) self.bad_request_and_disconnect({"msg_type": 0, "piece": "1"}) self.bad_request_and_disconnect({"msg_type": 0, "piece": [1, 2]}) self.bad_request_and_disconnect({"msg_type": 0, "PIECE": 1}) def bad_request_and_disconnect(self, payload): conn = BTConnection("localhost", self.session.get_listen_port(), user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) conn.send(EXTEND + chr(metadata_id) + bencode(payload)) self.read_extend_metadata_close(conn)
class TestMagnetFakePeer(TestAsServer, MagnetHelpers): """ A MiniBitTorrent instance is used to connect to BitTorrent clients and download the info part from the metadata. """ def setUp(self): # listener for incoming connections from MiniBitTorrent self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.server.bind(("", self.session.get_listen_port())) self.server.listen(5) TestAsServer.setUp(self) # the metadata that we want to transfer self.tdef = TorrentDef() self.tdef.add_content(os.path.join(TESTS_API_DIR, "video.avi")) self.tdef.set_tracker("http://fake.net/announce") # we use a small piece length to obtain multiple pieces self.tdef.set_piece_length(1) self.tdef.finalize() MagnetHelpers.__init__(self, self.tdef) def setUpPreSession(self): TestAsServer.setUpPreSession(self) self.config.set_libtorrent(True) def create_good_url(self, infohash=None, title=None, tracker=None): url = "magnet:?xt=urn:btih:" if infohash: assert isinstance(infohash, str) url += hexlify(infohash) else: url += hexlify(self.tdef.get_infohash()) if title: assert isinstance(title, str) url += "&dn=" + title if tracker: assert isinstance(tracker, str) url += "&tr=" + tracker return url @skip("not working, seems to return binary data") def test_good_transfer(self): def torrentdef_retrieved(meta_info): tags["metainfo"] = meta_info tags["retrieved"].set() tags = {"retrieved": threading.Event()} self.session.lm.ltmgr.get_metainfo(self.create_good_url(), torrentdef_retrieved, timeout=60) def do_supply(): # supply fake addresses (regular dht obviously wont work here) ltmgr = LibtorrentMgr.getInstance() for infohash in ltmgr.metainfo_requests: handle = ltmgr.ltsession.find_torrent( lt.big_number(infohash.decode('hex'))) handle.connect_peer( ("127.0.0.1", self.session.get_listen_port()), 0) self.session.lm.threadpool.add_task(do_supply, delay=5.0) # accept incoming connection # self.server.settimeout(10.0) sock, address = self.server.accept() assert sock, "No incoming connection" # handshakes conn = BTConnection(address[0], address[1], opensock=sock, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) # serve pieces for counter in xrange(len(self.metadata_list)): piece = self.read_extend_metadata_request(conn) assert 0 <= piece < len(self.metadata_list) conn.send( self.create_good_extend_metadata_reply(metadata_id, piece)) # no more metadata request may be send and the connection must # be closed self.read_extend_metadata_close(conn) assert tags["retrieved"].wait(5) assert tags["metainfo"]["info"] == self.tdef.get_metainfo()["info"]
def make_meta_file(srcpaths, params, userabortflag, progressCallback, torrentfilenameCallback): tdef = TorrentDef() basedir = None nrFiles = len([file for file in srcpaths if os.path.isfile(file)]) if nrFiles > 1: #outpaths should start with a common prefix, this prefix is the swarmname of the torrent #if srcpaths contain c:\a\1, c:\a\2 -> basepath should be c:\ and basedir a and outpaths should be a\1 and a\2 #if srcpaths contain c:\a\1, c:\a\2, c:\a\b\1, c:\a\b\2 -> basepath should be c:\ and outpaths should be a\1, a\2, a\b\1 and a\b\2 basepath = os.path.abspath(os.path.commonprefix(srcpaths)) basepath, basedir = os.path.split(basepath) for srcpath in srcpaths: if os.path.isfile(srcpath): outpath = os.path.relpath(srcpath, basepath) # h4x0r playtime if 'playtime' in params: tdef.add_content(srcpath, outpath, playtime=params['playtime']) else: tdef.add_content(srcpath, outpath) else: srcpaths = [file for file in srcpaths if os.path.isfile(file)] srcpath = srcpaths[0] basepath, _ = os.path.split(srcpath) if 'playtime' in params: tdef.add_content(srcpath, playtime=params['playtime']) else: tdef.add_content(srcpath) if params.get('urllist', False): tdef.set_urllist(params['urllist']) if params['name']: tdef.set_name(params['name']) if params['comment']: tdef.set_comment(params['comment']) if params['created by']: tdef.set_created_by(params['created by']) if params['announce']: tdef.set_tracker(params['announce']) if params['announce-list']: tdef.set_tracker_hierarchy(params['announce-list']) if params['nodes']: # mainline DHT tdef.set_dht_nodesmax(params['nodes']) if params['httpseeds']: tdef.set_httpseeds(params['httpseeds']) if params['encoding']: tdef.set_encoding(params['encoding']) if params['piece length']: tdef.set_piece_length(params['piece length']) if params['makehash_md5']: print >> sys.stderr, "TorrentMaker: make MD5" tdef.set_add_md5hash(params['makehash_md5']) if params['makehash_crc32']: print >> sys.stderr, "TorrentMaker: make CRC32" tdef.set_add_crc32(params['makehash_crc32']) if params['makehash_sha1']: print >> sys.stderr, "TorrentMaker: make SHA1" tdef.set_add_sha1hash(params['makehash_sha1']) if params['createmerkletorrent']: tdef.set_create_merkle_torrent(params['createmerkletorrent']) if params['torrentsigkeypairfilename']: tdef.set_signature_keypair_filename( params['torrentsigkeypairfilename']) if params['thumb']: tdef.set_thumbnail(params['thumb']) tdef.finalize(userabortflag=userabortflag, userprogresscallback=progressCallback) if params['createmerkletorrent']: postfix = TRIBLER_TORRENT_EXT else: postfix = '.torrent' if params.get('target', False): torrentfilename = os.path.join( params['target'], os.path.split(os.path.normpath(srcpath))[1] + postfix) else: torrentfilename = os.path.join(basepath, tdef.get_name() + postfix) tdef.save(torrentfilename) # Inform higher layer we created torrent torrentfilenameCallback(basepath, basedir, torrentfilename)
def lineReceived(self, line): anon_tunnel = self.anon_tunnel if line == 'threads': for thread in threading.enumerate(): logger.debug("%s \t %d", thread.name, thread.ident) elif line == 'c': logger.debug( "========\nCircuits\n========\nid\taddress\t\t\t\t\tgoal\thops\tIN (MB)\tOUT (MB)\tinfohash\ttype" ) for circuit_id, circuit in anon_tunnel.community.circuits.items(): info_hash = circuit.info_hash.encode( 'hex')[:10] if circuit.info_hash else '?' logger.debug( "%d\t%s:%d\t%d\t%d\t\t%.2f\t\t%.2f\t\t%s\t%s" % circuit_id, circuit.first_hop[0], circuit.first_hop[1], circuit.goal_hops, len(circuit.hops), circuit.bytes_down / 1024.0 / 1024.0, circuit.bytes_up / 1024.0 / 1024.0, info_hash, circuit.ctype) elif line.startswith('s'): cur_path = os.getcwd() line_split = line.split(' ') filename = 'test_file' if len(line_split) == 1 else line_split[1] if not os.path.exists(filename): logger.info("Creating torrent..") with open(filename, 'wb') as fp: fp.write(os.urandom(50 * 1024 * 1024)) tdef = TorrentDef() tdef.add_content(os.path.join(cur_path, filename)) tdef.set_tracker("udp://localhost/announce") tdef.set_private() tdef.finalize() tdef.save(os.path.join(cur_path, filename + '.torrent')) else: logger.info("Loading existing torrent..") tdef = TorrentDef.load(filename + '.torrent') logger.info("loading torrent done, infohash of torrent: %s" % (tdef.get_infohash().encode('hex')[:10])) defaultDLConfig = DefaultDownloadStartupConfig.getInstance() dscfg = defaultDLConfig.copy() dscfg.set_hops(1) dscfg.set_dest_dir(cur_path) reactor.callFromThread( anon_tunnel.session.start_download_from_tdef, tdef, dscfg) elif line.startswith('i'): # Introduce dispersy port from other main peer to this peer line_split = line.split(' ') to_introduce_ip = line_split[1] to_introduce_port = int(line_split[2]) self.anon_tunnel.community.add_discovered_candidate( Candidate((to_introduce_ip, to_introduce_port), tunnel=False)) elif line.startswith('d'): line_split = line.split(' ') filename = 'test_file' if len(line_split) == 1 else line_split[1] logger.info("Loading torrent..") tdef = TorrentDef.load(filename + '.torrent') logger.info("Loading torrent done") defaultDLConfig = DefaultDownloadStartupConfig.getInstance() dscfg = defaultDLConfig.copy() dscfg.set_hops(1) dscfg.set_dest_dir( os.path.join( os.getcwd(), 'downloader%s' % anon_tunnel.session.get_dispersy_port())) def start_download(): def cb(ds): logger.info( 'Download infohash=%s, down=%s, progress=%s, status=%s, seedpeers=%s, candidates=%d' % (tdef.get_infohash().encode('hex')[:10], ds.get_current_speed('down'), ds.get_progress(), dlstatus_strings[ds.get_status()], sum(ds.get_num_seeds_peers()), sum(1 for _ in anon_tunnel.community. dispersy_yield_verified_candidates()))) return 1.0, False download = anon_tunnel.session.start_download_from_tdef( tdef, dscfg) download.set_state_callback(cb) reactor.callFromThread(start_download) elif line == 'q': anon_tunnel.should_run = False elif line == 'r': logger.debug("circuit\t\t\tdirection\tcircuit\t\t\tTraffic (MB)") from_to = anon_tunnel.community.relay_from_to for key in from_to.keys(): relay = from_to[key] logger.info("%s-->\t%s\t\t%.2f" % ( (key[0], key[1]), (relay.sock_addr, relay.circuit_id), relay.bytes[1] / 1024.0 / 1024.0, ))
class TestVideoServerSession(TestAsServer): """ Class for testing HTTP-based video server in a session. Mainly HTTP range queries. """ @inlineCallbacks def setUp(self): """ unittest test setup code """ yield super(TestVideoServerSession, self).setUp() self.port = self.session.config.get_video_server_port() self.sourcefn = os.path.join(TESTS_DATA_DIR, "video.avi") self.sourcesize = os.path.getsize(self.sourcefn) self.tdef = None self.expsize = 0 yield self.start_vod_download() def setUpPreSession(self): TestAsServer.setUpPreSession(self) self.config.set_libtorrent_enabled(True) self.config.set_video_server_enabled(True) @trial_timeout(10) def test_specific_range(self): return self.range_check(115, 214) @trial_timeout(10) def test_last_100(self): return self.range_check(self.sourcesize - 100, None) @trial_timeout(10) def test_first_100(self): return self.range_check(None, 100) @trial_timeout(10) def test_combined(self): return self.range_check(115, 214, setset=True) def start_vod_download(self): self.tdef = TorrentDef() self.tdef.add_content(self.sourcefn) self.tdef.set_tracker("http://127.0.0.1:12/announce") self.tdef.finalize() dscfg = DownloadStartupConfig() dscfg.set_dest_dir(os.path.dirname(self.sourcefn)) download = self.session.start_download_from_tdef(self.tdef, dscfg) return download.get_handle() def get_std_header(self): msg = "GET /%s/0 HTTP/1.1\r\n" % binascii.hexlify(self.tdef.get_infohash()) msg += "Host: 127.0.0.1:" + str(self.port) + "\r\n" return msg @staticmethod def create_range_str(firstbyte, lastbyte): head = "" if firstbyte is not None: head += str(firstbyte) head += "-" if lastbyte is not None: head += str(lastbyte) return head def get_header(self, firstbyte, lastbyte, setset=False): head = self.get_std_header() head += "Range: bytes=" head += self.create_range_str(firstbyte, lastbyte) if setset: # Make into set of byte ranges, VideoHTTPServer should refuse. head += ",0-99" head += "\r\n" head += "Connection: close\r\n" return head + "\r\n" def range_check(self, firstbyte, lastbyte, setset=False): test_deferred = Deferred() self._logger.debug("range_test: %s %s %s setset %s", firstbyte, lastbyte, self.sourcesize, setset) if firstbyte is not None and lastbyte is None: exp_byte_range = (firstbyte, self.sourcesize - 1) elif firstbyte is None and lastbyte is not None: exp_byte_range = (self.sourcesize - lastbyte, self.sourcesize - 1) else: exp_byte_range = (firstbyte, lastbyte) # the amount of bytes actually requested. (Content-length) self.expsize = exp_byte_range[1] - exp_byte_range[0] + 1 f = open(self.sourcefn, "rb") f.seek(exp_byte_range[0]) expdata = f.read(self.expsize) f.close() def on_connected(p): p.sendMessage(self.get_header(firstbyte, lastbyte, setset)) endpoint = TCP4ClientEndpoint(reactor, "localhost", self.port) connectProtocol(endpoint, VideoServerProtocol(test_deferred, self.sourcesize, expdata, setset, exp_byte_range))\ .addCallback(on_connected) return test_deferred
class TestMerkleMessage(TestAsServer): """ Testing Merkle hashpiece messages for both: * Merkle BEP style * old Tribler <= 4.5.2 that did not use the Extention protocol (BEP 10). See BitTornado/BT1/Connecter.py """ def setUp(self): """ override TestAsServer """ TestAsServer.setUp(self) print >> sys.stderr, "test: Giving Session time to startup" time.sleep(5) print >> sys.stderr, "test: Session should have started up" def setUpPreSession(self): """ override TestAsServer """ TestAsServer.setUpPreSession(self) self.config.set_overlay(False) self.config.set_megacache(False) def setUpPostSession(self): """ override TestAsServer """ TestAsServer.setUpPostSession(self) # Let Tribler start downloading an non-functioning torrent, so # we can talk to a normal download engine. self.tdef = TorrentDef() self.sourcefn = os.path.join(os.getcwd(), "API", "file2.wmv") self.tdef.add_content(self.sourcefn) self.tdef.set_create_merkle_torrent(True) self.tdef.set_tracker("http://127.0.0.1:12/announce") self.tdef.finalize() self.torrentfn = os.path.join(self.session.get_state_dir(), "gen.torrent") self.tdef.save(self.torrentfn) dscfg = self.setUpDownloadConfig() self.session.start_download(self.tdef, dscfg) self.infohash = self.tdef.get_infohash() self.mylistenport = 4810 self.numpieces = (self.tdef.get_length() + self.tdef.get_piece_length() - 1) / self.tdef.get_piece_length() b = Bitfield(self.numpieces) for i in range(self.numpieces): b[i] = True self.assert_(b.complete()) self.seederbitfieldstr = b.tostring() # piece_hashes = ['\x01\x02\x03\x04\x05\x06\x07\x08\x07\x06\x05\x04\x03\x02\x01\x00\x01\x02\x03\x04' ] * npieces # Construct Merkle tree tdef2 = TorrentDef() tdef2.add_content(self.sourcefn) tdef2.set_create_merkle_torrent(False) tdef2.set_tracker("http://127.0.0.1:12/announce") tdef2.set_piece_length(self.tdef.get_piece_length()) tdef2.finalize() metainfo = tdef2.get_metainfo() piecesstr = metainfo['info']['pieces'] print >> sys.stderr, "test: pieces has len", len(piecesstr) piece_hashes = [] for i in range(0, len(piecesstr), 20): hash = piecesstr[i:i + 20] print >> sys.stderr, "test: piece", i / 20, "hash", repr(hash) piece_hashes.append(hash) print >> sys.stderr, "test: Putting", len(piece_hashes), "into MerkleTree, size", self.tdef.get_piece_length(), tdef2.get_piece_length() self.tree = MerkleTree(self.tdef.get_piece_length(), self.tdef.get_length(), None, piece_hashes) f = open(self.sourcefn, "rb") piece1 = f.read(2 ** 18) piece2 = f.read(2 ** 18) print >> sys.stderr, "read piece1", len(piece1) print >> sys.stderr, "read piece2", len(piece2) f.close() hash1 = sha(piece1).digest() hash2 = sha(piece2).digest() print >> sys.stderr, "hash piece1", repr(hash1) print >> sys.stderr, "hash piece2", repr(hash2) f2 = open("piece1.bin", "wb") f2.write(piece2) f2.close() def setUpDownloadConfig(self): dscfg = DownloadStartupConfig() print >> sys.stderr, "test: Downloading to", self.config_path dscfg.set_dest_dir(self.config_path) dscfg.set_breakup_seed_bitfield(False) return dscfg def tearDown(self): TestAsServer.tearDown(self) try: os.remove('piece1.bin') except: pass def singtest_good_hashpiece_bepstyle(self): self.subtest_good_hashpiece(False) def singtest_good_hashpiece_oldstyle(self): self.subtest_good_hashpiece(True) def singtest_good_request_bepstyle(self): # Let Session download file first self.subtest_good_hashpiece(False) # Now connect as different peer and download print >> sys.stderr, "\n\ntest: test_good_request: STARTING" self._test_good_request() def singtest_bad_hashpiece_bepstyle(self): self.subtest_bad_hashpiece(False) def singtest_bad_hashpiece_oldstyle(self): self.subtest_bad_hashpiece(True) # # Good hashpiece message # def subtest_good_hashpiece(self, oldstyle): print >> sys.stderr, "test: Testing good hashpiece, oldstyle", oldstyle if oldstyle: self._test_good(self.create_good_hashpiece, oldstyle, self.create_good_tribler_extend_hs, infohash=self.infohash) else: options = '\x00\x00\x00\x00\x00\x10\x00\x00' self._test_good(self.create_good_hashpiece, oldstyle, self.create_good_nontribler_extend_hs, options=options, infohash=self.infohash) def _test_good(self, msg_gen_func, oldstyle, extend_hs_gen_func, options=None, infohash=None): if options is None and infohash is None: s = BTConnection('localhost', self.hisport) elif options is None: s = BTConnection('localhost', self.hisport, user_infohash=infohash) elif infohash is None: s = BTConnection('localhost', self.hisport, user_option_pattern=options) else: s = BTConnection('localhost', self.hisport, user_option_pattern=options, user_infohash=infohash) print >> sys.stderr, "test: test_good: Create EXTEND HS" msg = extend_hs_gen_func() print >> sys.stderr, "test: test_good: Sending EXTEND HS", repr(msg) s.send(msg) print >> sys.stderr, "test: test_good: Waiting for BT HS" s.read_handshake_medium_rare() # Tribler should send an EXTEND message back try: print >> sys.stderr, "test: Waiting for reply" s.s.settimeout(10.0) resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply", getMessageName(resp[0]) self.assert_(resp[0] == EXTEND) self.check_tribler_extend_hs(resp[1:]) # 1. Pretend we're seeder: send BITFIELD and UNCHOKE msg = BITFIELD + self.seederbitfieldstr s.send(msg) msg = UNCHOKE s.send(msg) print >> sys.stderr, "test: Pretend we are seeder" while True: resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply2", getMessageName(resp[0]) self.assert_(resp[0] == REQUEST or resp[0] == INTERESTED or resp[0] == UNCHOKE or resp[0] == HAVE or resp[0] == NOT_INTERESTED) if resp[0] == REQUEST: chunkid = self.check_request(resp) # 2. Reply to REQUEST with HASHPIECE (oldstyle) or Tr_hashpiece msg = msg_gen_func(oldstyle, chunkid) s.send(msg) elif resp[0] == NOT_INTERESTED: break # s.close() except socket.timeout: print >> sys.stderr, "test: Timeout, bad, peer didn't reply in time" self.assert_(False) destfn = os.path.join(self.config_path, "file2.wmv") sf = open(self.sourcefn, "rb") df = open(destfn, "rb") n = self.tdef.get_piece_length() while True: sdata = sf.read(n) if len(sdata) == 0: break ddata = df.read(n) self.assert_(sdata == ddata) time.sleep(3) s.close() def create_good_nontribler_extend_hs(self): """ Merkle BEP style """ d = {} d['m'] = {'Tr_hashpiece': 250} d['p'] = self.mylistenport d['v'] = 'TestSweet 1.2.3.4' bd = bencode(d) return EXTEND + chr(0) + bd def create_good_tribler_extend_hs(self): """ old Tribler style """ d = {} d['m'] = {'Tr_OVERLAYSWARM': 253} d['p'] = self.mylistenport d['v'] = 'Tribler 3.5.1' bd = bencode(d) return EXTEND + chr(0) + bd def check_tribler_extend_hs(self, data): self.assert_(data[0] == chr(0)) d = bdecode(data[1:]) self.assert_(isinstance(d, DictType)) self.assert_('m' in d.keys()) m = d['m'] self.assert_(isinstance(m, DictType)) self.assert_('Tr_hashpiece' in m.keys()) val = m['Tr_hashpiece'] self.assert_(isinstance(val, IntType)) self.assert_(val == 250) def check_request(self, data): index = toint(data[1:5]) begin = toint(data[5:9]) length = toint(data[9:]) return (index, begin, length) def create_good_hashpiece(self, oldstyle, chunkid): index, begin, length = chunkid if begin == 0: ohlist = self.tree.get_hashes_for_piece(index) else: ohlist = [] chunk = self.read_chunk(index, begin, length) bohlist = bencode(ohlist) print >> sys.stderr, "test: create_good_hashpiece:", index, begin, length, "==len", len(chunk) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk if oldstyle: msg = HASHPIECE + payload else: # Offical: use the msg ID he defined in his handshake msg = EXTEND + HASHPIECE + payload return msg def read_chunk(self, index, begin, length): offset = index * self.tdef.get_piece_length() + begin f = open(self.sourcefn, "rb") f.seek(offset) chunk = f.read(length) f.close() return chunk # # Test whether Tribler sends good Tr_hashpiece on our requests # def _test_good_request(self): options = '\x00\x00\x00\x00\x00\x10\x00\x00' myid = Rand.rand_bytes(20) s = BTConnection('localhost', self.hisport, user_option_pattern=options, user_infohash=self.infohash, myid=myid) msg = self.create_good_nontribler_extend_hs() s.send(msg) s.read_handshake_medium_rare() # Tribler should send an EXTEND message back try: print >> sys.stderr, "test: Waiting for reply" s.s.settimeout(10.0) resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply", getMessageName(resp[0]) self.assert_(resp[0] == EXTEND) self.check_tribler_extend_hs(resp[1:]) # 1. Pretend we're leecher: send INTERESTED msg = INTERESTED s.send(msg) print >> sys.stderr, "test: Pretend we are leecher" while True: resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply2", getMessageName(resp[0]) if resp[0] == EXTEND: print >> sys.stderr, "test: Got EXTEND type", getMessageName(resp[1]) self.assert_(resp[0] == UNCHOKE or resp[0] == BITFIELD or resp[0] == EXTEND or resp[0] == HAVE) if resp[0] == UNCHOKE: # 2. Reply with REQUESTs for index in range(0, self.numpieces): plen = self.get_piece_length(index) for begin in range(0, plen, 2 ** 14): length = self.get_chunk_length(index, begin) print >> sys.stderr, "RETRIEVE", index, begin, length chunkid = (index, begin, length) msg = self.create_request(chunkid) s.send(msg) # s.send(NOT_INTERESTED) elif resp[0] == EXTEND and resp[1] == HASHPIECE: done = self.check_hashpiece(resp) if done: break elif resp[0] == BITFIELD: self.check_bitfield(resp) # s.close() except socket.timeout: print >> sys.stderr, "test: Timeout, bad, peer didn't reply in time" self.assert_(False) time.sleep(3) s.close() def get_piece_length(self, index): if index == (self.numpieces - 1): plen = self.tdef.get_length() % self.tdef.get_piece_length() else: plen = self.tdef.get_piece_length() return plen def get_chunk_length(self, index, begin): plen = self.get_piece_length(index) length = 2 ** 14 if index == (self.numpieces - 1): if (begin + 2 ** 14) > plen: length = plen - begin return length def create_request(self, chunkid): index, begin, length = chunkid return REQUEST + tobinary(index) + tobinary(begin) + tobinary(length) def check_hashpiece(self, resp): """ Merkle BEP style """ print >> sys.stderr, "test: good_request: check_hashpiece" self.assert_(resp[0] == EXTEND) self.assert_(resp[1] == HASHPIECE) index = toint(resp[2:2 + 4]) begin = toint(resp[6:6 + 4]) ohlen = toint(resp[10:10 + 4]) print >> sys.stderr, "test: good_request: check_hashpiece", index, begin, ohlen bohlist = resp[14:14 + ohlen] hisohlist = bdecode(bohlist) hischunk = resp[14 + ohlen:] if begin == 0: self.assert_(isinstance(hisohlist, ListType)) for oh in hisohlist: self.assert_(isinstance(oh, ListType)) self.assert_(len(oh) == 2) self.assert_(isinstance(oh[0], IntType)) self.assert_(isinstance(oh[1], StringType)) hisohlist.sort() print >> sys.stderr, "test: good_request: check_hashpiece", repr(hisohlist) myohlist = self.tree.get_hashes_for_piece(index) myohlist.sort() self.assert_(len(hisohlist) == len(myohlist)) for i in range(0, len(hisohlist)): hisoh = hisohlist[i] myoh = myohlist[i] self.assert_(hisoh == myoh) else: self.assert_(len(hisohlist) == 0) mylength = self.get_chunk_length(index, begin) mychunk = self.read_chunk(index, begin, mylength) self.assert_(hischunk == mychunk) return index == self.numpieces - 1 and mylength != 2 ** 14 def check_bitfield(self, data): self.assert_(data[0] == BITFIELD) bitmap = data[1:] self.assert_(len(bitmap) == 1) # Must have set_breakup_seed_bitfield() set to False self.assert_(bitmap == '\xc0') # # Bad EXTEND handshake message # def subtest_bad_hashpiece(self, oldstyle): if not oldstyle: # Test becomes equivalent to BT keep alive message (len 0, payload '') self._test_bad(self.create_empty, oldstyle) self._test_bad(self.create_ext_id_not_byte, oldstyle) self._test_bad(self.create_not_hashpiece, oldstyle) self._test_bad(self.create_not_index, oldstyle) self._test_bad(self.create_not_begin, oldstyle) self._test_bad(self.create_not_len_bohlist, oldstyle) self._test_bad(self.create_ohlist_not_bdecodable, oldstyle) self._test_bad(self.create_ohlist_wrong_no_hashes, oldstyle) self._test_bad(self.create_ohlist_wrong_no_root_hash, oldstyle) self._test_bad(self.create_ohlist_wrong_bad_offset, oldstyle) self._test_bad(self.create_ohlist_wrong_bad_hash, oldstyle) # TODO: need working peer kicking for that # self._test_bad(self.create_bad_chunk,oldstyle) # # Main test code for bad EXTEND handshake messages # def _test_bad(self, msg_gen_func, oldstyle): print >> sys.stderr, "test: test_BAD: Create EXTEND HS", repr(msg_gen_func), oldstyle if oldstyle: options = None exthsmsg = self.create_good_tribler_extend_hs() else: options = '\x00\x00\x00\x00\x00\x10\x00\x00' exthsmsg = self.create_good_nontribler_extend_hs() s = BTConnection('localhost', self.hisport, user_option_pattern=options, user_infohash=self.infohash) s.send(exthsmsg) s.read_handshake_medium_rare() # Tribler should send an EXTEND message back try: print >> sys.stderr, "test: Waiting for reply" s.s.settimeout(10.0) resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply", getMessageName(resp[0]) self.assert_(resp[0] == EXTEND) self.check_tribler_extend_hs(resp[1:]) # 1. Pretend we're seeder: send BITFIELD and UNCHOKE msg = BITFIELD + self.seederbitfieldstr s.send(msg) msg = UNCHOKE s.send(msg) print >> sys.stderr, "test: Pretend we are seeder" while True: resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply 2", getMessageName(resp[0]) self.assert_(resp[0] == REQUEST or resp[0] == INTERESTED or resp[0] == UNCHOKE or resp[0] == HAVE or resp[0] == NOT_INTERESTED) if resp[0] == REQUEST: chunkid = self.check_request(resp) # 2. Reply to REQUEST with *bad* HASHPIECE msg = msg_gen_func(chunkid) if oldstyle: if len(msg) == 1: msg = '' else: msg = msg[1:] # Strip EXTEND byte s.send(msg) break # s.close() except socket.timeout: print >> sys.stderr, "test: Timeout, bad, peer didn't reply in time" self.assert_(False) time.sleep(3) # Should have closed the connection try: s.send(UNCHOKE) self.assert_(False) except: print_exc() s.close() # # Bad message creators (all create Merkle BEP style, I strip first byte # later for oldstyle # def create_empty(self, chunkid): return EXTEND def create_ext_id_not_byte(self, chunkid): return EXTEND + 'Hallo kijkbuiskinderen' def create_not_hashpiece(self, chunkid): index, begin, length = chunkid ohlist = [] bohlist = bencode(ohlist) chunk = self.read_chunk(index, begin, length) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + chr(231) + payload def create_not_index(self, chunkid): payload = 'bla' return EXTEND + HASHPIECE + payload def create_not_begin(self, chunkid): index, begin, length = chunkid payload = tobinary(index) + 'bla' return EXTEND + HASHPIECE + payload def create_not_len_bohlist(self, chunkid): index, begin, length = chunkid payload = tobinary(index) + tobinary(begin) + 'bla' return EXTEND + HASHPIECE + payload def create_ohlist_not_bdecodable(self, chunkid): index, begin, length = chunkid bohlist = 'bla' chunk = '*' * (2 ** 14) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload def create_ohlist_wrong_no_hashes(self, chunkid): index, begin, length = chunkid ohlist = [(0, '#' * 20), (1, '$' * 20)] # should contain 3 for file2.wmv: own, sibling and root bohlist = bencode(ohlist) chunk = '*' * (2 ** 14) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload def create_ohlist_wrong_no_root_hash(self, chunkid): index, begin, length = chunkid ohlist = self.tree.get_hashes_for_piece(index) newohlist = [] # Remove root hash for oh in ohlist: if oh[0] != 0: newohlist.append(oh) ohlist = newohlist bohlist = bencode(ohlist) chunk = self.read_chunk(index, begin, length) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload def create_ohlist_wrong_bad_offset(self, chunkid): index, begin, length = chunkid ohlist = self.tree.get_hashes_for_piece(index) ohlist[1][0] = 481 bohlist = bencode(ohlist) chunk = self.read_chunk(index, begin, length) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload def create_ohlist_wrong_bad_hash(self, chunkid): index, begin, length = chunkid ohlist = self.tree.get_hashes_for_piece(index) ohlist[1][1] = '$' * 20 bohlist = bencode(ohlist) chunk = self.read_chunk(index, begin, length) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload def create_bad_chunk(self, chunkid): index, begin, length = chunkid ohlist = self.tree.get_hashes_for_piece(index) bohlist = bencode(ohlist) chunk = '*' * length payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload
class TestMagnetMiniBitTorrent(TestAsServer, MagnetHelpers): """ A MiniBitTorrent instance is used to connect to BitTorrent clients and download the info part from the metadata. """ def setUp(self): """ override TestAsServer """ # listener for incoming connections from MiniBitTorrent self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.server.bind(("localhost", LISTEN_PORT)) self.server.listen(1) # the metadata that we want to transfer self.tdef = TorrentDef() self.tdef.add_content(os.path.join(os.getcwd(), "API", "file.wmv")) self.tdef.set_tracker("http://fake.net/announce") # we use a small piece length to obtain multiple pieces self.tdef.set_piece_length(1) self.tdef.finalize() MagnetHelpers.__init__(self, self.tdef) # startup the client TestAsServer.setUp(self) print >>sys.stderr,"test: Giving MyLaunchMany time to startup" time.sleep(5) print >>sys.stderr,"test: MyLaunchMany should have started up" def create_good_url(self, infohash=None, title=None, tracker=None): url = "magnet:?xt=urn:btih:" if infohash: assert isinstance(infohash, str) url += hexlify(infohash) else: url += hexlify(self.tdef.get_infohash()) if title: assert isinstance(title, str) url += "&dn=" + title if tracker: assert isinstance(tracker, str) url += "&tr=" + tracker return url def test_good_transfer(self): def torrentdef_retrieved(tdef): tags["retrieved"] = True tags["metainfo"] = tdef.get_metainfo() tags = {"retrieved":False} assert TorrentDef.retrieve_from_magnet(self.create_good_url(), torrentdef_retrieved) # supply fake addresses (regular dht obviously wont work here) for magnetlink in MagnetHandler.get_instance().get_magnets(): magnetlink._swarm.add_potential_peers([("localhost", LISTEN_PORT)]) # accept incoming connection self.server.settimeout(10.0) sock, address = self.server.accept() assert sock, "No incoming connection" # handshakes conn = BTConnection(address[0], address[1], opensock=sock, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) # serve pieces for counter in xrange(len(self.metadata_list)): piece = self.read_extend_metadata_request(conn) assert 0 <= piece < len(self.metadata_list) conn.send(self.create_good_extend_metadata_reply(metadata_id, piece)) # no more metadata request may be send and the connection must # be closed self.read_extend_metadata_close(conn) time.sleep(5) assert tags["retrieved"] assert tags["metainfo"]["info"] == self.tdef.get_metainfo()["info"]
class TestVideoHTTPServer(TestAsServer): """ Class for testing HTTP-based video server. Mainly HTTP range queries. """ def setUp(self): """ unittest test setup code """ TestAsServer.setUp(self) self.port = self.session.get_videoplayer_port() self.sourcefn = os.path.join(TESTS_DATA_DIR, "video.avi") self.sourcesize = os.path.getsize(self.sourcefn) # wait 5s to allow server to start time.sleep(5) def setUpPreSession(self): TestAsServer.setUpPreSession(self) self.config.set_libtorrent(True) self.config.set_videoplayer(True) def tearDown(self): """ unittest test tear down code """ TestAsServer.tearDown(self) time.sleep(2) # # Tests # def test_specific_range(self): self.range_check(115, 214, self.sourcesize) def test_last_100(self): self.range_check(self.sourcesize - 100, None, self.sourcesize) def test_first_100(self): self.range_check(None, 100, self.sourcesize) def test_combined(self): self.range_check(115, 214, self.sourcesize, setset=True) # # Internal # def register_file_stream(self): self.tdef = TorrentDef() self.tdef.add_content(self.sourcefn) self.tdef.set_tracker("http://127.0.0.1:12/announce") self.tdef.finalize() dscfg = DownloadStartupConfig() dscfg.set_dest_dir(os.path.dirname(self.sourcefn)) download = self.session.start_download(self.tdef, dscfg) while not download.handle: time.sleep(1) def get_std_header(self): msg = "GET /%s/0 HTTP/1.1\r\n" % binascii.hexlify( self.tdef.get_infohash()) msg += "Host: 127.0.0.1:" + str(self.port) + "\r\n" return msg def create_range_str(self, firstbyte, lastbyte): head = "" if firstbyte is not None: head += str(firstbyte) head += "-" if lastbyte is not None: head += str(lastbyte) return head def range_check(self, firstbyte, lastbyte, sourcesize, setset=False): self._logger.debug("range_test: %s %s %s setset %s", firstbyte, lastbyte, sourcesize, setset) self.register_file_stream() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect(('127.0.0.1', self.port)) head = self.get_std_header() head += "Range: bytes=" head += self.create_range_str(firstbyte, lastbyte) if setset: # Make into set of byte ranges, VideoHTTPServer should refuse. head += ",0-99" head += "\r\n" head += "Connection: close\r\n" head += "\r\n" if firstbyte is not None and lastbyte is None: # 100- expfirstbyte = firstbyte explastbyte = self.sourcesize - 1 elif firstbyte is None and lastbyte is not None: # -100 expfirstbyte = self.sourcesize - lastbyte explastbyte = self.sourcesize - 1 else: expfirstbyte = firstbyte explastbyte = lastbyte # the amount of bytes actually requested. (Content-length) expsize = explastbyte - expfirstbyte + 1 self._logger.debug("Expecting first %s last %s size %s ", expfirstbyte, explastbyte, sourcesize) s.send(head) # Parse header s.settimeout(10.0) while True: line = self.readline(s) if DEBUG: self._logger.debug("Got line: %s", repr(line)) if len(line) == 0: if DEBUG: self._logger.debug("server closed conn") self.assert_(False) return if line.startswith("HTTP"): if not setset: # Python returns "HTTP/1.0 206 Partial Content\r\n" HTTP 1.0??? self.assert_(line.startswith("HTTP/1.")) self.assert_(line.find("206") != -1) # Partial content else: self.assert_(line.startswith("HTTP/1.")) self.assert_(line.find("416") != -1) # Requested Range Not Satisfiable return elif line.startswith("Content-Range:"): expline = "Content-Range: bytes " + self.create_range_str( expfirstbyte, explastbyte) + "/" + str(sourcesize) + "\r\n" self.assertEqual(expline, line) elif line.startswith("Content-Type:"): self.assertEqual(line, "Content-Type: video/x-msvideo\r\n") elif line.startswith("Content-Length:"): self.assertEqual(line, "Content-Length: " + str(expsize) + "\r\n") elif line.endswith("\r\n") and len(line) == 2: # End of header break data = s.recv(expsize) if len(data) == 0: if DEBUG: self._logger.debug("server closed conn2") self.assert_(False) return else: f = open(self.sourcefn, "rb") if firstbyte is not None: f.seek(firstbyte) else: f.seek(lastbyte, os.SEEK_END) expdata = f.read(expsize) f.close() self.assert_(data, expdata) try: # Read body, reading more should EOF (we disabled persist conn) data = s.recv(10240) self.assert_(len(data) == 0) except socket.timeout: if DEBUG: self._logger.debug( "Timeout, video server didn't respond with requested bytes, possibly bug in Python impl of HTTP" ) print_exc() def readline(self, s): line = '' while True: data = s.recv(1) if len(data) == 0: return line else: line = line + data if data == '\n' and len(line) >= 2 and line[-2:] == '\r\n': return line
class TestVideoOnDemand(TestAsServer): """ Testing Merkle hashpiece messages for both: * Merkle BEP style * old Tribler <= 4.5.2 that did not use the Extention protocol (BEP 10). See BitTornado/BT1/Connecter.py """ def setUp(self): """ override TestAsServer """ TestAsServer.setUp(self) self.vodstarted = False def setUpPreSession(self): TestAsServer.setUpPreSession(self) self.config.set_libtorrent(True) def create_torrent(self): [srchandle, self.sourcefn] = mkstemp() self.content = Rand.rand_bytes(self.contentlen) os.write(srchandle, self.content) os.close(srchandle) self.tdef = TorrentDef() self.tdef.add_content(self.sourcefn) self.tdef.set_piece_length(self.piecelen) self.tdef.set_tracker("http://127.0.0.1:12/announce") self.tdef.finalize() self.torrentfn = os.path.join(self.session.get_state_dir(), "gen.torrent") self.tdef.save(self.torrentfn) dscfg = DownloadStartupConfig() destdir = os.path.dirname(self.sourcefn) dscfg.set_dest_dir(destdir) dscfg.set_video_event_callback(self.sesscb_vod_event_callback) self.session.set_download_states_callback(self.states_callback) self.session.start_download(self.tdef, dscfg) def states_callback(self, dslist): ds = dslist[0] d = ds.get_download() # print >>sys.stderr,`d.get_def().get_name()`,dlstatus_strings[ds.get_status()],ds.get_progress(),"%",ds.get_error(),"up",ds.get_current_speed(UPLOAD),"down",ds.get_current_speed(DOWNLOAD) print >> sys.stderr, '%s %s %5.2f%% %s up %8.2fKB/s down %8.2fKB/s' % \ (d.get_def().get_name(), dlstatus_strings[ds.get_status()], ds.get_progress() * 100, ds.get_error(), ds.get_current_speed(UPLOAD), ds.get_current_speed(DOWNLOAD)) return (1.0, []) def sesscb_vod_event_callback(self, d, event, params): if self.vodstarted: return self.vodstarted = True print >> sys.stderr, "Test: vod_event_callback", event, params if event == VODEVENT_START: stream = params['stream'] # Read last piece lastpieceoff = ((self.contentlen - 1) / self.piecelen) * self.piecelen lastpiecesize = self.contentlen - lastpieceoff print >> sys.stderr, "Test: stream: lastpieceoff", lastpieceoff, lastpiecesize self.stream_read(stream, lastpieceoff, lastpiecesize, self.piecelen) # Read second,3rd,4th byte, only secoff = 1 secsize = 3 blocksize = 3 self.stream_read(stream, secoff, secsize, blocksize) # Read last byte lastoff = self.contentlen - 1 lastsize = 1 self.stream_read(stream, lastoff, lastsize, self.piecelen) def stream_read(self, stream, off, size, blocksize): stream.seek(off) data = stream.read(blocksize) print >> sys.stderr, "Test: stream: Got data", len(data) self.assertEquals(len(data), size) self.assertEquals(data, self.content[off:off + size]) def test_99(self): self.contentlen = 99 self.piecelen = 10 self.create_torrent() print >> sys.stderr, "Test: Letting network thread create Download, sleeping" time.sleep(5) dlist = self.session.get_downloads() d = dlist[0] vs = d.get_vod_info()['status'] if vs: goodrange = ((0, 0), (9, 8)) self.assertEqual(vs.movie_range, goodrange) self.assertEqual(vs.first_piecelen, 10) self.assertEqual(vs.last_piecelen, 9) self.assertEqual(vs.first_piece, 0) self.assertEqual(vs.last_piece, 9) self.assertEqual(vs.movie_numpieces, 10) def test_100(self): self.contentlen = 100 self.piecelen = 10 self.create_torrent() print >> sys.stderr, "Test: Letting network thread create Download, sleeping" time.sleep(5) dlist = self.session.get_downloads() d = dlist[0] vs = d.get_vod_info()['status'] if vs: goodrange = ((0, 0), (9, 9)) self.assertEqual(vs.movie_range, goodrange) self.assertEqual(vs.first_piecelen, 10) self.assertEqual(vs.last_piecelen, 10) self.assertEqual(vs.first_piece, 0) self.assertEqual(vs.last_piece, 9) self.assertEqual(vs.movie_numpieces, 10) def test_101(self): self.contentlen = 101 self.piecelen = 10 self.create_torrent() print >> sys.stderr, "Test: Letting network thread create Download, sleeping" time.sleep(5) dlist = self.session.get_downloads() d = dlist[0] vs = d.get_vod_info()['status'] if vs: goodrange = ((0, 0), (10, 0)) self.assertEqual(vs.movie_range, goodrange) self.assertEqual(vs.first_piecelen, 10) self.assertEqual(vs.last_piecelen, 1) self.assertEqual(vs.first_piece, 0) self.assertEqual(vs.last_piece, 10) self.assertEqual(vs.movie_numpieces, 11)
class TestMerkleMessage(TestAsServer): """ Testing Merkle hashpiece messages for both: * Merkle BEP style * old Tribler <= 4.5.2 that did not use the Extention protocol (BEP 10). See BitTornado/BT1/Connecter.py """ def setUp(self): """ override TestAsServer """ TestAsServer.setUp(self) print >> sys.stderr, "test: Giving Session time to startup" time.sleep(5) print >> sys.stderr, "test: Session should have started up" def setUpPreSession(self): """ override TestAsServer """ TestAsServer.setUpPreSession(self) self.config.set_overlay(False) self.config.set_megacache(False) def setUpPostSession(self): """ override TestAsServer """ TestAsServer.setUpPostSession(self) # Let Tribler start downloading an non-functioning torrent, so # we can talk to a normal download engine. self.tdef = TorrentDef() self.sourcefn = os.path.join(os.getcwd(), "API", "file2.wmv") self.tdef.add_content(self.sourcefn) self.tdef.set_create_merkle_torrent(True) self.tdef.set_tracker("http://127.0.0.1:12/announce") self.tdef.finalize() self.torrentfn = os.path.join(self.session.get_state_dir(), "gen.torrent") self.tdef.save(self.torrentfn) dscfg = self.setUpDownloadConfig() self.session.start_download(self.tdef, dscfg) self.infohash = self.tdef.get_infohash() self.mylistenport = 4810 self.numpieces = (self.tdef.get_length() + self.tdef.get_piece_length() - 1) / self.tdef.get_piece_length() b = Bitfield(self.numpieces) for i in range(self.numpieces): b[i] = True self.assert_(b.complete()) self.seederbitfieldstr = b.tostring() # piece_hashes = ['\x01\x02\x03\x04\x05\x06\x07\x08\x07\x06\x05\x04\x03\x02\x01\x00\x01\x02\x03\x04' ] * npieces # Construct Merkle tree tdef2 = TorrentDef() tdef2.add_content(self.sourcefn) tdef2.set_create_merkle_torrent(False) tdef2.set_tracker("http://127.0.0.1:12/announce") tdef2.set_piece_length(self.tdef.get_piece_length()) tdef2.finalize() metainfo = tdef2.get_metainfo() piecesstr = metainfo["info"]["pieces"] print >> sys.stderr, "test: pieces has len", len(piecesstr) piece_hashes = [] for i in range(0, len(piecesstr), 20): hash = piecesstr[i : i + 20] print >> sys.stderr, "test: piece", i / 20, "hash", repr(hash) piece_hashes.append(hash) print >> sys.stderr, "test: Putting", len( piece_hashes ), "into MerkleTree, size", self.tdef.get_piece_length(), tdef2.get_piece_length() self.tree = MerkleTree(self.tdef.get_piece_length(), self.tdef.get_length(), None, piece_hashes) f = open(self.sourcefn, "rb") piece1 = f.read(2 ** 18) piece2 = f.read(2 ** 18) print >> sys.stderr, "read piece1", len(piece1) print >> sys.stderr, "read piece2", len(piece2) f.close() hash1 = sha(piece1).digest() hash2 = sha(piece2).digest() print >> sys.stderr, "hash piece1", repr(hash1) print >> sys.stderr, "hash piece2", repr(hash2) f2 = open("piece1.bin", "wb") f2.write(piece2) f2.close() def setUpDownloadConfig(self): dscfg = DownloadStartupConfig() print >> sys.stderr, "test: Downloading to", self.config_path dscfg.set_dest_dir(self.config_path) dscfg.set_breakup_seed_bitfield(False) return dscfg def tearDown(self): TestAsServer.tearDown(self) try: os.remove("piece1.bin") except: pass def singtest_good_hashpiece_bepstyle(self): self.subtest_good_hashpiece(False) def singtest_good_hashpiece_oldstyle(self): self.subtest_good_hashpiece(True) def singtest_good_request_bepstyle(self): # Let Session download file first self.subtest_good_hashpiece(False) # Now connect as different peer and download print >> sys.stderr, "\n\ntest: test_good_request: STARTING" self._test_good_request() def singtest_bad_hashpiece_bepstyle(self): self.subtest_bad_hashpiece(False) def singtest_bad_hashpiece_oldstyle(self): self.subtest_bad_hashpiece(True) # # Good hashpiece message # def subtest_good_hashpiece(self, oldstyle): print >> sys.stderr, "test: Testing good hashpiece, oldstyle", oldstyle if oldstyle: self._test_good( self.create_good_hashpiece, oldstyle, self.create_good_tribler_extend_hs, infohash=self.infohash ) else: options = "\x00\x00\x00\x00\x00\x10\x00\x00" self._test_good( self.create_good_hashpiece, oldstyle, self.create_good_nontribler_extend_hs, options=options, infohash=self.infohash, ) def _test_good(self, msg_gen_func, oldstyle, extend_hs_gen_func, options=None, infohash=None): if options is None and infohash is None: s = BTConnection("localhost", self.hisport) elif options is None: s = BTConnection("localhost", self.hisport, user_infohash=infohash) elif infohash is None: s = BTConnection("localhost", self.hisport, user_option_pattern=options) else: s = BTConnection("localhost", self.hisport, user_option_pattern=options, user_infohash=infohash) print >> sys.stderr, "test: test_good: Create EXTEND HS" msg = extend_hs_gen_func() print >> sys.stderr, "test: test_good: Sending EXTEND HS", repr(msg) s.send(msg) print >> sys.stderr, "test: test_good: Waiting for BT HS" s.read_handshake_medium_rare() # Tribler should send an EXTEND message back try: print >> sys.stderr, "test: Waiting for reply" s.s.settimeout(10.0) resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply", getMessageName(resp[0]) self.assert_(resp[0] == EXTEND) self.check_tribler_extend_hs(resp[1:]) # 1. Pretend we're seeder: send BITFIELD and UNCHOKE msg = BITFIELD + self.seederbitfieldstr s.send(msg) msg = UNCHOKE s.send(msg) print >> sys.stderr, "test: Pretend we are seeder" while True: resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply2", getMessageName(resp[0]) self.assert_( resp[0] == REQUEST or resp[0] == INTERESTED or resp[0] == UNCHOKE or resp[0] == HAVE or resp[0] == NOT_INTERESTED ) if resp[0] == REQUEST: chunkid = self.check_request(resp) # 2. Reply to REQUEST with HASHPIECE (oldstyle) or Tr_hashpiece msg = msg_gen_func(oldstyle, chunkid) s.send(msg) elif resp[0] == NOT_INTERESTED: break # s.close() except socket.timeout: print >> sys.stderr, "test: Timeout, bad, peer didn't reply in time" self.assert_(False) destfn = os.path.join(self.config_path, "file2.wmv") sf = open(self.sourcefn, "rb") df = open(destfn, "rb") n = self.tdef.get_piece_length() while True: sdata = sf.read(n) if len(sdata) == 0: break ddata = df.read(n) self.assert_(sdata == ddata) time.sleep(3) s.close() def create_good_nontribler_extend_hs(self): """ Merkle BEP style """ d = {} d["m"] = {"Tr_hashpiece": 250} d["p"] = self.mylistenport d["v"] = "TestSweet 1.2.3.4" bd = bencode(d) return EXTEND + chr(0) + bd def create_good_tribler_extend_hs(self): """ old Tribler style """ d = {} d["m"] = {"Tr_OVERLAYSWARM": 253} d["p"] = self.mylistenport d["v"] = "Tribler 3.5.1" bd = bencode(d) return EXTEND + chr(0) + bd def check_tribler_extend_hs(self, data): self.assert_(data[0] == chr(0)) d = bdecode(data[1:]) self.assert_(isinstance(d, DictType)) self.assert_("m" in d.keys()) m = d["m"] self.assert_(isinstance(m, DictType)) self.assert_("Tr_hashpiece" in m.keys()) val = m["Tr_hashpiece"] self.assert_(isinstance(val, IntType)) self.assert_(val == 250) def check_request(self, data): index = toint(data[1:5]) begin = toint(data[5:9]) length = toint(data[9:]) return (index, begin, length) def create_good_hashpiece(self, oldstyle, chunkid): index, begin, length = chunkid if begin == 0: ohlist = self.tree.get_hashes_for_piece(index) else: ohlist = [] chunk = self.read_chunk(index, begin, length) bohlist = bencode(ohlist) print >> sys.stderr, "test: create_good_hashpiece:", index, begin, length, "==len", len(chunk) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk if oldstyle: msg = HASHPIECE + payload else: # Offical: use the msg ID he defined in his handshake msg = EXTEND + HASHPIECE + payload return msg def read_chunk(self, index, begin, length): offset = index * self.tdef.get_piece_length() + begin f = open(self.sourcefn, "rb") f.seek(offset) chunk = f.read(length) f.close() return chunk # # Test whether Tribler sends good Tr_hashpiece on our requests # def _test_good_request(self): options = "\x00\x00\x00\x00\x00\x10\x00\x00" myid = Rand.rand_bytes(20) s = BTConnection("localhost", self.hisport, user_option_pattern=options, user_infohash=self.infohash, myid=myid) msg = self.create_good_nontribler_extend_hs() s.send(msg) s.read_handshake_medium_rare() # Tribler should send an EXTEND message back try: print >> sys.stderr, "test: Waiting for reply" s.s.settimeout(10.0) resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply", getMessageName(resp[0]) self.assert_(resp[0] == EXTEND) self.check_tribler_extend_hs(resp[1:]) # 1. Pretend we're leecher: send INTERESTED msg = INTERESTED s.send(msg) print >> sys.stderr, "test: Pretend we are leecher" while True: resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply2", getMessageName(resp[0]) if resp[0] == EXTEND: print >> sys.stderr, "test: Got EXTEND type", getMessageName(resp[1]) self.assert_(resp[0] == UNCHOKE or resp[0] == BITFIELD or resp[0] == EXTEND or resp[0] == HAVE) if resp[0] == UNCHOKE: # 2. Reply with REQUESTs for index in range(0, self.numpieces): plen = self.get_piece_length(index) for begin in range(0, plen, 2 ** 14): length = self.get_chunk_length(index, begin) print >> sys.stderr, "RETRIEVE", index, begin, length chunkid = (index, begin, length) msg = self.create_request(chunkid) s.send(msg) # s.send(NOT_INTERESTED) elif resp[0] == EXTEND and resp[1] == HASHPIECE: done = self.check_hashpiece(resp) if done: break elif resp[0] == BITFIELD: self.check_bitfield(resp) # s.close() except socket.timeout: print >> sys.stderr, "test: Timeout, bad, peer didn't reply in time" self.assert_(False) time.sleep(3) s.close() def get_piece_length(self, index): if index == (self.numpieces - 1): plen = self.tdef.get_length() % self.tdef.get_piece_length() else: plen = self.tdef.get_piece_length() return plen def get_chunk_length(self, index, begin): plen = self.get_piece_length(index) length = 2 ** 14 if index == (self.numpieces - 1): if (begin + 2 ** 14) > plen: length = plen - begin return length def create_request(self, chunkid): index, begin, length = chunkid return REQUEST + tobinary(index) + tobinary(begin) + tobinary(length) def check_hashpiece(self, resp): """ Merkle BEP style """ print >> sys.stderr, "test: good_request: check_hashpiece" self.assert_(resp[0] == EXTEND) self.assert_(resp[1] == HASHPIECE) index = toint(resp[2 : 2 + 4]) begin = toint(resp[6 : 6 + 4]) ohlen = toint(resp[10 : 10 + 4]) print >> sys.stderr, "test: good_request: check_hashpiece", index, begin, ohlen bohlist = resp[14 : 14 + ohlen] hisohlist = bdecode(bohlist) hischunk = resp[14 + ohlen :] if begin == 0: self.assert_(isinstance(hisohlist, ListType)) for oh in hisohlist: self.assert_(isinstance(oh, ListType)) self.assert_(len(oh) == 2) self.assert_(isinstance(oh[0], IntType)) self.assert_(isinstance(oh[1], StringType)) hisohlist.sort() print >> sys.stderr, "test: good_request: check_hashpiece", repr(hisohlist) myohlist = self.tree.get_hashes_for_piece(index) myohlist.sort() self.assert_(len(hisohlist) == len(myohlist)) for i in range(0, len(hisohlist)): hisoh = hisohlist[i] myoh = myohlist[i] self.assert_(hisoh == myoh) else: self.assert_(len(hisohlist) == 0) mylength = self.get_chunk_length(index, begin) mychunk = self.read_chunk(index, begin, mylength) self.assert_(hischunk == mychunk) return index == self.numpieces - 1 and mylength != 2 ** 14 def check_bitfield(self, data): self.assert_(data[0] == BITFIELD) bitmap = data[1:] self.assert_(len(bitmap) == 1) # Must have set_breakup_seed_bitfield() set to False self.assert_(bitmap == "\xc0") # # Bad EXTEND handshake message # def subtest_bad_hashpiece(self, oldstyle): if not oldstyle: # Test becomes equivalent to BT keep alive message (len 0, payload '') self._test_bad(self.create_empty, oldstyle) self._test_bad(self.create_ext_id_not_byte, oldstyle) self._test_bad(self.create_not_hashpiece, oldstyle) self._test_bad(self.create_not_index, oldstyle) self._test_bad(self.create_not_begin, oldstyle) self._test_bad(self.create_not_len_bohlist, oldstyle) self._test_bad(self.create_ohlist_not_bdecodable, oldstyle) self._test_bad(self.create_ohlist_wrong_no_hashes, oldstyle) self._test_bad(self.create_ohlist_wrong_no_root_hash, oldstyle) self._test_bad(self.create_ohlist_wrong_bad_offset, oldstyle) self._test_bad(self.create_ohlist_wrong_bad_hash, oldstyle) # TODO: need working peer kicking for that # self._test_bad(self.create_bad_chunk,oldstyle) # # Main test code for bad EXTEND handshake messages # def _test_bad(self, msg_gen_func, oldstyle): print >> sys.stderr, "test: test_BAD: Create EXTEND HS", repr(msg_gen_func), oldstyle if oldstyle: options = None exthsmsg = self.create_good_tribler_extend_hs() else: options = "\x00\x00\x00\x00\x00\x10\x00\x00" exthsmsg = self.create_good_nontribler_extend_hs() s = BTConnection("localhost", self.hisport, user_option_pattern=options, user_infohash=self.infohash) s.send(exthsmsg) s.read_handshake_medium_rare() # Tribler should send an EXTEND message back try: print >> sys.stderr, "test: Waiting for reply" s.s.settimeout(10.0) resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply", getMessageName(resp[0]) self.assert_(resp[0] == EXTEND) self.check_tribler_extend_hs(resp[1:]) # 1. Pretend we're seeder: send BITFIELD and UNCHOKE msg = BITFIELD + self.seederbitfieldstr s.send(msg) msg = UNCHOKE s.send(msg) print >> sys.stderr, "test: Pretend we are seeder" while True: resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply 2", getMessageName(resp[0]) self.assert_( resp[0] == REQUEST or resp[0] == INTERESTED or resp[0] == UNCHOKE or resp[0] == HAVE or resp[0] == NOT_INTERESTED ) if resp[0] == REQUEST: chunkid = self.check_request(resp) # 2. Reply to REQUEST with *bad* HASHPIECE msg = msg_gen_func(chunkid) if oldstyle: if len(msg) == 1: msg = "" else: msg = msg[1:] # Strip EXTEND byte s.send(msg) break # s.close() except socket.timeout: print >> sys.stderr, "test: Timeout, bad, peer didn't reply in time" self.assert_(False) time.sleep(3) # Should have closed the connection try: s.send(UNCHOKE) self.assert_(False) except: print_exc() s.close() # # Bad message creators (all create Merkle BEP style, I strip first byte # later for oldstyle # def create_empty(self, chunkid): return EXTEND def create_ext_id_not_byte(self, chunkid): return EXTEND + "Hallo kijkbuiskinderen" def create_not_hashpiece(self, chunkid): index, begin, length = chunkid ohlist = [] bohlist = bencode(ohlist) chunk = self.read_chunk(index, begin, length) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + chr(231) + payload def create_not_index(self, chunkid): payload = "bla" return EXTEND + HASHPIECE + payload def create_not_begin(self, chunkid): index, begin, length = chunkid payload = tobinary(index) + "bla" return EXTEND + HASHPIECE + payload def create_not_len_bohlist(self, chunkid): index, begin, length = chunkid payload = tobinary(index) + tobinary(begin) + "bla" return EXTEND + HASHPIECE + payload def create_ohlist_not_bdecodable(self, chunkid): index, begin, length = chunkid bohlist = "bla" chunk = "*" * (2 ** 14) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload def create_ohlist_wrong_no_hashes(self, chunkid): index, begin, length = chunkid ohlist = [(0, "#" * 20), (1, "$" * 20)] # should contain 3 for file2.wmv: own, sibling and root bohlist = bencode(ohlist) chunk = "*" * (2 ** 14) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload def create_ohlist_wrong_no_root_hash(self, chunkid): index, begin, length = chunkid ohlist = self.tree.get_hashes_for_piece(index) newohlist = [] # Remove root hash for oh in ohlist: if oh[0] != 0: newohlist.append(oh) ohlist = newohlist bohlist = bencode(ohlist) chunk = self.read_chunk(index, begin, length) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload def create_ohlist_wrong_bad_offset(self, chunkid): index, begin, length = chunkid ohlist = self.tree.get_hashes_for_piece(index) ohlist[1][0] = 481 bohlist = bencode(ohlist) chunk = self.read_chunk(index, begin, length) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload def create_ohlist_wrong_bad_hash(self, chunkid): index, begin, length = chunkid ohlist = self.tree.get_hashes_for_piece(index) ohlist[1][1] = "$" * 20 bohlist = bencode(ohlist) chunk = self.read_chunk(index, begin, length) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload def create_bad_chunk(self, chunkid): index, begin, length = chunkid ohlist = self.tree.get_hashes_for_piece(index) bohlist = bencode(ohlist) chunk = "*" * length payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload