def test_add_content_file_and_copy(self): """ Add a single file to a TorrentDef """ t = TorrentDef() fn = os.path.join(TESTS_DATA_DIR, self.VIDEO_FILE_NAME) t.add_content(fn) t.set_tracker(TRACKER) t.finalize() s = os.path.getsize(fn) metainfo = t.get_metainfo() self.general_check(metainfo) self.assertEqual(metainfo['info']['name'], self.VIDEO_FILE_NAME) self.assertEqual(metainfo['info']['length'], s) self.assertTrue(t.get_pieces()) self.assertEqual(len(t.get_infohash()), INFOHASH_LENGTH) self.assertTrue(t.get_name()) # test copy constructor nt = TorrentDef(t.input, t.metainfo, t.infohash) self.assertEqual(nt.input, t.input) self.assertEqual(nt.metainfo, t.metainfo) self.assertEqual(nt.infohash, t.infohash) # test removing content nt.remove_content("/test123") self.assertEqual(len(nt.input['files']), 1) nt.remove_content(unicode(fn)) self.assertEqual(len(nt.input['files']), 0) nt.remove_content(unicode(fn))
def test_dlstates_cb_error(self): """ Testing whether a download is stopped on error in the download states callback in LaunchManyCore """ error_stop_deferred = Deferred() def mocked_stop(): error_stop_deferred.callback(None) error_tdef = TorrentDef() error_tdef.get_infohash = lambda: 'aaaa' fake_error_download = MockObject() fake_error_download.get_def = lambda: error_tdef fake_error_download.get_def().get_name_as_unicode = lambda: "test.iso" fake_error_download.stop = mocked_stop fake_error_state = MockObject() fake_error_state.get_infohash = lambda: 'aaaa' fake_error_state.get_error = lambda: "test error" fake_error_state.get_status = lambda: DLSTATUS_STOPPED_ON_ERROR fake_error_state.get_download = lambda: fake_error_download self.lm.downloads = {'aaaa': fake_error_download} self.lm.sesscb_states_callback([fake_error_state]) return error_stop_deferred
def test_add_content_file_and_copy(self): """ Add a single file to a TorrentDef """ t = TorrentDef() fn = os.path.join(TESTS_DATA_DIR, self.VIDEO_FILE_NAME) t.add_content(fn) t.set_tracker(TRACKER) t.finalize() s = os.path.getsize(fn) metainfo = t.get_metainfo() self.general_check(metainfo) self.assertEqual(metainfo['info']['name'], self.VIDEO_FILE_NAME) self.assertEqual(metainfo['info']['length'], s) self.assertTrue(t.get_pieces()) self.assertEqual(len(t.get_infohash()), INFOHASH_LENGTH) self.assertTrue(t.get_name()) # test copy constructor nt = TorrentDef(t.input, t.metainfo, t.infohash) self.assertEqual(nt.input, t.input) self.assertEqual(nt.metainfo, t.metainfo) self.assertEqual(nt.infohash, t.infohash) # test removing content nt.remove_content("/test123") self.assertEqual(len(nt.input['files']), 1) nt.remove_content(unicode(fn)) self.assertEqual(len(nt.input['files']), 0) nt.remove_content(unicode(fn))
def test_readd_download_safe_seeding(self): """ Test whether a download is re-added when doing safe seeding """ readd_deferred = Deferred() def mocked_update_download_hops(*_): readd_deferred.callback(None) self.lm.update_download_hops = mocked_update_download_hops tdef = TorrentDef() tdef.get_infohash = lambda: 'aaaa' fake_download = MockObject() fake_download.get_def = lambda: tdef fake_download.get_def().get_name_as_unicode = lambda: "test.iso" fake_download.get_hops = lambda: 0 fake_download.get_safe_seeding = lambda: True dl_state = MockObject() dl_state.get_infohash = lambda: 'aaaa' dl_state.get_status = lambda: DLSTATUS_SEEDING dl_state.get_download = lambda: fake_download self.lm.downloads = {'aaaa': fake_download} self.lm.sesscb_states_callback([dl_state]) return readd_deferred
def test_dlstates_cb_seeding(self): """ Testing whether a download is readded when safe seeding in the download states callback in LaunchManyCore """ readd_deferred = Deferred() def mocked_start_download(tdef, dscfg): self.assertEqual(tdef, seed_tdef) self.assertEqual(dscfg, seed_download) readd_deferred.callback(None) def mocked_remove_download(download): self.assertEqual(download, seed_download) self.lm.session.start_download_from_tdef = mocked_start_download self.lm.session.remove_download = mocked_remove_download seed_tdef = TorrentDef() seed_tdef.get_infohash = lambda: 'aaaa' seed_download = MockObject() seed_download.get_def = lambda: seed_tdef seed_download.get_def().get_name_as_unicode = lambda: "test.iso" seed_download.get_hops = lambda: 0 seed_download.get_safe_seeding = lambda: True seed_download.copy = lambda: seed_download seed_download.set_hops = lambda _: None fake_seed_download_state = MockObject() fake_seed_download_state.get_infohash = lambda: 'aaaa' fake_seed_download_state.get_status = lambda: DLSTATUS_SEEDING fake_seed_download_state.get_download = lambda: seed_download self.lm.sesscb_states_callback([fake_seed_download_state]) return readd_deferred
def create_fake_download_and_state(): """ Create a fake download and state which can be passed to the global download callback. """ tdef = TorrentDef() tdef.get_infohash = lambda: 'aaaa' fake_peer = {'extended_version': 'Tribler', 'id': 'a' * 20, 'dtotal': 10 * 1024 * 1024} fake_download = MockObject() fake_download.get_def = lambda: tdef fake_download.get_def().get_name_as_unicode = lambda: "test.iso" fake_download.get_hops = lambda: 0 fake_download.get_safe_seeding = lambda: True fake_download.get_peerlist = lambda: [fake_peer] dl_state = MockObject() dl_state.get_infohash = lambda: 'aaaa' dl_state.get_status = lambda: DLSTATUS_SEEDING dl_state.get_download = lambda: fake_download return fake_download, dl_state
def create_fake_download_and_state(): """ Create a fake download and state which can be passed to the global download callback. """ tdef = TorrentDef() tdef.get_infohash = lambda: 'aaaa' fake_peer = { 'extended_version': 'Tribler', 'id': 'a' * 20, 'dtotal': 10 * 1024 * 1024 } fake_download = MockObject() fake_download.get_def = lambda: tdef fake_download.get_def().get_name_as_unicode = lambda: "test.iso" fake_download.get_hops = lambda: 0 fake_download.get_safe_seeding = lambda: True fake_download.get_peerlist = lambda: [fake_peer] dl_state = MockObject() dl_state.get_infohash = lambda: 'aaaa' dl_state.get_status = lambda: DLSTATUS_SEEDING dl_state.get_download = lambda: fake_download return fake_download, dl_state
class TestMetadataFakePeer(TestAsServer, MagnetHelpers): """ Once we are downloading a torrent, our client should respond to the ut_metadata extention message. This allows other clients to obtain the info part of the metadata from us. """ def setUp(self): TestAsServer.setUp(self) # the metadata that we want to transfer self.tdef = TorrentDef() self.tdef.add_content(os.path.join(TESTS_API_DIR, "file.wmv")) self.tdef.set_tracker("http://fake.net/announce") # we use a small piece length to obtain multiple pieces self.tdef.set_piece_length(1) self.tdef.finalize() self.setup_seeder() MagnetHelpers.__init__(self, self.tdef) def setUpPreSession(self): TestAsServer.setUpPreSession(self) self.config.set_libtorrent(True) self.config2 = self.config.copy() self.config2.set_state_dir(self.getStateDir(2)) def tearDown(self): self.teardown_seeder() TestAsServer.tearDown(self) def setup_seeder(self): self.seeder_setup_complete = threading.Event() self.dscfg = DownloadStartupConfig() self.dscfg.set_dest_dir(TESTS_API_DIR) self.download = self.session.start_download(self.tdef, self.dscfg) self.download.set_state_callback(self.seeder_state_callback) assert self.seeder_setup_complete.wait(30) def teardown_seeder(self): self.session.remove_download(self.download) def seeder_state_callback(self, ds): if ds.get_status() == DLSTATUS_SEEDING: self.seeder_setup_complete.set() d = ds.get_download() self._logger.debug("seeder: %s %s %s", repr(d.get_def().get_name()), dlstatus_strings[ds.get_status()], ds.get_progress()) return 1.0, False def test_good_request(self): conn = BTConnection("localhost", self.session.get_listen_port(), user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) # request metadata block 0, 2, 3, and the last conn.send(self.create_good_extend_metadata_request(metadata_id, 0)) conn.send(self.create_good_extend_metadata_request(metadata_id, 2)) conn.send(self.create_good_extend_metadata_request(metadata_id, 3)) conn.send( self.create_good_extend_metadata_request( metadata_id, len(self.metadata_list) - 1)) self.read_extend_metadata_reply(conn, 0) self.read_extend_metadata_reply(conn, 2) self.read_extend_metadata_reply(conn, 3) self.read_extend_metadata_reply(conn, len(self.metadata_list) - 1) def test_good_flood(self): conn = BTConnection("localhost", self.session.get_listen_port(), user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) for counter in xrange(len(self.metadata_list) * 2): piece = counter % len(self.metadata_list) conn.send( self.create_good_extend_metadata_request(metadata_id, piece)) if counter > len(self.metadata_list): self.read_extend_metadata_reject(conn, piece) else: self.read_extend_metadata_reply(conn, piece) def test_bad_request(self): self.bad_request_and_disconnect({ "msg_type": 0, "piece": len(self.metadata_list) }) self.bad_request_and_disconnect({"msg_type": 0, "piece": -1}) self.bad_request_and_disconnect({"msg_type": 0, "piece": "1"}) self.bad_request_and_disconnect({"msg_type": 0, "piece": [1, 2]}) self.bad_request_and_disconnect({"msg_type": 0, "PIECE": 1}) def bad_request_and_disconnect(self, payload): conn = BTConnection("localhost", self.session.get_listen_port(), user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) conn.send(EXTEND + chr(metadata_id) + bencode(payload)) self.read_extend_metadata_close(conn)
class TestMagnetFakePeer(TestAsServer, MagnetHelpers): """ A MiniBitTorrent instance is used to connect to BitTorrent clients and download the info part from the metadata. """ def setUp(self): # listener for incoming connections from MiniBitTorrent self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.server.bind(("", self.session.get_listen_port())) self.server.listen(5) TestAsServer.setUp(self) # the metadata that we want to transfer self.tdef = TorrentDef() self.tdef.add_content(os.path.join(TESTS_API_DIR, "video.avi")) self.tdef.set_tracker("http://fake.net/announce") # we use a small piece length to obtain multiple pieces self.tdef.set_piece_length(1) self.tdef.finalize() MagnetHelpers.__init__(self, self.tdef) def setUpPreSession(self): TestAsServer.setUpPreSession(self) self.config.set_libtorrent(True) def create_good_url(self, infohash=None, title=None, tracker=None): url = "magnet:?xt=urn:btih:" if infohash: assert isinstance(infohash, str) url += hexlify(infohash) else: url += hexlify(self.tdef.get_infohash()) if title: assert isinstance(title, str) url += "&dn=" + title if tracker: assert isinstance(tracker, str) url += "&tr=" + tracker return url @skip("not working, seems to return binary data") def test_good_transfer(self): def torrentdef_retrieved(meta_info): tags["metainfo"] = meta_info tags["retrieved"].set() tags = {"retrieved": threading.Event()} self.session.lm.ltmgr.get_metainfo(self.create_good_url(), torrentdef_retrieved, timeout=60) def do_supply(): # supply fake addresses (regular dht obviously wont work here) ltmgr = LibtorrentMgr.getInstance() for infohash in ltmgr.metainfo_requests: handle = ltmgr.ltsession.find_torrent( lt.big_number(infohash.decode('hex'))) handle.connect_peer( ("127.0.0.1", self.session.get_listen_port()), 0) self.session.lm.threadpool.add_task(do_supply, delay=5.0) # accept incoming connection # self.server.settimeout(10.0) sock, address = self.server.accept() assert sock, "No incoming connection" # handshakes conn = BTConnection(address[0], address[1], opensock=sock, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) # serve pieces for counter in xrange(len(self.metadata_list)): piece = self.read_extend_metadata_request(conn) assert 0 <= piece < len(self.metadata_list) conn.send( self.create_good_extend_metadata_reply(metadata_id, piece)) # no more metadata request may be send and the connection must # be closed self.read_extend_metadata_close(conn) assert tags["retrieved"].wait(5) assert tags["metainfo"]["info"] == self.tdef.get_metainfo()["info"]
class TestVideoServerSession(TestAsServer): """ Class for testing HTTP-based video server in a session. Mainly HTTP range queries. """ @blocking_call_on_reactor_thread @inlineCallbacks def setUp(self, autoload_discovery=True): """ unittest test setup code """ yield TestAsServer.setUp(self, autoload_discovery=autoload_discovery) self.port = self.session.config.get_video_server_port() self.sourcefn = os.path.join(TESTS_DATA_DIR, "video.avi") self.sourcesize = os.path.getsize(self.sourcefn) self.tdef = None self.expsize = 0 yield self.start_vod_download() def setUpPreSession(self): TestAsServer.setUpPreSession(self) self.config.set_libtorrent_enabled(True) self.config.set_video_server_enabled(True) # # Tests # @deferred(timeout=10) def test_specific_range(self): return self.range_check(115, 214) @deferred(timeout=10) def test_last_100(self): return self.range_check(self.sourcesize - 100, None) @deferred(timeout=10) def test_first_100(self): return self.range_check(None, 100) @deferred(timeout=10) def test_combined(self): return self.range_check(115, 214, setset=True) def start_vod_download(self): self.tdef = TorrentDef() self.tdef.add_content(self.sourcefn) self.tdef.set_tracker("http://127.0.0.1:12/announce") self.tdef.finalize() dscfg = DownloadStartupConfig() dscfg.set_dest_dir(os.path.dirname(self.sourcefn)) download = self.session.start_download_from_tdef(self.tdef, dscfg) return download.get_handle() def get_std_header(self): msg = "GET /%s/0 HTTP/1.1\r\n" % binascii.hexlify( self.tdef.get_infohash()) msg += "Host: 127.0.0.1:" + str(self.port) + "\r\n" return msg @staticmethod def create_range_str(firstbyte, lastbyte): head = "" if firstbyte is not None: head += str(firstbyte) head += "-" if lastbyte is not None: head += str(lastbyte) return head def get_header(self, firstbyte, lastbyte, setset=False): head = self.get_std_header() head += "Range: bytes=" head += self.create_range_str(firstbyte, lastbyte) if setset: # Make into set of byte ranges, VideoHTTPServer should refuse. head += ",0-99" head += "\r\n" head += "Connection: close\r\n" return head + "\r\n" def range_check(self, firstbyte, lastbyte, setset=False): test_deferred = Deferred() self._logger.debug("range_test: %s %s %s setset %s", firstbyte, lastbyte, self.sourcesize, setset) if firstbyte is not None and lastbyte is None: exp_byte_range = (firstbyte, self.sourcesize - 1) elif firstbyte is None and lastbyte is not None: exp_byte_range = (self.sourcesize - lastbyte, self.sourcesize - 1) else: exp_byte_range = (firstbyte, lastbyte) # the amount of bytes actually requested. (Content-length) self.expsize = exp_byte_range[1] - exp_byte_range[0] + 1 f = open(self.sourcefn, "rb") f.seek(exp_byte_range[0]) expdata = f.read(self.expsize) f.close() def on_connected(p): p.sendMessage(self.get_header(firstbyte, lastbyte, setset)) endpoint = TCP4ClientEndpoint(reactor, "localhost", self.port) connectProtocol(endpoint, VideoServerProtocol(test_deferred, self.sourcesize, expdata, setset, exp_byte_range))\ .addCallback(on_connected) return test_deferred
class TestMagnetMiniBitTorrent(TestAsServer, MagnetHelpers): """ A MiniBitTorrent instance is used to connect to BitTorrent clients and download the info part from the metadata. """ def setUp(self): """ override TestAsServer """ # listener for incoming connections from MiniBitTorrent self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.server.bind(("localhost", LISTEN_PORT)) self.server.listen(1) # the metadata that we want to transfer self.tdef = TorrentDef() self.tdef.add_content(os.path.join(os.getcwd(), "API", "file.wmv")) self.tdef.set_tracker("http://fake.net/announce") # we use a small piece length to obtain multiple pieces self.tdef.set_piece_length(1) self.tdef.finalize() MagnetHelpers.__init__(self, self.tdef) # startup the client TestAsServer.setUp(self) print >> sys.stderr, "test: Giving MyLaunchMany time to startup" time.sleep(5) print >> sys.stderr, "test: MyLaunchMany should have started up" def create_good_url(self, infohash=None, title=None, tracker=None): url = "magnet:?xt=urn:btih:" if infohash: assert isinstance(infohash, str) url += hexlify(infohash) else: url += hexlify(self.tdef.get_infohash()) if title: assert isinstance(title, str) url += "&dn=" + title if tracker: assert isinstance(tracker, str) url += "&tr=" + tracker return url def test_good_transfer(self): def torrentdef_retrieved(tdef): tags["retrieved"] = True tags["metainfo"] = tdef.get_metainfo() tags = {"retrieved": False} assert TorrentDef.retrieve_from_magnet(self.create_good_url(), torrentdef_retrieved) # supply fake addresses (regular dht obviously wont work here) for magnetlink in MagnetHandler.get_instance().get_magnets(): magnetlink._swarm.add_potential_peers([("localhost", LISTEN_PORT)]) # accept incoming connection self.server.settimeout(10.0) sock, address = self.server.accept() assert sock, "No incoming connection" # handshakes conn = BTConnection(address[0], address[1], opensock=sock, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) # serve pieces for counter in xrange(len(self.metadata_list)): piece = self.read_extend_metadata_request(conn) assert 0 <= piece < len(self.metadata_list) conn.send( self.create_good_extend_metadata_reply(metadata_id, piece)) # no more metadata request may be send and the connection must # be closed self.read_extend_metadata_close(conn) time.sleep(5) assert tags["retrieved"] assert tags["metainfo"]["info"] == self.tdef.get_metainfo()["info"]
class TestVideoHTTPServer(TestAsServer): """ Class for testing HTTP-based video server. Mainly HTTP range queries. """ def setUp(self): """ unittest test setup code """ TestAsServer.setUp(self) self.port = self.session.get_videoplayer_port() self.sourcefn = os.path.join(TESTS_DATA_DIR, "video.avi") self.sourcesize = os.path.getsize(self.sourcefn) # wait 5s to allow server to start time.sleep(5) def setUpPreSession(self): TestAsServer.setUpPreSession(self) self.config.set_libtorrent(True) self.config.set_videoplayer(True) def tearDown(self): """ unittest test tear down code """ TestAsServer.tearDown(self) time.sleep(2) # # Tests # def test_specific_range(self): self.range_check(115, 214, self.sourcesize) def test_last_100(self): self.range_check(self.sourcesize - 100, None, self.sourcesize) def test_first_100(self): self.range_check(None, 100, self.sourcesize) def test_combined(self): self.range_check(115, 214, self.sourcesize, setset=True) # # Internal # def register_file_stream(self): self.tdef = TorrentDef() self.tdef.add_content(self.sourcefn) self.tdef.set_tracker("http://127.0.0.1:12/announce") self.tdef.finalize() dscfg = DownloadStartupConfig() dscfg.set_dest_dir(os.path.dirname(self.sourcefn)) download = self.session.start_download(self.tdef, dscfg) while not download.handle: time.sleep(1) def get_std_header(self): msg = "GET /%s/0 HTTP/1.1\r\n" % binascii.hexlify(self.tdef.get_infohash()) msg += "Host: 127.0.0.1:" + str(self.port) + "\r\n" return msg def create_range_str(self, firstbyte, lastbyte): head = "" if firstbyte is not None: head += str(firstbyte) head += "-" if lastbyte is not None: head += str(lastbyte) return head def range_check(self, firstbyte, lastbyte, sourcesize, setset=False): self._logger.debug("range_test: %s %s %s setset %s", firstbyte, lastbyte, sourcesize, setset) self.register_file_stream() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect(('127.0.0.1', self.port)) head = self.get_std_header() head += "Range: bytes=" head += self.create_range_str(firstbyte, lastbyte) if setset: # Make into set of byte ranges, VideoHTTPServer should refuse. head += ",0-99" head += "\r\n" head += "Connection: close\r\n" head += "\r\n" if firstbyte is not None and lastbyte is None: # 100- expfirstbyte = firstbyte explastbyte = self.sourcesize - 1 elif firstbyte is None and lastbyte is not None: # -100 expfirstbyte = self.sourcesize - lastbyte explastbyte = self.sourcesize - 1 else: expfirstbyte = firstbyte explastbyte = lastbyte # the amount of bytes actually requested. (Content-length) expsize = explastbyte - expfirstbyte + 1 self._logger.debug("Expecting first %s last %s size %s ", expfirstbyte, explastbyte, sourcesize) s.send(head) # Parse header s.settimeout(10.0) while True: line = self.readline(s) if DEBUG: self._logger.debug("Got line: %s", repr(line)) if len(line) == 0: if DEBUG: self._logger.debug("server closed conn") self.assert_(False) return if line.startswith("HTTP"): if not setset: # Python returns "HTTP/1.0 206 Partial Content\r\n" HTTP 1.0??? self.assert_(line.startswith("HTTP/1.")) self.assert_(line.find("206") != -1) # Partial content else: self.assert_(line.startswith("HTTP/1.")) self.assert_(line.find("416") != -1) # Requested Range Not Satisfiable return elif line.startswith("Content-Range:"): expline = "Content-Range: bytes " + self.create_range_str( expfirstbyte, explastbyte) + "/" + str(sourcesize) + "\r\n" self.assertEqual(expline, line) elif line.startswith("Content-Type:"): self.assertEqual(line, "Content-Type: video/x-msvideo\r\n") elif line.startswith("Content-Length:"): self.assertEqual(line, "Content-Length: " + str(expsize) + "\r\n") elif line.endswith("\r\n") and len(line) == 2: # End of header break data = s.recv(expsize) if len(data) == 0: if DEBUG: self._logger.debug("server closed conn2") self.assert_(False) return else: f = open(self.sourcefn, "rb") if firstbyte is not None: f.seek(firstbyte) else: f.seek(lastbyte, os.SEEK_END) expdata = f.read(expsize) f.close() self.assert_(data, expdata) try: # Read body, reading more should EOF (we disabled persist conn) data = s.recv(10240) self.assert_(len(data) == 0) except socket.timeout: if DEBUG: self._logger.debug( "Timeout, video server didn't respond with requested bytes, possibly bug in Python impl of HTTP") print_exc() def readline(self, s): line = '' while True: data = s.recv(1) if len(data) == 0: return line else: line = line + data if data == '\n' and len(line) >= 2 and line[-2:] == '\r\n': return line
class TestMetadata(TestAsServer, MagnetHelpers): """ Once we are downloading a torrent, our client should respond to the ut_metadata extention message. This allows other clients to obtain the info part of the metadata from us. """ def setUp(self): """ override TestAsServer """ TestAsServer.setUp(self) print >>sys.stderr,"test: Giving MyLaunchMany time to startup" time.sleep(5) print >>sys.stderr,"test: MyLaunchMany should have started up" # the metadata that we want to transfer self.tdef = TorrentDef() self.tdef.add_content(os.path.join(os.getcwd(), "API", "file.wmv")) self.tdef.set_tracker(self.session.get_internal_tracker_url()) # we use a small piece length to obtain multiple pieces self.tdef.set_piece_length(1) self.tdef.finalize() # self.tdef.save(os.path.join(self.session.get_state_dir(), "gen.torrent")) MagnetHelpers.__init__(self, self.tdef) def setup_seeder(self): self.seeder_setup_complete = False self.seeder_teardown_complete = False self.seeder_teardown = False self.dscfg = DownloadStartupConfig() self.dscfg.set_dest_dir(os.getcwd()) self.download = self.session.start_download(self.tdef, self.dscfg) self.download.set_state_callback(self.seeder_state_callback) counter = 0 while not self.seeder_setup_complete: counter += 1 time.sleep(1) assert counter < 30, "timeout" print >> sys.stderr, "test: setup_seeder() complete" def teardown_seeder(self): self.seeder_teardown_complete = False self.session.remove_download(self.download) counter = 0 while not self.seeder_setup_complete: counter += 1 time.sleep(1) assert counter < 30, "timeout" print >> sys.stderr, "test: teardown_seeder() complete" def seeder_state_callback(self,ds): assert not self.seeder_teardown_complete self.seeder_setup_complete = (ds.get_status() == DLSTATUS_DOWNLOADING) d = ds.get_download() print >> sys.stderr, "test: seeder:", `d.get_def().get_name()`, dlstatus_strings[ds.get_status()], ds.get_progress() if self.seeder_teardown: self.seeder_teardown_complete = True else: return (1.0, False) def test_all(self): self.setup_seeder() try: self.subtest_good_flood() finally: self.teardown_seeder() self.setup_seeder() try: self.subtest_good_request() self.subtest_bad_request() finally: self.teardown_seeder() def subtest_good_request(self): conn = BTConnection("localhost", self.hisport, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) # request metadata block 0, 2, 3, and the last conn.send(self.create_good_extend_metadata_request(metadata_id, 0)) conn.send(self.create_good_extend_metadata_request(metadata_id, 2)) conn.send(self.create_good_extend_metadata_request(metadata_id, 3)) conn.send(self.create_good_extend_metadata_request(metadata_id, len(self.metadata_list) - 1)) self.read_extend_metadata_reply(conn, 0) self.read_extend_metadata_reply(conn, 2) self.read_extend_metadata_reply(conn, 3) self.read_extend_metadata_reply(conn, len(self.metadata_list) - 1) def subtest_good_flood(self): conn = BTConnection("localhost", self.hisport, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) for counter in xrange(len(self.metadata_list) * 2): piece = counter % len(self.metadata_list) conn.send(self.create_good_extend_metadata_request(metadata_id, piece)) if counter > len(self.metadata_list): self.read_extend_metadata_reject(conn, piece) else: self.read_extend_metadata_reply(conn, piece) def subtest_bad_request(self): self.bad_request_and_disconnect({"msg_type":0, "piece":len(self.metadata_list)}) self.bad_request_and_disconnect({"msg_type":0, "piece":-1}) self.bad_request_and_disconnect({"msg_type":0, "piece":"1"}) self.bad_request_and_disconnect({"msg_type":0, "piece":[1,2]}) self.bad_request_and_disconnect({"msg_type":0, "PIECE":1}) def bad_request_and_disconnect(self, payload): conn = BTConnection("localhost", self.hisport, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) conn.send(EXTEND + chr(metadata_id) + bencode(payload)) self.read_extend_metadata_close(conn)
class TestQueryReplyActive(TestAsServer): """ Testing QUERY_REPLY message of Query extension V1 This test checks how the Tribler code responds to good and bad QUERY_REPLY messages. I.e. the Tribler client initiates the dialback by connecting to us and sending a QUERY and we reply with good and bad messages. This test allows authoritative answers from superpeers. WARNING: Each of the test_ methods should be tested by running the TestCase in a separate Python interpreter to prevent problems with our singleton classes, e.g. SuperPeerDB, etc. """ def setUpPreSession(self): """ override TestAsServer """ print >> sys.stderr,"test: Pre Tribler Init" TestAsServer.setUpPreSession(self) print >> sys.stderr,"test: Pre Tribler Init: config_path",self.config_path # Enable remote querying self.config.set_remote_query(True) def setUpPostSession(self): """ override TestAsServer """ TestAsServer.setUpPostSession(self) self.hispermid = str(self.his_keypair.pub().get_der()) self.my_permid = str(self.my_keypair.pub().get_der()) def pretest_simple(self,keyword): self.pretest_q('SIMPLE',keyword) def pretest_simpleplustorrents(self,keyword): self.pretest_q('SIMPLE+METADATA',keyword) def pretest_q(self,queryprefix,keyword): query = queryprefix+' '+keyword self.content_name = keyword.upper()+' S22E44' self.tdef = TorrentDef() self.tdef.set_tracker('http://localhost:0/announce') self.tdef.set_piece_length(2 ** 15) self.tdef.create_live(self.content_name,2 ** 16) self.tdef.finalize() # 1. First connect to Tribler self.openconn = OLConnection(self.my_keypair,'localhost',self.hisport) sleep(3) # 2. Make Tribler send query self.query = query self.session.query_connected_peers(query,self.query_usercallback,max_peers_to_query=10) def query_usercallback(self,permid,query,hits): print >>sys.stderr,"test: query_usercallback:",`permid`,`query`,`hits` self.assert_(query == self.query) self.assert_(permid == self.my_permid) self.check_good_qreply(hits) # TODO: if SIMPLE+METADATA: check torrent now in db. # # Good SIMPLE QUERY, builds on TestQueryReply code # def singtest_good_simple_reply(self): self.pretest_simple('hallo') self._test_qreply(self.create_good_simple_reply,True) # # Good SIMPLE+METADATA QUERY, builds on TestQueryReply code # def singtest_good_simpleplustorrents_reply(self): self.pretest_simpleplustorrents('hallo') self._test_qreply(self.create_good_simpleplustorrents_reply,True) # # Good SIMPLE QUERY Unicode, builds on TestQueryReply code # def singtest_good_simple_reply_unicode(self): self.pretest_simple(u'Ch\u00e8rie') self._test_qreply(self.create_good_simple_reply,True) # # Good SIMPLE+METADATA QUERY Unicode, builds on TestQueryReply code # def singtest_good_simpleplustorrents_reply_unicode(self): self.pretest_simpleplustorrents(u'Ch\u00e8rie') self._test_qreply(self.create_good_simpleplustorrents_reply,True) # # Bad QUERY, builds on TestQueryReply code # def singtest_bad_not_bdecodable(self): self.pretest_simple('hallo') self._test_qreply(self.create_not_bdecodable,False) # # Bad SIMPLE+METADATA QUERY, builds on TestQueryReply code # def singtest_bad_not_bdecodable_torrentfile(self): self.pretest_simpleplustorrents('hallo') self._test_qreply(self.create_not_bdecodable_torrentfile,False) ### TODO: send different valid answers so consensus not reached # # Main test code # def _test_qreply(self,gen_qreply,good): print >> sys.stderr,"test: waiting for reply" s = self.openconn msg = s.recv() self.assert_(len(msg) > 0) print >> sys.stderr,"test: Received overlay message",getMessageName(msg[0]) self.assert_(msg[0] == QUERY) id = self.check_rquery(msg[1:]) resp = gen_qreply(id) print >> sys.stderr,"test: sending QUERY_REPLY" s.send(resp) if good: time.sleep(10) # the other side should not have closed the connection, as # this is all valid, so this should not throw an exception: s.send('bla') s.close() else: # the other side should not like this and close the connection self.assert_(len(s.recv())==0) s.close() def create_good_simple_reply_dict(self,id): r = {} r['content_name'] = self.content_name.encode("UTF-8") r['length'] = LENGTH r['leecher'] = LEECHERS r['seeder'] = SEEDERS r['category'] = CATEGORY # OLPROTO_PROTO_ELEVENTH # set later r['torrent_size'] = 42 r['channel_permid'] = '$' * 83 r['channel_name'] = 'Nitin Channel' d2 = {} d2[self.tdef.get_infohash()] = r d = {} d['id'] = id d['a'] = d2 return d def create_good_simple_reply(self,id): d = self.create_good_simple_reply_dict(id) bmetainfo = bencode(self.tdef.get_metainfo()) d['a'][self.tdef.get_infohash()]['torrent_size'] = len(bmetainfo) b = bencode(d) return QUERY_REPLY+b def create_good_simpleplustorrents_reply(self,id): d = self.create_good_simple_reply_dict(id) bmetainfo = bencode(self.tdef.get_metainfo()) d['a'][self.tdef.get_infohash()]['torrent_size'] = len(bmetainfo) d['a'][self.tdef.get_infohash()]['metatype'] = 'application/x-tribler-stream' d['a'][self.tdef.get_infohash()]['metadata'] = bmetainfo b = bencode(d) return QUERY_REPLY+b def check_good_qreply(self,hits): self.assert_(len(hits) == 1) self.assert_(hits.keys()[0] == self.tdef.get_infohash()) hit = hits[self.tdef.get_infohash()] self.assert_(hit['content_name'] == self.content_name) self.assert_(hit['length'] == LENGTH) self.assert_(hit['leecher'] == LEECHERS) self.assert_(hit['seeder'] == SEEDERS) self.assert_(hit['category'] == CATEGORY) # OLPROTO_VERSION_ELEVENTH bmetainfo = bencode(self.tdef.get_metainfo()) self.assert_(hit['torrent_size'] == len(bmetainfo)) if self.query.startswith('SIMPLE+METADATA'): self.assert_(hit['metadata'] == bmetainfo) def create_not_bdecodable(self,id): return QUERY_REPLY+"bla" def create_not_bdecodable_torrentfile(self,id): d = self.create_good_simple_reply_dict(id) d['a'][self.tdef.get_infohash()]['torrent_size'] = 3 # consistent with metadata. Should be named "metasize" d['a'][self.tdef.get_infohash()]['metadata'] = 'bla' b = bencode(d) return QUERY_REPLY+b def check_rquery(self,data): d = bdecode(data) self.assert_(type(d) == DictType) self.assert_(d.has_key('q')) q = d['q'] self.assert_(type(q) == StringType) self.assert_(d.has_key('id')) id = d['id'] self.assert_(type(id) == StringType) self.assert_(q == self.query.encode("UTF-8")) return d['id']
class TestVideoServerSession(TestAsServer): """ Class for testing HTTP-based video server in a session. Mainly HTTP range queries. """ @inlineCallbacks def setUp(self): """ unittest test setup code """ yield super(TestVideoServerSession, self).setUp() self.port = self.session.config.get_video_server_port() self.sourcefn = os.path.join(TESTS_DATA_DIR, "video.avi") self.sourcesize = os.path.getsize(self.sourcefn) self.tdef = None self.expsize = 0 yield self.start_vod_download() def setUpPreSession(self): TestAsServer.setUpPreSession(self) self.config.set_libtorrent_enabled(True) self.config.set_video_server_enabled(True) @trial_timeout(10) def test_specific_range(self): return self.range_check(115, 214) @trial_timeout(10) def test_last_100(self): return self.range_check(self.sourcesize - 100, None) @trial_timeout(10) def test_first_100(self): return self.range_check(None, 100) @trial_timeout(10) def test_combined(self): return self.range_check(115, 214, setset=True) def start_vod_download(self): self.tdef = TorrentDef() self.tdef.add_content(self.sourcefn) self.tdef.set_tracker("http://127.0.0.1:12/announce") self.tdef.finalize() dscfg = DownloadStartupConfig() dscfg.set_dest_dir(os.path.dirname(self.sourcefn)) download = self.session.start_download_from_tdef(self.tdef, dscfg) return download.get_handle() def get_std_header(self): msg = "GET /%s/0 HTTP/1.1\r\n" % binascii.hexlify(self.tdef.get_infohash()) msg += "Host: 127.0.0.1:" + str(self.port) + "\r\n" return msg @staticmethod def create_range_str(firstbyte, lastbyte): head = "" if firstbyte is not None: head += str(firstbyte) head += "-" if lastbyte is not None: head += str(lastbyte) return head def get_header(self, firstbyte, lastbyte, setset=False): head = self.get_std_header() head += "Range: bytes=" head += self.create_range_str(firstbyte, lastbyte) if setset: # Make into set of byte ranges, VideoHTTPServer should refuse. head += ",0-99" head += "\r\n" head += "Connection: close\r\n" return head + "\r\n" def range_check(self, firstbyte, lastbyte, setset=False): test_deferred = Deferred() self._logger.debug("range_test: %s %s %s setset %s", firstbyte, lastbyte, self.sourcesize, setset) if firstbyte is not None and lastbyte is None: exp_byte_range = (firstbyte, self.sourcesize - 1) elif firstbyte is None and lastbyte is not None: exp_byte_range = (self.sourcesize - lastbyte, self.sourcesize - 1) else: exp_byte_range = (firstbyte, lastbyte) # the amount of bytes actually requested. (Content-length) self.expsize = exp_byte_range[1] - exp_byte_range[0] + 1 f = open(self.sourcefn, "rb") f.seek(exp_byte_range[0]) expdata = f.read(self.expsize) f.close() def on_connected(p): p.sendMessage(self.get_header(firstbyte, lastbyte, setset)) endpoint = TCP4ClientEndpoint(reactor, "localhost", self.port) connectProtocol(endpoint, VideoServerProtocol(test_deferred, self.sourcesize, expdata, setset, exp_byte_range))\ .addCallback(on_connected) return test_deferred
class TestMetadataFakePeer(TestAsServer, MagnetHelpers): """ Once we are downloading a torrent, our client should respond to the ut_metadata extention message. This allows other clients to obtain the info part of the metadata from us. """ def setUp(self): TestAsServer.setUp(self) # the metadata that we want to transfer self.tdef = TorrentDef() self.tdef.add_content(os.path.join(BASE_DIR, "API", "file.wmv")) self.tdef.set_tracker("http://fake.net/announce") # we use a small piece length to obtain multiple pieces self.tdef.set_piece_length(1) self.tdef.finalize() self.setup_seeder() MagnetHelpers.__init__(self, self.tdef) def setUpPreSession(self): TestAsServer.setUpPreSession(self) self.config.set_libtorrent(True) self.config2 = self.config.copy() self.config2.set_state_dir(self.getStateDir(2)) self.config2.set_listen_port(4810) def tearDown(self): self.teardown_seeder() TestAsServer.tearDown(self) def setup_seeder(self): self.seeder_setup_complete = threading.Event() self.dscfg = DownloadStartupConfig() self.dscfg.set_dest_dir(os.path.join(BASE_DIR, "API")) self.download = self.session.start_download(self.tdef, self.dscfg) self.download.set_state_callback(self.seeder_state_callback) assert self.seeder_setup_complete.wait(30) def teardown_seeder(self): self.session.remove_download(self.download) def seeder_state_callback(self, ds): if ds.get_status() == DLSTATUS_SEEDING: self.seeder_setup_complete.set() d = ds.get_download() print >> sys.stderr, "test: seeder:", repr(d.get_def().get_name()), dlstatus_strings[ds.get_status()], ds.get_progress() return (1.0, False) def test_good_request(self): conn = BTConnection("localhost", self.hisport, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) # request metadata block 0, 2, 3, and the last conn.send(self.create_good_extend_metadata_request(metadata_id, 0)) conn.send(self.create_good_extend_metadata_request(metadata_id, 2)) conn.send(self.create_good_extend_metadata_request(metadata_id, 3)) conn.send(self.create_good_extend_metadata_request(metadata_id, len(self.metadata_list) - 1)) self.read_extend_metadata_reply(conn, 0) self.read_extend_metadata_reply(conn, 2) self.read_extend_metadata_reply(conn, 3) self.read_extend_metadata_reply(conn, len(self.metadata_list) - 1) def test_good_flood(self): conn = BTConnection("localhost", self.hisport, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) for counter in xrange(len(self.metadata_list) * 2): piece = counter % len(self.metadata_list) conn.send(self.create_good_extend_metadata_request(metadata_id, piece)) if counter > len(self.metadata_list): self.read_extend_metadata_reject(conn, piece) else: self.read_extend_metadata_reply(conn, piece) def test_bad_request(self): self.bad_request_and_disconnect({"msg_type": 0, "piece": len(self.metadata_list)}) self.bad_request_and_disconnect({"msg_type": 0, "piece":-1}) self.bad_request_and_disconnect({"msg_type": 0, "piece": "1"}) self.bad_request_and_disconnect({"msg_type": 0, "piece": [1, 2]}) self.bad_request_and_disconnect({"msg_type": 0, "PIECE": 1}) def bad_request_and_disconnect(self, payload): conn = BTConnection("localhost", self.hisport, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) conn.send(EXTEND + chr(metadata_id) + bencode(payload)) self.read_extend_metadata_close(conn)
def lineReceived(self, line): anon_tunnel = self.anon_tunnel profile = self.profile if line == 'threads': for thread in threading.enumerate(): print "%s \t %d" % (thread.name, thread.ident) elif line == 'p': if profile: for func_stats in yappi.get_func_stats().sort("subtime")[:50]: print "YAPPI: %10dx %10.3fs" % (func_stats.ncall, func_stats.tsub), func_stats.name else: logger.error("Profiling disabled!") elif line == 'P': if profile: filename = 'callgrindc_%d.yappi' % anon_tunnel.dispersy.lan_address[1] yappi.get_func_stats().save(filename, type='callgrind') else: logger.error("Profiling disabled!") elif line == 't': if profile: yappi.get_thread_stats().sort("totaltime").print_all() else: logger.error("Profiling disabled!") elif line == 'c': print "========\nCircuits\n========\nid\taddress\t\t\t\t\tgoal\thops\tIN (MB)\tOUT (MB)\tinfohash\ttype" for circuit_id, circuit in anon_tunnel.community.circuits.items(): info_hash = circuit.info_hash.encode('hex')[:10] if circuit.info_hash else '?' print "%d\t%s:%d\t%d\t%d\t\t%.2f\t\t%.2f\t\t%s\t%s" % (circuit_id, circuit.first_hop[0], circuit.first_hop[1], circuit.goal_hops, len(circuit.hops), circuit.bytes_down / 1024.0 / 1024.0, circuit.bytes_up / 1024.0 / 1024.0, info_hash, circuit.ctype) elif line.startswith('s'): cur_path = os.getcwd() line_split = line.split(' ') filename = 'test_file' if len(line_split) == 1 else line_split[1] if not os.path.exists(filename): logger.info("Creating torrent..") with open(filename, 'wb') as fp: fp.write(os.urandom(50 * 1024 * 1024)) tdef = TorrentDef() tdef.add_content(os.path.join(cur_path, filename)) tdef.set_tracker("udp://fake.net/announce") tdef.set_private() tdef.finalize() tdef.save(os.path.join(cur_path, filename + '.torrent')) else: logger.info("Loading existing torrent..") tdef = TorrentDef.load(filename + '.torrent') logger.info("loading torrent done, infohash of torrent: %s" % (tdef.get_infohash().encode('hex')[:10])) defaultDLConfig = DefaultDownloadStartupConfig.getInstance() dscfg = defaultDLConfig.copy() dscfg.set_hops(1) dscfg.set_dest_dir(cur_path) anon_tunnel.session.lm.threadpool.call(0, anon_tunnel.session.start_download, tdef, dscfg) elif line.startswith('i'): # Introduce dispersy port from other main peer to this peer line_split = line.split(' ') to_introduce_ip = line_split[1] to_introduce_port = int(line_split[2]) self.anon_tunnel.community.add_discovered_candidate(Candidate((to_introduce_ip, to_introduce_port), tunnel=False)) elif line.startswith('d'): line_split = line.split(' ') filename = 'test_file' if len(line_split) == 1 else line_split[1] logger.info("Loading torrent..") tdef = TorrentDef.load(filename + '.torrent') logger.info("Loading torrent done") defaultDLConfig = DefaultDownloadStartupConfig.getInstance() dscfg = defaultDLConfig.copy() dscfg.set_hops(1) dscfg.set_dest_dir(os.path.join(os.getcwd(), 'downloader%s' % anon_tunnel.session.get_dispersy_port())) def start_download(): def cb(ds): logger.info('Download infohash=%s, down=%s, progress=%s, status=%s, seedpeers=%s, candidates=%d' % (tdef.get_infohash().encode('hex')[:10], ds.get_current_speed('down'), ds.get_progress(), dlstatus_strings[ds.get_status()], sum(ds.get_num_seeds_peers()), sum(1 for _ in anon_tunnel.community.dispersy_yield_verified_candidates()))) return 1.0, False download = anon_tunnel.session.start_download(tdef, dscfg) download.set_state_callback(cb, delay=1) anon_tunnel.session.lm.threadpool.call(0, start_download) elif line == 'q': anon_tunnel.stop() return elif line == 'r': print "circuit\t\t\tdirection\tcircuit\t\t\tTraffic (MB)" from_to = anon_tunnel.community.relay_from_to for key in from_to.keys(): relay = from_to[key] logger.info("%s-->\t%s\t\t%.2f" % ((key[0], key[1]), (relay.sock_addr, relay.circuit_id), relay.bytes[1] / 1024.0 / 1024.0,))
class TestMagnetFakePeer(TestAsServer, MagnetHelpers): """ A MiniBitTorrent instance is used to connect to BitTorrent clients and download the info part from the metadata. """ def setUp(self): # listener for incoming connections from MiniBitTorrent self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.server.bind(("", LISTEN_PORT)) self.server.listen(5) TestAsServer.setUp(self) # the metadata that we want to transfer self.tdef = TorrentDef() self.tdef.add_content(os.path.join(BASE_DIR, "API", "file.wmv")) self.tdef.set_tracker("http://fake.net/announce") # we use a small piece length to obtain multiple pieces self.tdef.set_piece_length(1) self.tdef.finalize() MagnetHelpers.__init__(self, self.tdef) def setUpPreSession(self): TestAsServer.setUpPreSession(self) self.config.set_libtorrent(True) def create_good_url(self, infohash=None, title=None, tracker=None): url = "magnet:?xt=urn:btih:" if infohash: assert isinstance(infohash, str) url += hexlify(infohash) else: url += hexlify(self.tdef.get_infohash()) if title: assert isinstance(title, str) url += "&dn=" + title if tracker: assert isinstance(tracker, str) url += "&tr=" + tracker return url @skip("not working, seems to return binary data") def test_good_transfer(self): def torrentdef_retrieved(tdef): tags["retrieved"].set() tags["metainfo"] = tdef.get_metainfo() tags = {"retrieved": threading.Event()} assert TorrentDef.retrieve_from_magnet(self.create_good_url(), torrentdef_retrieved, timeout=60) def do_supply(): # supply fake addresses (regular dht obviously wont work here) ltmgr = LibtorrentMgr.getInstance() for infohash in ltmgr.metainfo_requests: handle = ltmgr.ltsession.find_torrent(lt.big_number(infohash.decode('hex'))) handle.connect_peer(("127.0.0.1", LISTEN_PORT), 0) self.session.lm.rawserver.add_task(do_supply, delay=5.0) # accept incoming connection # self.server.settimeout(10.0) sock, address = self.server.accept() assert sock, "No incoming connection" # handshakes conn = BTConnection(address[0], address[1], opensock=sock, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) # serve pieces for counter in xrange(len(self.metadata_list)): piece = self.read_extend_metadata_request(conn) assert 0 <= piece < len(self.metadata_list) conn.send(self.create_good_extend_metadata_reply(metadata_id, piece)) # no more metadata request may be send and the connection must # be closed self.read_extend_metadata_close(conn) assert tags["retrieved"].wait(5) assert tags["metainfo"]["info"] == self.tdef.get_metainfo()["info"]
class TestQueryReplyActive(TestAsServer): """ Testing QUERY_REPLY message of Query extension V1 This test checks how the Tribler code responds to good and bad QUERY_REPLY messages. I.e. the Tribler client initiates the dialback by connecting to us and sending a QUERY and we reply with good and bad messages. This test allows authoritative answers from superpeers. WARNING: Each of the test_ methods should be tested by running the TestCase in a separate Python interpreter to prevent problems with our singleton classes, e.g. SuperPeerDB, etc. """ def setUpPreSession(self): """ override TestAsServer """ print >> sys.stderr,"test: Pre Tribler Init" TestAsServer.setUpPreSession(self) print >> sys.stderr,"test: Pre Tribler Init: config_path",self.config_path # Enable remote querying self.config.set_remote_query(True) def setUpPostSession(self): """ override TestAsServer """ TestAsServer.setUpPostSession(self) self.hispermid = str(self.his_keypair.pub().get_der()) self.my_permid = str(self.my_keypair.pub().get_der()) def pretest_simple(self,keyword): self.pretest_q('SIMPLE',keyword) def pretest_simpleplustorrents(self,keyword): self.pretest_q('SIMPLE+METADATA',keyword) def pretest_q(self,queryprefix,keyword): query = queryprefix+' '+keyword self.content_name = keyword.upper()+' S22E44' self.tdef = TorrentDef() self.tdef.set_tracker('http://localhost:0/announce') self.tdef.set_piece_length(2 ** 15) self.tdef.create_live(self.content_name,2 ** 16) self.tdef.finalize() # 1. First connect to Tribler self.openconn = OLConnection(self.my_keypair,'localhost',self.hisport) sleep(3) # 2. Make Tribler send query self.query = query self.session.query_connected_peers(query,self.query_usercallback,max_peers_to_query=10) def query_usercallback(self,permid,query,hits): print >>sys.stderr,"test: query_usercallback:",`permid`,`query`,`hits` self.assert_(query == self.query) self.assert_(permid == self.my_permid) self.check_good_qreply(hits) # TODO: if SIMPLE+METADATA: check torrent now in db. # # Good SIMPLE QUERY, builds on TestQueryReply code # def singtest_good_simple_reply(self): self.pretest_simple('hallo') self._test_qreply(self.create_good_simple_reply,True) # # Good SIMPLE+METADATA QUERY, builds on TestQueryReply code # def singtest_good_simpleplustorrents_reply(self): self.pretest_simpleplustorrents('hallo') self._test_qreply(self.create_good_simpleplustorrents_reply,True) # # Good SIMPLE QUERY Unicode, builds on TestQueryReply code # def singtest_good_simple_reply_unicode(self): self.pretest_simple(u'Ch\u00e8rie') self._test_qreply(self.create_good_simple_reply,True) # # Good SIMPLE+METADATA QUERY Unicode, builds on TestQueryReply code # def singtest_good_simpleplustorrents_reply_unicode(self): self.pretest_simpleplustorrents(u'Ch\u00e8rie') self._test_qreply(self.create_good_simpleplustorrents_reply,True) # # Bad QUERY, builds on TestQueryReply code # def singtest_bad_not_bdecodable(self): self.pretest_simple('hallo') self._test_qreply(self.create_not_bdecodable,False) # # Bad SIMPLE+METADATA QUERY, builds on TestQueryReply code # def singtest_bad_not_bdecodable_torrentfile(self): self.pretest_simpleplustorrents('hallo') self._test_qreply(self.create_not_bdecodable_torrentfile,False) ### TODO: send different valid answers so consensus not reached # # Main test code # def _test_qreply(self,gen_qreply,good): print >> sys.stderr,"test: waiting for reply" s = self.openconn msg = s.recv() self.assert_(len(msg) > 0) print >> sys.stderr,"test: Received overlay message",getMessageName(msg[0]) self.assert_(msg[0] == QUERY) id = self.check_rquery(msg[1:]) resp = gen_qreply(id) print >> sys.stderr,"test: sending QUERY_REPLY" s.send(resp) if good: time.sleep(10) # the other side should not have closed the connection, as # this is all valid, so this should not throw an exception: s.send('bla') s.close() else: # the other side should not like this and close the connection self.assert_(len(s.recv())==0) s.close() def create_good_simple_reply_dict(self,id): r = {} r['content_name'] = self.content_name.encode("UTF-8") r['length'] = LENGTH r['leecher'] = LEECHERS r['seeder'] = SEEDERS r['category'] = CATEGORY # OLPROTO_PROTO_ELEVENTH # set later r['torrent_size'] = 42 r['channel_permid'] = '$' * 83 r['channel_name'] = 'Nitin Channel' d2 = {} d2[self.tdef.get_infohash()] = r d = {} d['id'] = id d['a'] = d2 return d def create_good_simple_reply(self,id): d = self.create_good_simple_reply_dict(id) bmetainfo = bencode(self.tdef.get_metainfo()) d['a'][self.tdef.get_infohash()]['torrent_size'] = len(bmetainfo) b = bencode(d) return QUERY_REPLY+b def create_good_simpleplustorrents_reply(self,id): d = self.create_good_simple_reply_dict(id) bmetainfo = bencode(self.tdef.get_metainfo()) d['a'][self.tdef.get_infohash()]['torrent_size'] = len(bmetainfo) d['a'][self.tdef.get_infohash()]['metatype'] = 'application/x-tribler-stream' d['a'][self.tdef.get_infohash()]['metadata'] = bmetainfo b = bencode(d) return QUERY_REPLY+b def check_good_qreply(self,hits): self.assert_(len(hits) == 1) self.assert_(hits.keys()[0] == self.tdef.get_infohash()) hit = hits[self.tdef.get_infohash()] self.assert_(hit['content_name'] == self.content_name) self.assert_(hit['length'] == LENGTH) self.assert_(hit['leecher'] == LEECHERS) self.assert_(hit['seeder'] == SEEDERS) self.assert_(hit['category'] == CATEGORY) # OLPROTO_VERSION_ELEVENTH bmetainfo = bencode(self.tdef.get_metainfo()) self.assert_(hit['torrent_size'] == len(bmetainfo)) if self.query.startswith('SIMPLE+METADATA'): self.assert_(hit['metadata'] == bmetainfo) def create_not_bdecodable(self,id): return QUERY_REPLY+"bla" def create_not_bdecodable_torrentfile(self,id): d = self.create_good_simple_reply_dict(id) d['a'][self.tdef.get_infohash()]['torrent_size'] = 3 # consistent with metadata. Should be named "metasize" d['a'][self.tdef.get_infohash()]['metadata'] = 'bla' b = bencode(d) return QUERY_REPLY+b def check_rquery(self,data): d = bdecode(data) self.assert_(type(d) == DictType) self.assert_(d.has_key('q')) q = d['q'] self.assert_(type(q) == StringType) self.assert_(d.has_key('id')) id = d['id'] self.assert_(type(id) == StringType) self.assert_(q == self.query.encode("UTF-8")) return d['id']
class TestMetadata(TestAsServer, MagnetHelpers): """ Once we are downloading a torrent, our client should respond to the ut_metadata extention message. This allows other clients to obtain the info part of the metadata from us. """ def setUp(self): """ override TestAsServer """ TestAsServer.setUp(self) print >> sys.stderr, "test: Giving MyLaunchMany time to startup" time.sleep(5) print >> sys.stderr, "test: MyLaunchMany should have started up" # the metadata that we want to transfer self.tdef = TorrentDef() self.tdef.add_content(os.path.join(os.getcwd(), "API", "file.wmv")) self.tdef.set_tracker(self.session.get_internal_tracker_url()) # we use a small piece length to obtain multiple pieces self.tdef.set_piece_length(1) self.tdef.finalize() # self.tdef.save(os.path.join(self.session.get_state_dir(), "gen.torrent")) MagnetHelpers.__init__(self, self.tdef) def setup_seeder(self): self.seeder_setup_complete = False self.seeder_teardown_complete = False self.seeder_teardown = False self.dscfg = DownloadStartupConfig() self.dscfg.set_dest_dir(os.getcwd()) self.download = self.session.start_download(self.tdef, self.dscfg) self.download.set_state_callback(self.seeder_state_callback) counter = 0 while not self.seeder_setup_complete: counter += 1 time.sleep(1) assert counter < 30, "timeout" print >> sys.stderr, "test: setup_seeder() complete" def teardown_seeder(self): self.seeder_teardown_complete = False self.session.remove_download(self.download) counter = 0 while not self.seeder_setup_complete: counter += 1 time.sleep(1) assert counter < 30, "timeout" print >> sys.stderr, "test: teardown_seeder() complete" def seeder_state_callback(self, ds): assert not self.seeder_teardown_complete self.seeder_setup_complete = (ds.get_status() == DLSTATUS_DOWNLOADING) d = ds.get_download() print >> sys.stderr, "test: seeder:", ` d.get_def().get_name( ) `, dlstatus_strings[ds.get_status()], ds.get_progress() if self.seeder_teardown: self.seeder_teardown_complete = True else: return (1.0, False) def test_all(self): self.setup_seeder() try: self.subtest_good_flood() finally: self.teardown_seeder() self.setup_seeder() try: self.subtest_good_request() self.subtest_bad_request() finally: self.teardown_seeder() def subtest_good_request(self): conn = BTConnection("localhost", self.hisport, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) # request metadata block 0, 2, 3, and the last conn.send(self.create_good_extend_metadata_request(metadata_id, 0)) conn.send(self.create_good_extend_metadata_request(metadata_id, 2)) conn.send(self.create_good_extend_metadata_request(metadata_id, 3)) conn.send( self.create_good_extend_metadata_request( metadata_id, len(self.metadata_list) - 1)) self.read_extend_metadata_reply(conn, 0) self.read_extend_metadata_reply(conn, 2) self.read_extend_metadata_reply(conn, 3) self.read_extend_metadata_reply(conn, len(self.metadata_list) - 1) def subtest_good_flood(self): conn = BTConnection("localhost", self.hisport, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) for counter in xrange(len(self.metadata_list) * 2): piece = counter % len(self.metadata_list) conn.send( self.create_good_extend_metadata_request(metadata_id, piece)) if counter > len(self.metadata_list): self.read_extend_metadata_reject(conn, piece) else: self.read_extend_metadata_reply(conn, piece) def subtest_bad_request(self): self.bad_request_and_disconnect({ "msg_type": 0, "piece": len(self.metadata_list) }) self.bad_request_and_disconnect({"msg_type": 0, "piece": -1}) self.bad_request_and_disconnect({"msg_type": 0, "piece": "1"}) self.bad_request_and_disconnect({"msg_type": 0, "piece": [1, 2]}) self.bad_request_and_disconnect({"msg_type": 0, "PIECE": 1}) def bad_request_and_disconnect(self, payload): conn = BTConnection("localhost", self.hisport, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) conn.send(EXTEND + chr(metadata_id) + bencode(payload)) self.read_extend_metadata_close(conn)
class TestMerkleMessage(TestAsServer): """ Testing Merkle hashpiece messages for both: * Merkle BEP style * old Tribler <= 4.5.2 that did not use the Extention protocol (BEP 10). See BitTornado/BT1/Connecter.py """ def setUp(self): """ override TestAsServer """ TestAsServer.setUp(self) print >> sys.stderr, "test: Giving Session time to startup" time.sleep(5) print >> sys.stderr, "test: Session should have started up" def setUpPreSession(self): """ override TestAsServer """ TestAsServer.setUpPreSession(self) self.config.set_overlay(False) self.config.set_megacache(False) def setUpPostSession(self): """ override TestAsServer """ TestAsServer.setUpPostSession(self) # Let Tribler start downloading an non-functioning torrent, so # we can talk to a normal download engine. self.tdef = TorrentDef() self.sourcefn = os.path.join(os.getcwd(), "API", "file2.wmv") self.tdef.add_content(self.sourcefn) self.tdef.set_create_merkle_torrent(True) self.tdef.set_tracker("http://127.0.0.1:12/announce") self.tdef.finalize() self.torrentfn = os.path.join(self.session.get_state_dir(), "gen.torrent") self.tdef.save(self.torrentfn) dscfg = self.setUpDownloadConfig() self.session.start_download(self.tdef, dscfg) self.infohash = self.tdef.get_infohash() self.mylistenport = 4810 self.numpieces = (self.tdef.get_length() + self.tdef.get_piece_length() - 1) / self.tdef.get_piece_length() b = Bitfield(self.numpieces) for i in range(self.numpieces): b[i] = True self.assert_(b.complete()) self.seederbitfieldstr = b.tostring() # piece_hashes = ['\x01\x02\x03\x04\x05\x06\x07\x08\x07\x06\x05\x04\x03\x02\x01\x00\x01\x02\x03\x04' ] * npieces # Construct Merkle tree tdef2 = TorrentDef() tdef2.add_content(self.sourcefn) tdef2.set_create_merkle_torrent(False) tdef2.set_tracker("http://127.0.0.1:12/announce") tdef2.set_piece_length(self.tdef.get_piece_length()) tdef2.finalize() metainfo = tdef2.get_metainfo() piecesstr = metainfo['info']['pieces'] print >> sys.stderr, "test: pieces has len", len(piecesstr) piece_hashes = [] for i in range(0, len(piecesstr), 20): hash = piecesstr[i:i + 20] print >> sys.stderr, "test: piece", i / 20, "hash", repr(hash) piece_hashes.append(hash) print >> sys.stderr, "test: Putting", len(piece_hashes), "into MerkleTree, size", self.tdef.get_piece_length(), tdef2.get_piece_length() self.tree = MerkleTree(self.tdef.get_piece_length(), self.tdef.get_length(), None, piece_hashes) f = open(self.sourcefn, "rb") piece1 = f.read(2 ** 18) piece2 = f.read(2 ** 18) print >> sys.stderr, "read piece1", len(piece1) print >> sys.stderr, "read piece2", len(piece2) f.close() hash1 = sha(piece1).digest() hash2 = sha(piece2).digest() print >> sys.stderr, "hash piece1", repr(hash1) print >> sys.stderr, "hash piece2", repr(hash2) f2 = open("piece1.bin", "wb") f2.write(piece2) f2.close() def setUpDownloadConfig(self): dscfg = DownloadStartupConfig() print >> sys.stderr, "test: Downloading to", self.config_path dscfg.set_dest_dir(self.config_path) dscfg.set_breakup_seed_bitfield(False) return dscfg def tearDown(self): TestAsServer.tearDown(self) try: os.remove('piece1.bin') except: pass def singtest_good_hashpiece_bepstyle(self): self.subtest_good_hashpiece(False) def singtest_good_hashpiece_oldstyle(self): self.subtest_good_hashpiece(True) def singtest_good_request_bepstyle(self): # Let Session download file first self.subtest_good_hashpiece(False) # Now connect as different peer and download print >> sys.stderr, "\n\ntest: test_good_request: STARTING" self._test_good_request() def singtest_bad_hashpiece_bepstyle(self): self.subtest_bad_hashpiece(False) def singtest_bad_hashpiece_oldstyle(self): self.subtest_bad_hashpiece(True) # # Good hashpiece message # def subtest_good_hashpiece(self, oldstyle): print >> sys.stderr, "test: Testing good hashpiece, oldstyle", oldstyle if oldstyle: self._test_good(self.create_good_hashpiece, oldstyle, self.create_good_tribler_extend_hs, infohash=self.infohash) else: options = '\x00\x00\x00\x00\x00\x10\x00\x00' self._test_good(self.create_good_hashpiece, oldstyle, self.create_good_nontribler_extend_hs, options=options, infohash=self.infohash) def _test_good(self, msg_gen_func, oldstyle, extend_hs_gen_func, options=None, infohash=None): if options is None and infohash is None: s = BTConnection('localhost', self.hisport) elif options is None: s = BTConnection('localhost', self.hisport, user_infohash=infohash) elif infohash is None: s = BTConnection('localhost', self.hisport, user_option_pattern=options) else: s = BTConnection('localhost', self.hisport, user_option_pattern=options, user_infohash=infohash) print >> sys.stderr, "test: test_good: Create EXTEND HS" msg = extend_hs_gen_func() print >> sys.stderr, "test: test_good: Sending EXTEND HS", repr(msg) s.send(msg) print >> sys.stderr, "test: test_good: Waiting for BT HS" s.read_handshake_medium_rare() # Tribler should send an EXTEND message back try: print >> sys.stderr, "test: Waiting for reply" s.s.settimeout(10.0) resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply", getMessageName(resp[0]) self.assert_(resp[0] == EXTEND) self.check_tribler_extend_hs(resp[1:]) # 1. Pretend we're seeder: send BITFIELD and UNCHOKE msg = BITFIELD + self.seederbitfieldstr s.send(msg) msg = UNCHOKE s.send(msg) print >> sys.stderr, "test: Pretend we are seeder" while True: resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply2", getMessageName(resp[0]) self.assert_(resp[0] == REQUEST or resp[0] == INTERESTED or resp[0] == UNCHOKE or resp[0] == HAVE or resp[0] == NOT_INTERESTED) if resp[0] == REQUEST: chunkid = self.check_request(resp) # 2. Reply to REQUEST with HASHPIECE (oldstyle) or Tr_hashpiece msg = msg_gen_func(oldstyle, chunkid) s.send(msg) elif resp[0] == NOT_INTERESTED: break # s.close() except socket.timeout: print >> sys.stderr, "test: Timeout, bad, peer didn't reply in time" self.assert_(False) destfn = os.path.join(self.config_path, "file2.wmv") sf = open(self.sourcefn, "rb") df = open(destfn, "rb") n = self.tdef.get_piece_length() while True: sdata = sf.read(n) if len(sdata) == 0: break ddata = df.read(n) self.assert_(sdata == ddata) time.sleep(3) s.close() def create_good_nontribler_extend_hs(self): """ Merkle BEP style """ d = {} d['m'] = {'Tr_hashpiece': 250} d['p'] = self.mylistenport d['v'] = 'TestSweet 1.2.3.4' bd = bencode(d) return EXTEND + chr(0) + bd def create_good_tribler_extend_hs(self): """ old Tribler style """ d = {} d['m'] = {'Tr_OVERLAYSWARM': 253} d['p'] = self.mylistenport d['v'] = 'Tribler 3.5.1' bd = bencode(d) return EXTEND + chr(0) + bd def check_tribler_extend_hs(self, data): self.assert_(data[0] == chr(0)) d = bdecode(data[1:]) self.assert_(isinstance(d, DictType)) self.assert_('m' in d.keys()) m = d['m'] self.assert_(isinstance(m, DictType)) self.assert_('Tr_hashpiece' in m.keys()) val = m['Tr_hashpiece'] self.assert_(isinstance(val, IntType)) self.assert_(val == 250) def check_request(self, data): index = toint(data[1:5]) begin = toint(data[5:9]) length = toint(data[9:]) return (index, begin, length) def create_good_hashpiece(self, oldstyle, chunkid): index, begin, length = chunkid if begin == 0: ohlist = self.tree.get_hashes_for_piece(index) else: ohlist = [] chunk = self.read_chunk(index, begin, length) bohlist = bencode(ohlist) print >> sys.stderr, "test: create_good_hashpiece:", index, begin, length, "==len", len(chunk) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk if oldstyle: msg = HASHPIECE + payload else: # Offical: use the msg ID he defined in his handshake msg = EXTEND + HASHPIECE + payload return msg def read_chunk(self, index, begin, length): offset = index * self.tdef.get_piece_length() + begin f = open(self.sourcefn, "rb") f.seek(offset) chunk = f.read(length) f.close() return chunk # # Test whether Tribler sends good Tr_hashpiece on our requests # def _test_good_request(self): options = '\x00\x00\x00\x00\x00\x10\x00\x00' myid = Rand.rand_bytes(20) s = BTConnection('localhost', self.hisport, user_option_pattern=options, user_infohash=self.infohash, myid=myid) msg = self.create_good_nontribler_extend_hs() s.send(msg) s.read_handshake_medium_rare() # Tribler should send an EXTEND message back try: print >> sys.stderr, "test: Waiting for reply" s.s.settimeout(10.0) resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply", getMessageName(resp[0]) self.assert_(resp[0] == EXTEND) self.check_tribler_extend_hs(resp[1:]) # 1. Pretend we're leecher: send INTERESTED msg = INTERESTED s.send(msg) print >> sys.stderr, "test: Pretend we are leecher" while True: resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply2", getMessageName(resp[0]) if resp[0] == EXTEND: print >> sys.stderr, "test: Got EXTEND type", getMessageName(resp[1]) self.assert_(resp[0] == UNCHOKE or resp[0] == BITFIELD or resp[0] == EXTEND or resp[0] == HAVE) if resp[0] == UNCHOKE: # 2. Reply with REQUESTs for index in range(0, self.numpieces): plen = self.get_piece_length(index) for begin in range(0, plen, 2 ** 14): length = self.get_chunk_length(index, begin) print >> sys.stderr, "RETRIEVE", index, begin, length chunkid = (index, begin, length) msg = self.create_request(chunkid) s.send(msg) # s.send(NOT_INTERESTED) elif resp[0] == EXTEND and resp[1] == HASHPIECE: done = self.check_hashpiece(resp) if done: break elif resp[0] == BITFIELD: self.check_bitfield(resp) # s.close() except socket.timeout: print >> sys.stderr, "test: Timeout, bad, peer didn't reply in time" self.assert_(False) time.sleep(3) s.close() def get_piece_length(self, index): if index == (self.numpieces - 1): plen = self.tdef.get_length() % self.tdef.get_piece_length() else: plen = self.tdef.get_piece_length() return plen def get_chunk_length(self, index, begin): plen = self.get_piece_length(index) length = 2 ** 14 if index == (self.numpieces - 1): if (begin + 2 ** 14) > plen: length = plen - begin return length def create_request(self, chunkid): index, begin, length = chunkid return REQUEST + tobinary(index) + tobinary(begin) + tobinary(length) def check_hashpiece(self, resp): """ Merkle BEP style """ print >> sys.stderr, "test: good_request: check_hashpiece" self.assert_(resp[0] == EXTEND) self.assert_(resp[1] == HASHPIECE) index = toint(resp[2:2 + 4]) begin = toint(resp[6:6 + 4]) ohlen = toint(resp[10:10 + 4]) print >> sys.stderr, "test: good_request: check_hashpiece", index, begin, ohlen bohlist = resp[14:14 + ohlen] hisohlist = bdecode(bohlist) hischunk = resp[14 + ohlen:] if begin == 0: self.assert_(isinstance(hisohlist, ListType)) for oh in hisohlist: self.assert_(isinstance(oh, ListType)) self.assert_(len(oh) == 2) self.assert_(isinstance(oh[0], IntType)) self.assert_(isinstance(oh[1], StringType)) hisohlist.sort() print >> sys.stderr, "test: good_request: check_hashpiece", repr(hisohlist) myohlist = self.tree.get_hashes_for_piece(index) myohlist.sort() self.assert_(len(hisohlist) == len(myohlist)) for i in range(0, len(hisohlist)): hisoh = hisohlist[i] myoh = myohlist[i] self.assert_(hisoh == myoh) else: self.assert_(len(hisohlist) == 0) mylength = self.get_chunk_length(index, begin) mychunk = self.read_chunk(index, begin, mylength) self.assert_(hischunk == mychunk) return index == self.numpieces - 1 and mylength != 2 ** 14 def check_bitfield(self, data): self.assert_(data[0] == BITFIELD) bitmap = data[1:] self.assert_(len(bitmap) == 1) # Must have set_breakup_seed_bitfield() set to False self.assert_(bitmap == '\xc0') # # Bad EXTEND handshake message # def subtest_bad_hashpiece(self, oldstyle): if not oldstyle: # Test becomes equivalent to BT keep alive message (len 0, payload '') self._test_bad(self.create_empty, oldstyle) self._test_bad(self.create_ext_id_not_byte, oldstyle) self._test_bad(self.create_not_hashpiece, oldstyle) self._test_bad(self.create_not_index, oldstyle) self._test_bad(self.create_not_begin, oldstyle) self._test_bad(self.create_not_len_bohlist, oldstyle) self._test_bad(self.create_ohlist_not_bdecodable, oldstyle) self._test_bad(self.create_ohlist_wrong_no_hashes, oldstyle) self._test_bad(self.create_ohlist_wrong_no_root_hash, oldstyle) self._test_bad(self.create_ohlist_wrong_bad_offset, oldstyle) self._test_bad(self.create_ohlist_wrong_bad_hash, oldstyle) # TODO: need working peer kicking for that # self._test_bad(self.create_bad_chunk,oldstyle) # # Main test code for bad EXTEND handshake messages # def _test_bad(self, msg_gen_func, oldstyle): print >> sys.stderr, "test: test_BAD: Create EXTEND HS", repr(msg_gen_func), oldstyle if oldstyle: options = None exthsmsg = self.create_good_tribler_extend_hs() else: options = '\x00\x00\x00\x00\x00\x10\x00\x00' exthsmsg = self.create_good_nontribler_extend_hs() s = BTConnection('localhost', self.hisport, user_option_pattern=options, user_infohash=self.infohash) s.send(exthsmsg) s.read_handshake_medium_rare() # Tribler should send an EXTEND message back try: print >> sys.stderr, "test: Waiting for reply" s.s.settimeout(10.0) resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply", getMessageName(resp[0]) self.assert_(resp[0] == EXTEND) self.check_tribler_extend_hs(resp[1:]) # 1. Pretend we're seeder: send BITFIELD and UNCHOKE msg = BITFIELD + self.seederbitfieldstr s.send(msg) msg = UNCHOKE s.send(msg) print >> sys.stderr, "test: Pretend we are seeder" while True: resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply 2", getMessageName(resp[0]) self.assert_(resp[0] == REQUEST or resp[0] == INTERESTED or resp[0] == UNCHOKE or resp[0] == HAVE or resp[0] == NOT_INTERESTED) if resp[0] == REQUEST: chunkid = self.check_request(resp) # 2. Reply to REQUEST with *bad* HASHPIECE msg = msg_gen_func(chunkid) if oldstyle: if len(msg) == 1: msg = '' else: msg = msg[1:] # Strip EXTEND byte s.send(msg) break # s.close() except socket.timeout: print >> sys.stderr, "test: Timeout, bad, peer didn't reply in time" self.assert_(False) time.sleep(3) # Should have closed the connection try: s.send(UNCHOKE) self.assert_(False) except: print_exc() s.close() # # Bad message creators (all create Merkle BEP style, I strip first byte # later for oldstyle # def create_empty(self, chunkid): return EXTEND def create_ext_id_not_byte(self, chunkid): return EXTEND + 'Hallo kijkbuiskinderen' def create_not_hashpiece(self, chunkid): index, begin, length = chunkid ohlist = [] bohlist = bencode(ohlist) chunk = self.read_chunk(index, begin, length) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + chr(231) + payload def create_not_index(self, chunkid): payload = 'bla' return EXTEND + HASHPIECE + payload def create_not_begin(self, chunkid): index, begin, length = chunkid payload = tobinary(index) + 'bla' return EXTEND + HASHPIECE + payload def create_not_len_bohlist(self, chunkid): index, begin, length = chunkid payload = tobinary(index) + tobinary(begin) + 'bla' return EXTEND + HASHPIECE + payload def create_ohlist_not_bdecodable(self, chunkid): index, begin, length = chunkid bohlist = 'bla' chunk = '*' * (2 ** 14) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload def create_ohlist_wrong_no_hashes(self, chunkid): index, begin, length = chunkid ohlist = [(0, '#' * 20), (1, '$' * 20)] # should contain 3 for file2.wmv: own, sibling and root bohlist = bencode(ohlist) chunk = '*' * (2 ** 14) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload def create_ohlist_wrong_no_root_hash(self, chunkid): index, begin, length = chunkid ohlist = self.tree.get_hashes_for_piece(index) newohlist = [] # Remove root hash for oh in ohlist: if oh[0] != 0: newohlist.append(oh) ohlist = newohlist bohlist = bencode(ohlist) chunk = self.read_chunk(index, begin, length) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload def create_ohlist_wrong_bad_offset(self, chunkid): index, begin, length = chunkid ohlist = self.tree.get_hashes_for_piece(index) ohlist[1][0] = 481 bohlist = bencode(ohlist) chunk = self.read_chunk(index, begin, length) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload def create_ohlist_wrong_bad_hash(self, chunkid): index, begin, length = chunkid ohlist = self.tree.get_hashes_for_piece(index) ohlist[1][1] = '$' * 20 bohlist = bencode(ohlist) chunk = self.read_chunk(index, begin, length) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload def create_bad_chunk(self, chunkid): index, begin, length = chunkid ohlist = self.tree.get_hashes_for_piece(index) bohlist = bencode(ohlist) chunk = '*' * length payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload
class TestSeeding(TestAsServer): """ Testing seeding via new tribler API: """ def setUp(self): """ override TestAsServer """ super(TestSeeding, self).setUp() self.session2 = None self.seeding_event = threading.Event() self.downloading_event = threading.Event() def setUpPreSession(self): """ override TestAsServer """ super(TestSeeding, self).setUpPreSession() self.config.set_libtorrent(True) self.config2 = self.config.copy() # not really necess self.config2.set_state_dir(self.getStateDir(2)) self.dscfg2 = DownloadStartupConfig() self.dscfg2.set_dest_dir(self.getDestDir(2)) def setUpPostSession(self): pass def tearDown(self): if self.session2: self._shutdown_session(self.session2) time.sleep(10) super(TestSeeding, self).tearDown() def setup_seeder(self, filename='video.avi'): self.tdef = TorrentDef() self.sourcefn = os.path.join(TESTS_API_DIR, filename) self.tdef.add_content(self.sourcefn) self.tdef.set_tracker("http://fake.net/announce") self.tdef.finalize() self.torrentfn = os.path.join(self.session.get_state_dir(), "gen.torrent") self.tdef.save(self.torrentfn) self._logger.debug("name is %s", self.tdef.metainfo['info']['name']) self.dscfg = DownloadStartupConfig() self.dscfg.set_dest_dir(TESTS_API_DIR) # basedir of the file we are seeding d = self.session.start_download(self.tdef, self.dscfg) d.set_state_callback(self.seeder_state_callback) self._logger.debug("starting to wait for download to reach seeding state") assert self.seeding_event.wait(60) def seeder_state_callback(self, ds): d = ds.get_download() self._logger.debug("seeder status: %s %s %s", repr(d.get_def().get_name()), dlstatus_strings[ds.get_status()], ds.get_progress()) if ds.get_status() == DLSTATUS_SEEDING: self.seeding_event.set() return 1.0, False def test_normal_torrent(self): self.setup_seeder() self.subtest_is_seeding() self.subtest_download() def subtest_is_seeding(self): infohash = self.tdef.get_infohash() s = BTConnection('localhost', self.session.get_listen_port(), user_infohash=infohash) s.read_handshake_medium_rare() s.send(CHOKE) try: s.s.settimeout(10.0) resp = s.recv() self.assert_(len(resp) > 0) self.assert_(resp[0] == EXTEND) except socket.timeout: print >> sys.stderr, "test: Timeout, peer didn't reply" self.assert_(False) s.close() def subtest_download(self): """ Now download the file via another Session """ self.session2 = Session(self.config2, ignore_singleton=True) upgrader = self.session2.prestart() while not upgrader.is_done: time.sleep(0.1) self.session2.start() time.sleep(1) time.sleep(5) tdef2 = TorrentDef.load(self.torrentfn) d = self.session2.start_download(tdef2, self.dscfg2) d.set_state_callback(self.downloader_state_callback) time.sleep(5) d.add_peer(("127.0.0.1", self.session.get_listen_port())) assert self.downloading_event.wait(60) def downloader_state_callback(self, ds): d = ds.get_download() self._logger.debug("download status: %s %s %s", repr(d.get_def().get_name()), dlstatus_strings[ds.get_status()], ds.get_progress()) if ds.get_status() == DLSTATUS_SEEDING: # File is in destfn = os.path.join(self.getDestDir(2), "video.avi") f = open(destfn, "rb") realdata = f.read() f.close() f = open(self.sourcefn, "rb") expdata = f.read() f.close() self.assert_(realdata == expdata) self.downloading_event.set() return 1.0, True return 1.0, False
def test_get_infohash(self): t = TorrentDef() t.get_infohash()
class TestMerkleMessage(TestAsServer): """ Testing Merkle hashpiece messages for both: * Merkle BEP style * old Tribler <= 4.5.2 that did not use the Extention protocol (BEP 10). See BitTornado/BT1/Connecter.py """ def setUp(self): """ override TestAsServer """ TestAsServer.setUp(self) print >> sys.stderr, "test: Giving Session time to startup" time.sleep(5) print >> sys.stderr, "test: Session should have started up" def setUpPreSession(self): """ override TestAsServer """ TestAsServer.setUpPreSession(self) self.config.set_overlay(False) self.config.set_megacache(False) def setUpPostSession(self): """ override TestAsServer """ TestAsServer.setUpPostSession(self) # Let Tribler start downloading an non-functioning torrent, so # we can talk to a normal download engine. self.tdef = TorrentDef() self.sourcefn = os.path.join(os.getcwd(), "API", "file2.wmv") self.tdef.add_content(self.sourcefn) self.tdef.set_create_merkle_torrent(True) self.tdef.set_tracker("http://127.0.0.1:12/announce") self.tdef.finalize() self.torrentfn = os.path.join(self.session.get_state_dir(), "gen.torrent") self.tdef.save(self.torrentfn) dscfg = self.setUpDownloadConfig() self.session.start_download(self.tdef, dscfg) self.infohash = self.tdef.get_infohash() self.mylistenport = 4810 self.numpieces = (self.tdef.get_length() + self.tdef.get_piece_length() - 1) / self.tdef.get_piece_length() b = Bitfield(self.numpieces) for i in range(self.numpieces): b[i] = True self.assert_(b.complete()) self.seederbitfieldstr = b.tostring() # piece_hashes = ['\x01\x02\x03\x04\x05\x06\x07\x08\x07\x06\x05\x04\x03\x02\x01\x00\x01\x02\x03\x04' ] * npieces # Construct Merkle tree tdef2 = TorrentDef() tdef2.add_content(self.sourcefn) tdef2.set_create_merkle_torrent(False) tdef2.set_tracker("http://127.0.0.1:12/announce") tdef2.set_piece_length(self.tdef.get_piece_length()) tdef2.finalize() metainfo = tdef2.get_metainfo() piecesstr = metainfo["info"]["pieces"] print >> sys.stderr, "test: pieces has len", len(piecesstr) piece_hashes = [] for i in range(0, len(piecesstr), 20): hash = piecesstr[i : i + 20] print >> sys.stderr, "test: piece", i / 20, "hash", repr(hash) piece_hashes.append(hash) print >> sys.stderr, "test: Putting", len( piece_hashes ), "into MerkleTree, size", self.tdef.get_piece_length(), tdef2.get_piece_length() self.tree = MerkleTree(self.tdef.get_piece_length(), self.tdef.get_length(), None, piece_hashes) f = open(self.sourcefn, "rb") piece1 = f.read(2 ** 18) piece2 = f.read(2 ** 18) print >> sys.stderr, "read piece1", len(piece1) print >> sys.stderr, "read piece2", len(piece2) f.close() hash1 = sha(piece1).digest() hash2 = sha(piece2).digest() print >> sys.stderr, "hash piece1", repr(hash1) print >> sys.stderr, "hash piece2", repr(hash2) f2 = open("piece1.bin", "wb") f2.write(piece2) f2.close() def setUpDownloadConfig(self): dscfg = DownloadStartupConfig() print >> sys.stderr, "test: Downloading to", self.config_path dscfg.set_dest_dir(self.config_path) dscfg.set_breakup_seed_bitfield(False) return dscfg def tearDown(self): TestAsServer.tearDown(self) try: os.remove("piece1.bin") except: pass def singtest_good_hashpiece_bepstyle(self): self.subtest_good_hashpiece(False) def singtest_good_hashpiece_oldstyle(self): self.subtest_good_hashpiece(True) def singtest_good_request_bepstyle(self): # Let Session download file first self.subtest_good_hashpiece(False) # Now connect as different peer and download print >> sys.stderr, "\n\ntest: test_good_request: STARTING" self._test_good_request() def singtest_bad_hashpiece_bepstyle(self): self.subtest_bad_hashpiece(False) def singtest_bad_hashpiece_oldstyle(self): self.subtest_bad_hashpiece(True) # # Good hashpiece message # def subtest_good_hashpiece(self, oldstyle): print >> sys.stderr, "test: Testing good hashpiece, oldstyle", oldstyle if oldstyle: self._test_good( self.create_good_hashpiece, oldstyle, self.create_good_tribler_extend_hs, infohash=self.infohash ) else: options = "\x00\x00\x00\x00\x00\x10\x00\x00" self._test_good( self.create_good_hashpiece, oldstyle, self.create_good_nontribler_extend_hs, options=options, infohash=self.infohash, ) def _test_good(self, msg_gen_func, oldstyle, extend_hs_gen_func, options=None, infohash=None): if options is None and infohash is None: s = BTConnection("localhost", self.hisport) elif options is None: s = BTConnection("localhost", self.hisport, user_infohash=infohash) elif infohash is None: s = BTConnection("localhost", self.hisport, user_option_pattern=options) else: s = BTConnection("localhost", self.hisport, user_option_pattern=options, user_infohash=infohash) print >> sys.stderr, "test: test_good: Create EXTEND HS" msg = extend_hs_gen_func() print >> sys.stderr, "test: test_good: Sending EXTEND HS", repr(msg) s.send(msg) print >> sys.stderr, "test: test_good: Waiting for BT HS" s.read_handshake_medium_rare() # Tribler should send an EXTEND message back try: print >> sys.stderr, "test: Waiting for reply" s.s.settimeout(10.0) resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply", getMessageName(resp[0]) self.assert_(resp[0] == EXTEND) self.check_tribler_extend_hs(resp[1:]) # 1. Pretend we're seeder: send BITFIELD and UNCHOKE msg = BITFIELD + self.seederbitfieldstr s.send(msg) msg = UNCHOKE s.send(msg) print >> sys.stderr, "test: Pretend we are seeder" while True: resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply2", getMessageName(resp[0]) self.assert_( resp[0] == REQUEST or resp[0] == INTERESTED or resp[0] == UNCHOKE or resp[0] == HAVE or resp[0] == NOT_INTERESTED ) if resp[0] == REQUEST: chunkid = self.check_request(resp) # 2. Reply to REQUEST with HASHPIECE (oldstyle) or Tr_hashpiece msg = msg_gen_func(oldstyle, chunkid) s.send(msg) elif resp[0] == NOT_INTERESTED: break # s.close() except socket.timeout: print >> sys.stderr, "test: Timeout, bad, peer didn't reply in time" self.assert_(False) destfn = os.path.join(self.config_path, "file2.wmv") sf = open(self.sourcefn, "rb") df = open(destfn, "rb") n = self.tdef.get_piece_length() while True: sdata = sf.read(n) if len(sdata) == 0: break ddata = df.read(n) self.assert_(sdata == ddata) time.sleep(3) s.close() def create_good_nontribler_extend_hs(self): """ Merkle BEP style """ d = {} d["m"] = {"Tr_hashpiece": 250} d["p"] = self.mylistenport d["v"] = "TestSweet 1.2.3.4" bd = bencode(d) return EXTEND + chr(0) + bd def create_good_tribler_extend_hs(self): """ old Tribler style """ d = {} d["m"] = {"Tr_OVERLAYSWARM": 253} d["p"] = self.mylistenport d["v"] = "Tribler 3.5.1" bd = bencode(d) return EXTEND + chr(0) + bd def check_tribler_extend_hs(self, data): self.assert_(data[0] == chr(0)) d = bdecode(data[1:]) self.assert_(isinstance(d, DictType)) self.assert_("m" in d.keys()) m = d["m"] self.assert_(isinstance(m, DictType)) self.assert_("Tr_hashpiece" in m.keys()) val = m["Tr_hashpiece"] self.assert_(isinstance(val, IntType)) self.assert_(val == 250) def check_request(self, data): index = toint(data[1:5]) begin = toint(data[5:9]) length = toint(data[9:]) return (index, begin, length) def create_good_hashpiece(self, oldstyle, chunkid): index, begin, length = chunkid if begin == 0: ohlist = self.tree.get_hashes_for_piece(index) else: ohlist = [] chunk = self.read_chunk(index, begin, length) bohlist = bencode(ohlist) print >> sys.stderr, "test: create_good_hashpiece:", index, begin, length, "==len", len(chunk) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk if oldstyle: msg = HASHPIECE + payload else: # Offical: use the msg ID he defined in his handshake msg = EXTEND + HASHPIECE + payload return msg def read_chunk(self, index, begin, length): offset = index * self.tdef.get_piece_length() + begin f = open(self.sourcefn, "rb") f.seek(offset) chunk = f.read(length) f.close() return chunk # # Test whether Tribler sends good Tr_hashpiece on our requests # def _test_good_request(self): options = "\x00\x00\x00\x00\x00\x10\x00\x00" myid = Rand.rand_bytes(20) s = BTConnection("localhost", self.hisport, user_option_pattern=options, user_infohash=self.infohash, myid=myid) msg = self.create_good_nontribler_extend_hs() s.send(msg) s.read_handshake_medium_rare() # Tribler should send an EXTEND message back try: print >> sys.stderr, "test: Waiting for reply" s.s.settimeout(10.0) resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply", getMessageName(resp[0]) self.assert_(resp[0] == EXTEND) self.check_tribler_extend_hs(resp[1:]) # 1. Pretend we're leecher: send INTERESTED msg = INTERESTED s.send(msg) print >> sys.stderr, "test: Pretend we are leecher" while True: resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply2", getMessageName(resp[0]) if resp[0] == EXTEND: print >> sys.stderr, "test: Got EXTEND type", getMessageName(resp[1]) self.assert_(resp[0] == UNCHOKE or resp[0] == BITFIELD or resp[0] == EXTEND or resp[0] == HAVE) if resp[0] == UNCHOKE: # 2. Reply with REQUESTs for index in range(0, self.numpieces): plen = self.get_piece_length(index) for begin in range(0, plen, 2 ** 14): length = self.get_chunk_length(index, begin) print >> sys.stderr, "RETRIEVE", index, begin, length chunkid = (index, begin, length) msg = self.create_request(chunkid) s.send(msg) # s.send(NOT_INTERESTED) elif resp[0] == EXTEND and resp[1] == HASHPIECE: done = self.check_hashpiece(resp) if done: break elif resp[0] == BITFIELD: self.check_bitfield(resp) # s.close() except socket.timeout: print >> sys.stderr, "test: Timeout, bad, peer didn't reply in time" self.assert_(False) time.sleep(3) s.close() def get_piece_length(self, index): if index == (self.numpieces - 1): plen = self.tdef.get_length() % self.tdef.get_piece_length() else: plen = self.tdef.get_piece_length() return plen def get_chunk_length(self, index, begin): plen = self.get_piece_length(index) length = 2 ** 14 if index == (self.numpieces - 1): if (begin + 2 ** 14) > plen: length = plen - begin return length def create_request(self, chunkid): index, begin, length = chunkid return REQUEST + tobinary(index) + tobinary(begin) + tobinary(length) def check_hashpiece(self, resp): """ Merkle BEP style """ print >> sys.stderr, "test: good_request: check_hashpiece" self.assert_(resp[0] == EXTEND) self.assert_(resp[1] == HASHPIECE) index = toint(resp[2 : 2 + 4]) begin = toint(resp[6 : 6 + 4]) ohlen = toint(resp[10 : 10 + 4]) print >> sys.stderr, "test: good_request: check_hashpiece", index, begin, ohlen bohlist = resp[14 : 14 + ohlen] hisohlist = bdecode(bohlist) hischunk = resp[14 + ohlen :] if begin == 0: self.assert_(isinstance(hisohlist, ListType)) for oh in hisohlist: self.assert_(isinstance(oh, ListType)) self.assert_(len(oh) == 2) self.assert_(isinstance(oh[0], IntType)) self.assert_(isinstance(oh[1], StringType)) hisohlist.sort() print >> sys.stderr, "test: good_request: check_hashpiece", repr(hisohlist) myohlist = self.tree.get_hashes_for_piece(index) myohlist.sort() self.assert_(len(hisohlist) == len(myohlist)) for i in range(0, len(hisohlist)): hisoh = hisohlist[i] myoh = myohlist[i] self.assert_(hisoh == myoh) else: self.assert_(len(hisohlist) == 0) mylength = self.get_chunk_length(index, begin) mychunk = self.read_chunk(index, begin, mylength) self.assert_(hischunk == mychunk) return index == self.numpieces - 1 and mylength != 2 ** 14 def check_bitfield(self, data): self.assert_(data[0] == BITFIELD) bitmap = data[1:] self.assert_(len(bitmap) == 1) # Must have set_breakup_seed_bitfield() set to False self.assert_(bitmap == "\xc0") # # Bad EXTEND handshake message # def subtest_bad_hashpiece(self, oldstyle): if not oldstyle: # Test becomes equivalent to BT keep alive message (len 0, payload '') self._test_bad(self.create_empty, oldstyle) self._test_bad(self.create_ext_id_not_byte, oldstyle) self._test_bad(self.create_not_hashpiece, oldstyle) self._test_bad(self.create_not_index, oldstyle) self._test_bad(self.create_not_begin, oldstyle) self._test_bad(self.create_not_len_bohlist, oldstyle) self._test_bad(self.create_ohlist_not_bdecodable, oldstyle) self._test_bad(self.create_ohlist_wrong_no_hashes, oldstyle) self._test_bad(self.create_ohlist_wrong_no_root_hash, oldstyle) self._test_bad(self.create_ohlist_wrong_bad_offset, oldstyle) self._test_bad(self.create_ohlist_wrong_bad_hash, oldstyle) # TODO: need working peer kicking for that # self._test_bad(self.create_bad_chunk,oldstyle) # # Main test code for bad EXTEND handshake messages # def _test_bad(self, msg_gen_func, oldstyle): print >> sys.stderr, "test: test_BAD: Create EXTEND HS", repr(msg_gen_func), oldstyle if oldstyle: options = None exthsmsg = self.create_good_tribler_extend_hs() else: options = "\x00\x00\x00\x00\x00\x10\x00\x00" exthsmsg = self.create_good_nontribler_extend_hs() s = BTConnection("localhost", self.hisport, user_option_pattern=options, user_infohash=self.infohash) s.send(exthsmsg) s.read_handshake_medium_rare() # Tribler should send an EXTEND message back try: print >> sys.stderr, "test: Waiting for reply" s.s.settimeout(10.0) resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply", getMessageName(resp[0]) self.assert_(resp[0] == EXTEND) self.check_tribler_extend_hs(resp[1:]) # 1. Pretend we're seeder: send BITFIELD and UNCHOKE msg = BITFIELD + self.seederbitfieldstr s.send(msg) msg = UNCHOKE s.send(msg) print >> sys.stderr, "test: Pretend we are seeder" while True: resp = s.recv() self.assert_(len(resp) > 0) print >> sys.stderr, "test: Got reply 2", getMessageName(resp[0]) self.assert_( resp[0] == REQUEST or resp[0] == INTERESTED or resp[0] == UNCHOKE or resp[0] == HAVE or resp[0] == NOT_INTERESTED ) if resp[0] == REQUEST: chunkid = self.check_request(resp) # 2. Reply to REQUEST with *bad* HASHPIECE msg = msg_gen_func(chunkid) if oldstyle: if len(msg) == 1: msg = "" else: msg = msg[1:] # Strip EXTEND byte s.send(msg) break # s.close() except socket.timeout: print >> sys.stderr, "test: Timeout, bad, peer didn't reply in time" self.assert_(False) time.sleep(3) # Should have closed the connection try: s.send(UNCHOKE) self.assert_(False) except: print_exc() s.close() # # Bad message creators (all create Merkle BEP style, I strip first byte # later for oldstyle # def create_empty(self, chunkid): return EXTEND def create_ext_id_not_byte(self, chunkid): return EXTEND + "Hallo kijkbuiskinderen" def create_not_hashpiece(self, chunkid): index, begin, length = chunkid ohlist = [] bohlist = bencode(ohlist) chunk = self.read_chunk(index, begin, length) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + chr(231) + payload def create_not_index(self, chunkid): payload = "bla" return EXTEND + HASHPIECE + payload def create_not_begin(self, chunkid): index, begin, length = chunkid payload = tobinary(index) + "bla" return EXTEND + HASHPIECE + payload def create_not_len_bohlist(self, chunkid): index, begin, length = chunkid payload = tobinary(index) + tobinary(begin) + "bla" return EXTEND + HASHPIECE + payload def create_ohlist_not_bdecodable(self, chunkid): index, begin, length = chunkid bohlist = "bla" chunk = "*" * (2 ** 14) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload def create_ohlist_wrong_no_hashes(self, chunkid): index, begin, length = chunkid ohlist = [(0, "#" * 20), (1, "$" * 20)] # should contain 3 for file2.wmv: own, sibling and root bohlist = bencode(ohlist) chunk = "*" * (2 ** 14) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload def create_ohlist_wrong_no_root_hash(self, chunkid): index, begin, length = chunkid ohlist = self.tree.get_hashes_for_piece(index) newohlist = [] # Remove root hash for oh in ohlist: if oh[0] != 0: newohlist.append(oh) ohlist = newohlist bohlist = bencode(ohlist) chunk = self.read_chunk(index, begin, length) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload def create_ohlist_wrong_bad_offset(self, chunkid): index, begin, length = chunkid ohlist = self.tree.get_hashes_for_piece(index) ohlist[1][0] = 481 bohlist = bencode(ohlist) chunk = self.read_chunk(index, begin, length) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload def create_ohlist_wrong_bad_hash(self, chunkid): index, begin, length = chunkid ohlist = self.tree.get_hashes_for_piece(index) ohlist[1][1] = "$" * 20 bohlist = bencode(ohlist) chunk = self.read_chunk(index, begin, length) payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload def create_bad_chunk(self, chunkid): index, begin, length = chunkid ohlist = self.tree.get_hashes_for_piece(index) bohlist = bencode(ohlist) chunk = "*" * length payload = tobinary(index) + tobinary(begin) + tobinary(len(bohlist)) + bohlist + chunk return EXTEND + HASHPIECE + payload
class TestMagnetMiniBitTorrent(TestAsServer, MagnetHelpers): """ A MiniBitTorrent instance is used to connect to BitTorrent clients and download the info part from the metadata. """ def setUp(self): """ override TestAsServer """ # listener for incoming connections from MiniBitTorrent self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.server.bind(("localhost", LISTEN_PORT)) self.server.listen(1) # the metadata that we want to transfer self.tdef = TorrentDef() self.tdef.add_content(os.path.join(os.getcwd(), "API", "file.wmv")) self.tdef.set_tracker("http://fake.net/announce") # we use a small piece length to obtain multiple pieces self.tdef.set_piece_length(1) self.tdef.finalize() MagnetHelpers.__init__(self, self.tdef) # startup the client TestAsServer.setUp(self) print >>sys.stderr,"test: Giving MyLaunchMany time to startup" time.sleep(5) print >>sys.stderr,"test: MyLaunchMany should have started up" def create_good_url(self, infohash=None, title=None, tracker=None): url = "magnet:?xt=urn:btih:" if infohash: assert isinstance(infohash, str) url += hexlify(infohash) else: url += hexlify(self.tdef.get_infohash()) if title: assert isinstance(title, str) url += "&dn=" + title if tracker: assert isinstance(tracker, str) url += "&tr=" + tracker return url def test_good_transfer(self): def torrentdef_retrieved(tdef): tags["retrieved"] = True tags["metainfo"] = tdef.get_metainfo() tags = {"retrieved":False} assert TorrentDef.retrieve_from_magnet(self.create_good_url(), torrentdef_retrieved) # supply fake addresses (regular dht obviously wont work here) for magnetlink in MagnetHandler.get_instance().get_magnets(): magnetlink._swarm.add_potential_peers([("localhost", LISTEN_PORT)]) # accept incoming connection self.server.settimeout(10.0) sock, address = self.server.accept() assert sock, "No incoming connection" # handshakes conn = BTConnection(address[0], address[1], opensock=sock, user_infohash=self.tdef.get_infohash()) conn.send(self.create_good_extend_handshake()) conn.read_handshake_medium_rare() metadata_id = self.read_extend_handshake(conn) # serve pieces for counter in xrange(len(self.metadata_list)): piece = self.read_extend_metadata_request(conn) assert 0 <= piece < len(self.metadata_list) conn.send(self.create_good_extend_metadata_reply(metadata_id, piece)) # no more metadata request may be send and the connection must # be closed self.read_extend_metadata_close(conn) time.sleep(5) assert tags["retrieved"] assert tags["metainfo"]["info"] == self.tdef.get_metainfo()["info"]
def lineReceived(self, line): anon_tunnel = self.anon_tunnel if line == 'threads': for thread in threading.enumerate(): logger.debug("%s \t %d", thread.name, thread.ident) elif line == 'c': logger.debug( "========\nCircuits\n========\nid\taddress\t\t\t\t\tgoal\thops\tIN (MB)\tOUT (MB)\tinfohash\ttype" ) for circuit_id, circuit in anon_tunnel.community.circuits.items(): info_hash = circuit.info_hash.encode( 'hex')[:10] if circuit.info_hash else '?' logger.debug( "%d\t%s:%d\t%d\t%d\t\t%.2f\t\t%.2f\t\t%s\t%s" % circuit_id, circuit.first_hop[0], circuit.first_hop[1], circuit.goal_hops, len(circuit.hops), circuit.bytes_down / 1024.0 / 1024.0, circuit.bytes_up / 1024.0 / 1024.0, info_hash, circuit.ctype) elif line.startswith('s'): cur_path = os.getcwd() line_split = line.split(' ') filename = 'test_file' if len(line_split) == 1 else line_split[1] if not os.path.exists(filename): logger.info("Creating torrent..") with open(filename, 'wb') as fp: fp.write(os.urandom(50 * 1024 * 1024)) tdef = TorrentDef() tdef.add_content(os.path.join(cur_path, filename)) tdef.set_tracker("udp://localhost/announce") tdef.set_private() tdef.finalize() tdef.save(os.path.join(cur_path, filename + '.torrent')) else: logger.info("Loading existing torrent..") tdef = TorrentDef.load(filename + '.torrent') logger.info("loading torrent done, infohash of torrent: %s" % (tdef.get_infohash().encode('hex')[:10])) defaultDLConfig = DefaultDownloadStartupConfig.getInstance() dscfg = defaultDLConfig.copy() dscfg.set_hops(1) dscfg.set_dest_dir(cur_path) reactor.callFromThread( anon_tunnel.session.start_download_from_tdef, tdef, dscfg) elif line.startswith('i'): # Introduce dispersy port from other main peer to this peer line_split = line.split(' ') to_introduce_ip = line_split[1] to_introduce_port = int(line_split[2]) self.anon_tunnel.community.add_discovered_candidate( Candidate((to_introduce_ip, to_introduce_port), tunnel=False)) elif line.startswith('d'): line_split = line.split(' ') filename = 'test_file' if len(line_split) == 1 else line_split[1] logger.info("Loading torrent..") tdef = TorrentDef.load(filename + '.torrent') logger.info("Loading torrent done") defaultDLConfig = DefaultDownloadStartupConfig.getInstance() dscfg = defaultDLConfig.copy() dscfg.set_hops(1) dscfg.set_dest_dir( os.path.join( os.getcwd(), 'downloader%s' % anon_tunnel.session.get_dispersy_port())) def start_download(): def cb(ds): logger.info( 'Download infohash=%s, down=%s, progress=%s, status=%s, seedpeers=%s, candidates=%d' % (tdef.get_infohash().encode('hex')[:10], ds.get_current_speed('down'), ds.get_progress(), dlstatus_strings[ds.get_status()], sum(ds.get_num_seeds_peers()), sum(1 for _ in anon_tunnel.community. dispersy_yield_verified_candidates()))) return 1.0, False download = anon_tunnel.session.start_download_from_tdef( tdef, dscfg) download.set_state_callback(cb) reactor.callFromThread(start_download) elif line == 'q': anon_tunnel.should_run = False elif line == 'r': logger.debug("circuit\t\t\tdirection\tcircuit\t\t\tTraffic (MB)") from_to = anon_tunnel.community.relay_from_to for key in from_to.keys(): relay = from_to[key] logger.info("%s-->\t%s\t\t%.2f" % ( (key[0], key[1]), (relay.sock_addr, relay.circuit_id), relay.bytes[1] / 1024.0 / 1024.0, ))
def test_get_infohash(self): t = TorrentDef() t.get_infohash()
class TestVideoHTTPServer(TestAsServer): """ Class for testing HTTP-based video server. Mainly HTTP range queries. """ def setUp(self): """ unittest test setup code """ TestAsServer.setUp(self) self.port = self.session.get_videoplayer_port() self.sourcefn = os.path.join(TESTS_DATA_DIR, "video.avi") self.sourcesize = os.path.getsize(self.sourcefn) # wait 5s to allow server to start time.sleep(5) def setUpPreSession(self): TestAsServer.setUpPreSession(self) self.config.set_libtorrent(True) self.config.set_videoplayer(True) def tearDown(self): """ unittest test tear down code """ TestAsServer.tearDown(self) time.sleep(2) # # Tests # def test_specific_range(self): self.range_check(115, 214, self.sourcesize) def test_last_100(self): self.range_check(self.sourcesize - 100, None, self.sourcesize) def test_first_100(self): self.range_check(None, 100, self.sourcesize) def test_combined(self): self.range_check(115, 214, self.sourcesize, setset=True) # # Internal # def register_file_stream(self): self.tdef = TorrentDef() self.tdef.add_content(self.sourcefn) self.tdef.set_tracker("http://127.0.0.1:12/announce") self.tdef.finalize() dscfg = DownloadStartupConfig() dscfg.set_dest_dir(os.path.dirname(self.sourcefn)) download = self.session.start_download(self.tdef, dscfg) while not download.handle: time.sleep(1) def get_std_header(self): msg = "GET /%s/0 HTTP/1.1\r\n" % binascii.hexlify( self.tdef.get_infohash()) msg += "Host: 127.0.0.1:" + str(self.port) + "\r\n" return msg def create_range_str(self, firstbyte, lastbyte): head = "" if firstbyte is not None: head += str(firstbyte) head += "-" if lastbyte is not None: head += str(lastbyte) return head def range_check(self, firstbyte, lastbyte, sourcesize, setset=False): self._logger.debug("range_test: %s %s %s setset %s", firstbyte, lastbyte, sourcesize, setset) self.register_file_stream() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect(('127.0.0.1', self.port)) head = self.get_std_header() head += "Range: bytes=" head += self.create_range_str(firstbyte, lastbyte) if setset: # Make into set of byte ranges, VideoHTTPServer should refuse. head += ",0-99" head += "\r\n" head += "Connection: close\r\n" head += "\r\n" if firstbyte is not None and lastbyte is None: # 100- expfirstbyte = firstbyte explastbyte = self.sourcesize - 1 elif firstbyte is None and lastbyte is not None: # -100 expfirstbyte = self.sourcesize - lastbyte explastbyte = self.sourcesize - 1 else: expfirstbyte = firstbyte explastbyte = lastbyte # the amount of bytes actually requested. (Content-length) expsize = explastbyte - expfirstbyte + 1 self._logger.debug("Expecting first %s last %s size %s ", expfirstbyte, explastbyte, sourcesize) s.send(head) # Parse header s.settimeout(10.0) while True: line = self.readline(s) if DEBUG: self._logger.debug("Got line: %s", repr(line)) if len(line) == 0: if DEBUG: self._logger.debug("server closed conn") self.assert_(False) return if line.startswith("HTTP"): if not setset: # Python returns "HTTP/1.0 206 Partial Content\r\n" HTTP 1.0??? self.assert_(line.startswith("HTTP/1.")) self.assert_(line.find("206") != -1) # Partial content else: self.assert_(line.startswith("HTTP/1.")) self.assert_(line.find("416") != -1) # Requested Range Not Satisfiable return elif line.startswith("Content-Range:"): expline = "Content-Range: bytes " + self.create_range_str( expfirstbyte, explastbyte) + "/" + str(sourcesize) + "\r\n" self.assertEqual(expline, line) elif line.startswith("Content-Type:"): self.assertEqual(line, "Content-Type: video/x-msvideo\r\n") elif line.startswith("Content-Length:"): self.assertEqual(line, "Content-Length: " + str(expsize) + "\r\n") elif line.endswith("\r\n") and len(line) == 2: # End of header break data = s.recv(expsize) if len(data) == 0: if DEBUG: self._logger.debug("server closed conn2") self.assert_(False) return else: f = open(self.sourcefn, "rb") if firstbyte is not None: f.seek(firstbyte) else: f.seek(lastbyte, os.SEEK_END) expdata = f.read(expsize) f.close() self.assert_(data, expdata) try: # Read body, reading more should EOF (we disabled persist conn) data = s.recv(10240) self.assert_(len(data) == 0) except socket.timeout: if DEBUG: self._logger.debug( "Timeout, video server didn't respond with requested bytes, possibly bug in Python impl of HTTP" ) print_exc() def readline(self, s): line = '' while True: data = s.recv(1) if len(data) == 0: return line else: line = line + data if data == '\n' and len(line) >= 2 and line[-2:] == '\r\n': return line
def lineReceived(self, line): anon_tunnel = self.anon_tunnel profile = self.profile if line == 'threads': for thread in threading.enumerate(): print "%s \t %d" % (thread.name, thread.ident) elif line == 'p': if profile: for func_stats in yappi.get_func_stats().sort("subtime")[:50]: print "YAPPI: %10dx %10.3fs" % ( func_stats.ncall, func_stats.tsub), func_stats.name else: logger.error("Profiling disabled!") elif line == 'P': if profile: filename = 'callgrindc_%d.yappi' % anon_tunnel.dispersy.lan_address[ 1] yappi.get_func_stats().save(filename, type='callgrind') else: logger.error("Profiling disabled!") elif line == 't': if profile: yappi.get_thread_stats().sort("totaltime").print_all() else: logger.error("Profiling disabled!") elif line == 'c': print "========\nCircuits\n========\nid\taddress\t\t\t\t\tgoal\thops\tIN (MB)\tOUT (MB)\tinfohash\ttype" for circuit_id, circuit in anon_tunnel.community.circuits.items(): info_hash = circuit.info_hash.encode( 'hex')[:10] if circuit.info_hash else '?' print "%d\t%s:%d\t%d\t%d\t\t%.2f\t\t%.2f\t\t%s\t%s" % ( circuit_id, circuit.first_hop[0], circuit.first_hop[1], circuit.goal_hops, len(circuit.hops), circuit.bytes_down / 1024.0 / 1024.0, circuit.bytes_up / 1024.0 / 1024.0, info_hash, circuit.ctype) elif line.startswith('s'): cur_path = os.getcwd() line_split = line.split(' ') filename = 'test_file' if len(line_split) == 1 else line_split[1] if not os.path.exists(filename): logger.info("Creating torrent..") with open(filename, 'wb') as fp: fp.write(os.urandom(50 * 1024 * 1024)) tdef = TorrentDef() tdef.add_content(os.path.join(cur_path, filename)) tdef.set_tracker("udp://fake.net/announce") tdef.set_private() tdef.finalize() tdef.save(os.path.join(cur_path, filename + '.torrent')) else: logger.info("Loading existing torrent..") tdef = TorrentDef.load(filename + '.torrent') logger.info("loading torrent done, infohash of torrent: %s" % (tdef.get_infohash().encode('hex')[:10])) defaultDLConfig = DefaultDownloadStartupConfig.getInstance() dscfg = defaultDLConfig.copy() dscfg.set_hops(1) dscfg.set_dest_dir(cur_path) anon_tunnel.session.lm.threadpool.call( 0, anon_tunnel.session.start_download, tdef, dscfg) elif line.startswith('i'): # Introduce dispersy port from other main peer to this peer line_split = line.split(' ') to_introduce_ip = line_split[1] to_introduce_port = int(line_split[2]) self.anon_tunnel.community.add_discovered_candidate( Candidate((to_introduce_ip, to_introduce_port), tunnel=False)) elif line.startswith('d'): line_split = line.split(' ') filename = 'test_file' if len(line_split) == 1 else line_split[1] logger.info("Loading torrent..") tdef = TorrentDef.load(filename + '.torrent') logger.info("Loading torrent done") defaultDLConfig = DefaultDownloadStartupConfig.getInstance() dscfg = defaultDLConfig.copy() dscfg.set_hops(1) dscfg.set_dest_dir( os.path.join( os.getcwd(), 'downloader%s' % anon_tunnel.session.get_dispersy_port())) def start_download(): def cb(ds): logger.info( 'Download infohash=%s, down=%s, progress=%s, status=%s, seedpeers=%s, candidates=%d' % (tdef.get_infohash().encode('hex')[:10], ds.get_current_speed('down'), ds.get_progress(), dlstatus_strings[ds.get_status()], sum(ds.get_num_seeds_peers()), sum(1 for _ in anon_tunnel.community. dispersy_yield_verified_candidates()))) return 1.0, False download = anon_tunnel.session.start_download(tdef, dscfg) download.set_state_callback(cb, delay=1) anon_tunnel.session.lm.threadpool.call(0, start_download) elif line == 'q': anon_tunnel.stop() return elif line == 'r': print "circuit\t\t\tdirection\tcircuit\t\t\tTraffic (MB)" from_to = anon_tunnel.community.relay_from_to for key in from_to.keys(): relay = from_to[key] logger.info("%s-->\t%s\t\t%.2f" % ( (key[0], key[1]), (relay.sock_addr, relay.circuit_id), relay.bytes[1] / 1024.0 / 1024.0, ))
class TestSeeding(TestAsServer): """ Testing seeding via new tribler API: """ def setUp(self): """ override TestAsServer """ super(TestSeeding, self).setUp() self.session2 = None self.seeding_event = threading.Event() self.downloading_event = threading.Event() def setUpPreSession(self): """ override TestAsServer """ super(TestSeeding, self).setUpPreSession() self.config.set_libtorrent(True) self.config2 = self.config.copy() # not really necess self.config2.set_state_dir(self.getStateDir(2)) self.dscfg2 = DownloadStartupConfig() self.dscfg2.set_dest_dir(self.getDestDir(2)) def setUpPostSession(self): pass def tearDown(self): if self.session2: self._shutdown_session(self.session2) time.sleep(10) super(TestSeeding, self).tearDown() def setup_seeder(self, filename='video.avi'): self.tdef = TorrentDef() self.sourcefn = os.path.join(TESTS_API_DIR, filename) self.tdef.add_content(self.sourcefn) self.tdef.set_tracker("http://fake.net/announce") self.tdef.finalize() self.torrentfn = os.path.join(self.session.get_state_dir(), "gen.torrent") self.tdef.save(self.torrentfn) self._logger.debug("name is %s", self.tdef.metainfo['info']['name']) self.dscfg = DownloadStartupConfig() self.dscfg.set_dest_dir( TESTS_API_DIR) # basedir of the file we are seeding d = self.session.start_download(self.tdef, self.dscfg) d.set_state_callback(self.seeder_state_callback) self._logger.debug( "starting to wait for download to reach seeding state") assert self.seeding_event.wait(60) def seeder_state_callback(self, ds): d = ds.get_download() self._logger.debug("seeder status: %s %s %s", repr(d.get_def().get_name()), dlstatus_strings[ds.get_status()], ds.get_progress()) if ds.get_status() == DLSTATUS_SEEDING: self.seeding_event.set() return 1.0, False def test_normal_torrent(self): self.setup_seeder() self.subtest_is_seeding() self.subtest_download() def subtest_is_seeding(self): infohash = self.tdef.get_infohash() s = BTConnection('localhost', self.session.get_listen_port(), user_infohash=infohash) s.read_handshake_medium_rare() s.send(CHOKE) try: s.s.settimeout(10.0) resp = s.recv() self.assert_(len(resp) > 0) self.assert_(resp[0] == EXTEND) except socket.timeout: print >> sys.stderr, "test: Timeout, peer didn't reply" self.assert_(False) s.close() def subtest_download(self): """ Now download the file via another Session """ self.session2 = Session(self.config2, ignore_singleton=True) upgrader = self.session2.prestart() while not upgrader.is_done: time.sleep(0.1) self.session2.start() time.sleep(1) time.sleep(5) tdef2 = TorrentDef.load(self.torrentfn) d = self.session2.start_download(tdef2, self.dscfg2) d.set_state_callback(self.downloader_state_callback) time.sleep(5) d.add_peer(("127.0.0.1", self.session.get_listen_port())) assert self.downloading_event.wait(60) def downloader_state_callback(self, ds): d = ds.get_download() self._logger.debug("download status: %s %s %s", repr(d.get_def().get_name()), dlstatus_strings[ds.get_status()], ds.get_progress()) if ds.get_status() == DLSTATUS_SEEDING: # File is in destfn = os.path.join(self.getDestDir(2), "video.avi") f = open(destfn, "rb") realdata = f.read() f.close() f = open(self.sourcefn, "rb") expdata = f.read() f.close() self.assert_(realdata == expdata) self.downloading_event.set() return 1.0, True return 1.0, False