def testECreateRichTorrents(self):
     _log.info("Create rich metadata torrents ------------")
     if not TestMetaGenerator.run_test:
         _log.debug("Test avoided.")
         return
     data = None
     self.assertTrue(TestMetaGenerator.data != None, "Test E depends on data gathered in test D, set testD attribute to True")
     if TestMetaGenerator.data != None:
         data = TestMetaGenerator.data
     else:
         return
     rmg = RichMetadataGenerator.getInstance()
     torrents = []
     for k, v in data.items():
         if v.content_base != None:
             # Choose only a small subset of content to download
             if v.content_base == "http://www.rtvslo.si/podcasts/gori_doli_naokoli.xml":
                 mm_file = tempfile.NamedTemporaryFile()
                 try:
                     urllib.urlretrieve(v.id, mm_file.name)
                     torrent_def = TorrentDef()
                     torrent_def.add_content(mm_file.name)
                     torrent_def.set_tracker("http://wanabe.a.tracker.com:8080")
                     torrent_def.set_metadata(k)
                     torrent_def.finalize()
                     torrent_def.save(mm_file.name + ".torrent")
                     torrents.append(mm_file.name + ".torrent")
                 except Exception, e:
                     _log.error("Failed to generate torrent file, reason: " + str(e))
def create_torrent(file, port, try_vod = True):
    '''
    Creates a torrent for the given file. Returns a string representing the
    path to the torrent file.
    If try_vod is True try to add the playtime to the torrent making it "streamable".
    '''
    
    # generate torrent    
    tdef = TorrentDef()
    url = "http://localhost:%s/announce/" % port
    tdef.set_tracker(url)
    #tdef.set_create_merkle_torrent(True)
    #tdef.set_piece_length(self._pieceSize)
    
    if try_vod:
        torrent_name = FileUtils.get_relative_filename(file) + constants.TORRENT_VOD_EXT
        # set playtime
        playtime = videostats_via_ffmpeg(file)['playtime']
        print "playtime", playtime
    else:
        torrent_name = FileUtils.get_relative_filename(file) + constants.TORRENT_DOWNLOAD_EXT
        playtime = None
        print "Create a non-streamable torrent"
            
    tdef.add_content(file, playtime=playtime)
    tdef.finalize()
    
    torrent_dir = os.getcwd()
    torrent = os.path.join(torrent_dir, torrent_name)
    tdef.save(torrent)
    
    print "created torrent file: %s" % torrent
    print "Tracker uses the announce URL: %s" % tdef.get_tracker()

    return tdef
class TestMetadata(TestAsServer, MagnetHelpers):
    """
    Once we are downloading a torrent, our client should respond to
    the ut_metadata extention message.  This allows other clients to
    obtain the info part of the metadata from us.
    """
    def setUp(self):
        """ override TestAsServer """
        TestAsServer.setUp(self)
        print >>sys.stderr,time.asctime(),'-', "test: Giving MyLaunchMany time to startup"
        time.sleep(5)
        print >>sys.stderr,time.asctime(),'-', "test: MyLaunchMany should have started up"
    
        # the metadata that we want to transfer
        self.tdef = TorrentDef()
        self.tdef.add_content(os.path.join(os.getcwd(), "API", "file.wmv"))
        self.tdef.set_tracker(self.session.get_internal_tracker_url())
        # we use a small piece length to obtain multiple pieces
        self.tdef.set_piece_length(1) 
        self.tdef.finalize()
        # self.tdef.save(os.path.join(self.session.get_state_dir(), "gen.torrent"))
        
        MagnetHelpers.__init__(self, self.tdef)

    def setup_seeder(self):
        self.seeder_setup_complete = False
        self.seeder_teardown_complete = False
        self.seeder_teardown = False

        self.dscfg = DownloadStartupConfig()
        self.dscfg.set_dest_dir(os.getcwd())
        self.download = self.session.start_download(self.tdef, self.dscfg)
        self.download.set_state_callback(self.seeder_state_callback)

        counter = 0
        while not self.seeder_setup_complete:
            counter += 1
            time.sleep(1)
            assert counter < 30, "timeout"

        print >> sys.stderr, time.asctime(),'-', "test: setup_seeder() complete"

    def teardown_seeder(self):
        self.seeder_teardown_complete = False
        self.session.remove_download(self.download)

        counter = 0
        while not self.seeder_setup_complete:
            counter += 1
            time.sleep(1)
            assert counter < 30, "timeout"

        print >> sys.stderr, time.asctime(),'-', "test: teardown_seeder() complete"

    def seeder_state_callback(self,ds):
        assert not self.seeder_teardown_complete
        self.seeder_setup_complete = (ds.get_status() == DLSTATUS_DOWNLOADING)
        d = ds.get_download()
        print >> sys.stderr, time.asctime(),'-', "test: seeder:", `d.get_def().get_name()`, dlstatus_strings[ds.get_status()], ds.get_progress()
        if self.seeder_teardown:
            self.seeder_teardown_complete = True
        else:
            return (1.0, False)

    def test_all(self):
        self.setup_seeder()
        try:
            self.subtest_good_flood()
        finally:
            self.teardown_seeder()

        self.setup_seeder()
        try:
            self.subtest_good_request()
            self.subtest_bad_request()
        finally:
            self.teardown_seeder()

    def subtest_good_request(self):
        conn = BTConnection("localhost", self.hisport, user_infohash=self.tdef.get_infohash())
        conn.send(self.create_good_extend_handshake())
        conn.read_handshake_medium_rare()
        metadata_id = self.read_extend_handshake(conn)

        # request metadata block 0, 2, 3, and the last
        conn.send(self.create_good_extend_metadata_request(metadata_id, 0))
        conn.send(self.create_good_extend_metadata_request(metadata_id, 2))
        conn.send(self.create_good_extend_metadata_request(metadata_id, 3))
        conn.send(self.create_good_extend_metadata_request(metadata_id, len(self.metadata_list) - 1))

        self.read_extend_metadata_reply(conn, 0)
        self.read_extend_metadata_reply(conn, 2)
        self.read_extend_metadata_reply(conn, 3)
        self.read_extend_metadata_reply(conn, len(self.metadata_list) - 1)

    def subtest_good_flood(self):
        conn = BTConnection("localhost", self.hisport, user_infohash=self.tdef.get_infohash())
        conn.send(self.create_good_extend_handshake())
        conn.read_handshake_medium_rare()
        metadata_id = self.read_extend_handshake(conn)

        for counter in xrange(len(self.metadata_list) * 2):
            piece = counter % len(self.metadata_list)
            conn.send(self.create_good_extend_metadata_request(metadata_id, piece))

            if counter > len(self.metadata_list):
                self.read_extend_metadata_reject(conn, piece)
            else:
                self.read_extend_metadata_reply(conn, piece)

    def subtest_bad_request(self):
        self.bad_request_and_disconnect({"msg_type":0, "piece":len(self.metadata_list)})
        self.bad_request_and_disconnect({"msg_type":0, "piece":-1})
        self.bad_request_and_disconnect({"msg_type":0, "piece":"1"})
        self.bad_request_and_disconnect({"msg_type":0, "piece":[1,2]})
        self.bad_request_and_disconnect({"msg_type":0, "PIECE":1})
        
    def bad_request_and_disconnect(self, payload):
        conn = BTConnection("localhost", self.hisport, user_infohash=self.tdef.get_infohash())
        conn.send(self.create_good_extend_handshake())
        conn.read_handshake_medium_rare()
        metadata_id = self.read_extend_handshake(conn)

        conn.send(EXTEND + chr(metadata_id) + bencode(payload))
        self.read_extend_metadata_close(conn)
class TestMagnetMiniBitTorrent(TestAsServer, MagnetHelpers):
    """
    A MiniBitTorrent instance is used to connect to BitTorrent clients
    and download the info part from the metadata. 
    """
    def setUp(self):
        """ override TestAsServer """
        # listener for incoming connections from MiniBitTorrent
        self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.server.bind(("localhost", LISTEN_PORT))
        self.server.listen(1)

        # the metadata that we want to transfer
        self.tdef = TorrentDef()
        self.tdef.add_content(os.path.join(os.getcwd(), "API", "file.wmv"))
        self.tdef.set_tracker("http://fake.net/announce")
        # we use a small piece length to obtain multiple pieces
        self.tdef.set_piece_length(1) 
        self.tdef.finalize()

        MagnetHelpers.__init__(self, self.tdef)

        # startup the client
        TestAsServer.setUp(self)
        print >>sys.stderr,time.asctime(),'-', "test: Giving MyLaunchMany time to startup"
        time.sleep(5)
        print >>sys.stderr,time.asctime(),'-', "test: MyLaunchMany should have started up"

    def create_good_url(self, infohash=None, title=None, tracker=None):
        url = "magnet:?xt=urn:btih:"
        if infohash:
            assert isinstance(infohash, str)
            url += hexlify(infohash)
        else:
            url += hexlify(self.tdef.get_infohash())
        if title:
            assert isinstance(title, str)
            url += "&dn=" + title
        if tracker:
            assert isinstance(tracker, str)
            url += "&tr=" + tracker
        return url

    def test_good_transfer(self):
        def torrentdef_retrieved(tdef):
            tags["retrieved"] = True
            tags["metainfo"] = tdef.get_metainfo()

        tags = {"retrieved":False}

        assert TorrentDef.retrieve_from_magnet(self.create_good_url(), torrentdef_retrieved)

        # supply fake addresses (regular dht obviously wont work here)
        for magnetlink in MagnetHandler.get_instance().get_magnets():
            magnetlink._swarm.add_potential_peers([("localhost", LISTEN_PORT)])

        # accept incoming connection
        self.server.settimeout(10.0)
        sock, address = self.server.accept()
        assert sock, "No incoming connection"

        # handshakes
        conn = BTConnection(address[0], address[1], opensock=sock, user_infohash=self.tdef.get_infohash())
        conn.send(self.create_good_extend_handshake())
        conn.read_handshake_medium_rare()
        metadata_id = self.read_extend_handshake(conn)

        # serve pieces
        for counter in xrange(len(self.metadata_list)):
            piece = self.read_extend_metadata_request(conn)
            assert 0 <= piece < len(self.metadata_list)
            conn.send(self.create_good_extend_metadata_reply(metadata_id, piece))

        # no more metadata request may be send and the connection must
        # be closed
        self.read_extend_metadata_close(conn)

        time.sleep(5)
        assert tags["retrieved"]
        assert tags["metainfo"]["info"] == self.tdef.get_metainfo()["info"]
Exemple #5
0
class TestVideoOnDemand(TestAsServer):
    """ 
    Testing Merkle hashpiece messages for both:
    * Merkle BEP style
    * old Tribler <= 4.5.2 that did not use the Extention protocol (BEP 10).
     
    See BitTornado/BT1/Connecter.py
    """

    def setUp(self):
        """ override TestAsServer """
        TestAsServer.setUp(self)
        self.vodstarted = False

    def setUpPreSession(self):
        """ override TestAsServer """
        TestAsServer.setUpPreSession(self)
        self.config.set_overlay(False)
        self.config.set_megacache(False)

    def setUpPostSession(self):
        """ override TestAsServer """
        TestAsServer.setUpPostSession(self)

    def tearDown(self):
        print >> sys.stderr, time.asctime(), "-", "Test: Sleep before tear down"
        time.sleep(10)

        TestAsServer.tearDown(self)

    def create_torrent(self):

        [srchandle, self.sourcefn] = mkstemp()
        self.content = Rand.rand_bytes(self.contentlen)
        os.write(srchandle, self.content)
        os.close(srchandle)

        self.tdef = TorrentDef()
        self.tdef.add_content(self.sourcefn)
        self.tdef.set_piece_length(self.piecelen)
        self.tdef.set_tracker("http://127.0.0.1:12/announce")
        self.tdef.finalize()

        self.torrentfn = os.path.join(self.session.get_state_dir(), "gen.torrent")
        self.tdef.save(self.torrentfn)

        dscfg = DownloadStartupConfig()
        destdir = os.path.dirname(self.sourcefn)
        dscfg.set_dest_dir(destdir)
        dscfg.set_video_event_callback(self.sesscb_vod_event_callback)

        self.session.set_download_states_callback(self.states_callback)
        self.session.start_download(self.tdef, dscfg)

    def states_callback(self, dslist):
        ds = dslist[0]
        d = ds.get_download()
        #    print >>sys.stderr,time.asctime(),'-', `d.get_def().get_name()`,dlstatus_strings[ds.get_status()],ds.get_progress(),"%",ds.get_error(),"up",ds.get_current_speed(UPLOAD),"down",ds.get_current_speed(DOWNLOAD)
        print >> sys.stderr, time.asctime(), "-", "%s %s %5.2f%% %s up %8.2fKB/s down %8.2fKB/s" % (
            d.get_def().get_name(),
            dlstatus_strings[ds.get_status()],
            ds.get_progress() * 100,
            ds.get_error(),
            ds.get_current_speed(UPLOAD),
            ds.get_current_speed(DOWNLOAD),
        )

        return (1.0, False)

    def sesscb_vod_event_callback(self, d, event, params):

        if self.vodstarted:
            return
        self.vodstarted = True

        print >> sys.stderr, time.asctime(), "-", "Test: vod_event_callback", event, params
        if event == VODEVENT_START:
            stream = params["stream"]

            # Read last piece
            lastpieceoff = ((self.contentlen - 1) / self.piecelen) * self.piecelen
            lastpiecesize = self.contentlen - lastpieceoff
            print >> sys.stderr, time.asctime(), "-", "Test: stream: lastpieceoff", lastpieceoff, lastpiecesize
            self.stream_read(stream, lastpieceoff, lastpiecesize, self.piecelen)

            # Read second,3rd,4th byte, only
            secoff = 1
            secsize = 3
            blocksize = 3
            self.stream_read(stream, secoff, secsize, blocksize)

            # Read last byte
            lastoff = self.contentlen - 1
            lastsize = 1
            self.stream_read(stream, lastoff, lastsize, self.piecelen)

            print >> sys.stderr, time.asctime(), "-", "Test: stream: Passed?"

    def stream_read(self, stream, off, size, blocksize):
        stream.seek(off)
        data = stream.read(blocksize)
        print >> sys.stderr, time.asctime(), "-", "Test: stream: Got data", len(data)
        self.assertEquals(len(data), size)
        self.assertEquals(data, self.content[off : off + size])

    def singtest_99(self):
        self.contentlen = 99
        self.piecelen = 10
        self.create_torrent()

        print >> sys.stderr, time.asctime(), "-", "Test: Letting network thread create Download, sleeping"
        time.sleep(5)

        dlist = self.session.get_downloads()
        d = dlist[0]
        vs = d.sd.videostatus

        goodrange = ((0, 0), (9, 8))
        self.assertEqual(vs.movie_range, goodrange)
        self.assertEqual(vs.first_piecelen, 10)
        self.assertEqual(vs.last_piecelen, 9)
        self.assertEqual(vs.first_piece, 0)
        self.assertEqual(vs.last_piece, 9)
        self.assertEqual(vs.movie_numpieces, 10)

        print >> sys.stderr, time.asctime(), "-", "Test: status: Passed? ****************************************************************"

    def singtest_100(self):
        self.contentlen = 100
        self.piecelen = 10
        self.create_torrent()

        print >> sys.stderr, time.asctime(), "-", "Test: Letting network thread create Download, sleeping"
        time.sleep(5)

        dlist = self.session.get_downloads()
        d = dlist[0]
        vs = d.sd.videostatus

        goodrange = ((0, 0), (9, 9))
        self.assertEqual(vs.movie_range, goodrange)
        self.assertEqual(vs.first_piecelen, 10)
        self.assertEqual(vs.last_piecelen, 10)
        self.assertEqual(vs.first_piece, 0)
        self.assertEqual(vs.last_piece, 9)
        self.assertEqual(vs.movie_numpieces, 10)

        print >> sys.stderr, time.asctime(), "-", "Test: status: Passed? ****************************************************************"

    def singtest_101(self):
        self.contentlen = 101
        self.piecelen = 10
        self.create_torrent()

        print >> sys.stderr, time.asctime(), "-", "Test: Letting network thread create Download, sleeping"
        time.sleep(5)

        dlist = self.session.get_downloads()
        d = dlist[0]
        vs = d.sd.videostatus

        goodrange = ((0, 0), (10, 0))
        self.assertEqual(vs.movie_range, goodrange)
        self.assertEqual(vs.first_piecelen, 10)
        self.assertEqual(vs.last_piecelen, 1)
        self.assertEqual(vs.first_piece, 0)
        self.assertEqual(vs.last_piece, 10)
        self.assertEqual(vs.movie_numpieces, 11)

        print >> sys.stderr, time.asctime(), "-", "Test: status: Passed? ****************************************************************"
class TestQueryReplyActive(TestAsServer):

    """  
    Testing QUERY_REPLY message of Query extension V1 

    This test checks how the Tribler code responds to good and bad 
    QUERY_REPLY messages. I.e. the Tribler client initiates
    the dialback by connecting to us and sending a QUERY and we
    reply with good and bad messages.

    This test allows authoritative answers from superpeers.

    WARNING: Each of the test_ methods should be tested by running the TestCase 
    in a separate Python interpreter to prevent problems with our singleton 
    classes, e.g. SuperPeerDB, etc.
    """

    def setUpPreSession(self):
        """ override TestAsServer """
        print >> sys.stderr,"test: Pre Tribler Init"
        TestAsServer.setUpPreSession(self)
        print >> sys.stderr,"test: Pre Tribler Init: config_path",self.config_path
        # Enable remote querying
        self.config.set_remote_query(True)

    def setUpPostSession(self):
        """ override TestAsServer """
        TestAsServer.setUpPostSession(self)
        self.hispermid = str(self.his_keypair.pub().get_der())
        self.my_permid = str(self.my_keypair.pub().get_der())

        self.content_name = 'Hallo S22E44'
        self.tdef = TorrentDef()
        self.tdef.set_tracker('http://localhost:0/announce')
        self.tdef.set_piece_length(2 ** 15)
        self.tdef.create_live(self.content_name,2 ** 16)
        self.tdef.finalize()


    def pretest_simple(self):
        self.pretest_q('SIMPLE hallo')

    def pretest_simpleplustorrents(self):
        self.pretest_q('SIMPLE+METADATA hallo')

    def pretest_q(self,query):
        # 1. First connect to Tribler
        self.openconn = OLConnection(self.my_keypair,'localhost',self.hisport)
        sleep(3)
        
        # 2. Make Tribler send query
        self.query = query
        self.session.query_connected_peers(query,self.query_usercallback,max_peers_to_query=10)

    def query_usercallback(self,permid,query,hits):
        
        print >>sys.stderr,"test: query_usercallback:",`permid`,`query`,`hits`
        
        self.assert_(query == self.query)
        self.assert_(permid == self.my_permid)
        self.check_good_qreply(hits)
        
        # TODO: if SIMPLE+METADATA: check torrent now in db.
        

    #
    # Good SIMPLE QUERY, builds on TestQueryReply code
    #    
    def singtest_good_simple_reply(self):
        self.pretest_simple()
        self._test_qreply(self.create_good_simple_reply,True)

    #
    # Good SIMPLE+METADATA QUERY, builds on TestQueryReply code
    #    
    def singtest_good_simpleplustorrents_reply(self):
        self.pretest_simpleplustorrents()
        self._test_qreply(self.create_good_simpleplustorrents_reply,True)


    #
    # Bad QUERY, builds on TestQueryReply code
    #    
    def singtest_bad_not_bdecodable(self):
        self.pretest_simple()
        self._test_qreply(self.create_not_bdecodable,False)

    #
    # Bad SIMPLE+METADATA QUERY, builds on TestQueryReply code
    #    
    def singtest_bad_not_bdecodable_torrentfile(self):
        self.pretest_simpleplustorrents()
        self._test_qreply(self.create_not_bdecodable_torrentfile,False)


    ### TODO: send different valid answers so consensus not reached

    #
    # Main test code
    #
    def _test_qreply(self,gen_qreply,good):
        print >> sys.stderr,"test: waiting for reply"
        s = self.openconn

        msg = s.recv()
        self.assert_(len(msg) > 0)
        print >> sys.stderr,"test: Received overlay message",getMessageName(msg[0])
        self.assert_(msg[0] == QUERY)
        id = self.check_rquery(msg[1:])
        
        resp = gen_qreply(id)
        print >> sys.stderr,"test: sending QUERY_REPLY"
        s.send(resp)
        if good:
            time.sleep(10)
            # the other side should not have closed the connection, as
            # this is all valid, so this should not throw an exception:
            s.send('bla')
            s.close()
        else:
            # the other side should not like this and close the connection
            self.assert_(len(s.recv())==0)
            s.close()


    def create_good_simple_reply_dict(self,id):
        r = {}
        r['content_name'] = self.content_name
        r['length'] = LENGTH
        r['leecher'] = LEECHERS
        r['seeder'] = SEEDERS
        r['category'] = CATEGORY
        # OLPROTO_PROTO_ELEVENTH
        # set later r['torrent_size'] = 42
        r['channel_permid'] = '$' * 83
        r['channel_name'] = 'Nitin Channel' 
        
        d2 = {}
        d2[self.tdef.get_infohash()] = r
        
        d = {}
        d['id'] = id
        d['a'] = d2
        return d
        
    def create_good_simple_reply(self,id):
        d = self.create_good_simple_reply_dict(id)
        bmetainfo = bencode(self.tdef.get_metainfo())
        d['a'][self.tdef.get_infohash()]['torrent_size'] = len(bmetainfo) 
        b = bencode(d)
        return QUERY_REPLY+b

    def create_good_simpleplustorrents_reply(self,id):
        d = self.create_good_simple_reply_dict(id)
        bmetainfo = bencode(self.tdef.get_metainfo())
        d['a'][self.tdef.get_infohash()]['torrent_size'] = len(bmetainfo)
        d['a'][self.tdef.get_infohash()]['metatype'] = 'application/x-tribler-stream' 
        d['a'][self.tdef.get_infohash()]['metadata'] = bmetainfo 
        b = bencode(d)
        return QUERY_REPLY+b

    

    def check_good_qreply(self,hits):
        self.assert_(len(hits) == 1)
        self.assert_(hits.keys()[0] == self.tdef.get_infohash())
        hit = hits[self.tdef.get_infohash()]
        self.assert_(hit['content_name'] == self.content_name)
        self.assert_(hit['length'] == LENGTH)
        self.assert_(hit['leecher'] == LEECHERS)
        self.assert_(hit['seeder'] == SEEDERS)
        self.assert_(hit['category'] ==  CATEGORY)
    
        # OLPROTO_VERSION_ELEVENTH
        bmetainfo = bencode(self.tdef.get_metainfo())
        self.assert_(hit['torrent_size'] == len(bmetainfo))
        if self.query.startswith('SIMPLE+METADATA'):
            self.assert_(hit['metadata'] == bmetainfo)

    def create_not_bdecodable(self,id):
        return QUERY_REPLY+"bla"

    def create_not_bdecodable_torrentfile(self,id):
        d = self.create_good_simple_reply_dict(id)
        d['a'][self.tdef.get_infohash()]['torrent_size'] = 3 # consistent with metadata. Should be named "metasize"
        d['a'][self.tdef.get_infohash()]['metadata'] = 'bla'
        b = bencode(d)
        return QUERY_REPLY+b

    def check_rquery(self,data):
        d = bdecode(data)
        self.assert_(type(d) == DictType)
        self.assert_(d.has_key('q'))
        q = d['q']
        self.assert_(type(q) == StringType)
        self.assert_(d.has_key('id'))
        id = d['id']
        self.assert_(type(id) == StringType)

        self.assert_(q == self.query)
        return d['id']
Exemple #7
0
    def setUpPostSession(self):
        """ override TestAsServer """
        TestAsServer.setUpPostSession(self)

        # Let Tribler start downloading an non-functioning torrent, so
        # we can talk to a normal download engine.
        self.tdef = TorrentDef()
        self.sourcefn = os.path.join(os.getcwd(),"API","file2.wmv")
        self.tdef.add_content(self.sourcefn)
        self.tdef.set_create_merkle_torrent(True)
        self.tdef.set_tracker("http://127.0.0.1:12/announce")
        self.tdef.finalize()

        self.torrentfn = os.path.join(self.session.get_state_dir(),"gen.torrent")
        self.tdef.save(self.torrentfn)
        
        dscfg = self.setUpDownloadConfig()
        
        self.session.start_download(self.tdef,dscfg)

        self.infohash = self.tdef.get_infohash()
        self.mylistenport = 4810
        
        self.numpieces = (self.tdef.get_length()+self.tdef.get_piece_length()-1) / self.tdef.get_piece_length()
        b = Bitfield(self.numpieces)
        for i in range(self.numpieces):
            b[i] = True
        self.assert_(b.complete())
        self.seederbitfieldstr = b.tostring()

        #piece_hashes = ['\x01\x02\x03\x04\x05\x06\x07\x08\x07\x06\x05\x04\x03\x02\x01\x00\x01\x02\x03\x04' ] * npieces
        # Construct Merkle tree
        tdef2 = TorrentDef()
        tdef2.add_content(self.sourcefn)
        tdef2.set_create_merkle_torrent(False)
        tdef2.set_tracker("http://127.0.0.1:12/announce")
        tdef2.set_piece_length(self.tdef.get_piece_length())
        tdef2.finalize()
        metainfo = tdef2.get_metainfo()
        
        piecesstr = metainfo['info']['pieces']
        print >>sys.stderr,"test: pieces has len",len(piecesstr)
        piece_hashes = []
        for i in range(0,len(piecesstr),20):
            hash = piecesstr[i:i+20]
            print >>sys.stderr,"test: piece",i/20,"hash",`hash`
            piece_hashes.append(hash)
            
        print >>sys.stderr,"test: Putting",len(piece_hashes),"into MerkleTree, size",self.tdef.get_piece_length(),tdef2.get_piece_length()
        
        self.tree = MerkleTree(self.tdef.get_piece_length(),self.tdef.get_length(),None,piece_hashes)
        
        
        f = open(self.sourcefn,"rb")
        piece1 = f.read(2 ** 18)
        piece2 = f.read(2 ** 18)
        print >>sys.stderr,"read piece1",len(piece1)
        print >>sys.stderr,"read piece2",len(piece2)
        f.close()
        hash1 = sha(piece1).digest()
        hash2 = sha(piece2).digest()
        print >>sys.stderr,"hash piece1",`hash1`
        print >>sys.stderr,"hash piece2",`hash2`
        f2 = open("piece1.bin","wb")
        f2.write(piece2)
        f2.close()
Exemple #8
0
class TestMerkleMessage(TestAsServer):
    """ 
    Testing Merkle hashpiece messages for both:
    * Merkle BEP style
    * old Tribler <= 4.5.2 that did not use the Extention protocol (BEP 10).
     
    See BitTornado/BT1/Connecter.py
    """

    def setUp(self):
        """ override TestAsServer """
        TestAsServer.setUp(self)
        print >>sys.stderr,"test: Giving Session time to startup"
        time.sleep(5)
        print >>sys.stderr,"test: Session should have started up"

    def setUpPreSession(self):
        """ override TestAsServer """
        TestAsServer.setUpPreSession(self)
        self.config.set_overlay(False)
        self.config.set_megacache(False)

    
    def setUpPostSession(self):
        """ override TestAsServer """
        TestAsServer.setUpPostSession(self)

        # Let Tribler start downloading an non-functioning torrent, so
        # we can talk to a normal download engine.
        self.tdef = TorrentDef()
        self.sourcefn = os.path.join(os.getcwd(),"API","file2.wmv")
        self.tdef.add_content(self.sourcefn)
        self.tdef.set_create_merkle_torrent(True)
        self.tdef.set_tracker("http://127.0.0.1:12/announce")
        self.tdef.finalize()

        self.torrentfn = os.path.join(self.session.get_state_dir(),"gen.torrent")
        self.tdef.save(self.torrentfn)
        
        dscfg = self.setUpDownloadConfig()
        
        self.session.start_download(self.tdef,dscfg)

        self.infohash = self.tdef.get_infohash()
        self.mylistenport = 4810
        
        self.numpieces = (self.tdef.get_length()+self.tdef.get_piece_length()-1) / self.tdef.get_piece_length()
        b = Bitfield(self.numpieces)
        for i in range(self.numpieces):
            b[i] = True
        self.assert_(b.complete())
        self.seederbitfieldstr = b.tostring()

        #piece_hashes = ['\x01\x02\x03\x04\x05\x06\x07\x08\x07\x06\x05\x04\x03\x02\x01\x00\x01\x02\x03\x04' ] * npieces
        # Construct Merkle tree
        tdef2 = TorrentDef()
        tdef2.add_content(self.sourcefn)
        tdef2.set_create_merkle_torrent(False)
        tdef2.set_tracker("http://127.0.0.1:12/announce")
        tdef2.set_piece_length(self.tdef.get_piece_length())
        tdef2.finalize()
        metainfo = tdef2.get_metainfo()
        
        piecesstr = metainfo['info']['pieces']
        print >>sys.stderr,"test: pieces has len",len(piecesstr)
        piece_hashes = []
        for i in range(0,len(piecesstr),20):
            hash = piecesstr[i:i+20]
            print >>sys.stderr,"test: piece",i/20,"hash",`hash`
            piece_hashes.append(hash)
            
        print >>sys.stderr,"test: Putting",len(piece_hashes),"into MerkleTree, size",self.tdef.get_piece_length(),tdef2.get_piece_length()
        
        self.tree = MerkleTree(self.tdef.get_piece_length(),self.tdef.get_length(),None,piece_hashes)
        
        
        f = open(self.sourcefn,"rb")
        piece1 = f.read(2 ** 18)
        piece2 = f.read(2 ** 18)
        print >>sys.stderr,"read piece1",len(piece1)
        print >>sys.stderr,"read piece2",len(piece2)
        f.close()
        hash1 = sha(piece1).digest()
        hash2 = sha(piece2).digest()
        print >>sys.stderr,"hash piece1",`hash1`
        print >>sys.stderr,"hash piece2",`hash2`
        f2 = open("piece1.bin","wb")
        f2.write(piece2)
        f2.close()
        

    def setUpDownloadConfig(self):
        dscfg = DownloadStartupConfig()
        print >>sys.stderr,"test: Downloading to",self.config_path
        dscfg.set_dest_dir(self.config_path)
        dscfg.set_breakup_seed_bitfield(False)
        
        return dscfg        
        
    def tearDown(self):
        TestAsServer.tearDown(self)
        try:
            os.remove('piece1.bin')
        except:
            pass

        
    def singtest_good_hashpiece_bepstyle(self):
        self.subtest_good_hashpiece(False)

    def singtest_good_hashpiece_oldstyle(self):
        self.subtest_good_hashpiece(True)

    def singtest_good_request_bepstyle(self):
        # Let Session download file first
        self.subtest_good_hashpiece(False)
        # Now connect as different peer and download
        print >>sys.stderr,"\n\ntest: test_good_request: STARTING"
        self._test_good_request()

    def singtest_bad_hashpiece_bepstyle(self):
        self.subtest_bad_hashpiece(False)

    def singtest_bad_hashpiece_oldstyle(self):
        self.subtest_bad_hashpiece(True)

    #
    # Good hashpiece message
    #
    def subtest_good_hashpiece(self,oldstyle):
        print >>sys.stderr,"test: Testing good hashpiece, oldstyle",oldstyle
        if oldstyle:
            self._test_good(self.create_good_hashpiece,oldstyle,self.create_good_tribler_extend_hs,infohash=self.infohash)
        else:
            options = '\x00\x00\x00\x00\x00\x10\x00\x00'
            self._test_good(self.create_good_hashpiece,oldstyle,self.create_good_nontribler_extend_hs,options=options,infohash=self.infohash)

    def _test_good(self,msg_gen_func,oldstyle,extend_hs_gen_func,options=None,infohash=None):
        if options is None and infohash is None:
            s = BTConnection('localhost',self.hisport)
        elif options is None:
            s = BTConnection('localhost',self.hisport,user_infohash=infohash)
        elif infohash is None:
            s = BTConnection('localhost',self.hisport,user_option_pattern=options)
        else:
            s = BTConnection('localhost',self.hisport,user_option_pattern=options,user_infohash=infohash)
        print >>sys.stderr,"test: test_good: Create EXTEND HS"
        msg = extend_hs_gen_func()
        print >>sys.stderr,"test: test_good: Sending EXTEND HS",`msg`
        s.send(msg)
        print >>sys.stderr,"test: test_good: Waiting for BT HS"
        s.read_handshake_medium_rare()

        # Tribler should send an EXTEND message back
        try:
            print >>sys.stderr,"test: Waiting for reply"
            s.s.settimeout(10.0)
            resp = s.recv()
            self.assert_(len(resp) > 0)
            print >>sys.stderr,"test: Got reply",getMessageName(resp[0])
            self.assert_(resp[0] == EXTEND)
            self.check_tribler_extend_hs(resp[1:])
            
            # 1. Pretend we're seeder: send BITFIELD and UNCHOKE
            msg = BITFIELD+self.seederbitfieldstr
            s.send(msg)
            msg = UNCHOKE
            s.send(msg)
            print >>sys.stderr,"test: Pretend we are seeder"
            while True:
                resp = s.recv()
                self.assert_(len(resp) > 0)
                print >>sys.stderr,"test: Got reply2",getMessageName(resp[0])
                self.assert_(resp[0] == REQUEST or resp[0] == INTERESTED or resp[0] == UNCHOKE or resp[0] == HAVE or resp[0] == NOT_INTERESTED)
                if resp[0] == REQUEST:
                    chunkid = self.check_request(resp)
            
                    # 2. Reply to REQUEST with HASHPIECE (oldstyle) or Tr_hashpiece
                    msg = msg_gen_func(oldstyle,chunkid)
                    s.send(msg)
                elif resp[0] == NOT_INTERESTED:
                    break
            
            #s.close()
        except socket.timeout:
            print >> sys.stderr,"test: Timeout, bad, peer didn't reply in time"
            self.assert_(False)
        
        destfn = os.path.join(self.config_path,"file2.wmv")
        sf = open(self.sourcefn,"rb")
        df = open(destfn,"rb")
        n = self.tdef.get_piece_length()
        while True:
            sdata = sf.read(n)
            if len(sdata) == 0:
                break
            ddata = df.read(n)
            self.assert_(sdata == ddata)

        time.sleep(3)
        s.close()

    def create_good_nontribler_extend_hs(self):
        """ Merkle BEP style """
        d = {}
        d['m'] = {'Tr_hashpiece':250}
        d['p'] = self.mylistenport
        d['v'] = 'TestSweet 1.2.3.4'
        bd = bencode(d)
        return EXTEND+chr(0)+bd

    def create_good_tribler_extend_hs(self):
        """ old Tribler style """
        d = {}
        d['m'] = {'Tr_OVERLAYSWARM':253}
        d['p'] = self.mylistenport
        d['v'] = 'Tribler 3.5.1'
        bd = bencode(d)
        return EXTEND+chr(0)+bd


    def check_tribler_extend_hs(self,data):
        self.assert_(data[0] == chr(0))
        d = bdecode(data[1:])
        self.assert_(type(d) == DictType)
        self.assert_('m' in d.keys())
        m = d['m']
        self.assert_(type(m) == DictType)
        self.assert_('Tr_hashpiece' in m.keys())
        val = m['Tr_hashpiece']
        self.assert_(type(val) == IntType)
        self.assert_(val == 250)

    def check_request(self,data):
        index = toint(data[1:5])
        begin = toint(data[5:9])
        length = toint(data[9:])
        return (index,begin,length)

    def create_good_hashpiece(self,oldstyle,chunkid):
        index, begin, length = chunkid
        if begin == 0:
            ohlist = self.tree.get_hashes_for_piece(index)
        else:
            ohlist = []

        chunk = self.read_chunk(index,begin,length)
        bohlist = bencode(ohlist)
        
        print >>sys.stderr,"test: create_good_hashpiece:",index,begin,length,"==len",len(chunk)

        payload = tobinary(index)+tobinary(begin)+tobinary(len(bohlist))+bohlist+chunk
        if oldstyle:
            msg = HASHPIECE+payload
        else:
            # Offical: use the msg ID he defined in his handshake
            msg = EXTEND+HASHPIECE+payload
        return msg

    def read_chunk(self,index,begin,length):
        offset = index*self.tdef.get_piece_length() + begin
        f = open(self.sourcefn,"rb")
        f.seek(offset)
        chunk = f.read(length)
        f.close()
        return chunk



    #
    # Test whether Tribler sends good Tr_hashpiece on our requests
    #
    def _test_good_request(self):
        options = '\x00\x00\x00\x00\x00\x10\x00\x00'
        myid = Rand.rand_bytes(20)
         
        s = BTConnection('localhost',self.hisport,user_option_pattern=options,user_infohash=self.infohash,myid=myid)
        msg = self.create_good_nontribler_extend_hs()
        s.send(msg)
        s.read_handshake_medium_rare()

        # Tribler should send an EXTEND message back
        try:
            print >>sys.stderr,"test: Waiting for reply"
            s.s.settimeout(10.0)
            resp = s.recv()
            self.assert_(len(resp) > 0)
            print >>sys.stderr,"test: Got reply",getMessageName(resp[0])
            self.assert_(resp[0] == EXTEND)
            self.check_tribler_extend_hs(resp[1:])
            
            # 1. Pretend we're leecher: send INTERESTED
            msg = INTERESTED
            s.send(msg)
            print >>sys.stderr,"test: Pretend we are leecher"
            while True:
                resp = s.recv()
                self.assert_(len(resp) > 0)
                print >>sys.stderr,"test: Got reply2",getMessageName(resp[0])
                if resp[0] == EXTEND:
                    print >>sys.stderr,"test: Got EXTEND type",getMessageName(resp[1])
                self.assert_(resp[0] == UNCHOKE or resp[0] == BITFIELD or resp[0] == EXTEND or resp[0] == HAVE)
                if resp[0] == UNCHOKE:
                    # 2. Reply with REQUESTs
                    for index in range(0,self.numpieces):
                        plen = self.get_piece_length(index)
                            
                        for begin in range(0,plen,2 ** 14):
                            length = self.get_chunk_length(index,begin)
                            print >>sys.stderr,"RETRIEVE",index,begin,length
                            chunkid = (index,begin,length)
                            msg = self.create_request(chunkid)
                            s.send(msg)
                            
                    #s.send(NOT_INTERESTED)
                        
                elif resp[0] == EXTEND and resp[1] == HASHPIECE:
                    done = self.check_hashpiece(resp)
                    if done:
                        break
                elif resp[0] == BITFIELD:
                    self.check_bitfield(resp)
            
            #s.close()
        except socket.timeout:
            print >> sys.stderr,"test: Timeout, bad, peer didn't reply in time"
            self.assert_(False)

        time.sleep(3)
        s.close()


    def get_piece_length(self,index):
        if index == (self.numpieces-1):
            plen = self.tdef.get_length() % self.tdef.get_piece_length()
        else:
            plen = self.tdef.get_piece_length()
        return plen
    
    def get_chunk_length(self,index,begin):
        plen = self.get_piece_length(index)
        length = 2 ** 14
        if index == (self.numpieces-1):
            if (begin+2 ** 14) > plen:
                length = plen - begin
        return length
                
        
    def create_request(self,chunkid):
        index,begin,length = chunkid
        return REQUEST+tobinary(index)+tobinary(begin)+tobinary(length)


    def check_hashpiece(self,resp):
        """ Merkle BEP style """
        print >>sys.stderr,"test: good_request: check_hashpiece"
        self.assert_(resp[0] == EXTEND)
        self.assert_(resp[1] == HASHPIECE)
        index = toint(resp[2:2+4])
        begin = toint(resp[6:6+4])
        ohlen = toint(resp[10:10+4])
        print >>sys.stderr,"test: good_request: check_hashpiece",index,begin,ohlen
        bohlist = resp[14:14+ohlen]
        hisohlist = bdecode(bohlist)
        hischunk = resp[14+ohlen:]
        
        if begin == 0:
            self.assert_(type(hisohlist) == ListType)
            for oh in hisohlist:
                self.assert_(type(oh) == ListType)
                self.assert_(len(oh) == 2)
                self.assert_(type(oh[0]) == IntType)
                self.assert_(type(oh[1]) == StringType)
            
            hisohlist.sort()
            print >>sys.stderr,"test: good_request: check_hashpiece",`hisohlist`
            myohlist = self.tree.get_hashes_for_piece(index)
            myohlist.sort()
            
            self.assert_(len(hisohlist) == len(myohlist))
            for i in range(0,len(hisohlist)):
                hisoh = hisohlist[i]
                myoh = myohlist[i]
                self.assert_(hisoh == myoh)
        else:
            self.assert_(len(hisohlist) == 0)
        
        mylength = self.get_chunk_length(index,begin)
        mychunk = self.read_chunk(index,begin,mylength)
        
        self.assert_(hischunk == mychunk)

        return index == self.numpieces-1 and mylength != 2 ** 14

    def check_bitfield(self,data):
        self.assert_(data[0] == BITFIELD)
        bitmap = data[1:]
        self.assert_(len(bitmap) == 1)
        # Must have set_breakup_seed_bitfield() set to False
        self.assert_(bitmap == '\xc0')


    #
    # Bad EXTEND handshake message
    #    
    def subtest_bad_hashpiece(self,oldstyle):
        if not oldstyle:
            # Test becomes equivalent to BT keep alive message (len 0, payload '')
            self._test_bad(self.create_empty,oldstyle)
        self._test_bad(self.create_ext_id_not_byte,oldstyle)
        self._test_bad(self.create_not_hashpiece,oldstyle)
        self._test_bad(self.create_not_index,oldstyle)
        self._test_bad(self.create_not_begin,oldstyle)
        self._test_bad(self.create_not_len_bohlist,oldstyle)
        self._test_bad(self.create_ohlist_not_bdecodable,oldstyle)
        self._test_bad(self.create_ohlist_wrong_no_hashes,oldstyle)
        self._test_bad(self.create_ohlist_wrong_no_root_hash,oldstyle)
        self._test_bad(self.create_ohlist_wrong_bad_offset,oldstyle)
        self._test_bad(self.create_ohlist_wrong_bad_hash,oldstyle)
        # TODO: need working peer kicking for that
        ##self._test_bad(self.create_bad_chunk,oldstyle)
    
    #
    # Main test code for bad EXTEND handshake messages
    #
    def _test_bad(self,msg_gen_func,oldstyle):
        print >>sys.stderr,"test: test_BAD: Create EXTEND HS",`msg_gen_func`,oldstyle
        if oldstyle:
            options = None
            exthsmsg = self.create_good_tribler_extend_hs()
        else:
            options = '\x00\x00\x00\x00\x00\x10\x00\x00'
            exthsmsg = self.create_good_nontribler_extend_hs()
        
        
        s = BTConnection('localhost',self.hisport,user_option_pattern=options,user_infohash=self.infohash)
        s.send(exthsmsg)
        s.read_handshake_medium_rare()

        # Tribler should send an EXTEND message back
        try:
            print >>sys.stderr,"test: Waiting for reply"
            s.s.settimeout(10.0)
            resp = s.recv()
            self.assert_(len(resp) > 0)
            print >>sys.stderr,"test: Got reply",getMessageName(resp[0])
            self.assert_(resp[0] == EXTEND)
            self.check_tribler_extend_hs(resp[1:])
            
            # 1. Pretend we're seeder: send BITFIELD and UNCHOKE
            msg = BITFIELD+self.seederbitfieldstr
            s.send(msg)
            msg = UNCHOKE
            s.send(msg)
            print >>sys.stderr,"test: Pretend we are seeder"
            while True:
                resp = s.recv()
                self.assert_(len(resp) > 0)
                print >>sys.stderr,"test: Got reply 2",getMessageName(resp[0])
                self.assert_(resp[0] == REQUEST or resp[0] == INTERESTED or resp[0] == UNCHOKE or resp[0] == HAVE or resp[0] == NOT_INTERESTED)
                if resp[0] == REQUEST:
                    chunkid = self.check_request(resp)
            
                    # 2. Reply to REQUEST with *bad* HASHPIECE
                    msg = msg_gen_func(chunkid)
                    if oldstyle:
                        if len(msg) == 1:
                            msg = ''
                        else:
                            msg = msg[1:] # Strip EXTEND byte
                    s.send(msg)
                    break
                    
            
            #s.close()
        except socket.timeout:
            print >> sys.stderr,"test: Timeout, bad, peer didn't reply in time"
            self.assert_(False)


        time.sleep(3)
        # Should have closed the connection
        try:
            s.send(UNCHOKE)
            self.assert_(False)
        except:
            print_exc()

        s.close()
            

    #
    # Bad message creators (all create Merkle BEP style, I strip first byte
    # later for oldstyle
    # 
    def create_empty(self,chunkid):
        return EXTEND

    def create_ext_id_not_byte(self,chunkid):
        return EXTEND+'Hallo kijkbuiskinderen'

    def create_not_hashpiece(self,chunkid):
        index,begin,length = chunkid
        ohlist = []
        bohlist = bencode(ohlist)
        chunk = self.read_chunk(index,begin,length)
        payload = tobinary(index)+tobinary(begin)+tobinary(len(bohlist))+bohlist+chunk
        return EXTEND+chr(231)+payload

    def create_not_index(self,chunkid):
        payload = 'bla'
        return EXTEND+HASHPIECE+payload

    def create_not_begin(self,chunkid):
        index,begin,length = chunkid
        payload = tobinary(index)+'bla'
        return EXTEND+HASHPIECE+payload

    def create_not_len_bohlist(self,chunkid):
        index,begin,length = chunkid
        payload = tobinary(index)+tobinary(begin)+'bla'
        return EXTEND+HASHPIECE+payload

    def create_ohlist_not_bdecodable(self,chunkid):
        index,begin,length = chunkid
        bohlist = 'bla'
        chunk = '*' * (2 ** 14)
        payload = tobinary(index)+tobinary(begin)+tobinary(len(bohlist))+bohlist+chunk
        return EXTEND+HASHPIECE+payload

    def create_ohlist_wrong_no_hashes(self,chunkid):
        index,begin,length = chunkid
        ohlist = [ (0,'#' * 20),(1,'$' * 20)]  # should contain 3 for file2.wmv: own, sibling and root
        bohlist = bencode(ohlist)
        chunk = '*' * (2 ** 14)
        payload = tobinary(index)+tobinary(begin)+tobinary(len(bohlist))+bohlist+chunk
        return EXTEND+HASHPIECE+payload

    def create_ohlist_wrong_no_root_hash(self,chunkid):
        index,begin,length = chunkid
        ohlist = self.tree.get_hashes_for_piece(index)
        newohlist = []
        # Remove root hash
        for oh in ohlist:
            if oh[0] != 0:
                newohlist.append(oh)
        ohlist = newohlist
        bohlist = bencode(ohlist)
        chunk = self.read_chunk(index,begin,length)
        payload = tobinary(index)+tobinary(begin)+tobinary(len(bohlist))+bohlist+chunk
        return EXTEND+HASHPIECE+payload


    def create_ohlist_wrong_bad_offset(self,chunkid):
        index,begin,length = chunkid
        ohlist = self.tree.get_hashes_for_piece(index)
        ohlist[1][0] = 481
        bohlist = bencode(ohlist)
        chunk = self.read_chunk(index,begin,length)
        payload = tobinary(index)+tobinary(begin)+tobinary(len(bohlist))+bohlist+chunk
        return EXTEND+HASHPIECE+payload

    def create_ohlist_wrong_bad_hash(self,chunkid):
        index,begin,length = chunkid
        ohlist = self.tree.get_hashes_for_piece(index)
        ohlist[1][1] = '$' * 20
        bohlist = bencode(ohlist)
        chunk = self.read_chunk(index,begin,length)
        payload = tobinary(index)+tobinary(begin)+tobinary(len(bohlist))+bohlist+chunk
        return EXTEND+HASHPIECE+payload

    def create_bad_chunk(self,chunkid):
        index,begin,length = chunkid
        ohlist = self.tree.get_hashes_for_piece(index)
        bohlist = bencode(ohlist)
        chunk = '*' * length
        payload = tobinary(index)+tobinary(begin)+tobinary(len(bohlist))+bohlist+chunk
        return EXTEND+HASHPIECE+payload