Example #1
0
 def scenario_offline_beta(self, *classes):
     options = [(t[0], float(t[1]))
                for t in (s.split(":") for s in classes)]
     for class_, offline_beta in options:
         if class_ == self.__class:
             self.__offline_beta = offline_beta
     dprint("choosing offline beta ", self.__offline_beta)
Example #2
0
    def on_votecast(self, messages):
        if self.integrate_with_tribler:
            votelist = []
            for message in messages:
                if __debug__: dprint(message)
                dispersy_id = message.packet_id
                channel_id = getattr(message, "channel_id", 0)
                
                authentication_member = message.authentication.member
                if authentication_member == self._my_member:
                    peer_id = None
                    
                    #if channel_id is not found, then this is a manual join
                    #insert placeholder into database which will be replaced after channelmessage has been received
                    if not channel_id:
                        select_channel = "SELECT id FROM _Channels WHERE dispersy_cid = ?"
                        channel_id = self._channelcast_db._db.fetchone(select_channel, (buffer(message.payload.cid),))

                        if not channel_id:
                            insert_channel = "INSERT INTO _Channels (dispersy_cid, peer_id, name) VALUES (?, ?, ?); SELECT last_insert_rowid();"
                            channel_id = self._channelcast_db._db.fetchone(insert_channel, (buffer(message.payload.cid), -1, ''))
                else:
                    peer_id = self._peer_db.addOrGetPeerID(authentication_member.public_key)
                
                votelist.append((channel_id, peer_id, dispersy_id, message.payload.vote, message.payload.timestamp))
                
                if DEBUG:
                    print >> sys.stderr, "AllChannelCommunity: got votecast message"
                    
            self._votecast_db.on_votes_from_dispersy(votelist)

            # this might be a response to a dispersy-missing-message
            self._dispersy.handle_missing_messages(messages, MissingMessageCache)
Example #3
0
    def set(self, origin):
        """
        """
        assert isinstance(origin, float)
        assert self._origin <= origin
        difference = int((origin - self._origin) / CYCLE_SIZE)
        if __debug__:
            dprint("difference ", difference, " (", origin - self._origin,
                   "s)")
        if difference:
            self._origin = origin

            # shift
            self._bits <<= difference

            # remove now obsolete bits
            self._bits &= (2**self._size - 1)

            # set last bit ACTIVE
            self._bits |= 1

            return True

        else:
            if self._bits & 1:
                return False
            else:
                self._bits |= 1
                return True
Example #4
0
    def _observation(self, candidate, member, now, update_record=True):
        if not isinstance(candidate, BootstrapCandidate):
            # try cache
            history = self._observations.get(member.database_id)
            if not history:
                # fetch from database
                try:
                    timestamp, bytes_ = next(self._database.execute(u"SELECT timestamp, effort FROM observation WHERE member = ?",
                                                                    (member.database_id,)))
                except StopIteration:
                    # first observation: create new history
                    history = EffortHistory(now)
                else:
                    history = EffortHistory(str(bytes_), float(timestamp))

                # store in cache
                self._observations[member.database_id] = history
                if len(self._observations) > self._observations_length:
                    key, value = self._observations.popitem(False)
                    self._database.execute(u"INSERT OR REPLACE INTO observation (member, timestamp, effort) VALUES (?, ?, ?)",
                                           (key, value.origin, buffer(value.bytes)))

            if update_record:
                changed = history.set(now)
                if changed:
                    self.try_adding_to_slope(candidate, member, history)
                if __debug__: dprint("c", int(now / CYCLE_SIZE), " ", candidate)

            return history
Example #5
0
    def on_votecast(self, messages):
        if self.integrate_with_tribler:
            votelist = []
            for message in messages:
                if __debug__: dprint(message)
                dispersy_id = message.packet_id
                channel_id = getattr(message, "channel_id", 0)

                authentication_member = message.authentication.member
                if authentication_member == self._my_member:
                    peer_id = None

                    #if channel_id is not found, then this is a manual join
                    #insert placeholder into database which will be replaced after channelmessage has been received
                    if not channel_id:
                        select_channel = "SELECT id FROM _Channels WHERE dispersy_cid = ?"
                        channel_id = self._channelcast_db._db.fetchone(select_channel, (buffer(message.payload.cid),))

                        if not channel_id:
                            insert_channel = "INSERT INTO _Channels (dispersy_cid, peer_id, name) VALUES (?, ?, ?); SELECT last_insert_rowid();"
                            channel_id = self._channelcast_db._db.fetchone(insert_channel, (buffer(message.payload.cid), -1, ''))
                else:
                    peer_id = self._peer_db.addOrGetPeerID(authentication_member.public_key)

                votelist.append((channel_id, peer_id, dispersy_id, message.payload.vote, message.payload.timestamp))

                if DEBUG:
                    print >> sys.stderr, "AllChannelCommunity: got votecast message"

            self._votecast_db.on_votes_from_dispersy(votelist)

            # this might be a response to a dispersy-missing-message
            self._dispersy.handle_missing_messages(messages, MissingMessageCache)
Example #6
0
 def dispersy_cleanup_community(self, message):
     if __debug__: dprint()
     self._has_been_killed = True
     # remove all data from the local database
     self._database.cleanup()
     # re-classify to prevent loading
     return super(EffortCommunity, self).dispersy_cleanup_community(message)
Example #7
0
 def dispersy_cleanup_community(self, message):
     if __debug__: dprint()
     self._has_been_killed = True
     # remove all data from the local database
     self._database.cleanup()
     # re-classify to prevent loading
     return super(EffortCommunity, self).dispersy_cleanup_community(message)
    def set(self, origin):
        """
        """
        assert isinstance(origin, float)
        assert self._origin <= origin
        difference = int((origin - self._origin) / CYCLE_SIZE)
        if __debug__: dprint("difference ", difference, " (", origin - self._origin, "s)")
        if difference:
            self._origin = origin

            # shift
            self._bits <<= difference

            # remove now obsolete bits
            self._bits &= (2**self._size - 1)

            # set last bit ACTIVE
            self._bits |= 1

            return True

        else:
            if self._bits & 1:
                return False
            else:
                self._bits |= 1
                return True
Example #9
0
    def _observation(self, candidate, member, now, update_record=True):
        if not isinstance(candidate, BootstrapCandidate):
            # try cache
            history = self._observations.get(member.database_id)
            if not history:
                # fetch from database
                try:
                    timestamp, bytes_ = next(self._database.execute(u"SELECT timestamp, effort FROM observation WHERE member = ?",
                                                                    (member.database_id,)))
                except StopIteration:
                    # first observation: create new history
                    history = EffortHistory(now)
                else:
                    history = EffortHistory(str(bytes_), float(timestamp))

                # store in cache
                self._observations[member.database_id] = history
                if len(self._observations) > self._observations_length:
                    key, value = self._observations.popitem(False)
                    self._database.execute(u"INSERT OR REPLACE INTO observation (member, timestamp, effort) VALUES (?, ?, ?)",
                                           (key, value.origin, buffer(value.bytes)))

            if update_record:
                changed = history.set(now)
                if changed:
                    self.try_adding_to_slope(candidate, member, history)
                if __debug__: dprint("c", int(now / CYCLE_SIZE), " ", candidate)

            return history
Example #10
0
 def scenario_online_beta(self, *classes):
     options = [(t[0], float(t[1])) for t in (s.split(":") for s in classes)]
     for class_, online_beta in options:
         if class_ == self.__class:
             self.__online_beta = online_beta
             break
     dprint("choosing online beta ", self.__online_beta)
Example #11
0
    def unload_community(self):
        if __debug__: dprint()
        super(EffortCommunity, self).unload_community()

        # unsubscribe from events.  22/03/13 Boudewijn: Dispersy is not allowed to call swift
        # because it may cause database locks if people incorrectly keep the session locked.
        temp_thread = Callback()
        temp_thread.register(self._swift.set_subscribe_channel_close, ("ALL", False, self.i2ithread_channel_close))
        temp_thread.register(temp_thread.stop)
        temp_thread.start("Temporary-EffortCommunity")

        # cancel outstanding pings
        for ping_candidate in self._slope.itervalues():
            self._dispersy.callback.unregister(ping_candidate.callback_id)
        self._slope = {}

        # store all cached observations
        self._database.executemany(u"INSERT OR REPLACE INTO observation (member, timestamp, effort) VALUES (?, ?, ?)",
                                   [(database_id, history.origin, buffer(history.bytes)) for database_id, history in self._observations.iteritems()])

        # update all up and download values
        self.download_state_callback([], closing=True)

        # store all cached bandwidth guesses
        self._database.executemany(u"INSERT OR REPLACE INTO bandwidth_guess (ip, member, timestamp, upload, download) VALUES (?, ?, ?, ?, ?)",
                                   [(unicode(ip), guess.member.database_id if guess.member else 0, guess.timestamp, int(guess.upload), int(guess.download)) for ip, guess in self._bandwidth_guesses.iteritems()])
Example #12
0
 def on_torrent_collect_request(self, messages):
     if __debug__: dprint(len(messages))
     candidates = [message.candidate for message in messages]
     identifiers = [message.payload.identifier for message in messages]
     
     self._create_pingpong(u"torrent-collect-response", candidates, identifiers)
     self.on_torrent_collect_response(messages, verifyRequest = False)
    def on_torrent_collect_request(self, messages):
        if __debug__: dprint(len(messages))
        candidates = [message.candidate for message in messages]
        identifiers = [message.payload.identifier for message in messages]

        self._create_pingpong(u"torrent-collect-response", candidates, identifiers)
        self.on_torrent_collect_response(messages, verifyRequest = False)
Example #14
0
 def scenario_classes(self, *classes):
     options = [(t[0], float(t[1])) for t in (s.split(":") for s in classes)]
     value = random() * sum(weight for _, weight in options)
     for class_, weight in options:
         value -= weight
         if value <= 0.0:
             self.__class = class_
             break
     dprint("choosing class ", self.__class)
    def _get_channel_community(self, cid):
        assert isinstance(cid, str)
        assert len(cid) == 20

        try:
            return self._dispersy.get_community(cid, True)
        except KeyError:
            if __debug__: dprint("join preview community ", cid.encode("HEX"))
            return PreviewChannelCommunity.join_community(DummyMember(cid), self._my_member, self.integrate_with_tribler)
Example #16
0
 def _get_channel_community(self, cid):
     assert isinstance(cid, str)
     assert len(cid) == 20
     
     try:
         return self._dispersy.get_community(cid, True)
     except KeyError:
         if __debug__: dprint("join preview community ", cid.encode("HEX"))
         return PreviewChannelCommunity.join_community(DummyMember(cid), self._my_member, self.integrate_with_tribler) 
Example #17
0
    def _channel_close(self, roothash_hex, address, raw_bytes_up, raw_bytes_down, cooked_bytes_up, cooked_bytes_down):
        assert self._dispersy.callback.is_current_thread, "Must be called on the dispersy.callback thread"

        dprint("swift channel close ", address[0], ":", address[1], " with +", cooked_bytes_up, " -", cooked_bytes_down, force=1)
        guess = self._get_bandwidth_guess_from_ip(address[0])
        guess.upload += cooked_bytes_up / 1024.0
        guess.download += cooked_bytes_down / 1024.0
        guess.tmp_upload = 0.0
        guess.tmp_download = 0.0
Example #18
0
    def set_online(self):
        """ Restore on_incoming_packets and _send functions of
        dispersy back to normal.

        This simulates a node coming online, since it's able to send
        and receive messages.
        """
        dprint("Going online")
        self._dispersy.on_incoming_packets = self.original_on_incoming_packets
        self._dispersy._send = self.original_send
Example #19
0
 def scenario_classes(self, *classes):
     options = [(t[0], float(t[1]))
                for t in (s.split(":") for s in classes)]
     value = random() * sum(weight for _, weight in options)
     for class_, weight in options:
         value -= weight
         if value <= 0.0:
             self.__class = class_
             break
     dprint("choosing class ", self.__class)
Example #20
0
    def set_online(self):
        """ Restore on_incoming_packets and _send functions of
        dispersy back to normal.

        This simulates a node coming online, since it's able to send
        and receive messages.
        """
        dprint("Going online")
        self._dispersy.on_incoming_packets = self.original_on_incoming_packets
        self._dispersy._send = self.original_send
Example #21
0
    def create_effort_record(self, second_member, history, forward=True):
        """
        Create a dispersy-signature-request that encapsulates an effort-record.
        """
        if __debug__: dprint(second_member.mid.encode("HEX"), " = ", bin(history.long))

        meta = self.get_meta_message(u"effort-record")
        record = meta.impl(authentication=([self._my_member, second_member],),
                           distribution=(self.claim_global_time(),),
                           payload=(history.origin, 0.0, history))
        return self.create_dispersy_signature_request(record, self.on_signature_response, forward=forward)
Example #22
0
 def on_introduction_request(self, messages):
     try:
         return self._original_on_introduction_request(messages)
     finally:
         now = time()
         for message in messages:
             if not isinstance(message.candidate, BootstrapCandidate):
                 for member in message.candidate.get_members(self):
                     history = self._get_or_create_history(member, now)
                     changed = history.set(now)
                     if __debug__: dprint("introduction-request from ", message.candidate, " - ", bin(history.long))
Example #23
0
    def unload_preview(self):
        while True:
            yield 60.0

            cleanpoint = time() - 300
            inactive = [community for community in self.dispersy._communities.itervalues() if isinstance(community, PreviewChannelCommunity) and community.init_timestamp < cleanpoint]
            if __debug__:
                dprint("cleaning ", len(inactive), "/", len(self.dispersy._communities), " previewchannel communities")

            for community in inactive:
                community.unload_community()
Example #24
0
    def unload_preview(self):
        while True:
            yield 60.0

            cleanpoint = time() - 300
            inactive = [community for community in self.dispersy._communities.itervalues() if isinstance(community, PreviewChannelCommunity) and community.init_timestamp < cleanpoint]
            if __debug__:
                dprint("cleaning ", len(inactive), "/", len(self.dispersy._communities), " previewchannel communities")
                
            for community in inactive:
                community.unload_community()
Example #25
0
    def _watchdog(self):
        while True:
            try:
                yield 60.0

                # flush changes to disk every 1 minutes
                self._database.commit()
            except GeneratorExit:
                bz2log("effort.log", "unload")
                if __debug__: dprint("shutdown")
                self._database.commit()
                break
Example #26
0
    def set_offline(self):
        """ Replace on_incoming_packets and _sends functions of
        dispersy with dummies

        This simulates a node going offline, since it's not able to
        send or receive any messages
        """
        def dummy_function(*params):
            return
        dprint("Going offline")
        self._dispersy.on_incoming_packets = dummy_function
        self._dispersy._send = dummy_function
Example #27
0
    def set_offline(self):
        """ Replace on_incoming_packets and _sends functions of
        dispersy with dummies

        This simulates a node going offline, since it's not able to
        send or receive any messages
        """
        def dummy_function(*params):
            return

        dprint("Going offline")
        self._dispersy.on_incoming_packets = dummy_function
        self._dispersy._send = dummy_function
Example #28
0
    def on_signature_response(self, old_message, new_message, changed):
        """
        A dispersy-signature-response has been received.

        Return True or False to either accept or decline the message.
        """
        if __debug__: dprint(new_message)

        if new_message:
            if old_message.meta == new_message.meta:
                return True, True, False

        return False, False, False
Example #29
0
    def on_signature_response(self, message, request, retry):
        """ Handle a newly created double signed message or a timeout while signing

        When request for signing times out I just return. When I receive the signature for the
        message that I created then I send it to myself and then store and forward it to others of
        the community
        """
        if __debug__: dprint(message)

        if message:
            assert message.name == u"barter-record"
            assert message.authentication.is_signed

            if __debug__: dprint(message)

            # store, update, and forward to the community
            self._dispersy.store_update_forward([message], True, True, self.barter_forward_record_on_creation)
            log("dispersy.log", "created-barter-record") # TODO: maybe move to barter.log

        elif retry < 5:
            # signature timeout
            # retry
            if __debug__:
                log("barter.log", "barter-community-signature-request-timeout", retry=retry)
                dprint("Signature request timeout. Retry!")
                self.create_dispersy_signature_request(request, self.on_signature_response, (request, retry + 1))

        else:
            # close the transfer
            if __debug__:
                log("barter.log", "barter-community-signature-request-timeout", retry=retry)
                dprint("Signature request timeout")
Example #30
0
    def allow_signature_request(self, message):
        """ Decide whether to reply or not to a signature request

        Currently I always sign for testing proposes
        """
        assert message.name == u"barter-record"
        assert not message.authentication.is_signed
        if __debug__: dprint(message)

        is_signed, other_member = message.authentication.signed_members[0]
        if other_member == self._my_member:
            if __debug__: dprint("refuse signature: we should be the first signer", level="warning")
            return False

        if not is_signed:
            if __debug__: dprint("refuse signature: the other member did not sign it", level="warning")
            return False

        my_member = message.authentication.members[1]
        if not my_member == self._my_member:
            if __debug__: dprint("refuse signature: we should be the second signer")
            return False

        # todo: decide if we want to use add our signature to this # record
        if True:
            # recalculate the limits of what we want to upload to member.authentication.members[0]
            return True

        # we will not add our signature
        return False
Example #31
0
    def join_community(self, my_member):
        self.my_member = my_member
        
        master_key = "3081a7301006072a8648ce3d020106052b81040027038192000403cbbfd2dfb67a7db66c88988df56f93fa6e7f982f9a6a0fa8898492c8b8cae23e10b159ace60b7047012082a5aa4c6e221d7e58107bb550436d57e046c11ab4f51f0ab18fa8f58d0346cc12d1cc2b61fc86fe5ed192309152e11e3f02489e30c7c971dd989e1ce5030ea0fb77d5220a92cceb567cbc94bc39ba246a42e215b55e9315b543ddeff0209e916f77c0d747".decode("HEX")
        master = Member.get_instance(master_key)

        assert my_member.public_key
        assert my_member.private_key
        assert master.public_key
        assert not master.private_key

        dprint("-master- ", master.database_id, " ", id(master), " ", master.mid.encode("HEX"), force=1)
        dprint("-my member- ", my_member.database_id, " ", id(my_member), " ", my_member.mid.encode("HEX"), force=1)

        return AllChannelCommunity.join_community(master, my_member, my_member, integrate_with_tribler = False)
Example #32
0
    def on_introduction_response(self, messages):
        try:
            return self._original_on_introduction_response(messages)
        finally:
            now = time()
            for message in messages:
                if not isinstance(message.candidate, BootstrapCandidate):
                    for member in message.candidate.get_members(self):
                        history = self._get_or_create_history(member, now)
                        changed = history.set(now)
                        if __debug__: dprint("introduction-response from ", message.candidate, " - ", bin(history.long))

                        if changed:
                            if __debug__: dprint("changed! (", member.mid.encode("HEX"), ", ", bin(self._histories[member].long), ")")
                            self.create_effort_record(member, history)
Example #33
0
    def on_barter_record(self, messages):
        """ I handle received barter records

        I create or update a row in my database
        """
        if __debug__: dprint("storing ", len(messages), " records")
        execute = self._database.execute
        for message in messages:
            if __debug__: log("dispersy.log", "handled-barter-record") # TODO: maybe move to barter.log
            # check if there is already a record about this pair
            try:
                first_member, second_member, global_time = \
                              execute(u"SELECT first_member, second_member, global_time FROM \
                              record WHERE (first_member = ? AND second_member = ?) OR \
                              (first_member = ? AND second_member = ?)",
                                      (message.authentication.members[0].database_id,
                                       message.authentication.members[1].database_id,
                                       message.authentication.members[1].database_id,
                                       message.authentication.members[0].database_id)).next()
            except StopIteration:
                global_time = -1
                first_member = message.authentication.members[0].database_id
                second_member = message.authentication.members[1].database_id

            if global_time >= message.distribution.global_time:
                # ignore the message
                if __debug__:
                    dprint("Ignoring older message")
            else:
                self._database.execute(u"INSERT OR REPLACE INTO \
                record(community, global_time, first_member, second_member, \
                upload_first_member, upload_second_member) \
                VALUES(?, ?, ?, ?, ?, ?)",
                                       (self._database_id,
                                        message.distribution.global_time,
                                        first_member,
                                        second_member,
                                        message.payload.first_upload,
                                        message.payload.second_upload))
                if __debug__:
                    peer1_id = self._peer_ids.get(message.authentication.members[0].public_key, -1)
                    peer2_id = self._peer_ids.get(message.authentication.members[1].public_key, -1)
                    peer1_upload = message.payload.first_upload
                    peer2_upload = message.payload.second_upload
                    if peer1_id > peer2_id:
                        peer1_id, peer2_id = peer2_id, peer1_id
                        peer1_upload, peer2_upload = peer2_upload, peer1_upload
                    log("barter.log", "barter-record", first=peer1_id, second=peer2_id, first_upload=peer1_upload, second_upload=peer2_upload)
Example #34
0
 def _get_or_create_history(self, member, now):
     if __debug__:
         if member in self._histories:
             dprint("member in histories (", member.mid.encode("HEX"), ", ", bin(self._histories[member].long), ")")
         else:
             dprint("member NOT in histories")
     try:
         return self._histories[member]
     except KeyError:
         try:
             timestamp, bytes_ = self._database.execute(u"SELECT timestamp, effort FROM observation WHERE community = ? AND member = ?",
                                                        (self._database_id, member.database_id)).next()
         except StopIteration:
             self._histories[member] = history = EffortHistory(8*64, now)
         else:
             self._histories[member] = history = EffortHistory(bytes_, len(bytes) * 8, float(timestamp))
         return history
Example #35
0
    def unload_community(self):
        if __debug__: dprint()
        super(EffortCommunity, self).unload_community()

        # cancel outstanding pings
        for ping_candidate in self._slope.itervalues():
            self._dispersy.callback.unregister(ping_candidate.callback_id)
        self._slope = {}

        # store all cached observations
        self._database.executemany(u"INSERT OR REPLACE INTO observation (member, timestamp, effort) VALUES (?, ?, ?)",
                                   [(database_id, history.origin, buffer(history.bytes)) for database_id, history in self._observations.iteritems()])

        # update all up and download values
        self.download_state_callback([])

        # store all cached bandwidth guesses
        self._database.executemany(u"INSERT OR REPLACE INTO bandwidth_guess (ip, member, timestamp, upload, download) VALUES (?, ?, ?, ?, ?)",
                                   [(unicode(ip), guess.member.database_id if guess.member else 0, guess.timestamp, int(guess.upload), int(guess.download)) for ip, guess in self._bandwidth_guesses.iteritems()])
Example #36
0
    def check_votecast(self, messages):
        with self._dispersy.database:
            communities = {}
            for cid in set([message.payload.cid for message in messages]):
                channel_id = self._get_channel_id(cid)
                if not channel_id:
                    communities[cid] = self._get_channel_community(message.payload.cid)
            
            for message in messages:
                if __debug__: dprint(message)
                
                community = communities.get(message.payload.cid)
                if community:
                    yield DelayMessageReqChannelMessage(message, community, includeSnapshot = message.payload.vote > 0) #request torrents if positive vote
                else:
                    yield message

            # ensure that no commits occur
            raise IgnoreCommits()
Example #37
0
    def download_state_callback(self, states):
        assert self._dispersy.callback.is_current_thread, "Must be called on the dispersy.callback thread"
        assert isinstance(states, list)
        timestamp = int(time())

        # get all swift downloads that have peers
        active = dict((state.get_download().get_def().get_id(), state)
                      for state
                      in states
                      if state.get_download().get_def().get_def_type() == "swift" and state.get_peerlist())

        # get global up and download for swift
        for state in active.itervalues():
            stats = state.stats["stats"]
            self._swift_raw_bytes_up = stats.rawUpTotal
            self._swift_raw_bytes_down = stats.rawDownTotal

        # OLD is used to determine stopped downloads and peers that left.  NEW will become the next OLD
        old = self._download_states
        new = self._download_states = dict()

        # find downloads that stopped
        for identifier in set(old.iterkeys()).difference(set(active.iterkeys())):
            for ip, (up, down) in old[identifier].iteritems():
                if __debug__: dprint(identifier.encode("HEX"), "] ", ip, " +", up, " +", down)
                guess = self._get_bandwidth_guess_from_ip(ip)
                guess.timestamp = timestamp
                guess.upload += up
                guess.download += down

        for identifier, state in active.iteritems():
            if identifier in old:
                # find peers that left
                for ip in set(old[identifier]).difference(set(peer["ip"] for peer in state.get_peerlist())):
                    up, down = old[identifier][ip]
                    if __debug__: dprint(identifier.encode("HEX"), "] ", ip, " +", up, " +", down)
                    guess = self._get_bandwidth_guess_from_ip(ip)
                    guess.timestamp = timestamp
                    guess.upload += up
                    guess.download += down

            # set OLD for the next call to DOWNLOAD_STATE_CALLBACK
            new[identifier] = dict((peer["ip"], (peer["utotal"], peer["dtotal"])) for peer in state.get_peerlist() if peer["utotal"] > 0.0 or peer["dtotal"] > 0.0)
Example #38
0
    def on_effort_record(self, messages):
        def ordering(message):
            # print "RAW_RECORD_IN", message.packet.encode("HEX")
            # print \
            #     "RECORD_IN", \
            #     bin(message.payload.history.long), \
            #     message.distribution.global_time, \
            #     int(message.payload.history.origin / CYCLE_SIZE), \
            #     message.payload.first_timestamp, \
            #     message.payload.second_timestamp, \
            #     message.authentication.members[0].mid.encode("HEX"), \
            #     message.authentication.members[1].mid.encode("HEX")

            if message.authentication.members[0].database_id < message.authentication.members[1].database_id:
                return (message.packet_id,
                        message.distribution.global_time,
                        message.authentication.members[0].database_id,
                        message.authentication.members[1].database_id,
                        int(message.payload.first_timestamp),
                        int(message.payload.second_timestamp),
                        buffer(message.payload.history.bytes),
                        message.payload.first_up,
                        message.payload.first_down,
                        message.payload.second_up,
                        message.payload.second_down)

            else:
                return (message.packet_id,
                        message.distribution.global_time,
                        message.authentication.members[1].database_id,
                        message.authentication.members[0].database_id,
                        int(message.payload.second_timestamp),
                        int(message.payload.first_timestamp),
                        buffer(message.payload.history.bytes),
                        message.payload.first_up,
                        message.payload.first_down,
                        message.payload.second_up,
                        message.payload.second_down)

        if __debug__: dprint("storing ", len(messages), " effort records")
        self._database.executemany(u"INSERT OR REPLACE INTO record (sync, global_time, first_member, second_member, first_timestamp, second_timestamp, effort, first_upload, first_download, second_upload, second_download) VALUES (?, ?, ?, ?, ?, ?, ? ,?, ?, ?, ?)",
                                   (ordering(message) for message in messages))
Example #39
0
    def on_effort_record(self, messages):
        def ordering(message):
            # print "RAW_RECORD_IN", message.packet.encode("HEX")
            # print \
            #     "RECORD_IN", \
            #     bin(message.payload.history.long), \
            #     message.distribution.global_time, \
            #     int(message.payload.history.origin / CYCLE_SIZE), \
            #     message.payload.first_timestamp, \
            #     message.payload.second_timestamp, \
            #     message.authentication.members[0].mid.encode("HEX"), \
            #     message.authentication.members[1].mid.encode("HEX")

            if message.authentication.members[0].database_id < message.authentication.members[1].database_id:
                return (message.packet_id,
                        message.distribution.global_time,
                        message.authentication.members[0].database_id,
                        message.authentication.members[1].database_id,
                        int(message.payload.first_timestamp),
                        int(message.payload.second_timestamp),
                        buffer(message.payload.history.bytes),
                        message.payload.first_up,
                        message.payload.first_down,
                        message.payload.second_up,
                        message.payload.second_down)

            else:
                return (message.packet_id,
                        message.distribution.global_time,
                        message.authentication.members[1].database_id,
                        message.authentication.members[0].database_id,
                        int(message.payload.second_timestamp),
                        int(message.payload.first_timestamp),
                        buffer(message.payload.history.bytes),
                        message.payload.first_up,
                        message.payload.first_down,
                        message.payload.second_up,
                        message.payload.second_down)

        if __debug__: dprint("storing ", len(messages), " effort records")
        self._database.executemany(u"INSERT OR REPLACE INTO record (sync, global_time, first_member, second_member, first_timestamp, second_timestamp, effort, first_upload, first_download, second_upload, second_download) VALUES (?, ?, ?, ?, ?, ?, ? ,?, ?, ?, ?)",
                                   (ordering(message) for message in messages))
Example #40
0
    def create_effort_record(self, second_member, history):
        """
        Create a dispersy-signature-request that encapsulates an effort-record.
        """
        if __debug__: dprint("asking ", second_member.mid.encode("HEX"), " to sign ", bin(history.long))
        guess = self._try_bandwidth_guess_from_member(second_member)
        if guess:
            first_up = int(guess.upload)
            first_down = int(guess.download)
        else:
            first_up = 0
            first_down = 0

        self._statistic_outgoing_signature_request += 1
        meta = self.get_meta_message(u"effort-record")
        record = meta.impl(authentication=([self._my_member, second_member],),
                           distribution=(self.claim_global_time(),),
                           payload=(history.origin, history.origin, history, first_up, first_down, 0, 0),
                           sign=False)
        return self.create_dispersy_signature_request(record, self.on_signature_response)
Example #41
0
    def __init__(self, *args, **kargs):
        super(WalktestCommunity, self).__init__(*args, **kargs)
        if __debug__:
            dprint("cid: ", self.cid.encode("HEX"), force=1)
            dprint("mid: ", self.my_member.mid.encode("HEX"), force=1)

        try:
            hostname = open("/etc/hostname", "r").readline().strip()
        except:
            hostname = "unknown"

        bz2log("walktest.log",
               "load",
               mid=self.my_member.mid,
               hostname=hostname,
               **self._default_log())

        # redirect introduction-response timeout
        self._origional__introduction_response_timeout = self._dispersy.introduction_response_timeout
        self._dispersy.introduction_response_timeout = self._replacement__introduction_response_timeout
Example #42
0
    def create_effort_record(self, second_member, history):
        """
        Create a dispersy-signature-request that encapsulates an effort-record.
        """
        if __debug__: dprint("asking ", second_member.mid.encode("HEX"), " to sign ", bin(history.long))
        guess = self._try_bandwidth_guess_from_member(second_member)
        if guess:
            first_up = int(guess.upload + guess.tmp_upload)
            first_down = int(guess.download + guess.tmp_download)
        else:
            first_up = 0
            first_down = 0

        self._statistic_outgoing_signature_request += 1
        meta = self.get_meta_message(u"effort-record")
        record = meta.impl(authentication=([self._my_member, second_member],),
                           distribution=(self.claim_global_time(),),
                           payload=(history.origin, history.origin, history, first_up, first_down, 0, 0),
                           sign=False)
        return self.create_dispersy_signature_request(record, self.on_signature_response)
Example #43
0
    def __init__(self, *args, **kargs):
        super(WalktestCommunity, self).__init__(*args, **kargs)
        if __debug__:
            dprint("cid: ", self.cid.encode("HEX"), force=1)
            dprint("mid: ", self.my_member.mid.encode("HEX"), force=1)

        try:
            hostname = open("/etc/hostname", "r").readline().strip()
        except:
            hostname = "unknown"

        bz2log("walktest.log",
               "load",
               mid=self.my_member.mid,
               hostname=hostname,
               **self._default_log())

        # redirect introduction-response timeout
        self._origional__introduction_response_timeout = self._dispersy.introduction_response_timeout
        self._dispersy.introduction_response_timeout = self._replacement__introduction_response_timeout
Example #44
0
    def on_signature_response(self, cache, new_message, changed):
        """
        A dispersy-signature-response has been received.

        Return True or False to either accept or decline the message.
        """
        if __debug__: dprint(new_message)

        # TODO: we should ensure that new_message is correct (i.e. all checks made above)

        if new_message:
            self._statistic_outgoing_signature_request_success += 1
            self._observation(new_message.candidate, cache.members[0], time())

            assert cache.request.payload.message.meta == new_message.meta
            return True

        else:
            self._statistic_outgoing_signature_request_timeout += 1
            self.remove_from_slope(cache.members[0])
            return False
Example #45
0
    def on_signature_response(self, cache, new_message, changed):
        """
        A dispersy-signature-response has been received.

        Return True or False to either accept or decline the message.
        """
        if __debug__: dprint(new_message)

        # TODO: we should ensure that new_message is correct (i.e. all checks made above)

        if new_message:
            self._statistic_outgoing_signature_request_success += 1
            self._observation(new_message.candidate, cache.members[0], time())

            assert cache.request.payload.message.meta == new_message.meta
            return True

        else:
            self._statistic_outgoing_signature_request_timeout += 1
            self.remove_from_slope(cache.members[0])
            return False
    def on_torrent_collect_response(self, messages, verifyRequest = True):
        if __debug__: dprint(len(messages))
        toInsert = {}
        toCollect = {}
        toPopularity = {}
        for message in messages:
            if verifyRequest:
                pong_request = self._dispersy.request_cache.pop(message.payload.identifier, SearchCommunity.PingRequestCache)
                if __debug__: dprint("pop", pong_request.helper_candidate if pong_request else " (unknown)")
            else:
                if __debug__: dprint("no-pop")
                pong_request = True

            if pong_request and message.payload.hashtype == SWIFT_INFOHASHES:
                for roothash, infohash, seeders, leechers, ago in message.payload.torrents:
                    toInsert[infohash] = [infohash, roothash]
                    toPopularity[infohash] = [seeders, leechers, time() - (ago * 60)]
                    toCollect.setdefault(infohash, []).append(message.candidate)

        if len(toInsert) > 0:
            self._torrent_db.on_torrent_collect_response(toInsert.values())

        hashes = [hash_ for hash_ in toCollect.keys() if hash_]
        if hashes:
            hashesToCollect = self._torrent_db.selectSwiftTorrentsToCollect(hashes)
            for infohash, roothash in hashesToCollect[:5]:
                for candidate in toCollect[infohash]:
                    if DEBUG:
                        from Tribler.Core.CacheDB.sqlitecachedb import bin2str
                        print >> sys.stderr, "SearchCommunity: requesting .torrent after receiving ping/pong ", candidate, bin2str(infohash), bin2str(roothash)

                    self._rtorrent_handler.download_torrent(candidate, infohash, roothash, prio = LOW_PRIO_COLLECTING, timeout = CANDIDATE_WALK_LIFETIME)
Example #47
0
    def allow_signature_request(self, message):
        """
        A dispersy-signature-request has been received.

        Return None or a Message.Implementation.
        """
        assert message.name == u"effort-record"
        assert not message.authentication.is_signed
        if __debug__: dprint(message)

        _, first_member = message.authentication.signed_members[0]
        _, second_member = message.authentication.signed_members[1]
        global_time = message.distribution.global_time if message.distribution.global_time <= self.global_time else self.claim_global_time()

        if first_member == self._my_member:
            local_history = self._get_or_create_history(second_member, time())
            first_timestamp = local_history.origin
            second_timestamp = message.payload.second_timestamp

        else:
            assert second_member == self._my_member
            local_history = self._get_or_create_history(first_member, time())
            first_timestamp = message.payload.first_timestamp
            second_timestamp = local_history.origin

        if __debug__: dprint("time diff:", abs(first_timestamp - second_timestamp), "; bits diff:", local_history.long ^ message.payload.history.long)

        # TODO shift history and origin for a match
        history = EffortHistory(local_history.long & message.payload.history.long, local_history.size, local_history.origin)

        bz2log("effort.log", "diff", local=bin(history.long), remote=bin(message.payload.history.long), propose=bin(history.long), time_diff=int(abs(first_timestamp - second_timestamp)))

        # return the modified effort-record we propose
        meta = self.get_meta_message(u"effort-record")
        return meta.impl(authentication=([first_member, second_member],),
                         distribution=(global_time,),
                         payload=(first_timestamp, second_timestamp, history))
Example #48
0
    def check_votecast(self, messages):
        with self._dispersy.database:
            communities = {}
            for cid in set([message.payload.cid for message in messages]):
                channel_id = self._get_channel_id(cid)
                if not channel_id:
                    communities[cid] = self._get_channel_community(
                        message.payload.cid)

            for message in messages:
                if __debug__: dprint(message)

                community = communities.get(message.payload.cid)
                if community:
                    yield DelayMessageReqChannelMessage(
                        message,
                        community,
                        includeSnapshot=message.payload.vote >
                        0)  #request torrents if positive vote
                else:
                    yield message

            # ensure that no commits occur
            raise IgnoreCommits()