def join_community(self, my_member): self.my_member = my_member master_key = "3081a7301006072a8648ce3d020106052b81040027038192000404f10c33b03d2a09943d6d6a4b2cf4fe3129e5dce1df446a27d0ce00d48c845a4eff8102ef3becd6bc07c65953c824d227ebc110016d5ba71163bf6fb83fde7cdccf164bb007e27d07da952c47d30cf9c843034dc7a4603af3a84f8997e5d046e6a5f1ad489add6878898079a4663ade502829577c7d1e27302a3d5ea0ae06e83641a093a87465fdd4a3b43e031a9555".decode("HEX") master = Member(master_key) log(self._logfile, "joining community with kargs", kargs=self.community_kargs) if self.community_type == 'search': community = SearchCommunity.join_community(master, self.my_member, self.my_member, integrate_with_tribler=False, log_searches=True, **self.community_kargs) elif self.community_type == 'hsearch': community = HSearchCommunity.join_community(master, self.my_member, self.my_member, integrate_with_tribler=False, log_searches=True, **self.community_kargs) else: community = PSearchCommunity.join_community(master, self.my_member, self.my_member, integrate_with_tribler=False, log_searches=True, **self.community_kargs) self._add_taste_buddies = community.add_taste_buddies community.add_taste_buddies = self.log_taste_buddies self._manual_create_introduction_request = community.create_introduction_request if self.manual_connect: community.create_introduction_request = lambda destination, allow_sync: self._manual_create_introduction_request(destination, False) if int(self._my_name) <= self.late_join: self._create_introduction_request = community.create_introduction_request community.create_introduction_request = lambda *args: None self._dispersy.callback.register(self.monitor_taste_buddy, delay=1.0) return community
def execute_scenario_cmds(self, commands): for command in commands: cur_command = command.split() if cur_command[0] == 'download': infohash = cur_command[1] infohash = infohash + " "* (20 - len(infohash)) log(self._logfile, "registering download %s" % infohash) self._community._mypref_db.addMyPreference(infohash, {}) elif cur_command[0] == 'testset': infohash = cur_command[1] infohash = infohash + " "* (20 - len(infohash)) self.test_set.add(infohash) self._community._mypref_db.addTestPreference(infohash) elif cur_command[0] == 'taste_buddy': peer_id = int(cur_command[1]) ip, port = self.get_peer_ip_port(peer_id) self.taste_buddies.add((ip, port)) self.not_connected_taste_buddies.add((ip, port)) # connect to first 10 if len(self.taste_buddies) <= (10 * self.bootstrap_percentage): log(self._logfile, "new taste buddy %s:%d" % (ip, port)) if int(self._my_name) > self.late_join: self._dispersy.callback.register(self.connect_to_taste_buddy, args=((ip, port),), delay=1.0)
def execute_scenario_cmds(self, commands): for command in commands: cur_command = command.split() if cur_command[0] == 'download': infohash = cur_command[1] infohash = infohash + " " * (20 - len(infohash)) log(self._logfile, "registering download %s" % infohash) self._community._mypref_db.addMyPreference(infohash, {}) elif cur_command[0] == 'testset': infohash = cur_command[1] infohash = infohash + " " * (20 - len(infohash)) self.test_set.add(infohash) self._community._mypref_db.addTestPreference(infohash) elif cur_command[0] == 'taste_buddy': peer_id = int(cur_command[1]) ip, port = self.get_peer_ip_port(peer_id) self.taste_buddies.add((ip, port)) self.not_connected_taste_buddies.add((ip, port)) # connect to first 10 if len(self.taste_buddies) <= (10 * self.bootstrap_percentage): log(self._logfile, "new taste buddy %s:%d" % (ip, port)) if int(self._my_name) > self.late_join: self._dispersy.callback.register( self.connect_to_taste_buddy, args=((ip, port), ), delay=1.0)
def execute_scenario_cmds(self, commands): for command in commands: cur_command = command.split() if cur_command[0] == 'publish': log(self._logfile, "creating-text") text = u''.join(choice(letters) for _ in xrange(100)) self._community.create_text(text)
def execute_scenario_cmds(self, commands): for command in commands: cur_command = command.split() if cur_command[0] == 'publish': log(self._logfile, "creating-text") text = u''.join(choice(letters) for _ in xrange(100)) self._community.create_text(text)
def execute_scenario_cmds(self, commands): torrents = [] for command in commands: cur_command = command.split() if cur_command[0] == 'create': log(self._logfile, "creating-community") self.my_channel = ChannelCommunity.create_community(self._dispersy, self.my_member, integrate_with_tribler=False) log(self._logfile, "creating-channel-message") self.my_channel.create_channel(u'', u'') elif cur_command[0] == 'publish': if self.my_channel: infohash = str(self.torrentindex) infohash += ''.join(choice(letters) for _ in xrange(20 - len(infohash))) name = u''.join(choice(letters) for _ in xrange(100)) files = [] for _ in range(10): files.append((u''.join(choice(letters) for _ in xrange(30)), 123455)) trackers = [] for _ in range(10): trackers.append(''.join(choice(letters) for _ in xrange(30))) files = tuple(files) trackers = tuple(trackers) self.torrentindex += 1 torrents.append((infohash, int(time()), name, files, trackers)) elif cur_command[0] == 'post': if self.joined_community: text = ''.join(choice(letters) for i in xrange(160)) self.joined_community._disp_create_comment(text, int(time()), None, None, None, None) elif cur_command[0] == 'join': self.want_to_join = True if self.want_to_join: log(self._logfile, "trying-to-join-community") cid = self._community._channelcast_db.getChannelIdFromDispersyCID(None) if cid: community = self._community._get_channel_community(cid) if community._channel_id: self._community.disp_create_votecast(community.cid, 2, int(time())) log(self._logfile, "joining-community") self.joined_community = community self.want_to_join = False if len(torrents) > 0: log(self._logfile, "creating-torrents") self.my_channel._disp_create_torrents(torrents)
def log_search_response(self, keywords, results, candidate): for result in results: if result[0] in self.test_set: self.test_reply.add(result[0]) recall = len(self.test_reply) / float(len(self.test_set)) if results: log(self._logfile, "results", recall=recall, keywords=keywords, candidate=str(candidate), results=results) else: log(self._logfile, "no results", recall=recall, keywords=keywords, candidate=str(candidate))
def log_taste_buddies(self, new_taste_buddies): self._add_taste_buddies(new_taste_buddies) for taste_buddy in new_taste_buddies: log(self._logfile, "new taste buddy", sim=taste_buddy[0], sock=str(taste_buddy[-1]), is_tb=self._community.is_taste_buddy(taste_buddy[-1])) if taste_buddy[-1].sock_addr in self.not_connected_taste_buddies and not self._community.is_taste_buddy(taste_buddy[-1]): log(self._logfile, "currentlist", list=[map(str, tup) for tup in self._community.taste_buddies]) self.did_reply.add(taste_buddy[-1].sock_addr)
def connect_to_taste_buddy(self, sock_addr): candidate = self._dispersy.get_candidate(sock_addr, replace=False) if not candidate: candidate = self._community.create_candidate(sock_addr, False, sock_addr, sock_addr, u"unknown") while not self._community.is_taste_buddy(candidate): log(self._logfile, "sending introduction request to %s" % str(candidate)) self._manual_create_introduction_request(candidate, True) yield IntroductionRequestCache.timeout_delay + IntroductionRequestCache.cleanup_delay
def perform_searches(self): for infohash in (self.test_set - self.test_reply): candidates, local_results = self._community.create_search([unicode(infohash)], self.log_search_response) candidates = map(str, candidates) log(self._logfile, "send search query for '%s' to %d candidates" % (infohash, len(candidates)), candidates=candidates) if local_results: self.log_search_response([unicode(infohash)], local_results, None) yield 15.0
def connect_to_taste_buddy(self, sock_addr): candidate = self._community.get_candidate(sock_addr, replace=False) if not candidate: candidate = self._community.create_candidate( sock_addr, False, sock_addr, sock_addr, u"unknown") while not self._community.is_taste_buddy(candidate): log(self._logfile, "sending introduction request to %s" % str(candidate)) self._manual_create_introduction_request(candidate, True) yield IntroductionRequestCache.timeout_delay + IntroductionRequestCache.cleanup_delay
def join_community(self, my_member): self.my_member = my_member master_key = "3081a7301006072a8648ce3d020106052b81040027038192000404f10c33b03d2a09943d6d6a4b2cf4fe3129e5dce1df446a27d0ce00d48c845a4eff8102ef3becd6bc07c65953c824d227ebc110016d5ba71163bf6fb83fde7cdccf164bb007e27d07da952c47d30cf9c843034dc7a4603af3a84f8997e5d046e6a5f1ad489add6878898079a4663ade502829577c7d1e27302a3d5ea0ae06e83641a093a87465fdd4a3b43e031a9555".decode( "HEX") master = Member(master_key) log(self._logfile, "joining community with kargs", kargs=self.community_kargs) if self.community_type == 'search': community = SearchCommunity.join_community( master, self.my_member, self.my_member, integrate_with_tribler=False, log_searches=True, **self.community_kargs) elif self.community_type == 'hsearch': community = HSearchCommunity.join_community( master, self.my_member, self.my_member, integrate_with_tribler=False, log_searches=True, **self.community_kargs) else: community = PSearchCommunity.join_community( master, self.my_member, self.my_member, integrate_with_tribler=False, log_searches=True, **self.community_kargs) self._add_taste_buddies = community.add_taste_buddies community.add_taste_buddies = self.log_taste_buddies self._manual_create_introduction_request = community.create_introduction_request if self.manual_connect: community.create_introduction_request = lambda destination, allow_sync: self._manual_create_introduction_request( destination, False) if int(self._my_name) <= self.late_join: self._create_introduction_request = community.create_introduction_request community.create_introduction_request = lambda *args: None self._dispersy.callback.register(self.monitor_taste_buddy, delay=1.0) return community
def perform_searches(self): for infohash in (self.test_set - self.test_reply): candidates, local_results = self._community.create_search( [unicode(infohash)], self.log_search_response) candidates = map(str, candidates) log(self._logfile, "send search query for '%s' to %d candidates" % (infohash, len(candidates)), candidates=candidates) if local_results: self.log_search_response([unicode(infohash)], local_results, None) yield 5.0
def on_channelcast(self, messages): for message in messages: if DEBUG: print >> sys.stderr, "AllChannelCommunity: received channelcast message" toCollect = {} for cid, torrents in message.payload.torrents.iteritems(): for infohash in self._selectTorrentsToCollect(cid, torrents): toCollect.setdefault(cid,set()).add(infohash) nr_requests = sum([len(torrents) for torrents in toCollect.values()]) if nr_requests > 0: self.create_channelcast_request(toCollect, message.candidate) if __debug__: if not self.integrate_with_tribler: log("dispersy.log", "requesting-torrents", nr_requests = nr_requests)
def on_channelcast(self, messages): for message in messages: if DEBUG: print >> sys.stderr, "AllChannelCommunity: received channelcast message" toCollect = {} for cid, torrents in message.payload.torrents.iteritems(): for infohash in self._selectTorrentsToCollect(cid, torrents): toCollect.setdefault(cid,set()).add(infohash) nr_requests = sum([len(torrents) for torrents in toCollect.values()]) if nr_requests > 0: self.create_channelcast_request(toCollect, message.candidate) if __debug__: if not self.integrate_with_tribler: log("dispersy.log", "requesting-torrents", nr_requests = nr_requests)
def log_statistics(self): while True: latejoin = taste_ratio = 0 if len(self.taste_buddies): connected_taste_buddies = len(self.taste_buddies) - len(self.not_connected_taste_buddies) ratio = connected_taste_buddies / min(10.0, float(len(self.taste_buddies))) if int(self._my_name) <= self.late_join: latejoin = ratio / float(self.late_join) else: taste_ratio = ratio / float(self._nr_peers - self.late_join) recall = len(self.test_reply) / float(len(self.test_set)) recall /= float(self.do_search) log("dispersy.log", "scenario-statistics", bootstrapped=taste_ratio, latejoin=latejoin, recall=recall, nr_search_=self.nr_search) log("dispersy.log", "scenario-debug", not_connected=list(self.not_connected_taste_buddies), search_forward=self._community.search_forward, search_forward_success=self._community.search_forward_success, search_forward_timeout=self._community.search_forward_timeout, search_endpoint=self._community.search_endpoint, search_cycle_detected=self._community.search_cycle_detected, search_megacachesize=self._community.search_megacachesize, create_time_encryption=self._community.create_time_encryption, create_time_decryption=self._community.create_time_decryption, receive_time_encryption=self._community.receive_time_encryption) yield 5.0
def log_search_response(self, keywords, results, candidate): for result in results: if result[0] in self.test_set: self.test_reply.add(result[0]) recall = len(self.test_reply) / float(len(self.test_set)) if results: log(self._logfile, "results", recall=recall, keywords=keywords, candidate=str(candidate), results=results) else: log(self._logfile, "no results", recall=recall, keywords=keywords, candidate=str(candidate))
def log_search_response(self, keywords, results, candidate): for result in results: if result[0] in self.test_set: ip, port = result[1].split() peer = self.get_peer_id(ip, port) self.test_reply[result[0]].append(peer) recall = len(self.test_reply) / float(len(self.test_set)) paths_found = sum(len(paths) for paths in self.test_reply.itervalues()) sources_found = 0 for infohash, peers in self.test_reply.iteritems(): sources_found += sum(peer in self.file_availability[infohash] for peer in set(peers)) unique_sources = float( sum([ len(self.file_availability[infohash]) for infohash in self.test_reply.iterkeys() ])) if unique_sources: sources_found = sources_found / unique_sources paths_found = paths_found / unique_sources from Tribler.dispersy.tool.lencoder import log if results: log("dispersy.log", "results", recall=recall, paths_found=paths_found, sources_found=sources_found, keywords=keywords, candidate=str(candidate), results=results, unique_sources=unique_sources) else: log("dispersy.log", "no results", recall=recall, paths_found=paths_found, sources_found=sources_found, keywords=keywords, candidate=str(candidate), unique_sources=unique_sources)
def log_taste_buddies(self, new_taste_buddies): self._add_taste_buddies(new_taste_buddies) for taste_buddy in new_taste_buddies: log(self._logfile, "new taste buddy", sim=taste_buddy[0], sock=str(taste_buddy[-1]), is_tb=self._community.is_taste_buddy(taste_buddy[-1])) if taste_buddy[ -1].sock_addr in self.not_connected_taste_buddies and not self._community.is_taste_buddy( taste_buddy[-1]): log(self._logfile, "currentlist", list=[ map(str, tup) for tup in self._community.taste_buddies ]) self.did_reply.add(taste_buddy[-1].sock_addr)
def log_statistics(self): while True: latejoin = taste_ratio = 0 if len(self.taste_buddies): connected_taste_buddies = len(self.taste_buddies) - len( self.not_connected_taste_buddies) ratio = connected_taste_buddies / min( 10.0, float(len(self.taste_buddies))) if int(self._my_name) <= self.late_join: latejoin = ratio / float(self.late_join) else: taste_ratio = ratio / float(self._nr_peers - self.late_join) recall = len(self.test_reply) / float(len(self.test_set)) recall /= float(self._nr_peers) log("dispersy.log", "scenario-statistics", bootstrapped=taste_ratio, latejoin=latejoin, recall=recall, nr_search_=self.nr_search) log("dispersy.log", "scenario-debug", not_connected=list(self.not_connected_taste_buddies), search_forward=self._community.search_forward, search_forward_success=self._community.search_forward_success, search_forward_timeout=self._community.search_forward_timeout, search_endpoint=self._community.search_endpoint, search_cycle_detected=self._community.search_cycle_detected, search_megacachesize=self._community.search_megacachesize, create_time_encryption=self._community.create_time_encryption, create_time_decryption=self._community.create_time_decryption, receive_time_encryption=self._community.receive_time_encryption ) yield 5.0
def log_search_response(self, keywords, results, candidate): for result in results: if result[0] in self.test_set: ip, port = result[1].split() peer = self.get_peer_id(ip, port) self.test_reply[result[0]].append(peer) recall = len(self.test_reply) / float(len(self.test_set)) paths_found = sum(len(paths) for paths in self.test_reply.itervalues()) sources_found = 0 for infohash, peers in self.test_reply.iteritems(): sources_found += sum(peer in self.file_availability[infohash] for peer in set(peers)) unique_sources = float(sum([len(self.file_availability[infohash]) for infohash in self.test_reply.iterkeys()])) if unique_sources: sources_found = sources_found / unique_sources paths_found = paths_found / unique_sources from Tribler.dispersy.tool.lencoder import log if results: log("dispersy.log", "results", recall=recall, paths_found=paths_found, sources_found=sources_found, keywords=keywords, candidate=str(candidate), results=results, unique_sources=unique_sources) else: log("dispersy.log", "no results", recall=recall, paths_found=paths_found, sources_found=sources_found, keywords=keywords, candidate=str(candidate), unique_sources=unique_sources)
def execute_scenario_cmds(self, commands): torrents = [] for command in commands: cur_command = command.split() if cur_command[0] == 'create': log(self._logfile, "creating-community") self.my_channel = ChannelCommunity.create_community( self._dispersy, self.my_member, integrate_with_tribler=False) log(self._logfile, "creating-channel-message") self.my_channel.create_channel(u'', u'') elif cur_command[0] == 'publish': if self.my_channel: infohash = str(self.torrentindex) infohash += ''.join( choice(letters) for _ in xrange(20 - len(infohash))) name = u''.join(choice(letters) for _ in xrange(100)) files = [] for _ in range(10): files.append( (u''.join(choice(letters) for _ in xrange(30)), 123455)) trackers = [] for _ in range(10): trackers.append(''.join( choice(letters) for _ in xrange(30))) files = tuple(files) trackers = tuple(trackers) self.torrentindex += 1 torrents.append( (infohash, int(time()), name, files, trackers)) elif cur_command[0] == 'post': if self.joined_community: text = ''.join(choice(letters) for i in xrange(160)) self.joined_community._disp_create_comment( text, int(time()), None, None, None, None) elif cur_command[0] == 'join': self.want_to_join = True if self.want_to_join: log(self._logfile, "trying-to-join-community") cid = self._community._channelcast_db.getChannelIdFromDispersyCID( None) if cid: community = self._community._get_channel_community(cid) if community._channel_id: self._community.disp_create_votecast( community.cid, 2, int(time())) log(self._logfile, "joining-community") self.joined_community = community self.want_to_join = False if len(torrents) > 0: log(self._logfile, "creating-torrents") self.my_channel._disp_create_torrents(torrents)
def on_text(self, messages): for message in messages: log("dispersy.log", "handled-record", type="text", global_time=message._distribution.global_time)
def create_channelcast(self): mychannel_id = None while True: try: now = time() favoriteTorrents = None normalTorrents = None #cleanup blocklist for candidate in self._blocklist.keys(): if self._blocklist[candidate] + CHANNELCAST_BLOCK_PERIOD < now: #unblock address self._blocklist.pop(candidate) #fetch mychannel_id if neccesary if mychannel_id == None: mychannel_id = self._channelcast_db.getMyChannelId() #loop through all candidates to see if we can find a non-blocked address for candidate in [candidate for candidate in self._iter_categories([u'walk', u'stumble'], once = True) if not candidate in self._blocklist]: if not candidate: continue didFavorite = False #only check if we actually have a channel if mychannel_id: peer_ids = set() for member in candidate.get_members(self): key = member.public_key peer_ids.add(self._peer_db.addOrGetPeerID(key)) #see if all members on this address are subscribed to my channel didFavorite = len(peer_ids) > 0 for peer_id in peer_ids: vote = self._votecast_db.getVoteForMyChannel(peer_id) if vote != 2: didFavorite = False break #Modify type of message depending on if all peers have marked my channels as their favorite if didFavorite: if not favoriteTorrents: favoriteTorrents = self._channelcast_db.getRecentAndRandomTorrents(0, 0, 25, 25 ,5) torrents = favoriteTorrents else: if not normalTorrents: normalTorrents = self._channelcast_db.getRecentAndRandomTorrents() torrents = normalTorrents if len(torrents) > 0: meta = self.get_meta_message(u"channelcast") message = meta.impl(authentication=(self._my_member,), distribution=(self.global_time,), destination=(candidate,), payload=(torrents,)) self._dispersy._forward([message]) #we've send something to this address, add to blocklist self._blocklist[candidate] = now if DEBUG: nr_torrents = sum(len(torrent) for torrent in torrents.values()) print >> sys.stderr, "AllChannelCommunity: sending channelcast message containing",nr_torrents,"torrents to",candidate.sock_addr,"didFavorite",didFavorite if __debug__: if not self.integrate_with_tribler: nr_torrents = sum(len(torrent) for torrent in torrents.values()) log("dispersy.log", "Sending channelcast message containing %d torrents to %s didFavorite %s"%(nr_torrents,candidate.sock_addr,didFavorite)) #we're done break else: if DEBUG: print >> sys.stderr, "AllChannelCommunity: no candidates to send channelcast message too" if __debug__: if not self.integrate_with_tribler: log("dispersy.log", "Could not send channelcast message, no candidates") except: print_exc() yield CHANNELCAST_INTERVAL
def on_text(self, messages): for message in messages: log("dispersy.log", "handled-record", type="text", global_time=message._distribution.global_time)
def create_channelcast(self): mychannel_id = None while True: try: now = time() favoriteTorrents = None normalTorrents = None #cleanup blocklist for candidate in self._blocklist.keys(): if self._blocklist[candidate] + CHANNELCAST_BLOCK_PERIOD < now: #unblock address self._blocklist.pop(candidate) #fetch mychannel_id if neccesary if mychannel_id == None: mychannel_id = self._channelcast_db.getMyChannelId() #loop through all candidates to see if we can find a non-blocked address for candidate in [candidate for candidate in self._iter_categories([u'walk', u'stumble'], once = True) if not candidate in self._blocklist]: if not candidate: continue didFavorite = False #only check if we actually have a channel if mychannel_id: peer_ids = set() for member in candidate.get_members(self): key = member.public_key peer_ids.add(self._peer_db.addOrGetPeerID(key)) #see if all members on this address are subscribed to my channel didFavorite = len(peer_ids) > 0 for peer_id in peer_ids: vote = self._votecast_db.getVoteForMyChannel(peer_id) if vote != 2: didFavorite = False break #Modify type of message depending on if all peers have marked my channels as their favorite if didFavorite: if not favoriteTorrents: favoriteTorrents = self._channelcast_db.getRecentAndRandomTorrents(0, 0, 25, 25 ,5) torrents = favoriteTorrents else: if not normalTorrents: normalTorrents = self._channelcast_db.getRecentAndRandomTorrents() torrents = normalTorrents if len(torrents) > 0: meta = self.get_meta_message(u"channelcast") message = meta.impl(authentication=(self._my_member,), distribution=(self.global_time,), destination=(candidate,), payload=(torrents,)) self._dispersy._forward([message]) #we've send something to this address, add to blocklist self._blocklist[candidate] = now if DEBUG: nr_torrents = sum(len(torrent) for torrent in torrents.values()) print >> sys.stderr, "AllChannelCommunity: sending channelcast message containing",nr_torrents,"torrents to",candidate.sock_addr,"didFavorite",didFavorite if __debug__: if not self.integrate_with_tribler: nr_torrents = sum(len(torrent) for torrent in torrents.values()) log("dispersy.log", "Sending channelcast message containing %d torrents to %s didFavorite %s"%(nr_torrents,candidate.sock_addr,didFavorite)) #we're done break else: if DEBUG: print >> sys.stderr, "AllChannelCommunity: no candidates to send channelcast message too" if __debug__: if not self.integrate_with_tribler: log("dispersy.log", "Could not send channelcast message, no candidates") except: print_exc() yield CHANNELCAST_INTERVAL