def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): """ Received a CRAWLER_DATABASE_QUERY reply. @param permid The Crawler permid @param selversion The overlay protocol version @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair @param error The error value. 0 indicates success. @param message The message payload @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload) """ if error: if DEBUG: print >> sys.stderr, "databasecrawler: handle_crawler_reply", error, message self._file.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(error), message, "\n"))) self._file.flush() else: if DEBUG: print >> sys.stderr, "databasecrawler: handle_crawler_reply", show_permid_short( permid), cPickle.loads(message) self._file.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(error), str(cPickle.loads(message)), "\n"))) self._file.flush()
def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): """ Received a CRAWLER_REPEX_QUERY reply. @param permid The Crawler permid @param selversion The overlay protocol version @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair @param error The error value. 0 indicates success. @param message The message payload @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload) """ if error: if DEBUG: print >> sys.stderr, time.asctime(),'-', "repexcrawler: handle_crawler_reply", error, message self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(error), message, "\n"))) self._file.flush() else: if DEBUG: print >> sys.stderr, time.asctime(),'-', "repexcrawler: handle_crawler_reply", show_permid_short(permid), cPickle.loads(message) # The message is pickled, which we will just write to file. # To make later parsing easier, we base64 encode it self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(error), base64.b64encode(message), "\n"))) self._file.flush()
def tryHolePunching(self): if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: first element in peerlist", self.peerlist[len(self.peerlist)-1] print >> sys.stderr, "NatCheckMsgHandler: second element in peerlist", self.peerlist[len(self.peerlist)-2] holePunchingPort = random.randrange(3200, 4200, 1) holePunchingAddr = (self.holePunchingIP, holePunchingPort) peer1 = self.peerlist[len(self.peerlist)-1] peer2 = self.peerlist[len(self.peerlist)-2] request_id = str(show_permid_short(peer1[0]) + show_permid_short(peer2[0]) + str(random.randrange(0, 1000, 1))) self.udpConnect(peer1[0], request_id, holePunchingAddr) self.udpConnect(peer2[0], request_id, holePunchingAddr) # Register peerinfo on file self._file2.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "REQUEST", request_id, show_permid(peer1[0]), str(peer1[1]), str(peer1[2]), str(self._secure_overlay.get_dns_from_peerdb(peer1[0])), show_permid(peer2[0]), str(peer2[1]), str(peer2[2]), str(self._secure_overlay.get_dns_from_peerdb(peer2[0])), "\n"))) self._file2.flush() self.trav[request_id] = (None, None) thread.start_new_thread(coordinateHolePunching, (peer1, peer2, holePunchingAddr))
def tryHolePunching(self): if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: first element in peerlist", self.peerlist[ len(self.peerlist) - 1] print >> sys.stderr, "NatCheckMsgHandler: second element in peerlist", self.peerlist[ len(self.peerlist) - 2] holePunchingPort = random.randrange(3200, 4200, 1) holePunchingAddr = (self.holePunchingIP, holePunchingPort) peer1 = self.peerlist[len(self.peerlist) - 1] peer2 = self.peerlist[len(self.peerlist) - 2] request_id = str( show_permid_short(peer1[0]) + show_permid_short(peer2[0]) + str(random.randrange(0, 1000, 1))) self.udpConnect(peer1[0], request_id, holePunchingAddr) self.udpConnect(peer2[0], request_id, holePunchingAddr) # Register peerinfo on file self._file2.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), "REQUEST", request_id, show_permid(peer1[0]), str(peer1[1]), str(peer1[2]), str(self._secure_overlay.get_dns_from_peerdb(peer1[0])), show_permid(peer2[0]), str(peer2[1]), str(peer2[2]), str(self._secure_overlay.get_dns_from_peerdb(peer2[0])), "\n"))) self._file2.flush() self.trav[request_id] = (None, None) thread.start_new_thread(coordinateHolePunching, (peer1, peer2, holePunchingAddr))
def handle_info_crawler_reply(self, permid, selversion, channel_id, error, message, request_callback): """ <<Crawler-side>> Received a CRAWLER_VIDEOPLAYBACK_INFO_QUERY reply. @param permid The Crawler permid @param selversion The overlay protocol version @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair @param error The error value. 0 indicates success. @param message The message payload @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload) """ if error: if DEBUG: print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message self._file.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), " INFO REPLY", show_permid(permid), str(error), message, "\n"))) self._file.flush() else: if DEBUG: print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short( permid), cPickle.loads(message) info = cPickle.loads(message) self._file.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), " INFO REPLY", show_permid(permid), str(error), str(info), "\n"))) self._file.flush() i = 0 for key, timestamp, piece_size, num_pieces, bitrate, nat in info: i += 1 # do not remove the first item. the list is ordered # DESC so the first item is the last that is added to # the database and we can't affored to remove it, as # it may cause exceptions in the running playback. if i == 1: sql = """ SELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50; DELETE FROM playback_event WHERE key = '%s'; """ % (key, key) else: sql = """ SELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50; DELETE FROM playback_event WHERE key = '%s'; DELETE FROM playback_info WHERE key = '%s'; """ % (key, key, key) # todo: optimize to not select key for each row request_callback(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, sql, channel_data=key, callback=self._after_event_request_callback, frequency=0)
def process_query(self, permid, d, selversion): hits = None p = None sendtorrents = False netwq = d['q'] if netwq.startswith("SIMPLE"): # remote query # Format: 'SIMPLE '+string of space separated keywords or # 'SIMPLE+METADATA' +string of space separated keywords # # In the future we could support full SQL queries: # SELECT infohash,torrent_name FROM torrent_db WHERE status = ALIVE if netwq.startswith('SIMPLE+METADATA'): q = d['q'][len('SIMPLE+METADATA '):] sendtorrents = True else: q = d['q'][len('SIMPLE '):] q = self.clean_netwq(q) q = dunno2unicode(q) kws = re.split(r'\W+', q.lower()) hits = self.search_torrents(kws, maxhits=MAX_RESULTS, sendtorrents=sendtorrents) p = self.create_remote_query_reply(d['id'], hits, selversion) elif netwq.startswith("CHANNEL"): # channel query q = d['q'][len('CHANNEL '):] q = self.clean_netwq(q) q = dunno2unicode(q) hits = self.channelcast_db.searchChannels(q) p = self.create_channel_query_reply(d['id'], hits, selversion) # log incoming query, if logfile is set if self.logfile: self.log(permid, q) m = QUERY_REPLY + p if self.overlay_log: nqueries = self.get_peer_nqueries(permid) # RECV_MSG PERMID OVERSION NUM_QUERIES MSG self.overlay_log('RECV_QRY', show_permid(permid), selversion, nqueries, repr(d)) # RPLY_QRY PERMID NUM_HITS MSG self.overlay_log('RPLY_QRY', show_permid(permid), len(hits), repr(p)) self.overlay_bridge.send(permid, m, self.send_callback) self.inc_peer_nqueries(permid)
def gotUdpConnectReply(self, permid, selversion, channel_id, channel_data, error, mhr_msg, request_callback): if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: gotMakeHoleReplyMessage" try: mhr_data = bdecode(mhr_msg) except: print_exc() print >> sys.stderr, "NatCheckMsgHandler: bad encoded data:", mhr_msg return False if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: message is", mhr_data try: request_id, reply = mhr_data.split(":") except: print_exc() print >> sys.stderr, "NatCheckMsgHandler: error in received data:", mhr_data return False if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: request_id is", request_id if request_id in self.trav: if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: request_id is in the list" peer, value = self.trav[request_id] if peer == None: # first peer reply if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: first peer reply" self.trav[request_id] = ( (permid, self._secure_overlay.get_dns_from_peerdb(permid)), reply ) elif type(peer) == TupleType: # second peer reply if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: second peer reply" # Register peerinfo on file self._file2.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " REPLY", request_id, show_permid(peer[0]), str(peer[1]), value, show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), reply, "\n"))) del self.trav[request_id] self._file2.flush()
def gotUdpConnectReply(self, permid, selversion, channel_id, channel_data, error, mhr_msg, request_callback): if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: gotMakeHoleReplyMessage" try: mhr_data = bdecode(mhr_msg) except: print_exc() print >> sys.stderr, "NatCheckMsgHandler: bad encoded data:", mhr_msg return False if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: message is", mhr_data try: request_id, reply = mhr_data.split(":") except: print_exc() print >> sys.stderr, "NatCheckMsgHandler: error in received data:", mhr_data return False if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: request_id is", request_id if request_id in self.trav: if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: request_id is in the list" peer, value = self.trav[request_id] if peer == None: # first peer reply if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: first peer reply" self.trav[request_id] = (( permid, self._secure_overlay.get_dns_from_peerdb(permid)), reply) elif type(peer) == TupleType: # second peer reply if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: second peer reply" # Register peerinfo on file self._file2.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), " REPLY", request_id, show_permid(peer[0]), str(peer[1]), value, show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), reply, "\n"))) del self.trav[request_id] self._file2.flush()
def process_query(self, permid, d, selversion): hits = None p = None sendtorrents = False netwq = d['q'] if netwq.startswith("SIMPLE"): # remote query # Format: 'SIMPLE '+string of space separated keywords or # 'SIMPLE+METADATA' +string of space separated keywords # # In the future we could support full SQL queries: # SELECT infohash,torrent_name FROM torrent_db WHERE status = ALIVE if netwq.startswith('SIMPLE+METADATA'): q = d['q'][len('SIMPLE+METADATA '):] sendtorrents = True else: q = d['q'][len('SIMPLE '):] q = self.clean_netwq(q) q = dunno2unicode(q) kws = re.split(r'\W+', q.lower()) hits = self.search_torrents(kws, maxhits=MAX_RESULTS,sendtorrents=sendtorrents) p = self.create_remote_query_reply(d['id'],hits,selversion) elif netwq.startswith("CHANNEL"): # channel query q = d['q'][len('CHANNEL '):] q = self.clean_netwq(q) q = dunno2unicode(q) hits = self.channelcast_db.searchChannels(q) p = self.create_channel_query_reply(d['id'],hits,selversion) # log incoming query, if logfile is set if self.logfile: self.log(permid, q) m = QUERY_REPLY+p if self.overlay_log: nqueries = self.get_peer_nqueries(permid) # RECV_MSG PERMID OVERSION NUM_QUERIES MSG self.overlay_log('RECV_QRY', show_permid(permid), selversion, nqueries, repr(d)) # RPLY_QRY PERMID NUM_HITS MSG self.overlay_log('RPLY_QRY', show_permid(permid), len(hits), repr(p)) self.overlay_bridge.send(permid, m, self.send_callback) self.inc_peer_nqueries(permid)
def handle_event_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): """ <<Crawler-side>> Received a CRAWLER_VIDEOPLAYBACK_EVENT_QUERY reply. @param permid The Crawler permid @param selversion The overlay protocol version @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair @param channel_data Data associated with the request @param error The error value. 0 indicates success. @param message The message payload @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload) """ if error: if DEBUG: print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message self._file.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), message, "\n"))) self._file.flush() elif selversion >= OLPROTO_VER_TENTH: # Overlay version 10 sends the reply pickled and zipped if DEBUG: print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short( permid), len(message), "bytes zipped" info = cPickle.loads(zlib.decompress(message)) self._file.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), str(info), "\n"))) self._file.flush() elif selversion >= OLPROTO_VER_EIGHTH: if DEBUG: print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short( permid), cPickle.loads(message) info = cPickle.loads(message) self._file.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), str(info), "\n"))) self._file.flush()
def formatForText(self, friends): lines = [] for friend in friends: permid = show_permid(friend['permid']) line = ', '.join([friend['name'], permid, friend['ip'], str(friend['port'])]) line += '\n' lines.append(line) return lines
def _after_request_callback(self, exc, permid): """ Called by the Crawler with the result of the request_callback call in the query_initiator method. """ if not exc: if DEBUG: print >>sys.stderr, time.asctime(),'-', "databasecrawler: request send to", show_permid_short(permid) self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "REQUEST", show_permid(permid), "\n"))) self._file.flush()
def _after_request_callback(self, exc, permid): """ Called by the Crawler with the result of the request_callback call in the query_initiator method. """ if not exc: if DEBUG: print >>sys.stderr, "repexcrawler: request sent to", show_permid_short(permid) self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "REQUEST", show_permid(permid), "\n"))) self._file.flush()
def _after_request_callback(self, exc, permid): """ Called by the Crawler with the result of the request_callback call in the query_initiator method. """ if not exc: if DEBUG: print >>sys.stderr, "puncturecrawler: request sent to", show_permid_short(permid) self._file.write("REQUEST %s %.2f\n" % (show_permid(permid), time.time())) self._file.flush()
def setUpPreSession(self): """ override TestAsServer """ print >> sys.stderr,"test: Pre Tribler Init" TestAsServer.setUpPreSession(self) print >> sys.stderr,"test: Pre Tribler Init: config_path",self.config_path # Enable dialback support self.config.set_dialback(True) self.config.set_buddycast(True) # make sure overlay connections are being made self.config.set_start_recommender(True) # Write superpeers.txt self.install_path = tempfile.mkdtemp() spdir = os.path.join(self.install_path, LIBRARYNAME, 'Core') os.makedirs(spdir) statsdir = os.path.join(self.install_path, LIBRARYNAME, 'Core', 'Statistics') os.makedirs(statsdir) superpeerfilename = os.path.join(spdir, 'superpeer.txt') print >> sys.stderr,"test: writing",self.NLISTENERS,"superpeers to",superpeerfilename f = open(superpeerfilename, "w") self.mylistenport = [] self.myss = [] self.mykeypairs = [] self.mypermids = [] for i in range(self.NLISTENERS): # Start our server side, to with Tribler will try to connect self.mylistenport.append(4810+i) self.myss.append(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) self.myss[i].setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.myss[i].bind(('', self.mylistenport[i])) self.myss[i].listen(1) self.mykeypairs.append(EC.gen_params(EC.NID_sect233k1)) self.mykeypairs[i].gen_key() self.mypermids.append(str(self.mykeypairs[i].pub().get_der())) content = '127.0.0.1, '+str(self.mylistenport[i])+', '+show_permid(self.mypermids[i])+', FakeSuperPeer\n' f.write(content) f.close() self.config.set_install_dir(self.install_path) srcfiles = [] srcfiles.append(os.path.join(LIBRARYNAME,"schema_sdb_v"+str(CURRENT_MAIN_DB_VERSION)+".sql")) srcfiles.append(os.path.join(LIBRARYNAME,"Core","Statistics","tribler_seedingstats_sdb.sql")) srcfiles.append(os.path.join(LIBRARYNAME,"Core","Statistics","tribler_friendship_stats_sdb.sql")) srcfiles.append(os.path.join(LIBRARYNAME,"Core","Statistics","tribler_videoplayback_stats.sql")) for srcfile in srcfiles: sfn = os.path.join('..','..',srcfile) dfn = os.path.join(self.install_path,srcfile) print >>sys.stderr,"test: copying",sfn,dfn shutil.copyfile(sfn,dfn) """
def formatForText(self, friends): lines = [] for friend in friends: permid = show_permid(friend['permid']) line = ', '.join( [friend['name'], permid, friend['ip'], str(friend['port'])]) line += '\n' lines.append(line) return lines
def _after_event_request_callback(self, exc, permid): """ <<Crawler-side>> Called by the Crawler with the result of the request_callback call in the handle_crawler_reply method. """ if not exc: if DEBUG: print >> sys.stderr, "videoplaybackcrawler: request send to", show_permid_short( permid) self._file.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), " INFO REQUEST", show_permid(permid), "\n"))) self._file.flush()
def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): """ Received a CRAWLER_CHANNEL_QUERY reply. @param permid The Crawler permid @param selversion The overlay protocol version @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair @param error The error value. 0 indicates success. @param message The message payload @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload) """ if error: if DEBUG: print >> sys.stderr, "channelcrawler: handle_crawler_reply", error, message self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(error), message, "\n"))) self._file.flush() else: if DEBUG: print >> sys.stderr, "channelcrawler: handle_crawler_reply", show_permid_short(permid), cPickle.loads(message) self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(error), str(cPickle.loads(message)), "\n"))) self._file.flush()
def doNatCheckCallback(self, exc, permid): if exc is not None: return False if DEBUG: print >> sys.stderr, "NATCHECK_REQUEST was sent to", show_permid_short( permid), exc # Register peerinfo on file self._file.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), "REQUEST", show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), "\n"))) self._file.flush() return True
def query_initiator(self, permid, selversion, request_callback): """ Established a new connection. Send a CRAWLER_DATABASE_QUERY request. @param permid The Tribler peer permid @param selversion The oberlay protocol version @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload) """ if DEBUG: print >> sys.stderr, "databasecrawler: query_initiator", show_permid_short( permid) sql = [] if selversion >= OLPROTO_VER_SEVENTH: sql.extend(("SELECT 'peer_count', count(*) FROM Peer", "SELECT 'torrent_count', count(*) FROM Torrent")) if selversion >= OLPROTO_VER_ELEVENTH: sql.extend(( "SELECT 'my_subscriptions', count(*) FROM VoteCast where voter_id='" + show_permid(permid) + "' and vote=2", "SELECT 'my_negative_votes', count(*) FROM VoteCast where voter_id='" + show_permid(permid) + "' and vote=-1", "SELECT 'my_channel_files', count(*) FROM ChannelCast where publisher_id='" + show_permid(permid) + "'", "SELECT 'all_subscriptions', count(*) FROM VoteCast where vote=2", "SELECT 'all_negative_votes', count(*) FROM VoteCast where vote=-1" )) # if OLPROTO_VER_EIGHTH <= selversion <= 11: # sql.extend(("SELECT 'moderations_count', count(*) FROM ModerationCast")) # if selversion >= OLPROTO_VER_EIGHTH: # sql.extend(("SELECT 'positive_votes_count', count(*) FROM Moderators where status=1", # "SELECT 'negative_votes_count', count(*) FROM Moderators where status=-1")) request_callback(CRAWLER_DATABASE_QUERY, ";".join(sql), callback=self._after_request_callback)
def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): """ Received a CRAWLER_UDPUNCTURE_QUERY reply. @param permid The Crawler permid @param selversion The overlay protocol version @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair @param error The error value. 0 indicates success. @param message The message payload @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload) """ try: if error: if DEBUG: print >> sys.stderr, "puncturecrawler: handle_crawler_reply", error, message self._file.write("ERROR %s %.2f %d %s\n" % (show_permid(permid), time.time(), error, message)) self._file.flush() else: if DEBUG: print >> sys.stderr, "puncturecrawler: handle_crawler_reply", show_permid_short(permid) # 25/05/10 Boudewijn: We found that, for unknown # reasons, the decompressed(message) contains many # gigabytes worth of \0 characters. For now we filter # them out until Gert Jan can determine the actual # cause. data = zlib.decompress(message) filtered = filter(lambda char: char != "\0", data) self._file.write("REPLY %s %.2f\n" % (show_permid(permid), time.time())) self._file.write("# reply sizes: on-the-wire=%d, decompressed=%d, filtered=%d\n" % (len(message), len(data), len(filtered))) self._file.write(filtered) self._file.flush() except: if DEBUG: print >>sys.stderr, "puncturecrawler: error writing to file"
def doNatCheckCallback(self, exc, permid): if exc is not None: return False if DEBUG: print >> sys.stderr, "NATCHECK_REQUEST was sent to", show_permid_short(permid), exc # Register peerinfo on file self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "REQUEST", show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), "\n"))) self._file.flush() return True
def udpConnectCallback(self, exc, permid): if exc is not None: if DEBUG: print >> sys.stderr, "NATTRAVERSAL_REQUEST failed to", show_permid_short(permid), exc # Register peerinfo on file self._file2.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "REQUEST FAILED", show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), "\n"))) return False if DEBUG: print >> sys.stderr, "NATTRAVERSAL_REQUEST was sent to", show_permid_short(permid), exc return True
def udpConnectCallback(self, exc, permid): if exc is not None: if DEBUG: print >> sys.stderr, "NATTRAVERSAL_REQUEST failed to", show_permid_short( permid), exc # Register peerinfo on file self._file2.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), "REQUEST FAILED", show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), "\n"))) return False if DEBUG: print >> sys.stderr, "NATTRAVERSAL_REQUEST was sent to", show_permid_short( permid), exc return True
def setUpPreSession(self): """ override TestAsServer """ print >> sys.stderr, "test: Pre Tribler Init" TestAsServer.setUpPreSession(self) print >> sys.stderr, "test: Pre Tribler Init: config_path", self.config_path # Enable dialback support self.config.set_dialback(True) self.config.set_buddycast( True) # make sure overlay connections are being made self.config.set_start_recommender(True) # Write superpeers.txt self.install_path = tempfile.mkdtemp() spdir = os.path.join(self.install_path, LIBRARYNAME, 'Core') os.makedirs(spdir) statsdir = os.path.join(self.install_path, LIBRARYNAME, 'Core', 'Statistics') os.makedirs(statsdir) superpeerfilename = os.path.join(spdir, 'superpeer.txt') print >> sys.stderr, "test: writing", self.NLISTENERS, "superpeers to", superpeerfilename f = open(superpeerfilename, "w") self.mylistenport = [] self.myss = [] self.mykeypairs = [] self.mypermids = [] for i in range(self.NLISTENERS): # Start our server side, to with Tribler will try to connect self.mylistenport.append(4810 + i) self.myss.append(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) self.myss[i].setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.myss[i].bind(('', self.mylistenport[i])) self.myss[i].listen(1) self.mykeypairs.append(EC.gen_params(EC.NID_sect233k1)) self.mykeypairs[i].gen_key() self.mypermids.append(str(self.mykeypairs[i].pub().get_der())) content = '127.0.0.1, ' + str( self.mylistenport[i]) + ', ' + show_permid( self.mypermids[i]) + ', FakeSuperPeer\n' f.write(content) f.close() self.config.set_install_dir(self.install_path) srcfiles = [] srcfiles.append(os.path.join(LIBRARYNAME, "schema_sdb_v4.sql")) srcfiles.append( os.path.join(LIBRARYNAME, "Core", "Statistics", "tribler_seedingstats_sdb.sql")) srcfiles.append( os.path.join(LIBRARYNAME, "Core", "Statistics", "tribler_friendship_stats_sdb.sql")) srcfiles.append( os.path.join(LIBRARYNAME, "Core", "Statistics", "tribler_videoplayback_stats.sql")) for srcfile in srcfiles: sfn = os.path.join('..', '..', srcfile) dfn = os.path.join(self.install_path, srcfile) print >> sys.stderr, "test: copying", sfn, dfn shutil.copyfile(sfn, dfn) """
def _after_info_request_callback(self, exc, permid): """ <<Crawler-side>> Called by the Crawler with the result of the request_callback call in the query_initiator method. """ if not exc: if DEBUG: print >>sys.stderr, "videoplaybackcrawler: request send to", show_permid_short(permid) self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "INFO REQUEST", show_permid(permid), "\n"))) self._file.flush()
def gotNatCheckReplyMessage(self, permid, selversion, channel_id, channel_data, error, payload, request_callback): """ The handle-reply callback """ if error: if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: gotNatCheckReplyMessage" print >> sys.stderr, "NatCheckMsgHandler: error", error # generic error: another crawler already obtained these results self._file.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), "ERROR(%d)" % error, payload, "\n"))) self._file.flush() else: try: recv_data = bdecode(payload) except: print_exc() print >> sys.stderr, "bad encoded data:", payload return False try: # check natCheckReply message self.validNatCheckReplyMsg(recv_data) except RuntimeError, e: print >> sys.stderr, e return False if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: received NAT_CHECK_REPLY message: ", recv_data # Register peerinfo on file self._file.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), ":".join([str(x) for x in recv_data]), "\n"))) self._file.flush() # for Tribler versions < 5.0 : do nothing if selversion < OLPROTO_VER_NINETH: if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: Tribler version too old for NATTRAVERSAL: do nothing" return True if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: do NATTRAVERSAL" # Save peer in peerlist if len(self.peerlist) == PEERLIST_LEN: del self.peerlist[0] self.peerlist.append([permid, recv_data[1], recv_data[2]]) if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: peerlist length is: ", len( self.peerlist) # Try to perform hole punching if len(self.peerlist) >= 2: self.tryHolePunching()
def handle_info_crawler_reply(self, permid, selversion, channel_id, error, message, request_callback): """ <<Crawler-side>> Received a CRAWLER_VIDEOPLAYBACK_INFO_QUERY reply. @param permid The Crawler permid @param selversion The overlay protocol version @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair @param error The error value. 0 indicates success. @param message The message payload @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload) """ if error: if DEBUG: print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " INFO REPLY", show_permid(permid), str(error), message, "\n"))) self._file.flush() else: if DEBUG: print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(permid), cPickle.loads(message) info = cPickle.loads(message) self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " INFO REPLY", show_permid(permid), str(error), str(info), "\n"))) self._file.flush() i = 0 for key, timestamp, piece_size, num_pieces, bitrate, nat in info: i += 1 # do not remove the first item. the list is ordered # DESC so the first item is the last that is added to # the database and we can't affored to remove it, as # it may cause exceptions in the running playback. if i == 1: sql = """ SELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50; DELETE FROM playback_event WHERE key = '%s'; """ % (key, key) else: sql = """ SELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50; DELETE FROM playback_event WHERE key = '%s'; DELETE FROM playback_info WHERE key = '%s'; """ % (key, key, key) # todo: optimize to not select key for each row request_callback(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, sql, channel_data=key, callback=self._after_event_request_callback, frequency=0)
def gotNatCheckReplyMessage(self, permid, selversion, channel_id, channel_data, error, payload, request_callback): """ The handle-reply callback """ if error: if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: gotNatCheckReplyMessage" print >> sys.stderr, "NatCheckMsgHandler: error", error # generic error: another crawler already obtained these results self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), "ERROR(%d)" % error, payload, "\n"))) self._file.flush() else: try: recv_data = bdecode(payload) except: print_exc() print >> sys.stderr, "bad encoded data:", payload return False try: # check natCheckReply message self.validNatCheckReplyMsg(recv_data) except RuntimeError, e: print >> sys.stderr, e return False if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: received NAT_CHECK_REPLY message: ", recv_data # Register peerinfo on file self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), ":".join([str(x) for x in recv_data]), "\n"))) self._file.flush() # for Tribler versions < 5.0 : do nothing if selversion < OLPROTO_VER_NINETH: if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: Tribler version too old for NATTRAVERSAL: do nothing" return True if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: do NATTRAVERSAL" # Save peer in peerlist if len(self.peerlist) == PEERLIST_LEN: del self.peerlist[0] self.peerlist.append([permid,recv_data[1],recv_data[2]]) if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: peerlist length is: ", len(self.peerlist) # Try to perform hole punching if len(self.peerlist) >= 2: self.tryHolePunching()
def query_initiator(self, permid, selversion, request_callback): """ Established a new connection. Send a CRAWLER_DATABASE_QUERY request. @param permid The Tribler peer permid @param selversion The oberlay protocol version @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload) """ if DEBUG: print >>sys.stderr, time.asctime(),'-', "databasecrawler: query_initiator", show_permid_short(permid) sql = [] if selversion >= OLPROTO_VER_SEVENTH: sql.extend(("SELECT 'peer_count', count(*) FROM Peer", "SELECT 'torrent_count', count(*) FROM Torrent")) if selversion >= OLPROTO_VER_ELEVENTH: sql.extend(("SELECT 'my_subscriptions', count(*) FROM VoteCast where voter_id='" + show_permid(permid) + "' and vote=2", "SELECT 'my_negative_votes', count(*) FROM VoteCast where voter_id='" + show_permid(permid) + "' and vote=-1", "SELECT 'my_channel_files', count(*) FROM ChannelCast where publisher_id='" + show_permid(permid) + "'", "SELECT 'all_subscriptions', count(*) FROM VoteCast where vote=2", "SELECT 'all_negative_votes', count(*) FROM VoteCast where vote=-1")) # if OLPROTO_VER_EIGHTH <= selversion <= 11: # sql.extend(("SELECT 'moderations_count', count(*) FROM ModerationCast")) # if selversion >= OLPROTO_VER_EIGHTH: # sql.extend(("SELECT 'positive_votes_count', count(*) FROM Moderators where status=1", # "SELECT 'negative_votes_count', count(*) FROM Moderators where status=-1")) request_callback(CRAWLER_DATABASE_QUERY, ";".join(sql), callback=self._after_request_callback)
def handle_event_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): """ <<Crawler-side>> Received a CRAWLER_VIDEOPLAYBACK_EVENT_QUERY reply. @param permid The Crawler permid @param selversion The overlay protocol version @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair @param channel_data Data associated with the request @param error The error value. 0 indicates success. @param message The message payload @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload) """ if error: if DEBUG: print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), message, "\n"))) self._file.flush() elif selversion >= OLPROTO_VER_TENTH: # Overlay version 10 sends the reply pickled and zipped if DEBUG: print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(permid), len(message), "bytes zipped" info = cPickle.loads(zlib.decompress(message)) self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), str(info), "\n"))) self._file.flush() elif selversion >= OLPROTO_VER_EIGHTH: if DEBUG: print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(permid), cPickle.loads(message) info = cPickle.loads(message) self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), str(info), "\n"))) self._file.flush()
def query_initiator(self, permid, selversion, request_callback): """ Established a new connection. Send a CRAWLER_CHANNEL_QUERY request. @param permid The Tribler peer permid @param selversion The oberlay protocol version @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload) """ if DEBUG: print >>sys.stderr, "channelcrawler: query_initiator", show_permid_short(permid) sql = [] if selversion >= OLPROTO_VER_THIRTEENTH: sql.extend(("SELECT 'channel_files', publisher_id, count(*) FROM ChannelCast group by publisher_id", "SELECT 'my_votes', mod_id, voter_id, vote, time_stamp FROM VoteCast where voter_id='" + show_permid(permid) + "' order by time_stamp desc limit 100", "SELECT 'other_votes', mod_id, voter_id, vote, time_stamp FROM VoteCast where voter_id<>'" + show_permid(permid) + "' order by time_stamp desc limit 100")) request_callback(CRAWLER_CHANNEL_QUERY, ";".join(sql), callback=self._after_request_callback)