def handle_event_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): if error: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', error, message self._file.write('; '.join( (strftime('%Y/%m/%d %H:%M:%S'), ' EVENT REPLY', show_permid(permid), str(error), str(channel_data), message, '\n'))) self._file.flush() elif selversion >= OLPROTO_VER_TENTH: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', show_permid_short( permid), len(message), 'bytes zipped' info = cPickle.loads(zlib.decompress(message)) self._file.write('; '.join( (strftime('%Y/%m/%d %H:%M:%S'), ' EVENT REPLY', show_permid(permid), str(error), str(channel_data), str(info), '\n'))) self._file.flush() elif selversion >= OLPROTO_VER_EIGHTH: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', show_permid_short( permid), cPickle.loads(message) info = cPickle.loads(message) self._file.write('; '.join( (strftime('%Y/%m/%d %H:%M:%S'), ' EVENT REPLY', show_permid(permid), str(error), str(channel_data), str(info), '\n'))) self._file.flush()
def process_query(self, permid, d, selversion): hits = None p = None sendtorrents = False netwq = d['q'] if netwq.startswith('SIMPLE'): if netwq.startswith('SIMPLE+METADATA'): q = d['q'][len('SIMPLE+METADATA '):] sendtorrents = True else: q = d['q'][len('SIMPLE '):] uq = self.clean_netwq(q) kws = split_into_keywords(uq) hits = self.search_torrents(kws, maxhits=MAX_RESULTS, sendtorrents=sendtorrents) p = self.create_remote_query_reply(d['id'], hits, selversion) if self.logfile: self.log(permid, q) if p: m = QUERY_REPLY + p if self.overlay_log: nqueries = self.get_peer_nqueries(permid) self.overlay_log('RECV_QRY', show_permid(permid), selversion, nqueries, repr(d)) self.overlay_log('RPLY_QRY', show_permid(permid), len(hits), repr(p)) self.overlay_bridge.send(permid, m, self.send_callback) self.inc_peer_nqueries(permid)
def handle_info_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): if error: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', error, message self._file.write('; '.join( (strftime('%Y/%m/%d %H:%M:%S'), ' INFO REPLY', show_permid(permid), str(error), message, '\n'))) self._file.flush() else: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', show_permid_short( permid), cPickle.loads(message) info = cPickle.loads(message) self._file.write('; '.join( (strftime('%Y/%m/%d %H:%M:%S'), ' INFO REPLY', show_permid(permid), str(error), str(info), '\n'))) self._file.flush() i = 0 for key, timestamp, piece_size, num_pieces, bitrate, nat in info: i += 1 if i == 1: sql = "\nSELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;\nDELETE FROM playback_event WHERE key = '%s';\n" % ( key, key) else: sql = "\nSELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;\nDELETE FROM playback_event WHERE key = '%s';\nDELETE FROM playback_info WHERE key = '%s';\n" % ( key, key, key) request_callback(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, sql, channel_data=key, callback=self._after_event_request_callback, frequency=0)
def tryHolePunching(self): if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: first element in peerlist', self.peerlist[len(self.peerlist) - 1] print >> sys.stderr, 'NatCheckMsgHandler: second element in peerlist', self.peerlist[len(self.peerlist) - 2] holePunchingPort = random.randrange(3200, 4200, 1) holePunchingAddr = (self.holePunchingIP, holePunchingPort) peer1 = self.peerlist[len(self.peerlist) - 1] peer2 = self.peerlist[len(self.peerlist) - 2] request_id = str(show_permid_short(peer1[0]) + show_permid_short(peer2[0]) + str(random.randrange(0, 1000, 1))) self.udpConnect(peer1[0], request_id, holePunchingAddr) self.udpConnect(peer2[0], request_id, holePunchingAddr) self._file2.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST', request_id, show_permid(peer1[0]), str(peer1[1]), str(peer1[2]), str(self._secure_overlay.get_dns_from_peerdb(peer1[0])), show_permid(peer2[0]), str(peer2[1]), str(peer2[2]), str(self._secure_overlay.get_dns_from_peerdb(peer2[0])), '\n'))) self._file2.flush() self.trav[request_id] = (None, None) thread.start_new_thread(coordinateHolePunching, (peer1, peer2, holePunchingAddr))
def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): if error: if DEBUG: print >>sys.stderr, "databasecrawler: handle_crawler_reply", error, message self._file.write( "; ".join((strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(error), message, "\n")) ) self._file.flush() else: if DEBUG: print >>sys.stderr, "databasecrawler: handle_crawler_reply", show_permid_short(permid), cPickle.loads( message ) self._file.write( "; ".join( ( strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(error), str(cPickle.loads(message)), "\n", ) ) ) self._file.flush()
def handle_event_crawler_reply( self, permid, selversion, channel_id, channel_data, error, message, request_callback ): if error: if DEBUG: print >>sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message self._file.write( "; ".join( ( strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), message, "\n", ) ) ) self._file.flush() elif selversion >= OLPROTO_VER_TENTH: if DEBUG: print >>sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(permid), len( message ), "bytes zipped" info = cPickle.loads(zlib.decompress(message)) self._file.write( "; ".join( ( strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), str(info), "\n", ) ) ) self._file.flush() elif selversion >= OLPROTO_VER_EIGHTH: if DEBUG: print >>sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short( permid ), cPickle.loads(message) info = cPickle.loads(message) self._file.write( "; ".join( ( strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), str(info), "\n", ) ) ) self._file.flush()
def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): try: if error: if DEBUG: print >> sys.stderr, 'puncturecrawler: handle_crawler_reply', error, message self._file.write( 'ERROR %s %.2f %d %s\n' % (show_permid(permid), time.time(), error, message)) self._file.flush() else: if DEBUG: print >> sys.stderr, 'puncturecrawler: handle_crawler_reply', show_permid_short( permid) data = zlib.decompress(message) filtered = filter(lambda char: char != '\x00', data) self._file.write('REPLY %s %.2f\n' % (show_permid(permid), time.time())) self._file.write( '# reply sizes: on-the-wire=%d, decompressed=%d, filtered=%d\n' % (len(message), len(data), len(filtered))) self._file.write(filtered) self._file.flush() except: if DEBUG: print >> sys.stderr, 'puncturecrawler: error writing to file'
def gotNatCheckReplyMessage(self, permid, selversion, channel_id, channel_data, error, payload, request_callback): if error: if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: gotNatCheckReplyMessage' print >> sys.stderr, 'NatCheckMsgHandler: error', error self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), ' REPLY', show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), 'ERROR(%d)' % error, payload, '\n'))) self._file.flush() else: try: recv_data = bdecode(payload) except: print_exc() print >> sys.stderr, 'bad encoded data:', payload return False try: self.validNatCheckReplyMsg(recv_data) except RuntimeError as e: print >> sys.stderr, e return False if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: received NAT_CHECK_REPLY message: ', recv_data self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), ' REPLY', show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), ':'.join([ str(x) for x in recv_data ]), '\n'))) self._file.flush() if selversion < OLPROTO_VER_NINETH: if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: ACEStream version too old for NATTRAVERSAL: do nothing' return True if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: do NATTRAVERSAL' if len(self.peerlist) == PEERLIST_LEN: del self.peerlist[0] self.peerlist.append([permid, recv_data[1], recv_data[2]]) if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: peerlist length is: ', len(self.peerlist) if len(self.peerlist) >= 2: self.tryHolePunching() return True
def _after_request_callback(self, exc, permid): if not exc: if DEBUG: print >> sys.stderr, 'puncturecrawler: request sent to', show_permid_short( permid) self._file.write('REQUEST %s %.2f\n' % (show_permid(permid), time.time())) self._file.flush()
def _after_request_callback(self, exc, permid): if not exc: if DEBUG: print >> sys.stderr, 'databasecrawler: request send to', show_permid_short( permid) self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST', show_permid(permid), '\n'))) self._file.flush()
def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): if error: if DEBUG: print >> sys.stderr, 'databasecrawler: handle_crawler_reply', error, message self._file.write('; '.join( (strftime('%Y/%m/%d %H:%M:%S'), ' REPLY', show_permid(permid), str(error), message, '\n'))) self._file.flush() else: if DEBUG: print >> sys.stderr, 'databasecrawler: handle_crawler_reply', show_permid_short( permid), cPickle.loads(message) self._file.write('; '.join( (strftime('%Y/%m/%d %H:%M:%S'), ' REPLY', show_permid(permid), str(error), str(cPickle.loads(message)), '\n'))) self._file.flush()
def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): if error: if DEBUG: print >> sys.stderr, 'usereventlogcrawler: handle_crawler_reply', error, message msg = '; '.join(['REPLY', show_permid(permid), str(error), str(message)]) self.__log(msg) else: if DEBUG: print >> sys.stderr, 'usereventlogcrawler: handle_crawler_reply', show_permid_short(permid), cPickle.loads(message) msg = '; '.join(['REPLY', show_permid(permid), str(error), str(cPickle.loads(message))]) self.__log(msg)
def _after_request_callback(self, exc, permid): if not exc: if DEBUG: print >> sys.stderr, 'repexcrawler: request sent to', show_permid_short(permid) self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST', show_permid(permid), '\n'))) self._file.flush()
def gotUdpConnectReply(self, permid, selversion, channel_id, channel_data, error, mhr_msg, request_callback): if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: gotMakeHoleReplyMessage' try: mhr_data = bdecode(mhr_msg) except: print_exc() print >> sys.stderr, 'NatCheckMsgHandler: bad encoded data:', mhr_msg return False if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: message is', mhr_data try: request_id, reply = mhr_data.split(':') except: print_exc() print >> sys.stderr, 'NatCheckMsgHandler: error in received data:', mhr_data return False if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: request_id is', request_id if request_id in self.trav: if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: request_id is in the list' peer, value = self.trav[request_id] if peer == None: if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: first peer reply' self.trav[request_id] = ((permid, self._secure_overlay.get_dns_from_peerdb(permid)), reply) elif type(peer) == TupleType: if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: second peer reply' self._file2.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), ' REPLY', request_id, show_permid(peer[0]), str(peer[1]), value, show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), reply, '\n'))) del self.trav[request_id] self._file2.flush()
def handle_info_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): if error: if DEBUG: print >>sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message self._file.write( "; ".join( (strftime("%Y/%m/%d %H:%M:%S"), " INFO REPLY", show_permid(permid), str(error), message, "\n") ) ) self._file.flush() else: if DEBUG: print >>sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short( permid ), cPickle.loads(message) info = cPickle.loads(message) self._file.write( "; ".join( (strftime("%Y/%m/%d %H:%M:%S"), " INFO REPLY", show_permid(permid), str(error), str(info), "\n") ) ) self._file.flush() i = 0 for key, timestamp, piece_size, num_pieces, bitrate, nat in info: i += 1 if i == 1: sql = ( "\nSELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;\nDELETE FROM playback_event WHERE key = '%s';\n" % (key, key) ) else: sql = ( "\nSELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;\nDELETE FROM playback_event WHERE key = '%s';\nDELETE FROM playback_info WHERE key = '%s';\n" % (key, key, key) ) request_callback( CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, sql, channel_data=key, callback=self._after_event_request_callback, frequency=0, )
def doNatCheckCallback(self, exc, permid): if exc is not None: return False if DEBUG: print >> sys.stderr, 'NATCHECK_REQUEST was sent to', show_permid_short(permid), exc self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST', show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), '\n'))) self._file.flush() return True
def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): if error: if DEBUG: print >> sys.stderr, 'repexcrawler: handle_crawler_reply', error, message self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), ' REPLY', show_permid(permid), str(error), message, '\n'))) self._file.flush() else: if DEBUG: print >> sys.stderr, 'repexcrawler: handle_crawler_reply', show_permid_short(permid), cPickle.loads(message) self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), ' REPLY', show_permid(permid), str(error), base64.b64encode(message), '\n'))) self._file.flush()
def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): try: if error: if DEBUG: print >> sys.stderr, 'puncturecrawler: handle_crawler_reply', error, message self._file.write('ERROR %s %.2f %d %s\n' % (show_permid(permid), time.time(), error, message)) self._file.flush() else: if DEBUG: print >> sys.stderr, 'puncturecrawler: handle_crawler_reply', show_permid_short(permid) data = zlib.decompress(message) filtered = filter(lambda char: char != '\x00', data) self._file.write('REPLY %s %.2f\n' % (show_permid(permid), time.time())) self._file.write('# reply sizes: on-the-wire=%d, decompressed=%d, filtered=%d\n' % (len(message), len(data), len(filtered))) self._file.write(filtered) self._file.flush() except: if DEBUG: print >> sys.stderr, 'puncturecrawler: error writing to file'
def udpConnectCallback(self, exc, permid): if exc is not None: if DEBUG: print >> sys.stderr, 'NATTRAVERSAL_REQUEST failed to', show_permid_short(permid), exc self._file2.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST FAILED', show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), '\n'))) return False if DEBUG: print >> sys.stderr, 'NATTRAVERSAL_REQUEST was sent to', show_permid_short(permid), exc return True
def query_initiator(self, permid, selversion, request_callback): if DEBUG: print >> sys.stderr, 'databasecrawler: query_initiator', show_permid_short( permid) sql = [] if selversion >= OLPROTO_VER_SEVENTH: sql.extend(("SELECT 'peer_count', count(*) FROM Peer", "SELECT 'torrent_count', count(*) FROM Torrent")) if selversion >= OLPROTO_VER_ELEVENTH: sql.extend(( "SELECT 'my_subscriptions', count(*) FROM VoteCast where voter_id='" + show_permid(permid) + "' and vote=2", "SELECT 'my_negative_votes', count(*) FROM VoteCast where voter_id='" + show_permid(permid) + "' and vote=-1", "SELECT 'my_channel_files', count(*) FROM ChannelCast where publisher_id='" + show_permid(permid) + "'", "SELECT 'all_subscriptions', count(*) FROM VoteCast where vote=2", "SELECT 'all_negative_votes', count(*) FROM VoteCast where vote=-1" )) request_callback(CRAWLER_DATABASE_QUERY, ';'.join(sql), callback=self._after_request_callback)
def query_initiator(self, permid, selversion, request_callback): if DEBUG: print >>sys.stderr, "databasecrawler: query_initiator", show_permid_short(permid) sql = [] if selversion >= OLPROTO_VER_SEVENTH: sql.extend(("SELECT 'peer_count', count(*) FROM Peer", "SELECT 'torrent_count', count(*) FROM Torrent")) if selversion >= OLPROTO_VER_ELEVENTH: sql.extend( ( "SELECT 'my_subscriptions', count(*) FROM VoteCast where voter_id='" + show_permid(permid) + "' and vote=2", "SELECT 'my_negative_votes', count(*) FROM VoteCast where voter_id='" + show_permid(permid) + "' and vote=-1", "SELECT 'my_channel_files', count(*) FROM ChannelCast where publisher_id='" + show_permid(permid) + "'", "SELECT 'all_subscriptions', count(*) FROM VoteCast where vote=2", "SELECT 'all_negative_votes', count(*) FROM VoteCast where vote=-1", ) ) request_callback(CRAWLER_DATABASE_QUERY, ";".join(sql), callback=self._after_request_callback)
def _after_info_request_callback(self, exc, permid): if not exc: if DEBUG: print >>sys.stderr, "videoplaybackcrawler: request send to", show_permid_short(permid) self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "INFO REQUEST", show_permid(permid), "\n"))) self._file.flush()
def _after_request_callback(self, exc, permid): if not exc: if DEBUG: print >> sys.stderr, 'usereventlogcrawler: request send to', show_permid_short(permid) msg = '; '.join(['REQUEST', show_permid(permid)]) self.__log(msg)
def query_initiator(self, permid, selversion, request_callback): if DEBUG: print >> sys.stderr, 'channelcrawler: query_initiator', show_permid_short(permid) sql = [] if selversion >= OLPROTO_VER_THIRTEENTH: sql.extend(("SELECT 'channel_files', publisher_id, count(*) FROM ChannelCast group by publisher_id", "SELECT 'my_votes', mod_id, voter_id, vote, time_stamp FROM VoteCast where voter_id='" + show_permid(permid) + "' order by time_stamp desc limit 100", "SELECT 'other_votes', mod_id, voter_id, vote, time_stamp FROM VoteCast where voter_id<>'" + show_permid(permid) + "' order by time_stamp desc limit 100")) request_callback(CRAWLER_CHANNEL_QUERY, ';'.join(sql), callback=self._after_request_callback)
def _after_request_callback(self, exc, permid): if not exc: if DEBUG: print >> sys.stderr, 'puncturecrawler: request sent to', show_permid_short(permid) self._file.write('REQUEST %s %.2f\n' % (show_permid(permid), time.time())) self._file.flush()