def receivedSubsRequest(self, permid, request, selversion): channel_id, infohash, languages = request allSubtitles = self.subtitlesDb.getAllSubtitles(channel_id, infohash) contentsList = {} for lang in sorted(languages): if lang in allSubtitles.keys(): if allSubtitles[lang].subtitleExists(): content = self._readSubContent(allSubtitles[lang].path) if content is not None: contentsList[lang] = content else: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'File not available for channel %s, infohash %s, lang %s' % ( show_permid_short(channel_id), bin2str(infohash), lang) self.subtitlesDb.updateSubtitlePath( channel_id, infohash, lang, None) elif DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Subtitle not available for channel %s, infohash %s, lang %s' % ( show_permid_short(channel_id), bin2str(infohash), lang) if len(contentsList) == 0: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'None of the requested subtitles were available. No answer will be sent to %s' % show_permid_short( permid) return True return self._subsMsgHndlr.sendSubtitleResponse( permid, (channel_id, infohash, contentsList), selversion)
def handle_event_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): if error: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', error, message self._file.write('; '.join( (strftime('%Y/%m/%d %H:%M:%S'), ' EVENT REPLY', show_permid(permid), str(error), str(channel_data), message, '\n'))) self._file.flush() elif selversion >= OLPROTO_VER_TENTH: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', show_permid_short( permid), len(message), 'bytes zipped' info = cPickle.loads(zlib.decompress(message)) self._file.write('; '.join( (strftime('%Y/%m/%d %H:%M:%S'), ' EVENT REPLY', show_permid(permid), str(error), str(channel_data), str(info), '\n'))) self._file.flush() elif selversion >= OLPROTO_VER_EIGHTH: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', show_permid_short( permid), cPickle.loads(message) info = cPickle.loads(message) self._file.write('; '.join( (strftime('%Y/%m/%d %H:%M:%S'), ' EVENT REPLY', show_permid(permid), str(error), str(channel_data), str(info), '\n'))) self._file.flush()
def handleRMetadata(self, sender_permid, channelCastMessage, fromQuery=False): metadataDTOs = self._splitChannelcastAndRichMetadataContents( channelCastMessage) if DEBUG: print >> sys.stderr, 'Handling rich metadata from %s...' % show_permid_short( sender_permid) for md_and_have in metadataDTOs: md = md_and_have[0] havemask = md_and_have[1] vote = self.votecastDB.getVote(bin2str(md.channel), bin2str(self.my_permid)) if DEBUG: id = 'RQ' if fromQuery else 'R' print >> sys.stderr, '%s, %s, %s, %s, %d' % ( id, md.channel, md.infohash, show_permid_short(sender_permid), md.timestamp) if vote == -1: continue isUpdate = self.rmdDb.insertMetadata(md) self.peerHaveManager.newHaveReceived(md.channel, md.infohash, sender_permid, havemask) if isUpdate is not None: md = self.rmdDb.getMetadata(md.channel, md.infohash) self._notifyRichMetadata(md, isUpdate) if vote == 2: if DEBUG: print >> sys.stderr, 'Subscribed to channel %s, trying to retrieveall subtitle contents' % ( show_permid_short(md.channel), ) self._getAllSubtitles(md)
def _get_subs_connect_callback(self, exception, dns, permid, selversion, channel_id, infohash, bitmask, msgSentCallback, usrCallback): if exception is not None: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'GET_SUBS not sent. Unable to connect to ' + utilities.show_permid_short(permid) else: if selversion > 0 and selversion < OLPROTO_VER_FOURTEENTH: msg = 'GET_SUBS not send, the other peers had an old protocol version: %d' % selversion if DEBUG: print >> sys.stderr, msg raise SubtitleMsgHandlerException(msg) if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'sending GET_SUBS to ' + utilities.show_permid_short(permid) try: message = self._createGETSUBSMessage(channel_id, infohash, bitmask) if DEBUG: print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ('SG', show_permid_short(permid), show_permid_short(channel_id), bin2str(infohash), bitmask, len(message)) self._overlay_bridge.send(permid, message, lambda exc, permid: self._sent_callback(exc, permid, channel_id, infohash, bitmask, msgSentCallback, usrCallback)) except Exception as e: print_exc() msg = 'GET_SUBS not sent: %s' % str(e) raise SubtitleMsgHandlerException(e)
def handleRMetadata(self, sender_permid, channelCastMessage, fromQuery = False): metadataDTOs = self._splitChannelcastAndRichMetadataContents(channelCastMessage) if DEBUG: print >> sys.stderr, 'Handling rich metadata from %s...' % show_permid_short(sender_permid) for md_and_have in metadataDTOs: md = md_and_have[0] havemask = md_and_have[1] vote = self.votecastDB.getVote(bin2str(md.channel), bin2str(self.my_permid)) if DEBUG: id = 'RQ' if fromQuery else 'R' print >> sys.stderr, '%s, %s, %s, %s, %d' % (id, md.channel, md.infohash, show_permid_short(sender_permid), md.timestamp) if vote == -1: continue isUpdate = self.rmdDb.insertMetadata(md) self.peerHaveManager.newHaveReceived(md.channel, md.infohash, sender_permid, havemask) if isUpdate is not None: md = self.rmdDb.getMetadata(md.channel, md.infohash) self._notifyRichMetadata(md, isUpdate) if vote == 2: if DEBUG: print >> sys.stderr, 'Subscribed to channel %s, trying to retrieveall subtitle contents' % (show_permid_short(md.channel),) self._getAllSubtitles(md)
def tryHolePunching(self): if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: first element in peerlist', self.peerlist[len(self.peerlist) - 1] print >> sys.stderr, 'NatCheckMsgHandler: second element in peerlist', self.peerlist[len(self.peerlist) - 2] holePunchingPort = random.randrange(3200, 4200, 1) holePunchingAddr = (self.holePunchingIP, holePunchingPort) peer1 = self.peerlist[len(self.peerlist) - 1] peer2 = self.peerlist[len(self.peerlist) - 2] request_id = str(show_permid_short(peer1[0]) + show_permid_short(peer2[0]) + str(random.randrange(0, 1000, 1))) self.udpConnect(peer1[0], request_id, holePunchingAddr) self.udpConnect(peer2[0], request_id, holePunchingAddr) self._file2.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST', request_id, show_permid(peer1[0]), str(peer1[1]), str(peer1[2]), str(self._secure_overlay.get_dns_from_peerdb(peer1[0])), show_permid(peer2[0]), str(peer2[1]), str(peer2[2]), str(self._secure_overlay.get_dns_from_peerdb(peer2[0])), '\n'))) self._file2.flush() self.trav[request_id] = (None, None) thread.start_new_thread(coordinateHolePunching, (peer1, peer2, holePunchingAddr))
def handle_connection(self, exc, permid, selversion, locally_initiated): if exc: if DEBUG: print >> sys.stderr, 'crawler: overlay connection lost', show_permid_short( permid), exc print >> sys.stderr, repr(permid) elif selversion >= OLPROTO_VER_SEVENTH: already_known = False for tup in self._initiator_deadlines: if tup[4] == permid: already_known = True break if not already_known: if DEBUG: print >> sys.stderr, 'crawler: new overlay connection', show_permid_short( permid) print >> sys.stderr, repr(permid) for initiator_callback, frequency, accept_frequency in self._crawl_initiators: self._initiator_deadlines.append([ 0, frequency, accept_frequency, initiator_callback, permid, selversion, 0 ]) self._initiator_deadlines.sort() self._check_deadlines(False) elif DEBUG: print >> sys.stderr, 'crawler: new overlay connection (can not use version %d)' % selversion, show_permid_short( permid) print >> sys.stderr, repr(permid)
def tryHolePunching(self): if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: first element in peerlist', self.peerlist[ len(self.peerlist) - 1] print >> sys.stderr, 'NatCheckMsgHandler: second element in peerlist', self.peerlist[ len(self.peerlist) - 2] holePunchingPort = random.randrange(3200, 4200, 1) holePunchingAddr = (self.holePunchingIP, holePunchingPort) peer1 = self.peerlist[len(self.peerlist) - 1] peer2 = self.peerlist[len(self.peerlist) - 2] request_id = str( show_permid_short(peer1[0]) + show_permid_short(peer2[0]) + str(random.randrange(0, 1000, 1))) self.udpConnect(peer1[0], request_id, holePunchingAddr) self.udpConnect(peer2[0], request_id, holePunchingAddr) self._file2.write('; '.join( (strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST', request_id, show_permid(peer1[0]), str(peer1[1]), str(peer1[2]), str(self._secure_overlay.get_dns_from_peerdb(peer1[0])), show_permid(peer2[0]), str(peer2[1]), str(peer2[2]), str(self._secure_overlay.get_dns_from_peerdb(peer2[0])), '\n'))) self._file2.flush() self.trav[request_id] = (None, None) thread.start_new_thread(coordinateHolePunching, (peer1, peer2, holePunchingAddr))
def _handleSUBS(self, permid, message, selversion): if selversion < OLPROTO_VER_FOURTEENTH: if DEBUG: print >> sys.stderr, 'The peer that sent the SUBS request has an oldprotcol version: this is strange. Dropping the msg' return False decoded = self._decodeSUBSMessage(message) if decoded is None: if DEBUG: print >> sys.stderr, 'Error decoding a SUBS message from %s' % utilities.show_permid_short(permid) return False channel_id, infohash, bitmask, contents = decoded if DEBUG: print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ('RS', show_permid_short(permid), show_permid_short(channel_id), bin2str(infohash), bitmask, len(message)) requestedSubs = self._checkRequestedSubtitles(channel_id, infohash, bitmask) if requestedSubs == 0: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Received a SUBS message that was not requested. Dropping' return False requestedSubsCodes = self._languagesUtility.maskToLangCodes(requestedSubs) for lang in contents.keys(): if lang not in requestedSubsCodes: del contents[lang] callbacks = self._removeFromRequestedSubtitles(channel_id, infohash, bitmask) tuple = (channel_id, infohash, contents) for listener in self._listenersList: listener.receivedSubsResponse(permid, tuple, callbacks, selversion) return True
def handle_connection(self, exc, permid, selversion, locally_initiated): if exc: if DEBUG: print >> sys.stderr, 'crawler: overlay connection lost', show_permid_short(permid), exc print >> sys.stderr, repr(permid) elif selversion >= OLPROTO_VER_SEVENTH: already_known = False for tup in self._initiator_deadlines: if tup[4] == permid: already_known = True break if not already_known: if DEBUG: print >> sys.stderr, 'crawler: new overlay connection', show_permid_short(permid) print >> sys.stderr, repr(permid) for initiator_callback, frequency, accept_frequency in self._crawl_initiators: self._initiator_deadlines.append([0, frequency, accept_frequency, initiator_callback, permid, selversion, 0]) self._initiator_deadlines.sort() self._check_deadlines(False) elif DEBUG: print >> sys.stderr, 'crawler: new overlay connection (can not use version %d)' % selversion, show_permid_short(permid) print >> sys.stderr, repr(permid)
def connect(self, permid, callback): if DEBUG: print >> sys.stderr, 'secover: connect', show_permid_short(permid), currentThread().getName() dns = self.get_dns_from_peerdb(permid) task = Task(self._connect, permid, dns, callback) if DEBUG: print >> sys.stderr, 'secover: connect', show_permid_short(permid), 'currently at', dns self.rawserver.add_task(task.start, 0)
def query_initiator(self, permid, selversion, request_callback): if selversion >= OLPROTO_VER_TENTH: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: query_initiator', show_permid_short(permid), 'version', selversion request_callback(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, 'SELECT key, timestamp, event FROM playback_event; DELETE FROM playback_event;', callback=self._after_event_request_callback) elif selversion >= OLPROTO_VER_EIGHTH: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: query_initiator', show_permid_short(permid), 'version', selversion request_callback(CRAWLER_VIDEOPLAYBACK_INFO_QUERY, 'SELECT key, timestamp, piece_size, num_pieces, bitrate, nat FROM playback_info ORDER BY timestamp DESC LIMIT 50', callback=self._after_info_request_callback) elif DEBUG: print >> sys.stderr, 'videoplaybackcrawler: query_info_initiator', show_permid_short(permid), 'unsupported overlay version'
def get_metadata_connect_callback(self, exc, dns, permid, selversion, infohash): if exc is None: if DEBUG: print >> sys.stderr, 'metadata: Sending GET_METADATA to', show_permid_short(permid) try: metadata_request = bencode(infohash) self.overlay_bridge.send(permid, GET_METADATA + metadata_request, self.get_metadata_send_callback) self.requested_torrents.add(infohash) except: print_exc() elif DEBUG: print >> sys.stderr, 'metadata: GET_METADATA: error connecting to', show_permid_short(permid)
def udpConnectCallback(self, exc, permid): if exc is not None: if DEBUG: print >> sys.stderr, 'NATTRAVERSAL_REQUEST failed to', show_permid_short(permid), exc self._file2.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST FAILED', show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), '\n'))) return False if DEBUG: print >> sys.stderr, 'NATTRAVERSAL_REQUEST was sent to', show_permid_short(permid), exc return True
def handleMessage(self, permid, selversion, message): t = message[0] if t == GET_METADATA: if DEBUG: print >> sys.stderr, 'metadata: Got GET_METADATA', len(message), show_permid_short(permid) return self.send_metadata(permid, message, selversion) elif t == METADATA: if DEBUG: print >> sys.stderr, 'metadata: Got METADATA', len(message), show_permid_short(permid), selversion, currentThread().getName() return self.got_metadata(permid, message, selversion) else: if DEBUG: print >> sys.stderr, 'metadata: UNKNOWN OVERLAY MESSAGE', ord(t) return False
def udpConnectCallback(self, exc, permid): if exc is not None: if DEBUG: print >> sys.stderr, 'NATTRAVERSAL_REQUEST failed to', show_permid_short( permid), exc self._file2.write('; '.join( (strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST FAILED', show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), '\n'))) return False if DEBUG: print >> sys.stderr, 'NATTRAVERSAL_REQUEST was sent to', show_permid_short( permid), exc return True
def handle_crawler_request(self, permid, selversion, channel_id, message, reply_callback): if DEBUG: print >> sys.stderr, 'puncturecrawler: handle_crawler_request', show_permid_short( permid), message SimpleFileReporter.lock.acquire() try: if not self.reporter.file: try: self.reporter.file = open(self.reporter.path, 'a+b') except Exception as e: reply_callback(str(e), error=1) return file = self.reporter.file try: file.seek(0) result = '%.2f CRAWL\n' % time.time() + file.read() result = zlib.compress(result) reply_callback(result) file.truncate(0) except Exception as e: reply_callback(str(e), error=1) try: file.seek(0, os.SEEK_END) except: pass finally: SimpleFileReporter.lock.release()
def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): try: if error: if DEBUG: print >> sys.stderr, 'puncturecrawler: handle_crawler_reply', error, message self._file.write( 'ERROR %s %.2f %d %s\n' % (show_permid(permid), time.time(), error, message)) self._file.flush() else: if DEBUG: print >> sys.stderr, 'puncturecrawler: handle_crawler_reply', show_permid_short( permid) data = zlib.decompress(message) filtered = filter(lambda char: char != '\x00', data) self._file.write('REPLY %s %.2f\n' % (show_permid(permid), time.time())) self._file.write( '# reply sizes: on-the-wire=%d, decompressed=%d, filtered=%d\n' % (len(message), len(data), len(filtered))) self._file.write(filtered) self._file.flush() except: if DEBUG: print >> sys.stderr, 'puncturecrawler: error writing to file'
def natCheckReplySendCallback(self, exc, permid): if DEBUG: print >> sys.stderr, 'NATCHECK_REPLY was sent to', show_permid_short( permid), exc if exc is not None: return False return True
def query_initiator(self, permid, selversion, request_callback): if DEBUG: print >> sys.stderr, 'channelcrawler: query_initiator', show_permid_short(permid) sql = [] if selversion >= OLPROTO_VER_THIRTEENTH: sql.extend(("SELECT 'channel_files', publisher_id, count(*) FROM ChannelCast group by publisher_id", "SELECT 'my_votes', mod_id, voter_id, vote, time_stamp FROM VoteCast where voter_id='" + show_permid(permid) + "' order by time_stamp desc limit 100", "SELECT 'other_votes', mod_id, voter_id, vote, time_stamp FROM VoteCast where voter_id<>'" + show_permid(permid) + "' order by time_stamp desc limit 100")) request_callback(CRAWLER_CHANNEL_QUERY, ';'.join(sql), callback=self._after_request_callback)
def udpConnectReplySendCallback(self, exc, permid): if DEBUG: print >> sys.stderr, 'NATTRAVERSAL_REPLY was sent to', show_permid_short( permid), exc if exc is not None: return False return True
def query_initiator(self, permid, selversion, request_callback): if DEBUG: print >> sys.stderr, 'repexcrawler: query_initiator', show_permid_short( permid) request_callback(CRAWLER_REPEX_QUERY, '', callback=self._after_request_callback)
def handle_info_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): if error: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', error, message self._file.write('; '.join( (strftime('%Y/%m/%d %H:%M:%S'), ' INFO REPLY', show_permid(permid), str(error), message, '\n'))) self._file.flush() else: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', show_permid_short( permid), cPickle.loads(message) info = cPickle.loads(message) self._file.write('; '.join( (strftime('%Y/%m/%d %H:%M:%S'), ' INFO REPLY', show_permid(permid), str(error), str(info), '\n'))) self._file.flush() i = 0 for key, timestamp, piece_size, num_pieces, bitrate, nat in info: i += 1 if i == 1: sql = "\nSELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;\nDELETE FROM playback_event WHERE key = '%s';\n" % ( key, key) else: sql = "\nSELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;\nDELETE FROM playback_event WHERE key = '%s';\nDELETE FROM playback_info WHERE key = '%s';\n" % ( key, key, key) request_callback(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, sql, channel_data=key, callback=self._after_event_request_callback, frequency=0)
def send_metadata_request(self, permid, infohash, selversion = -1, caller = 'BC'): if DEBUG: print >> sys.stderr, 'metadata: Connect to send GET_METADATA to', show_permid_short(permid) if not isValidInfohash(infohash): return False filename, metadata = self.torrent_exists(infohash) if filename is not None: if DEBUG: print >> sys.stderr, 'metadata: send_meta_req: Already on disk??!' self.notify_torrent_is_in(infohash, metadata, filename) return True if caller == 'dlhelp': self.requested_torrents.add(infohash) if self.min_free_space != 0 and self.free_space - self.avg_torrent_size < self.min_free_space: self.free_space = self.get_free_space() if self.free_space - self.avg_torrent_size < self.min_free_space: self.warn_disk_full() return True try: if selversion == -1: self.overlay_bridge.connect(permid, lambda e, d, p, s: self.get_metadata_connect_callback(e, d, p, s, infohash)) else: self.get_metadata_connect_callback(None, None, permid, selversion, infohash) except: print_exc() return False return True
def gotUdpConnectRequest(self, permid, selversion, channel_id, mh_msg, reply_callback): if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: gotUdpConnectRequest from', show_permid_short(permid) try: mh_data = bdecode(mh_msg) except: print_exc() print >> sys.stderr, 'NatCheckMsgHandler: bad encoded data:', mh_msg return False if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: gotUdpConnectRequest is', mh_data try: request_id, host, port = mh_data.split(':') except: print_exc() print >> sys.stderr, 'NatCheckMsgHandler: error in received data:', mh_data return False coordinator = (host, int(port)) if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: coordinator address is', coordinator mhr_data = request_id + ':' + tryConnect(coordinator) try: mhr_msg = bencode(mhr_data) except: print_exc() print >> sys.stderr, 'NatCheckMsgHandler: error in encoding data:', mhr_data return False reply_callback(mhr_msg, callback=self.udpConnectReplySendCallback)
def send_join_helpers(self, permid): if DEBUG: print 'helper: send_join_helpers: sending a join_helpers message to', show_permid_short( permid) olthread_send_join_helpers_lambda = lambda: self.olthread_send_join_helpers( ) self.overlay_bridge.add_task(olthread_send_join_helpers_lambda, 0)
def gotUdpConnectRequest(self, permid, selversion, channel_id, mh_msg, reply_callback): if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: gotUdpConnectRequest from', show_permid_short( permid) try: mh_data = bdecode(mh_msg) except: print_exc() print >> sys.stderr, 'NatCheckMsgHandler: bad encoded data:', mh_msg return False if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: gotUdpConnectRequest is', mh_data try: request_id, host, port = mh_data.split(':') except: print_exc() print >> sys.stderr, 'NatCheckMsgHandler: error in received data:', mh_data return False coordinator = (host, int(port)) if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: coordinator address is', coordinator mhr_data = request_id + ':' + tryConnect(coordinator) try: mhr_msg = bencode(mhr_data) except: print_exc() print >> sys.stderr, 'NatCheckMsgHandler: error in encoding data:', mhr_data return False reply_callback(mhr_msg, callback=self.udpConnectReplySendCallback)
def _after_request_callback(self, exc, permid): if not exc: if DEBUG: print >> sys.stderr, 'usereventlogcrawler: request send to', show_permid_short( permid) msg = '; '.join(['REQUEST', show_permid(permid)]) self.__log(msg)
def handle_crawler_request(self, permid, selversion, channel_id, message, reply_callback): if DEBUG: print >> sys.stderr, 'puncturecrawler: handle_crawler_request', show_permid_short(permid), message SimpleFileReporter.lock.acquire() try: if not self.reporter.file: try: self.reporter.file = open(self.reporter.path, 'a+b') except Exception as e: reply_callback(str(e), error=1) return file = self.reporter.file try: file.seek(0) result = '%.2f CRAWL\n' % time.time() + file.read() result = zlib.compress(result) reply_callback(result) file.truncate(0) except Exception as e: reply_callback(str(e), error=1) try: file.seek(0, os.SEEK_END) except: pass finally: SimpleFileReporter.lock.release()
def handle_info_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): if error: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', error, message self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), ' INFO REPLY', show_permid(permid), str(error), message, '\n'))) self._file.flush() else: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', show_permid_short(permid), cPickle.loads(message) info = cPickle.loads(message) self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), ' INFO REPLY', show_permid(permid), str(error), str(info), '\n'))) self._file.flush() i = 0 for key, timestamp, piece_size, num_pieces, bitrate, nat in info: i += 1 if i == 1: sql = "\nSELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;\nDELETE FROM playback_event WHERE key = '%s';\n" % (key, key) else: sql = "\nSELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;\nDELETE FROM playback_event WHERE key = '%s';\nDELETE FROM playback_info WHERE key = '%s';\n" % (key, key, key) request_callback(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, sql, channel_data=key, callback=self._after_event_request_callback, frequency=0)
def _send_request(self, permid, message_id, channel_id, payload, frequency=3600, callback=None): def _after_send_request(exc, permid): if DEBUG: if exc: print >> sys.stderr, 'crawler: could not send request to', show_permid_short( permid), exc if exc: self._release_channel_id(permid, channel_id) if callback: callback(exc, permid) if DEBUG: print >> sys.stderr, 'crawler: sending', getMessageName( CRAWLER_REQUEST + message_id), 'with', len( payload), 'bytes payload to', show_permid_short(permid) self._overlay_bridge.send( permid, ''.join((CRAWLER_REQUEST, message_id, chr(channel_id & 255), chr(frequency >> 8 & 255) + chr(frequency & 255), str(payload))), _after_send_request) return channel_id
def _after_send_reply(exc, permid): if DEBUG: if exc: print >> sys.stderr, 'crawler: could not send request', show_permid_short( permid), exc if callback: callback(exc, permid)
def _after_send_reply(exc, permid): if DEBUG: print >> sys.stderr, 'crawler: _after_send_reply', show_permid_short(permid), exc if not exc: self.send_reply(permid, message_id, channel_id, remaining_payload, error=error) if callback: callback(exc, permid)
def got_auth_connection(self, oc): if DEBUG: print >> sys.stderr, 'secover: got_auth_connection', show_permid_short(oc.get_auth_permid()), oc.get_ip(), oc.get_auth_listen_port(), currentThread().getName() if oc.is_locally_initiated() and oc.get_port() != oc.get_auth_listen_port(): if DEBUG: print >> sys.stderr, 'secover: got_auth_connection: closing because auth', 'listen port not as expected', oc.get_port(), oc.get_auth_listen_port() self.cleanup_admin_and_callbacks(oc, Exception('closing because auth listen port not as expected')) return False ret = True iplport = ip_and_port2str(oc.get_ip(), oc.get_auth_listen_port()) known = iplport in self.iplport2oc if not known: self.iplport2oc[iplport] = oc elif known and not oc.is_locally_initiated(): if DEBUG: print >> sys.stderr, 'secover: got_auth_connection:', 'closing because we already have a connection to', iplport self.cleanup_admin_and_callbacks(oc, Exception('closing because we already have a connection to peer')) ret = False if ret: if oc.is_auth_done(): hisdns = (oc.get_ip(), oc.get_auth_listen_port()) else: hisdns = None if self.userconnhandler is not None: try: self.userconnhandler(None, oc.get_auth_permid(), oc.get_sel_proto_ver(), oc.is_locally_initiated(), hisdns) except: print_exc() oc.dequeue_callbacks() return ret
def _after_request_callback(self, exc, permid): if not exc: if DEBUG: print >> sys.stderr, 'repexcrawler: request sent to', show_permid_short( permid) self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST', show_permid(permid), '\n'))) self._file.flush()
def send_proxy_have(self, aggregated_haves): if DEBUG: print 'helper: send_proxy_have: sending a proxy_have message to', show_permid_short( self.coordinator_permid) aggregated_string = aggregated_haves.tostring() olthread_send_proxy_have_lambda = lambda: self.olthread_send_proxy_have( aggregated_string) self.overlay_bridge.add_task(olthread_send_proxy_have_lambda, 0)
def _after_request_callback(self, exc, permid): if not exc: if DEBUG: print >> sys.stderr, 'puncturecrawler: request sent to', show_permid_short( permid) self._file.write('REQUEST %s %.2f\n' % (show_permid(permid), time.time())) self._file.flush()
def _after_send_request(exc, permid): if DEBUG: if exc: print >> sys.stderr, 'crawler: could not send request to', show_permid_short(permid), exc if exc: self._release_channel_id(permid, channel_id) if callback: callback(exc, permid)
def process_query_reply(self, permid, query, usercallback, d): if DEBUG: print >> sys.stderr, 'rquery: process_query_reply:', show_permid_short(permid), query, d if len(d['a']) > 0: self.unidecode_hits(query, d) remote_query_usercallback_lambda = lambda : usercallback(permid, query, d['a']) self.session.uch.perform_usercallback(remote_query_usercallback_lambda) elif DEBUG: print >> sys.stderr, 'rquery: QUERY_REPLY: no results found'
def _after_send_request(exc, permid): if DEBUG: if exc: print >> sys.stderr, 'crawler: could not send request to', show_permid_short( permid), exc if exc: self._release_channel_id(permid, channel_id) if callback: callback(exc, permid)
def handle_crawler_request(self, permid, selversion, channel_id, message, reply_callback): if DEBUG: print >> sys.stderr, 'repexcrawler: handle_crawler_request', show_permid_short(permid), message try: repexhistory = self._repexlog.getHistoryAndCleanup() except Exception as e: reply_callback(str(e), error=1) else: reply_callback(cPickle.dumps(repexhistory, 2))
def _after_connect(exc, dns, permid, selversion): self._post_connection_attempt(permid, not exc) if exc: if DEBUG: print >> sys.stderr, 'crawler: could not connect', dns, show_permid_short(permid), exc if callback: callback(exc, permid) else: self._send_reply(permid, message_id, channel_id, payload, error=error, callback=callback)
def handleMessage(self, permid, selversion, message): t = message[0] if t == GET_METADATA: if DEBUG: print >> sys.stderr, 'metadata: Got GET_METADATA', len( message), show_permid_short(permid) return self.send_metadata(permid, message, selversion) elif t == METADATA: if DEBUG: print >> sys.stderr, 'metadata: Got METADATA', len( message), show_permid_short( permid), selversion, currentThread().getName() return self.got_metadata(permid, message, selversion) else: if DEBUG: print >> sys.stderr, 'metadata: UNKNOWN OVERLAY MESSAGE', ord( t) return False
def _after_request_callback(self, exc, permid): if not exc: if DEBUG: print >> sys.stderr, 'channelcrawler: request send to', show_permid_short(permid) self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST', show_permid(permid), '\n'))) self._file.flush()
def get_metadata_connect_callback(self, exc, dns, permid, selversion, infohash): if exc is None: if DEBUG: print >> sys.stderr, 'metadata: Sending GET_METADATA to', show_permid_short( permid) try: metadata_request = bencode(infohash) self.overlay_bridge.send(permid, GET_METADATA + metadata_request, self.get_metadata_send_callback) self.requested_torrents.add(infohash) except: print_exc() elif DEBUG: print >> sys.stderr, 'metadata: GET_METADATA: error connecting to', show_permid_short( permid)
def query_initiator(self, permid, selversion, request_callback): if DEBUG: print >> sys.stderr, 'usereventlogcrawler: query_initiator', show_permid_short( permid) if selversion >= OLPROTO_VER_FIFTEENTH: sql = 'SELECT * FROM UserEventLog; DELETE FROM UserEventLog;' request_callback(CRAWLER_USEREVENTLOG_QUERY, sql, callback=self._after_request_callback)
def _sent_callback(self, exc, permid, channel_id, infohash, bitmask, msgSentCallback, usrCallback): if exc is not None: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Unable to send GET_SUBS to: ' + utilities.show_permid_short(permid) + ': ' + exc else: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'GET_SUBS sent to %s' % utilities.show_permid_short(permid) self._addToRequestedSubtitles(channel_id, infohash, bitmask, usrCallback) if msgSentCallback is not None: msgSentCallback(exc, permid, channel_id, infohash, bitmask)
def doNatCheckCallback(self, exc, permid): if exc is not None: return False if DEBUG: print >> sys.stderr, 'NATCHECK_REQUEST was sent to', show_permid_short( permid), exc self._file.write('; '.join( (strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST', show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), '\n'))) self._file.flush() return True
def send(self, permid, msg, callback): if DEBUG: print >> sys.stderr, 'olbridge: send', show_permid_short(permid), len(msg) def olbridge_send_callback(cexc, cpermid): if DEBUG: print >> sys.stderr, 'olbridge: send_callback', cexc, show_permid_short(cpermid) olbridge_send_callback_lambda = lambda : callback(cexc, cpermid) self.add_task(olbridge_send_callback_lambda, 0) self.secover.send(permid, msg, olbridge_send_callback)
def handle_crawler_request(self, permid, selversion, channel_id, message, reply_callback): if DEBUG: print >> sys.stderr, 'repexcrawler: handle_crawler_request', show_permid_short( permid), message try: repexhistory = self._repexlog.getHistoryAndCleanup() except Exception as e: reply_callback(str(e), error=1) else: reply_callback(cPickle.dumps(repexhistory, 2))
def connect(self, permid, callback): if DEBUG: print >> sys.stderr, 'olbridge: connect', show_permid_short(permid), currentThread().getName() def olbridge_connect_callback(cexc, cdns, cpermid, cselver): if DEBUG: print >> sys.stderr, 'olbridge: connect_callback', cexc, cdns, show_permid_short(cpermid), cselver, callback, currentThread().getName() olbridge_connect_callback_lambda = lambda : callback(cexc, cdns, cpermid, cselver) self.add_task(olbridge_connect_callback_lambda, 0) self.secover.connect(permid, olbridge_connect_callback)
def handle_crawler_request(self, permid, selversion, channel_id, message, reply_callback): if DEBUG: print >> sys.stderr, 'channelcrawler: handle_crawler_request', show_permid_short(permid), message try: cursor = self._sqlite_cache_db.execute_read(message) except Exception as e: reply_callback(str(e), error=1) else: if cursor: reply_callback(cPickle.dumps(list(cursor), 2)) else: reply_callback('error', error=2)
def doNatCheckCallback(self, exc, permid): if exc is not None: return False if DEBUG: print >> sys.stderr, 'NATCHECK_REQUEST was sent to', show_permid_short(permid), exc self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST', show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), '\n'))) self._file.flush() return True
def _close(self, permid): if DEBUG: print >> sys.stderr, 'secover: actual close', show_permid_short(permid) try: oc = self.get_oc_by_permid(permid) if not oc: if DEBUG: print >> sys.stderr, 'secover: error - actual close, but no connection to peer in admin' else: oc.close() except Exception as e: print_exc()
def _checkingUploadQueue(self): if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Checking the upload queue...' if not self._tokenBucket.upload_rate > 0: return if not len(self._uploadQueue) > 0: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Upload queue is empty.' while len(self._uploadQueue) > 0: responseData = self._uploadQueue[0] encodedMsg = self._createSingleResponseMessage(responseData) if encodedMsg is None: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Nothing to send' del self._uploadQueue[0] continue msgSize = len(encodedMsg) / 1024.0 if msgSize > self._tokenBucket.capacity: print >> sys.stderr, 'Warning:' + SUBS_LOG_PREFIX + 'SUBS message too big. Discarded!' del self._uploadQueue[0] continue if self._tokenBucket.consume(msgSize): if DEBUG: keys = responseData['subtitles'].keys() bitmask = self._languagesUtility.langCodesToMask(keys) print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ('SS', show_permid_short(responseData['permid']), show_permid_short(responseData['channel_id']), bin2str(responseData['infohash']), bitmask, int(msgSize * 1024)) self._doSendSubtitles(responseData['permid'], encodedMsg, responseData['selversion']) del self._uploadQueue[0] else: neededCapacity = max(0, msgSize - self._tokenBucket.tokens) delay = neededCapacity / self._tokenBucket.upload_rate self._nextUploadTime = time() + delay self.overlay_bridge.add_task(self._checkingUploadQueue, delay) return
def olbridge_handle_conn_func(): if DEBUG: print >> sys.stderr, 'olbridge: handle_conn_func', exc, show_permid_short(permid), selversion, locally_initiated, hisdns, currentThread().getName() try: if hisdns: self.secover.add_peer_to_db(permid, hisdns, selversion) if self.olappsconnhandler is not None: self.olappsconnhandler(exc, permid, selversion, locally_initiated) except: print_exc() if isinstance(exc, CloseException): self.secover.update_peer_status(permid, exc.was_auth_done())