def handle_event_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): if error: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', error, message self._file.write('; '.join( (strftime('%Y/%m/%d %H:%M:%S'), ' EVENT REPLY', show_permid(permid), str(error), str(channel_data), message, '\n'))) self._file.flush() elif selversion >= OLPROTO_VER_TENTH: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', show_permid_short( permid), len(message), 'bytes zipped' info = cPickle.loads(zlib.decompress(message)) self._file.write('; '.join( (strftime('%Y/%m/%d %H:%M:%S'), ' EVENT REPLY', show_permid(permid), str(error), str(channel_data), str(info), '\n'))) self._file.flush() elif selversion >= OLPROTO_VER_EIGHTH: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', show_permid_short( permid), cPickle.loads(message) info = cPickle.loads(message) self._file.write('; '.join( (strftime('%Y/%m/%d %H:%M:%S'), ' EVENT REPLY', show_permid(permid), str(error), str(channel_data), str(info), '\n'))) self._file.flush()
def _get_subs_connect_callback(self, exception, dns, permid, selversion, channel_id, infohash, bitmask, msgSentCallback, usrCallback): if exception is not None: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'GET_SUBS not sent. Unable to connect to ' + utilities.show_permid_short( permid) else: if selversion > 0 and selversion < OLPROTO_VER_FOURTEENTH: msg = 'GET_SUBS not send, the other peers had an old protocol version: %d' % selversion if DEBUG: print >> sys.stderr, msg raise SubtitleMsgHandlerException(msg) if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'sending GET_SUBS to ' + utilities.show_permid_short( permid) try: message = self._createGETSUBSMessage(channel_id, infohash, bitmask) if DEBUG: print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ( 'SG', show_permid_short(permid), show_permid_short(channel_id), bin2str(infohash), bitmask, len(message)) self._overlay_bridge.send( permid, message, lambda exc, permid: self._sent_callback( exc, permid, channel_id, infohash, bitmask, msgSentCallback, usrCallback)) except Exception as e: print_exc() msg = 'GET_SUBS not sent: %s' % str(e) raise SubtitleMsgHandlerException(e)
def query_initiator(self, permid, selversion, request_callback): if selversion >= OLPROTO_VER_TENTH: if DEBUG: print >>sys.stderr, "videoplaybackcrawler: query_initiator", show_permid_short( permid ), "version", selversion request_callback( CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, "SELECT key, timestamp, event FROM playback_event; DELETE FROM playback_event;", callback=self._after_event_request_callback, ) elif selversion >= OLPROTO_VER_EIGHTH: if DEBUG: print >>sys.stderr, "videoplaybackcrawler: query_initiator", show_permid_short( permid ), "version", selversion request_callback( CRAWLER_VIDEOPLAYBACK_INFO_QUERY, "SELECT key, timestamp, piece_size, num_pieces, bitrate, nat FROM playback_info ORDER BY timestamp DESC LIMIT 50", callback=self._after_info_request_callback, ) elif DEBUG: print >>sys.stderr, "videoplaybackcrawler: query_info_initiator", show_permid_short( permid ), "unsupported overlay version"
def _get_subs_connect_callback(self, exception, dns, permid, selversion, channel_id, infohash, bitmask, msgSentCallback, usrCallback): if exception is not None: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'GET_SUBS not sent. Unable to connect to ' + utilities.show_permid_short(permid) else: if selversion > 0 and selversion < OLPROTO_VER_FOURTEENTH: msg = 'GET_SUBS not send, the other peers had an old protocol version: %d' % selversion if DEBUG: print >> sys.stderr, msg raise SubtitleMsgHandlerException(msg) if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'sending GET_SUBS to ' + utilities.show_permid_short(permid) try: message = self._createGETSUBSMessage(channel_id, infohash, bitmask) if DEBUG: print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ('SG', show_permid_short(permid), show_permid_short(channel_id), bin2str(infohash), bitmask, len(message)) self._overlay_bridge.send(permid, message, lambda exc, permid: self._sent_callback(exc, permid, channel_id, infohash, bitmask, msgSentCallback, usrCallback)) except Exception as e: print_exc() msg = 'GET_SUBS not sent: %s' % str(e) raise SubtitleMsgHandlerException(e)
def _handleSUBS(self, permid, message, selversion): if selversion < OLPROTO_VER_FOURTEENTH: if DEBUG: print >> sys.stderr, 'The peer that sent the SUBS request has an oldprotcol version: this is strange. Dropping the msg' return False decoded = self._decodeSUBSMessage(message) if decoded is None: if DEBUG: print >> sys.stderr, 'Error decoding a SUBS message from %s' % utilities.show_permid_short(permid) return False channel_id, infohash, bitmask, contents = decoded if DEBUG: print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ('RS', show_permid_short(permid), show_permid_short(channel_id), bin2str(infohash), bitmask, len(message)) requestedSubs = self._checkRequestedSubtitles(channel_id, infohash, bitmask) if requestedSubs == 0: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Received a SUBS message that was not requested. Dropping' return False requestedSubsCodes = self._languagesUtility.maskToLangCodes(requestedSubs) for lang in contents.keys(): if lang not in requestedSubsCodes: del contents[lang] callbacks = self._removeFromRequestedSubtitles(channel_id, infohash, bitmask) tuple = (channel_id, infohash, contents) for listener in self._listenersList: listener.receivedSubsResponse(permid, tuple, callbacks, selversion) return True
def handleRMetadata(self, sender_permid, channelCastMessage, fromQuery=False): metadataDTOs = self._splitChannelcastAndRichMetadataContents( channelCastMessage) if DEBUG: print >> sys.stderr, 'Handling rich metadata from %s...' % show_permid_short( sender_permid) for md_and_have in metadataDTOs: md = md_and_have[0] havemask = md_and_have[1] vote = self.votecastDB.getVote(bin2str(md.channel), bin2str(self.my_permid)) if DEBUG: id = 'RQ' if fromQuery else 'R' print >> sys.stderr, '%s, %s, %s, %s, %d' % ( id, md.channel, md.infohash, show_permid_short(sender_permid), md.timestamp) if vote == -1: continue isUpdate = self.rmdDb.insertMetadata(md) self.peerHaveManager.newHaveReceived(md.channel, md.infohash, sender_permid, havemask) if isUpdate is not None: md = self.rmdDb.getMetadata(md.channel, md.infohash) self._notifyRichMetadata(md, isUpdate) if vote == 2: if DEBUG: print >> sys.stderr, 'Subscribed to channel %s, trying to retrieveall subtitle contents' % ( show_permid_short(md.channel), ) self._getAllSubtitles(md)
def tryHolePunching(self): if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: first element in peerlist', self.peerlist[len(self.peerlist) - 1] print >> sys.stderr, 'NatCheckMsgHandler: second element in peerlist', self.peerlist[len(self.peerlist) - 2] holePunchingPort = random.randrange(3200, 4200, 1) holePunchingAddr = (self.holePunchingIP, holePunchingPort) peer1 = self.peerlist[len(self.peerlist) - 1] peer2 = self.peerlist[len(self.peerlist) - 2] request_id = str(show_permid_short(peer1[0]) + show_permid_short(peer2[0]) + str(random.randrange(0, 1000, 1))) self.udpConnect(peer1[0], request_id, holePunchingAddr) self.udpConnect(peer2[0], request_id, holePunchingAddr) self._file2.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST', request_id, show_permid(peer1[0]), str(peer1[1]), str(peer1[2]), str(self._secure_overlay.get_dns_from_peerdb(peer1[0])), show_permid(peer2[0]), str(peer2[1]), str(peer2[2]), str(self._secure_overlay.get_dns_from_peerdb(peer2[0])), '\n'))) self._file2.flush() self.trav[request_id] = (None, None) thread.start_new_thread(coordinateHolePunching, (peer1, peer2, holePunchingAddr))
def handle_connection(self, exc, permid, selversion, locally_initiated): if exc: if DEBUG: print >> sys.stderr, 'crawler: overlay connection lost', show_permid_short(permid), exc print >> sys.stderr, repr(permid) elif selversion >= OLPROTO_VER_SEVENTH: already_known = False for tup in self._initiator_deadlines: if tup[4] == permid: already_known = True break if not already_known: if DEBUG: print >> sys.stderr, 'crawler: new overlay connection', show_permid_short(permid) print >> sys.stderr, repr(permid) for initiator_callback, frequency, accept_frequency in self._crawl_initiators: self._initiator_deadlines.append([0, frequency, accept_frequency, initiator_callback, permid, selversion, 0]) self._initiator_deadlines.sort() self._check_deadlines(False) elif DEBUG: print >> sys.stderr, 'crawler: new overlay connection (can not use version %d)' % selversion, show_permid_short(permid) print >> sys.stderr, repr(permid)
def _subs_send_callback(self, exc, permid): if exc is not None: print >> sys.stderr, 'Warning: Sending of SUBS message to %s failed: %s' % ( show_permid_short(permid), str(exc)) elif DEBUG: print >> sys.stderr, 'SUBS message succesfully sent to %s' % show_permid_short( permid)
def handleRMetadata(self, sender_permid, channelCastMessage, fromQuery = False): metadataDTOs = self._splitChannelcastAndRichMetadataContents(channelCastMessage) if DEBUG: print >> sys.stderr, 'Handling rich metadata from %s...' % show_permid_short(sender_permid) for md_and_have in metadataDTOs: md = md_and_have[0] havemask = md_and_have[1] vote = self.votecastDB.getVote(bin2str(md.channel), bin2str(self.my_permid)) if DEBUG: id = 'RQ' if fromQuery else 'R' print >> sys.stderr, '%s, %s, %s, %s, %d' % (id, md.channel, md.infohash, show_permid_short(sender_permid), md.timestamp) if vote == -1: continue isUpdate = self.rmdDb.insertMetadata(md) self.peerHaveManager.newHaveReceived(md.channel, md.infohash, sender_permid, havemask) if isUpdate is not None: md = self.rmdDb.getMetadata(md.channel, md.infohash) self._notifyRichMetadata(md, isUpdate) if vote == 2: if DEBUG: print >> sys.stderr, 'Subscribed to channel %s, trying to retrieveall subtitle contents' % (show_permid_short(md.channel),) self._getAllSubtitles(md)
def handle_event_crawler_reply( self, permid, selversion, channel_id, channel_data, error, message, request_callback ): if error: if DEBUG: print >>sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message self._file.write( "; ".join( ( strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), message, "\n", ) ) ) self._file.flush() elif selversion >= OLPROTO_VER_TENTH: if DEBUG: print >>sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(permid), len( message ), "bytes zipped" info = cPickle.loads(zlib.decompress(message)) self._file.write( "; ".join( ( strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), str(info), "\n", ) ) ) self._file.flush() elif selversion >= OLPROTO_VER_EIGHTH: if DEBUG: print >>sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short( permid ), cPickle.loads(message) info = cPickle.loads(message) self._file.write( "; ".join( ( strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), str(info), "\n", ) ) ) self._file.flush()
def _handleSUBS(self, permid, message, selversion): if selversion < OLPROTO_VER_FOURTEENTH: if DEBUG: print >> sys.stderr, 'The peer that sent the SUBS request has an oldprotcol version: this is strange. Dropping the msg' return False decoded = self._decodeSUBSMessage(message) if decoded is None: if DEBUG: print >> sys.stderr, 'Error decoding a SUBS message from %s' % utilities.show_permid_short( permid) return False channel_id, infohash, bitmask, contents = decoded if DEBUG: print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ( 'RS', show_permid_short(permid), show_permid_short(channel_id), bin2str(infohash), bitmask, len(message)) requestedSubs = self._checkRequestedSubtitles(channel_id, infohash, bitmask) if requestedSubs == 0: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Received a SUBS message that was not requested. Dropping' return False requestedSubsCodes = self._languagesUtility.maskToLangCodes( requestedSubs) for lang in contents.keys(): if lang not in requestedSubsCodes: del contents[lang] callbacks = self._removeFromRequestedSubtitles(channel_id, infohash, bitmask) tuple = (channel_id, infohash, contents) for listener in self._listenersList: listener.receivedSubsResponse(permid, tuple, callbacks, selversion) return True
def connect(self, permid, callback): if DEBUG: print >> sys.stderr, 'secover: connect', show_permid_short(permid), currentThread().getName() dns = self.get_dns_from_peerdb(permid) task = Task(self._connect, permid, dns, callback) if DEBUG: print >> sys.stderr, 'secover: connect', show_permid_short(permid), 'currently at', dns self.rawserver.add_task(task.start, 0)
def connect(self, permid, callback): if DEBUG: print >> sys.stderr, 'secover: connect', show_permid_short( permid), currentThread().getName() dns = self.get_dns_from_peerdb(permid) task = Task(self._connect, permid, dns, callback) if DEBUG: print >> sys.stderr, 'secover: connect', show_permid_short( permid), 'currently at', dns self.rawserver.add_task(task.start, 0)
def get_metadata_connect_callback(self, exc, dns, permid, selversion, infohash): if exc is None: if DEBUG: print >> sys.stderr, 'metadata: Sending GET_METADATA to', show_permid_short(permid) try: metadata_request = bencode(infohash) self.overlay_bridge.send(permid, GET_METADATA + metadata_request, self.get_metadata_send_callback) self.requested_torrents.add(infohash) except: print_exc() elif DEBUG: print >> sys.stderr, 'metadata: GET_METADATA: error connecting to', show_permid_short(permid)
def udpConnectCallback(self, exc, permid): if exc is not None: if DEBUG: print >> sys.stderr, 'NATTRAVERSAL_REQUEST failed to', show_permid_short(permid), exc self._file2.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST FAILED', show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), '\n'))) return False if DEBUG: print >> sys.stderr, 'NATTRAVERSAL_REQUEST was sent to', show_permid_short(permid), exc return True
def handleMessage(self, permid, selversion, message): t = message[0] if t == GET_METADATA: if DEBUG: print >> sys.stderr, 'metadata: Got GET_METADATA', len(message), show_permid_short(permid) return self.send_metadata(permid, message, selversion) elif t == METADATA: if DEBUG: print >> sys.stderr, 'metadata: Got METADATA', len(message), show_permid_short(permid), selversion, currentThread().getName() return self.got_metadata(permid, message, selversion) else: if DEBUG: print >> sys.stderr, 'metadata: UNKNOWN OVERLAY MESSAGE', ord(t) return False
def handle_crawler_request(self, permid, selversion, channel_id, message, reply_callback): if DEBUG: print >> sys.stderr, 'puncturecrawler: handle_crawler_request', show_permid_short(permid), message SimpleFileReporter.lock.acquire() try: if not self.reporter.file: try: self.reporter.file = open(self.reporter.path, 'a+b') except Exception as e: reply_callback(str(e), error=1) return file = self.reporter.file try: file.seek(0) result = '%.2f CRAWL\n' % time.time() + file.read() result = zlib.compress(result) reply_callback(result) file.truncate(0) except Exception as e: reply_callback(str(e), error=1) try: file.seek(0, os.SEEK_END) except: pass finally: SimpleFileReporter.lock.release()
def query_initiator(self, permid, selversion, request_callback): if DEBUG: print >> sys.stderr, 'channelcrawler: query_initiator', show_permid_short(permid) sql = [] if selversion >= OLPROTO_VER_THIRTEENTH: sql.extend(("SELECT 'channel_files', publisher_id, count(*) FROM ChannelCast group by publisher_id", "SELECT 'my_votes', mod_id, voter_id, vote, time_stamp FROM VoteCast where voter_id='" + show_permid(permid) + "' order by time_stamp desc limit 100", "SELECT 'other_votes', mod_id, voter_id, vote, time_stamp FROM VoteCast where voter_id<>'" + show_permid(permid) + "' order by time_stamp desc limit 100")) request_callback(CRAWLER_CHANNEL_QUERY, ';'.join(sql), callback=self._after_request_callback)
def got_auth_connection(self, oc): if DEBUG: print >> sys.stderr, 'secover: got_auth_connection', show_permid_short(oc.get_auth_permid()), oc.get_ip(), oc.get_auth_listen_port(), currentThread().getName() if oc.is_locally_initiated() and oc.get_port() != oc.get_auth_listen_port(): if DEBUG: print >> sys.stderr, 'secover: got_auth_connection: closing because auth', 'listen port not as expected', oc.get_port(), oc.get_auth_listen_port() self.cleanup_admin_and_callbacks(oc, Exception('closing because auth listen port not as expected')) return False ret = True iplport = ip_and_port2str(oc.get_ip(), oc.get_auth_listen_port()) known = iplport in self.iplport2oc if not known: self.iplport2oc[iplport] = oc elif known and not oc.is_locally_initiated(): if DEBUG: print >> sys.stderr, 'secover: got_auth_connection:', 'closing because we already have a connection to', iplport self.cleanup_admin_and_callbacks(oc, Exception('closing because we already have a connection to peer')) ret = False if ret: if oc.is_auth_done(): hisdns = (oc.get_ip(), oc.get_auth_listen_port()) else: hisdns = None if self.userconnhandler is not None: try: self.userconnhandler(None, oc.get_auth_permid(), oc.get_sel_proto_ver(), oc.is_locally_initiated(), hisdns) except: print_exc() oc.dequeue_callbacks() return ret
def send_join_helpers(self, permid): if DEBUG: print 'helper: send_join_helpers: sending a join_helpers message to', show_permid_short( permid) olthread_send_join_helpers_lambda = lambda: self.olthread_send_join_helpers( ) self.overlay_bridge.add_task(olthread_send_join_helpers_lambda, 0)
def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): if error: if DEBUG: print >>sys.stderr, "databasecrawler: handle_crawler_reply", error, message self._file.write( "; ".join((strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(error), message, "\n")) ) self._file.flush() else: if DEBUG: print >>sys.stderr, "databasecrawler: handle_crawler_reply", show_permid_short(permid), cPickle.loads( message ) self._file.write( "; ".join( ( strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(error), str(cPickle.loads(message)), "\n", ) ) ) self._file.flush()
def _send(self, permid, dns, message, callback): if DEBUG: print >> sys.stderr, 'secover: actual send', getMessageName( message[0]), 'to', show_permid_short( permid), currentThread().getName() try: if dns is None: callback(KeyError('IP address + port for permid unknown'), permid) else: iplport = ip_and_port2str(dns[0], dns[1]) oc = None try: oc = self.iplport2oc[iplport] except KeyError: pass if oc is None: callback(KeyError('Not connected to permid'), permid) elif oc.is_auth_done(): if oc.get_auth_permid() == permid: oc.send_message(message) callback(None, permid) else: callback( KeyError( 'Recorded IP address + port now of other permid' ), permid) else: callback(KeyError('Connection not yet established'), permid) except Exception as exc: if DEBUG: print_exc() callback(exc, permid)
def _connect(self, expectedpermid, dns, callback): if DEBUG: print >> sys.stderr, 'secover: actual connect', show_permid_short( expectedpermid), currentThread().getName() if expectedpermid == self.permid: callback(KeyError('The target permid is the same as my permid'), None, expectedpermid, 0) try: oc = self.get_oc_by_permid(expectedpermid) if oc is None: if dns is None: callback(KeyError('IP address + port for permid unknown'), dns, expectedpermid, 0) else: self._connect_dns( dns, lambda exc, dns2, peerpermid, selver: self. _whoishe_callback(exc, dns2, peerpermid, selver, expectedpermid, callback)) else: self._whoishe_callback( None, (oc.get_ip(), oc.get_auth_listen_port()), expectedpermid, oc.get_sel_proto_ver(), expectedpermid, callback) except Exception as exc: if DEBUG: print_exc() callback(exc, None, expectedpermid, 0)
def gotUdpConnectRequest(self, permid, selversion, channel_id, mh_msg, reply_callback): if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: gotUdpConnectRequest from', show_permid_short(permid) try: mh_data = bdecode(mh_msg) except: print_exc() print >> sys.stderr, 'NatCheckMsgHandler: bad encoded data:', mh_msg return False if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: gotUdpConnectRequest is', mh_data try: request_id, host, port = mh_data.split(':') except: print_exc() print >> sys.stderr, 'NatCheckMsgHandler: error in received data:', mh_data return False coordinator = (host, int(port)) if DEBUG: print >> sys.stderr, 'NatCheckMsgHandler: coordinator address is', coordinator mhr_data = request_id + ':' + tryConnect(coordinator) try: mhr_msg = bencode(mhr_data) except: print_exc() print >> sys.stderr, 'NatCheckMsgHandler: error in encoding data:', mhr_data return False reply_callback(mhr_msg, callback=self.udpConnectReplySendCallback)
def handle_info_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): if error: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', error, message self._file.write('; '.join( (strftime('%Y/%m/%d %H:%M:%S'), ' INFO REPLY', show_permid(permid), str(error), message, '\n'))) self._file.flush() else: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', show_permid_short( permid), cPickle.loads(message) info = cPickle.loads(message) self._file.write('; '.join( (strftime('%Y/%m/%d %H:%M:%S'), ' INFO REPLY', show_permid(permid), str(error), str(info), '\n'))) self._file.flush() i = 0 for key, timestamp, piece_size, num_pieces, bitrate, nat in info: i += 1 if i == 1: sql = "\nSELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;\nDELETE FROM playback_event WHERE key = '%s';\n" % ( key, key) else: sql = "\nSELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;\nDELETE FROM playback_event WHERE key = '%s';\nDELETE FROM playback_info WHERE key = '%s';\n" % ( key, key, key) request_callback(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, sql, channel_data=key, callback=self._after_event_request_callback, frequency=0)
def _after_send_reply(exc, permid): if DEBUG: print >> sys.stderr, 'crawler: _after_send_reply', show_permid_short(permid), exc if not exc: self.send_reply(permid, message_id, channel_id, remaining_payload, error=error) if callback: callback(exc, permid)
def handle_crawler_request(self, permid, selversion, channel_id, message, reply_callback): if DEBUG: print >> sys.stderr, 'puncturecrawler: handle_crawler_request', show_permid_short( permid), message SimpleFileReporter.lock.acquire() try: if not self.reporter.file: try: self.reporter.file = open(self.reporter.path, 'a+b') except Exception as e: reply_callback(str(e), error=1) return file = self.reporter.file try: file.seek(0) result = '%.2f CRAWL\n' % time.time() + file.read() result = zlib.compress(result) reply_callback(result) file.truncate(0) except Exception as e: reply_callback(str(e), error=1) try: file.seek(0, os.SEEK_END) except: pass finally: SimpleFileReporter.lock.release()
def send_metadata_request(self, permid, infohash, selversion = -1, caller = 'BC'): if DEBUG: print >> sys.stderr, 'metadata: Connect to send GET_METADATA to', show_permid_short(permid) if not isValidInfohash(infohash): return False filename, metadata = self.torrent_exists(infohash) if filename is not None: if DEBUG: print >> sys.stderr, 'metadata: send_meta_req: Already on disk??!' self.notify_torrent_is_in(infohash, metadata, filename) return True if caller == 'dlhelp': self.requested_torrents.add(infohash) if self.min_free_space != 0 and self.free_space - self.avg_torrent_size < self.min_free_space: self.free_space = self.get_free_space() if self.free_space - self.avg_torrent_size < self.min_free_space: self.warn_disk_full() return True try: if selversion == -1: self.overlay_bridge.connect(permid, lambda e, d, p, s: self.get_metadata_connect_callback(e, d, p, s, infohash)) else: self.get_metadata_connect_callback(None, None, permid, selversion, infohash) except: print_exc() return False return True
def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): try: if error: if DEBUG: print >> sys.stderr, 'puncturecrawler: handle_crawler_reply', error, message self._file.write( 'ERROR %s %.2f %d %s\n' % (show_permid(permid), time.time(), error, message)) self._file.flush() else: if DEBUG: print >> sys.stderr, 'puncturecrawler: handle_crawler_reply', show_permid_short( permid) data = zlib.decompress(message) filtered = filter(lambda char: char != '\x00', data) self._file.write('REPLY %s %.2f\n' % (show_permid(permid), time.time())) self._file.write( '# reply sizes: on-the-wire=%d, decompressed=%d, filtered=%d\n' % (len(message), len(data), len(filtered))) self._file.write(filtered) self._file.flush() except: if DEBUG: print >> sys.stderr, 'puncturecrawler: error writing to file'
def _after_request_callback(self, exc, permid): if not exc: if DEBUG: print >> sys.stderr, 'puncturecrawler: request sent to', show_permid_short( permid) self._file.write('REQUEST %s %.2f\n' % (show_permid(permid), time.time())) self._file.flush()
def _after_send_request(exc, permid): if DEBUG: if exc: print >> sys.stderr, 'crawler: could not send request to', show_permid_short(permid), exc if exc: self._release_channel_id(permid, channel_id) if callback: callback(exc, permid)
def send_proxy_have(self, aggregated_haves): if DEBUG: print 'helper: send_proxy_have: sending a proxy_have message to', show_permid_short( self.coordinator_permid) aggregated_string = aggregated_haves.tostring() olthread_send_proxy_have_lambda = lambda: self.olthread_send_proxy_have( aggregated_string) self.overlay_bridge.add_task(olthread_send_proxy_have_lambda, 0)
def _after_request_callback(self, exc, permid): if not exc: if DEBUG: print >> sys.stderr, 'databasecrawler: request send to', show_permid_short( permid) self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST', show_permid(permid), '\n'))) self._file.flush()
def handleMessage(self, permid, selversion, message): t = message[0] if t == GET_SUBS: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Got GET_SUBS len: %s from %s' % ( len(message), show_permid_short(permid)) return self._handleGETSUBS(permid, message, selversion) elif t == SUBS: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Got SUBS len: %s from %s' % ( len(message), show_permid_short(permid)) return self._handleSUBS(permid, message, selversion) else: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Unknown Overlay Message %d' % ord( t) return False
def handle_crawler_request(self, permid, selversion, channel_id, message, reply_callback): if DEBUG: print >> sys.stderr, 'repexcrawler: handle_crawler_request', show_permid_short(permid), message try: repexhistory = self._repexlog.getHistoryAndCleanup() except Exception as e: reply_callback(str(e), error=1) else: reply_callback(cPickle.dumps(repexhistory, 2))
def _after_connect(exc, dns, permid, selversion): self._post_connection_attempt(permid, not exc) if exc: if DEBUG: print >> sys.stderr, 'crawler: could not connect', dns, show_permid_short(permid), exc if callback: callback(exc, permid) else: self._send_reply(permid, message_id, channel_id, payload, error=error, callback=callback)
def handleMessage(self, permid, selversion, message): t = message[0] if t == GET_METADATA: if DEBUG: print >> sys.stderr, 'metadata: Got GET_METADATA', len( message), show_permid_short(permid) return self.send_metadata(permid, message, selversion) elif t == METADATA: if DEBUG: print >> sys.stderr, 'metadata: Got METADATA', len( message), show_permid_short( permid), selversion, currentThread().getName() return self.got_metadata(permid, message, selversion) else: if DEBUG: print >> sys.stderr, 'metadata: UNKNOWN OVERLAY MESSAGE', ord( t) return False
def get_metadata_connect_callback(self, exc, dns, permid, selversion, infohash): if exc is None: if DEBUG: print >> sys.stderr, 'metadata: Sending GET_METADATA to', show_permid_short( permid) try: metadata_request = bencode(infohash) self.overlay_bridge.send(permid, GET_METADATA + metadata_request, self.get_metadata_send_callback) self.requested_torrents.add(infohash) except: print_exc() elif DEBUG: print >> sys.stderr, 'metadata: GET_METADATA: error connecting to', show_permid_short( permid)
def _after_request_callback(self, exc, permid): if not exc: if DEBUG: print >> sys.stderr, 'repexcrawler: request sent to', show_permid_short(permid) self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST', show_permid(permid), '\n'))) self._file.flush()
def process_query_reply(self, permid, query, usercallback, d): if DEBUG: print >> sys.stderr, 'rquery: process_query_reply:', show_permid_short(permid), query, d if len(d['a']) > 0: self.unidecode_hits(query, d) remote_query_usercallback_lambda = lambda : usercallback(permid, query, d['a']) self.session.uch.perform_usercallback(remote_query_usercallback_lambda) elif DEBUG: print >> sys.stderr, 'rquery: QUERY_REPLY: no results found'
def _sent_callback(self, exc, permid, channel_id, infohash, bitmask, msgSentCallback, usrCallback): if exc is not None: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Unable to send GET_SUBS to: ' + utilities.show_permid_short(permid) + ': ' + exc else: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'GET_SUBS sent to %s' % utilities.show_permid_short(permid) self._addToRequestedSubtitles(channel_id, infohash, bitmask, usrCallback) if msgSentCallback is not None: msgSentCallback(exc, permid, channel_id, infohash, bitmask)
def query_initiator(self, permid, selversion, request_callback): if selversion >= OLPROTO_VER_TENTH: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: query_initiator', show_permid_short( permid), 'version', selversion request_callback( CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, 'SELECT key, timestamp, event FROM playback_event; DELETE FROM playback_event;', callback=self._after_event_request_callback) elif selversion >= OLPROTO_VER_EIGHTH: if DEBUG: print >> sys.stderr, 'videoplaybackcrawler: query_initiator', show_permid_short( permid), 'version', selversion request_callback( CRAWLER_VIDEOPLAYBACK_INFO_QUERY, 'SELECT key, timestamp, piece_size, num_pieces, bitrate, nat FROM playback_info ORDER BY timestamp DESC LIMIT 50', callback=self._after_info_request_callback) elif DEBUG: print >> sys.stderr, 'videoplaybackcrawler: query_info_initiator', show_permid_short( permid), 'unsupported overlay version'
def connect(self, permid, callback): if DEBUG: print >> sys.stderr, 'olbridge: connect', show_permid_short(permid), currentThread().getName() def olbridge_connect_callback(cexc, cdns, cpermid, cselver): if DEBUG: print >> sys.stderr, 'olbridge: connect_callback', cexc, cdns, show_permid_short(cpermid), cselver, callback, currentThread().getName() olbridge_connect_callback_lambda = lambda : callback(cexc, cdns, cpermid, cselver) self.add_task(olbridge_connect_callback_lambda, 0) self.secover.connect(permid, olbridge_connect_callback)
def send(self, permid, msg, callback): if DEBUG: print >> sys.stderr, 'olbridge: send', show_permid_short(permid), len(msg) def olbridge_send_callback(cexc, cpermid): if DEBUG: print >> sys.stderr, 'olbridge: send_callback', cexc, show_permid_short(cpermid) olbridge_send_callback_lambda = lambda : callback(cexc, cpermid) self.add_task(olbridge_send_callback_lambda, 0) self.secover.send(permid, msg, olbridge_send_callback)
def doNatCheckCallback(self, exc, permid): if exc is not None: return False if DEBUG: print >> sys.stderr, 'NATCHECK_REQUEST was sent to', show_permid_short(permid), exc self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST', show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), '\n'))) self._file.flush() return True
def _close(self, permid): if DEBUG: print >> sys.stderr, 'secover: actual close', show_permid_short(permid) try: oc = self.get_oc_by_permid(permid) if not oc: if DEBUG: print >> sys.stderr, 'secover: error - actual close, but no connection to peer in admin' else: oc.close() except Exception as e: print_exc()
def handle_crawler_request(self, permid, selversion, channel_id, message, reply_callback): if DEBUG: print >> sys.stderr, 'usereventlogcrawler: handle_crawler_request', show_permid_short(permid), message try: cursor = self._sqlite_cache_db.execute_read(message) except Exception as e: reply_callback(str(e), error=1) else: if cursor: reply_callback(cPickle.dumps(list(cursor), 2)) else: reply_callback('error', error=2)
def process_query_reply(self, permid, query, usercallback, d): if DEBUG: print >> sys.stderr, 'rquery: process_query_reply:', show_permid_short( permid), query, d if len(d['a']) > 0: self.unidecode_hits(query, d) remote_query_usercallback_lambda = lambda: usercallback( permid, query, d['a']) self.session.uch.perform_usercallback( remote_query_usercallback_lambda) elif DEBUG: print >> sys.stderr, 'rquery: QUERY_REPLY: no results found'
def handle_info_crawler_request(self, permid, selversion, channel_id, message, reply_callback): if DEBUG: print >>sys.stderr, "videoplaybackcrawler: handle_info_crawler_request", show_permid_short(permid), message try: cursor = self._event_db._db.execute_read(message) except Exception as e: reply_callback(str(e), error=1) else: if cursor: reply_callback(zlib.compress(cPickle.dumps(list(cursor), 2), 9)) else: reply_callback("error", error=2)
def _handleGETSUBS(self, permid, message, selversion): if selversion < OLPROTO_VER_FOURTEENTH: if DEBUG: print >> sys.stderr, 'The peer that sent the GET_SUBS request has an oldprotcol version: this is strange. Dropping the msg' return False decoded = self._decodeGETSUBSMessage(message) if decoded is None: if DEBUG: print >> sys.stderr, 'Error decoding a GET_SUBS message from %s' % utilities.show_permid_short( permid) return False if DEBUG: channel_id, infohash, languages = decoded bitmask = self._languagesUtility.langCodesToMask(languages) print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ( 'RG', show_permid_short(permid), show_permid_short(channel_id), bin2str(infohash), bitmask, len(message)) for listener in self._listenersList: listener.receivedSubsRequest(permid, decoded, selversion) return True