def query_initiator(self, permid, selversion, request_callback): """ <<Crawler-side>> Established a new connection. Send a CRAWLER_VIDEOPLAYBACK_INFO_QUERY request. @param permid The Tribler peer permid @param selversion The oberlay protocol version @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload) """ if selversion >= OLPROTO_VER_TENTH: if DEBUG: print >>sys.stderr, "videoplaybackcrawler: query_initiator", show_permid_short(permid), "version", selversion # Overlay version 10 provided a simplification in the VOD # stats collecting. We now have only one database table: # playback_event that has only 3 columns: key, timestamp, # and event. request_callback(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, "SELECT key, timestamp, event FROM playback_event; DELETE FROM playback_event;", callback=self._after_event_request_callback) elif selversion >= OLPROTO_VER_EIGHTH: if DEBUG: print >>sys.stderr, "videoplaybackcrawler: query_initiator", show_permid_short(permid), "version", selversion # boudewijn: order the result DESC! From the resulting # list we will not remove the first entries from the # database because this (being the last item added) may # still be actively used. request_callback(CRAWLER_VIDEOPLAYBACK_INFO_QUERY, "SELECT key, timestamp, piece_size, num_pieces, bitrate, nat FROM playback_info ORDER BY timestamp DESC LIMIT 50", callback=self._after_info_request_callback) else: if DEBUG: print >>sys.stderr, "videoplaybackcrawler: query_info_initiator", show_permid_short(permid), "unsupported overlay version"
def receivedSubsRequest(self, permid, request, selversion): """ Reads a received GET_SUBS message and possibly sends a response. @param permid: the permid of the sender of the GET_SUBS message @param request: a tuple made of channel_id, infohash, language code @param selversion: the protocol version of the requesting peer @return: False if the message had something wrong. (a return value of False makes the caller close the connection). Otherwise True """ assert self.registered, SUBS_LOG_PREFIX + "Handler not yet registered" channel_id, infohash, languages = request #happily unpacking #diction {lang : Subtitle} allSubtitles = self.subtitlesDb.getAllSubtitles(channel_id, infohash) contentsList = {} #{langCode : path} #for each requested language check if the corresponding subtitle #is available for lang in sorted(languages): if lang in allSubtitles.keys(): if allSubtitles[lang].subtitleExists(): content = self._readSubContent(allSubtitles[lang].path) if content is not None: contentsList[lang] = content else: if DEBUG: print >> sys.stderr, time.asctime(),'-', SUBS_LOG_PREFIX + "File not available for " + \ "channel %s, infohash %s, lang %s" % \ (show_permid_short(channel_id), bin2str(infohash), lang) self.subtitlesDb.updateSubtitlePath(channel_id,infohash,lang,None) else: if DEBUG: print >> sys.stderr, time.asctime(),'-', SUBS_LOG_PREFIX + "Subtitle not available for " + \ "channel %s, infohash %s, lang %s" % \ (show_permid_short(channel_id), bin2str(infohash), lang) if len(contentsList) == 0: #pathlist is empty if DEBUG: print >> sys.stderr, time.asctime(),'-', SUBS_LOG_PREFIX + "None of the requested subtitles " + \ " was available. No answer will be sent to %s" % \ show_permid_short(permid) return True return self._subsMsgHndlr.sendSubtitleResponse(permid, (channel_id,infohash,contentsList), selversion)
def _handleGETSUBS(self,permid, message, selversion): if selversion < OLPROTO_VER_FOURTEENTH: if DEBUG: print >> sys.stderr, "The peer that sent the GET_SUBS request has an old" \ "protcol version: this is strange. Dropping the msg" return False decoded = self._decodeGETSUBSMessage(message) if decoded is None: if DEBUG: print >> sys.stderr, "Error decoding a GET_SUBS message from %s" %\ utilities.show_permid_short(permid) return False if DEBUG: channel_id, infohash, languages = decoded bitmask = self._languagesUtility.langCodesToMask(languages) print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % ("RG", show_permid_short(permid), show_permid_short(channel_id), bin2str(infohash), bitmask, len(message)) # no synch on _listenersList since both this method # and the registerListener method are called by # the OLThread for listener in self._listenersList: listener.receivedSubsRequest(permid, decoded, selversion) return True
def handleRMetadata(self, sender_permid, channelCastMessage, fromQuery = False): ''' Handles the reception of rich metadata. Called when an "erniched" channelCastMessage (v14) is received. @param sender_permid: the PermId of the peer who sent the message @param channelCastMessage: the received message @return: None ''' metadataDTOs, sizeList = \ self._splitChannelcastAndRichMetadataContents(channelCastMessage) if DEBUG: print >> sys.stderr, "Handling rich metadata from %s..." % show_permid_short(sender_permid) i=0 for md_and_have in metadataDTOs: md = md_and_have[0] havemask = md_and_have[1] vote = self.votecastDB.getVote(bin2str(md.channel), bin2str(self.my_permid)) # the next if may seem useless, but since sizeList is defined only when # logging is enabled for debug, I get an error without this conditional statement # because the argument for the debug() call getsEvaluated before the logging # system understands that debug is disabled #if announceStatsLog.isEnabledFor(logging.INFO): if DEBUG: id = "RQ" if fromQuery else "R" print >> sys.stderr, "%c, %s, %s, %s, %d, %d" % \ (id, md.channel, md.infohash, \ show_permid_short(sender_permid), md.timestamp, sizeList[i]) #format "R|S (R: received - S: sent), channel, infohash, sender|destination,metadataCreationTimestamp" # 30-06-2010: "RQ" as received from query i += 1 # check if the record belongs to a channel # who we have "reported spam" (negative vote) if vote == -1: # if so, ignore the incoming record continue isUpdate =self.rmdDb.insertMetadata(md) self.peerHaveManager.newHaveReceived(md.channel,md.infohash,sender_permid,havemask) if isUpdate is not None: #retrieve the metadataDTO from the database in the case it is an update md = self.rmdDb.getMetadata(md.channel,md.infohash) self._notifyRichMetadata(md, isUpdate) # if I am a subscriber send immediately a GET_SUBS to the # sender if vote == 2: if DEBUG: print >> sys.stderr, "Subscribed to channel %s, trying to retrieve" \ "all subtitle contents" % (show_permid_short(md.channel),) self._getAllSubtitles(md)
def handle_connection(self, exc, permid, selversion, locally_initiated): """ Called when overlay received a connection. Note that this method is only registered with OverlayApps when the command line option 'crawl' is used. """ if exc: # connection lost if DEBUG: print >>sys.stderr, "crawler: overlay connection lost", show_permid_short(permid), exc elif selversion >= OLPROTO_VER_SEVENTH: # verify that we do not already have deadlines for this permid already_known = False for tup in self._initiator_deadlines: if tup[4] == permid: already_known = True break if not already_known: if DEBUG: print >>sys.stderr, "crawler: new overlay connection", show_permid_short(permid) for initiator_callback, frequency, accept_frequency in self._crawl_initiators: self._initiator_deadlines.append([0, frequency, accept_frequency, initiator_callback, permid, selversion, 0]) self._initiator_deadlines.sort() # Start sending crawler requests self._check_deadlines(False) else: if DEBUG: print >>sys.stderr, "crawler: new overlay connection (can not use version %d)" % selversion, show_permid_short(permid)
def tryHolePunching(self): if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: first element in peerlist", self.peerlist[ len(self.peerlist) - 1] print >> sys.stderr, "NatCheckMsgHandler: second element in peerlist", self.peerlist[ len(self.peerlist) - 2] holePunchingPort = random.randrange(3200, 4200, 1) holePunchingAddr = (self.holePunchingIP, holePunchingPort) peer1 = self.peerlist[len(self.peerlist) - 1] peer2 = self.peerlist[len(self.peerlist) - 2] request_id = str( show_permid_short(peer1[0]) + show_permid_short(peer2[0]) + str(random.randrange(0, 1000, 1))) self.udpConnect(peer1[0], request_id, holePunchingAddr) self.udpConnect(peer2[0], request_id, holePunchingAddr) # Register peerinfo on file self._file2.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), "REQUEST", request_id, show_permid(peer1[0]), str(peer1[1]), str(peer1[2]), str(self._secure_overlay.get_dns_from_peerdb(peer1[0])), show_permid(peer2[0]), str(peer2[1]), str(peer2[2]), str(self._secure_overlay.get_dns_from_peerdb(peer2[0])), "\n"))) self._file2.flush() self.trav[request_id] = (None, None) thread.start_new_thread(coordinateHolePunching, (peer1, peer2, holePunchingAddr))
def tryHolePunching(self): if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: first element in peerlist", self.peerlist[len(self.peerlist)-1] print >> sys.stderr, "NatCheckMsgHandler: second element in peerlist", self.peerlist[len(self.peerlist)-2] holePunchingPort = random.randrange(3200, 4200, 1) holePunchingAddr = (self.holePunchingIP, holePunchingPort) peer1 = self.peerlist[len(self.peerlist)-1] peer2 = self.peerlist[len(self.peerlist)-2] request_id = str(show_permid_short(peer1[0]) + show_permid_short(peer2[0]) + str(random.randrange(0, 1000, 1))) self.udpConnect(peer1[0], request_id, holePunchingAddr) self.udpConnect(peer2[0], request_id, holePunchingAddr) # Register peerinfo on file self._file2.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "REQUEST", request_id, show_permid(peer1[0]), str(peer1[1]), str(peer1[2]), str(self._secure_overlay.get_dns_from_peerdb(peer1[0])), show_permid(peer2[0]), str(peer2[1]), str(peer2[2]), str(self._secure_overlay.get_dns_from_peerdb(peer2[0])), "\n"))) self._file2.flush() self.trav[request_id] = (None, None) thread.start_new_thread(coordinateHolePunching, (peer1, peer2, holePunchingAddr))
def setUp(self): self.config_path = tempfile.mkdtemp() config = {} config['state_dir'] = self.config_path config['install_dir'] = os.path.join('..','..') config['torrent_collecting_dir'] = self.config_path config['peer_icon_path'] = os.path.join(self.config_path,'peer_icons') config['superpeer'] = False sqlitecachedb.init(config, self.rawserver_fatalerrorfunc) secover1 = SecureOverlay.getInstance() secover1.resetSingleton() secover2 = SecureOverlay.getInstance() secover2.resetSingleton() self.peer1 = Peer(self,1234,secover1) self.peer2 = Peer(self,5678,secover2) self.peer1.start() self.peer2.start() self.wanted = False self.wanted2 = False self.got = False self.got2 = False self.first = True print >>sys.stderr,"test: setUp: peer1 permid is",show_permid_short(self.peer1.my_permid) print >>sys.stderr,"test: setUp: peer2 permid is",show_permid_short(self.peer2.my_permid) sleep(2) # let server threads start
def setUp(self): print >>sys.stderr,"test: TestOverlayThreadingBridge.setUp()" secover1 = SecureOverlay.getInstance() secover1.resetSingleton() secover2 = SecureOverlay.getInstance() secover2.resetSingleton() overbridge1 = OverlayThreadingBridge() overbridge1.register_bridge(secover1,None) overbridge2 = OverlayThreadingBridge() overbridge2.register_bridge(secover2,None) self.peer1 = Peer(self,1234,overbridge1) self.peer2 = Peer(self,5678,overbridge2) self.peer1.start() self.peer2.start() self.wanted = False self.wanted2 = False self.got = False self.got2 = False self.first = True print >>sys.stderr,"test: setUp: peer1 permid is",show_permid_short(self.peer1.my_permid) print >>sys.stderr,"test: setUp: peer2 permid is",show_permid_short(self.peer2.my_permid) time.sleep(2) # let server threads start
def handle_event_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): """ <<Crawler-side>> Received a CRAWLER_VIDEOPLAYBACK_EVENT_QUERY reply. @param permid The Crawler permid @param selversion The overlay protocol version @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair @param channel_data Data associated with the request @param error The error value. 0 indicates success. @param message The message payload @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload) """ if error: if DEBUG: print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), message, "\n"))) self._file.flush() elif selversion >= OLPROTO_VER_TENTH: # Overlay version 10 sends the reply pickled and zipped if DEBUG: print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(permid), len(message), "bytes zipped" info = cPickle.loads(zlib.decompress(message)) self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), str(info), "\n"))) self._file.flush() elif selversion >= OLPROTO_VER_EIGHTH: if DEBUG: print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(permid), cPickle.loads(message) info = cPickle.loads(message) self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), str(info), "\n"))) self._file.flush()
def olthread_pieces_reserved_send_callback(self,exc,permid): if exc is not None: if DEBUG: print >> sys.stderr,"dlhelp: PIECES_RESERVED: error sending to",show_permid_short(permid),exc else: if DEBUG: print >> sys.stderr,"dlhelp: PIECES_RESERVED: Successfully sent to",show_permid_short(permid)
def setUp(self): self.config_path = tempfile.mkdtemp() config = {} config['state_dir'] = self.config_path config['install_dir'] = os.path.join('..', '..') config['peer_icon_path'] = os.path.join(self.config_path, 'peer_icons') config['superpeer'] = False sqlitecachedb.init(config, self.rawserver_fatalerrorfunc) secover1 = SecureOverlay.getInstance() secover1.resetSingleton() secover2 = SecureOverlay.getInstance() secover2.resetSingleton() self.peer1 = Peer(self, 1234, secover1) self.peer2 = Peer(self, 5678, secover2) self.peer1.start() self.peer2.start() self.wanted = False self.wanted2 = False self.got = False self.got2 = False self.first = True print >> sys.stderr, "test: setUp: peer1 permid is", show_permid_short( self.peer1.my_permid) print >> sys.stderr, "test: setUp: peer2 permid is", show_permid_short( self.peer2.my_permid) sleep(2) # let server threads start
def _handleSUBS(self, permid, message, selversion): if selversion < OLPROTO_VER_FOURTEENTH: if DEBUG: print >> sys.stderr, "The peer that sent the SUBS request has an old" \ "protcol version: this is strange. Dropping the msg" return False decoded = self._decodeSUBSMessage(message) if decoded is None: if DEBUG: print >> sys.stderr, "Error decoding a SUBS message from %s" %\ utilities.show_permid_short(permid) return False channel_id, infohash, bitmask,contents = decoded #if no subtitle was requested drop the whole message if DEBUG: print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % ("RS", show_permid_short(permid), show_permid_short(channel_id), bin2str(infohash), bitmask, len(message)) requestedSubs = self._checkRequestedSubtitles(channel_id,infohash,bitmask) if requestedSubs == 0: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + "Received a SUBS message that was not"\ " requested. Dropping" return False requestedSubsCodes = self._languagesUtility.maskToLangCodes(requestedSubs) #drop from the contents subtitles that where not requested for lang in contents.keys(): if lang not in requestedSubsCodes: del contents[lang] #remove the received subtitles from the requested callbacks = \ self._removeFromRequestedSubtitles(channel_id, infohash, bitmask) #the receiver does not need the bitmask tuple = channel_id, infohash, contents # no synch on _listenersList since both this method # and the registerListener method are called by # the OLThread for listener in self._listenersList: listener.receivedSubsResponse(permid, tuple, callbacks, selversion) return True
def _subs_send_callback(self, exc, permid): ''' Called by the OLThread when a SUBS message is succesfully sent ''' if exc is not None: print >> sys.stderr, "Warning: Sending of SUBS message to %s failed: %s" % \ (show_permid_short(permid), str(exc)) else: if DEBUG: print >> sys.stderr, "SUBS message succesfully sent to %s" % show_permid_short(permid)
def olthread_request_help_connect_callback(self,exc,dns,permid,selversion): if exc is None: if DEBUG: print >> sys.stderr,"dlhelp: Coordinator sending to",show_permid_short(permid) ## Create message according to protocol version dlhelp_request = self.infohash self.overlay_bridge.send(permid, DOWNLOAD_HELP + dlhelp_request,self.olthread_request_help_send_callback) else: if DEBUG: print >> sys.stderr,"dlhelp: DOWNLOAD_HELP: error connecting to",show_permid_short(permid),exc self.olthread_remove_unreachable_helper(permid)
def _get_subs_connect_callback(self, exception, dns, permid, selversion, channel_id, infohash, bitmask, msgSentCallback, usrCallback): """ Called by the Overlay Thread when a connection with permid is established. Performs the actual action of sending a GET_SUBS request to the peer identified by permid. It is called by the OLThread when a connection with that peer is established. """ if exception is not None: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + \ "GET_SUBS not sent. Unable to connect to " + \ utilities.show_permid_short(permid) else: if (selversion > 0 and selversion < OLPROTO_VER_FOURTEENTH): msg = "GET_SUBS not send, the other peers had an old protocol version: %d" %\ selversion if DEBUG: print >> sys.stderr, msg raise SubtitleMsgHandlerException(msg) if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + "sending GET_SUBS to " + \ utilities.show_permid_short(permid) try : message = self._createGETSUBSMessage(channel_id, infohash, bitmask) if DEBUG: # Format: # SS|SG, destination, channel, infohash, bitmask, size print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % ("SG",show_permid_short(permid), show_permid_short(channel_id), bin2str(infohash),bitmask,len(message)) self._overlay_bridge.send(permid, message, lambda exc, permid: \ self._sent_callback(exc,permid, channel_id, infohash, bitmask, msgSentCallback, usrCallback)) except Exception,e: print_exc() msg = "GET_SUBS not sent: %s" % str(e) raise SubtitleMsgHandlerException(e)
def get_metadata_connect_callback(self,exc,dns,permid,selversion,infohash): if exc is None: if DEBUG: print >> sys.stderr,"metadata: Sending GET_METADATA to",show_permid_short(permid) ## Create metadata_request according to protocol version try: metadata_request = bencode(infohash) self.overlay_bridge.send(permid, GET_METADATA + metadata_request,self.get_metadata_send_callback) self.requested_torrents.add(infohash) except: print_exc() elif DEBUG: print >> sys.stderr,"metadata: GET_METADATA: error connecting to",show_permid_short(permid)
def setUpMyListenSocket(self): self.dest_keypair = EC.gen_params(EC.NID_sect233k1) self.dest_keypair.gen_key() self.destpermid = str(self.dest_keypair.pub().get_der()) self.destport = 4810 # Start our server side, to with Tribler will try to connect self.destss = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.destss.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.destss.bind(('', self.destport)) self.destss.listen(1) print >>sys.stderr,"test: my permid",show_permid_short(self.mypermid) print >>sys.stderr,"test: his permid",show_permid_short(self.hispermid) print >>sys.stderr,"test: dest permid",show_permid_short(self.destpermid)
def _sent_callback(self,exc,permid,channel_id,infohash,bitmask, msgSentCallback, usrCallback): """ Called by the OverlayThread after a GET_SUBS request has been sent. """ if exc is not None: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + "Unable to send GET_SUBS to: " + \ utilities.show_permid_short(permid) + ": " + exc else: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + "GET_SUBS sent to %s" % \ (utilities.show_permid_short(permid)) self._addToRequestedSubtitles(channel_id, infohash, bitmask, usrCallback) if msgSentCallback is not None: msgSentCallback(exc,permid,channel_id,infohash,bitmask)
def olthread_request_help_connect_callback(self, exc, dns, permid, selversion): if exc is None: if DEBUG: print >> sys.stderr, "dlhelp: Coordinator sending to", show_permid_short( permid) ## Create message according to protocol version dlhelp_request = self.infohash self.overlay_bridge.send(permid, DOWNLOAD_HELP + dlhelp_request, self.olthread_request_help_send_callback) else: if DEBUG: print >> sys.stderr, "dlhelp: DOWNLOAD_HELP: error connecting to", show_permid_short( permid), exc self.olthread_remove_unreachable_helper(permid)
def setUpMyListenSocket(self): self.dest_keypair = EC.gen_params(EC.NID_sect233k1) self.dest_keypair.gen_key() self.destpermid = str(self.dest_keypair.pub().get_der()) self.destport = 4810 # Start our server side, to with Tribler will try to connect self.destss = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.destss.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.destss.bind(('', self.destport)) self.destss.listen(1) print >>sys.stderr,time.asctime(),'-', "test: my permid",show_permid_short(self.mypermid) print >>sys.stderr,time.asctime(),'-', "test: his permid",show_permid_short(self.hispermid) print >>sys.stderr,time.asctime(),'-', "test: dest permid",show_permid_short(self.destpermid)
def olthread_reserve_pieces_connect_callback(self,exc,dns,permid,selversion,pieces,all_or_nothing): if exc is None: ## Create message according to protocol version if all_or_nothing: all_or_nothing = chr(1) else: all_or_nothing = chr(0) payload = self.torrent_hash + all_or_nothing + bencode(pieces) if DEBUG: print >> sys.stderr,"helper: RESERVE_PIECES: Sending!!!!!!!!!!!!!",show_permid_short(permid) self.overlay_bridge.send(permid, RESERVE_PIECES + payload,self.olthread_reserve_pieces_send_callback) elif DEBUG: print >> sys.stderr,"helper: RESERVE_PIECES: error connecting to",show_permid_short(permid),exc
def handle_event_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): """ <<Crawler-side>> Received a CRAWLER_VIDEOPLAYBACK_EVENT_QUERY reply. @param permid The Crawler permid @param selversion The overlay protocol version @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair @param channel_data Data associated with the request @param error The error value. 0 indicates success. @param message The message payload @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload) """ if error: if DEBUG: print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message self._file.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), message, "\n"))) self._file.flush() elif selversion >= OLPROTO_VER_TENTH: # Overlay version 10 sends the reply pickled and zipped if DEBUG: print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short( permid), len(message), "bytes zipped" info = cPickle.loads(zlib.decompress(message)) self._file.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), str(info), "\n"))) self._file.flush() elif selversion >= OLPROTO_VER_EIGHTH: if DEBUG: print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short( permid), cPickle.loads(message) info = cPickle.loads(message) self._file.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), str(info), "\n"))) self._file.flush()
def send_metadata_request(self, permid, infohash, selversion=-1, caller="BC"): if DEBUG: print >> sys.stderr,"metadata: Connect to send GET_METADATA to",show_permid_short(permid) if not isValidInfohash(infohash): return False filename,metadata = self.torrent_exists(infohash) if filename is not None: # torrent already exists on disk if DEBUG: print >> sys.stderr,"metadata: send_meta_req: Already on disk??!" self.notify_torrent_is_in(infohash, metadata, filename) return True if caller == "dlhelp": self.requested_torrents.add(infohash) if self.min_free_space != 0 and (self.free_space - self.avg_torrent_size < self.min_free_space): # no space to collect self.free_space = self.get_free_space() if self.free_space - self.avg_torrent_size < self.min_free_space: self.warn_disk_full() return True try: # Optimization: don't connect if we're connected, although it won't # do any harm. if selversion == -1: # not currently connected self.overlay_bridge.connect(permid,lambda e,d,p,s:self.get_metadata_connect_callback(e,d,p,s,infohash)) else: self.get_metadata_connect_callback(None,None,permid,selversion,infohash) except: print_exc() return False return True
def handleConnection(self,exc,permid,selversion,locally_initiated,hisdns): """ Called by NetworkThread """ # called by SecureOverlay.got_auth_connection() or cleanup_admin_and_callbacks() if DEBUG: print >>sys.stderr,"olbridge: handleConnection",exc,show_permid_short(permid),selversion,locally_initiated,hisdns,currentThread().getName() def olbridge_handle_conn_func(): # Called by OverlayThread if DEBUG: print >>sys.stderr,"olbridge: handle_conn_func",exc,show_permid_short(permid),selversion,locally_initiated,hisdns,currentThread().getName() try: if hisdns: self.secover.add_peer_to_db(permid,hisdns,selversion) if self.olappsconnhandler is not None: # self.olappsconnhandler = OverlayApps.handleConnection self.olappsconnhandler(exc,permid,selversion,locally_initiated) except: print_exc() if isinstance(exc,CloseException): self.secover.update_peer_status(permid,exc.was_auth_done()) self.tqueue.add_task(olbridge_handle_conn_func,0)
def handleMessage(self,permid,selversion,message): """ Called by NetworkThread """ #ProxyService_ # # DEBUG #print "### olbridge: handleMessage", show_permid_short(permid), selversion, getMessageName(message[0]), currentThread().getName() # #_ProxyService if DEBUG: print >>sys.stderr,"olbridge: handleMessage",show_permid_short(permid),selversion,getMessageName(message[0]),currentThread().getName() def olbridge_handle_msg_func(): # Called by OverlayThread if DEBUG: print >>sys.stderr,"olbridge: handle_msg_func",show_permid_short(permid),selversion,getMessageName(message[0]),currentThread().getName() try: if self.olappsmsghandler is None: ret = True else: ret = self.olappsmsghandler(permid,selversion,message) except: print_exc() ret = False if ret == False: if DEBUG: print >>sys.stderr,"olbridge: olbridge_handle_msg_func closing!",show_permid_short(permid),selversion,getMessageName(message[0]),currentThread().getName() self.close(permid) self.tqueue.add_task(olbridge_handle_msg_func,0) return True
def natCheckReplySendCallback(self, exc, permid): if DEBUG: print >> sys.stderr, "NATCHECK_REPLY was sent to", show_permid_short( permid), exc if exc is not None: return False return True
def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback): """ Received a CRAWLER_DATABASE_QUERY reply. @param permid The Crawler permid @param selversion The overlay protocol version @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair @param error The error value. 0 indicates success. @param message The message payload @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload) """ if error: if DEBUG: print >> sys.stderr, "databasecrawler: handle_crawler_reply", error, message self._file.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(error), message, "\n"))) self._file.flush() else: if DEBUG: print >> sys.stderr, "databasecrawler: handle_crawler_reply", show_permid_short( permid), cPickle.loads(message) self._file.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(error), str(cPickle.loads(message)), "\n"))) self._file.flush()
def handleMessage(self,permid,selversion,message): t = message[0] if t == GET_METADATA: # the other peer requests a torrent if DEBUG: print >> sys.stderr,"metadata: Got GET_METADATA",len(message),show_permid_short(permid) return self.send_metadata(permid, message, selversion) elif t == METADATA: # the other peer sends me a torrent if DEBUG: print >> sys.stderr,"metadata: Got METADATA",len(message),show_permid_short(permid),selversion, currentThread().getName() return self.got_metadata(permid, message, selversion) else: if DEBUG: print >> sys.stderr,"metadata: UNKNOWN OVERLAY MESSAGE", ord(t) return False
def handle_crawler_request(self, permid, selversion, channel_id, message, reply_callback): """ Received a CRAWLER_UDPUNCTURE_QUERY request. @param permid The Crawler permid @param selversion The overlay protocol version @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair @param message The message payload @param reply_callback Call this function once to send the reply: reply_callback(payload [, error=123]) """ if DEBUG: print >> sys.stderr, "puncturecrawler: handle_crawler_request", show_permid_short(permid), message SimpleFileReporter.lock.acquire() try: if not self.reporter.file: try: self.reporter.file = open(self.reporter.path, 'a+b') except Exception, e: reply_callback(str(e), error=1) return file = self.reporter.file try: file.seek(0) result = ("%.2f CRAWL\n" % time.time()) + file.read() result = zlib.compress(result) reply_callback(result) file.truncate(0) except Exception, e: reply_callback(str(e), error=1)
def handleConnection(self, exc, permid, selversion, locally_initiated, hisdns): """ Called by NetworkThread """ # called by SecureOverlay.got_auth_connection() or cleanup_admin_and_callbacks() if DEBUG: print >> sys.stderr, "olbridge: handleConnection", exc, show_permid_short( permid), selversion, locally_initiated, hisdns, currentThread( ).getName() def olbridge_handle_conn_func(): # Called by OverlayThread if DEBUG: print >> sys.stderr, "olbridge: handle_conn_func", exc, show_permid_short( permid ), selversion, locally_initiated, hisdns, currentThread( ).getName() try: if hisdns: self.secover.add_peer_to_db(permid, hisdns, selversion) if self.olappsconnhandler is not None: # self.olappsconnhandler = OverlayApps.handleConnection self.olappsconnhandler(exc, permid, selversion, locally_initiated) except: print_exc() if isinstance(exc, CloseException): self.secover.update_peer_status(permid, exc.was_auth_done()) self.tqueue.add_task(olbridge_handle_conn_func, 0)
def udpConnectReplySendCallback(self, exc, permid): if DEBUG: print >> sys.stderr, "NATTRAVERSAL_REPLY was sent to", show_permid_short(permid), exc if exc is not None: return False return True
def process_query_reply(self,permid,query,usercallback,d): if DEBUG: print >>sys.stderr,"rquery: process_query_reply:",show_permid_short(permid),query,d if len(d['a']) > 0: self.unidecode_hits(query,d) if query.startswith("CHANNEL"): # 13-04-2010 Andrea: The gotRemoteHits in SearchGridManager is too slow. # Since it is run by the GUIThread when there are too many hits the GUI # gets freezed. # dropping some random results if they are too many. # It is just an hack, a better method to improve performance should be found. if len(d['a']) > self.max_channel_query_results: if DEBUG: print >> sys.stderr, "DROPPING some answers: they where %d" % len(d['a']) newAnswers = {} newKeys = d['a'].keys()[:self.max_channel_query_results] for key in newKeys: newAnswers[key] = d['a'][key] d['a'] = newAnswers # Andrea 05-06-2010: updates the database through channelcast. Before this was # done by the GUIThread in SearchGridManager self.bc_fac.channelcast_core.updateChannel(permid,query,d['a']) # Inform user of remote channel hits remote_query_usercallback_lambda = lambda:usercallback(permid,query,d['a']) else: remote_query_usercallback_lambda = lambda:usercallback(permid,query,d['a']) self.session.uch.perform_usercallback(remote_query_usercallback_lambda) elif DEBUG: print >>sys.stderr,"rquery: QUERY_REPLY: no results found"
def handleMessage(self, permid, selversion, message): """ Called by NetworkThread """ if DEBUG: print >> sys.stderr, "olbridge: handleMessage", show_permid_short( permid), selversion, getMessageName( message[0]), currentThread().getName() def olbridge_handle_msg_func(): # Called by OverlayThread if DEBUG: print >> sys.stderr, "olbridge: handle_msg_func", show_permid_short( permid), selversion, getMessageName( message[0]), currentThread().getName() try: if self.olappsmsghandler is None: ret = True else: ret = self.olappsmsghandler(permid, selversion, message) except: print_exc() ret = False if ret == False: self.close(permid) self.tqueue.add_task(olbridge_handle_msg_func, 0) return True
def got_conn_outgoing_conns_callback(self,exc,permid,selversion,locally_initiated,hisdns=None): print >> sys.stderr,"test: got_conn_outgoing_conns_callback",exc,show_permid_short(permid) self.assert_(exc is None) self.assert_(permid == self.peer2.my_permid) self.assert_(selversion == OLPROTO_VER_CURRENT) self.assert_(locally_initiated) self.got = True
def get_metadata_send_callback(self,exc,permid): if exc is not None: if DEBUG: print >> sys.stderr,"metadata: GET_METADATA: error sending to",show_permid_short(permid),exc pass else: pass
def do_help(self, infohash, torrent_data, permid): basename = binascii.hexlify( infohash) + '.torrent' # ignore .tribe stuff, not vital torrentfilename = os.path.join(self.helpdir, basename) tfile = open(torrentfilename, "wb") tfile.write(torrent_data) tfile.close() if DEBUG: print >> sys.stderr, "helpmsg: Got metadata required for helping", show_permid_short( permid) print >> sys.stderr, "helpmsg: torrent: ", torrentfilename tdef = TorrentDef.load(torrentfilename) if self.dlconfig is None: dscfg = DownloadStartupConfig() else: dscfg = DownloadStartupConfig(self.dlconfig) dscfg.set_coopdl_coordinator_permid(permid) dscfg.set_dest_dir(self.helpdir) # Start new download self.session.start_download(tdef, dscfg)
def olthread_stop_help_connect_callback(self,exc,dns,permid,selversion): if exc is None: ## Create message according to protocol version stop_request = self.infohash self.overlay_bridge.send(permid,STOP_DOWNLOAD_HELP + stop_request,self.olthread_stop_help_send_callback) elif DEBUG: print >> sys.stderr,"dlhelp: STOP_DOWNLOAD_HELP: error connecting to",show_permid_short(permid),exc
def olthread_send_request_help(self, permidlist): for permid in permidlist: if DEBUG: print >> sys.stderr, "dlhelp: Coordinator connecting to", show_permid_short( permid), "for help" self.overlay_bridge.connect( permid, self.olthread_request_help_connect_callback)
def query_initiator(self, permid, selversion, request_callback): """ Established a new connection. Send a CRAWLER_DATABASE_QUERY request. @param permid The Tribler peer permid @param selversion The oberlay protocol version @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload) """ if DEBUG: print >>sys.stderr, "databasecrawler: query_initiator", show_permid_short(permid) sql = [] if selversion >= OLPROTO_VER_SEVENTH: sql.extend(("SELECT 'peer_count', count(*) FROM Peer", "SELECT 'torrent_count', count(*) FROM Torrent")) if selversion >= OLPROTO_VER_ELEVENTH: sql.extend(("SELECT 'my_subscriptions', count(*) FROM VoteCast where voter_id='" + show_permid(permid) + "' and vote=2", "SELECT 'my_negative_votes', count(*) FROM VoteCast where voter_id='" + show_permid(permid) + "' and vote=-1", "SELECT 'my_channel_files', count(*) FROM ChannelCast where publisher_id='" + show_permid(permid) + "'", "SELECT 'all_subscriptions', count(*) FROM VoteCast where vote=2", "SELECT 'all_negative_votes', count(*) FROM VoteCast where vote=-1")) # if OLPROTO_VER_EIGHTH <= selversion <= 11: # sql.extend(("SELECT 'moderations_count', count(*) FROM ModerationCast")) # if selversion >= OLPROTO_VER_EIGHTH: # sql.extend(("SELECT 'positive_votes_count', count(*) FROM Moderators where status=1", # "SELECT 'negative_votes_count', count(*) FROM Moderators where status=-1")) request_callback(CRAWLER_DATABASE_QUERY, ";".join(sql), callback=self._after_request_callback)
def _after_send_reply(exc, permid): if DEBUG: if exc: print >> sys.stderr, "crawler: could not send request", show_permid_short(permid), exc # call the optional callback supplied with send_request if callback: callback(exc, permid)
def udpConnect(self, permid, request_id, holePunchingAddr): if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: request UDP connection" mh_data = request_id + ":" + holePunchingAddr[0] + ":" + str( holePunchingAddr[1]) if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: udpConnect message is", mh_data try: mh_msg = bencode(mh_data) except: print_exc() if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: error mh_data:", mh_data return False # send the message self.crawler.send_request(permid, CRAWLER_NATTRAVERSAL, mh_msg, frequency=0, callback=self.udpConnectCallback) if DEBUG: print >> sys.stderr, "NatCheckMsgHandler: request for", show_permid_short( permid), "sent to crawler"
def get_metadata_send_callback(self, exc, permid): if exc is not None: if DEBUG: print >> sys.stderr, "metadata: GET_METADATA: error sending to", show_permid_short( permid), exc pass else: pass
def _after_send_reply(exc, permid): if DEBUG: if exc: print >> sys.stderr, "crawler: could not send request", show_permid_short( permid), exc # call the optional callback supplied with send_request if callback: callback(exc, permid)
def udpConnectReplySendCallback(self, exc, permid): if DEBUG: print >> sys.stderr, "NATTRAVERSAL_REPLY was sent to", show_permid_short( permid), exc if exc is not None: return False return True
def got_conn_outgoing_conns_callback(self, exc, permid, selversion, locally_initiated, hisdns): print >> sys.stderr, "test: got_conn_outgoing_conns_callback", exc, show_permid_short( permid) self.assert_(exc is None) self.assert_(permid == self.peer2.my_permid) self.assert_(selversion == OLPROTO_VER_CURRENT) self.assert_(locally_initiated) self.got = True
def olbridge_send_callback(cexc, cpermid): # Called by network thread if DEBUG: print >> sys.stderr, "olbridge: send_callback", cexc, show_permid_short( cpermid) olbridge_send_callback_lambda = lambda: callback(cexc, cpermid) self.add_task(olbridge_send_callback_lambda, 0)
def udpConnectCallback(self, exc, permid): if exc is not None: if DEBUG: print >> sys.stderr, "NATTRAVERSAL_REQUEST failed to", show_permid_short( permid), exc # Register peerinfo on file self._file2.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), "REQUEST FAILED", show_permid(permid), str(self._secure_overlay.get_dns_from_peerdb(permid)), "\n"))) return False if DEBUG: print >> sys.stderr, "NATTRAVERSAL_REQUEST was sent to", show_permid_short( permid), exc return True
def _after_request_callback(self, exc, permid): """ Called by the Crawler with the result of the request_callback call in the query_initiator method. """ if not exc: if DEBUG: print >>sys.stderr, "repexcrawler: request sent to", show_permid_short(permid) self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "REQUEST", show_permid(permid), "\n"))) self._file.flush()
def olthread_pieces_reserved_send_callback(self, exc, permid): if exc is not None: if DEBUG: print >> sys.stderr, "dlhelp: PIECES_RESERVED: error sending to", show_permid_short( permid), exc else: if DEBUG: print >> sys.stderr, "dlhelp: PIECES_RESERVED: Successfully sent to", show_permid_short( permid)
def get_metadata_connect_callback(self, exc, dns, permid, selversion, infohash): if exc is None: if DEBUG: print >> sys.stderr, "metadata: Sending GET_METADATA to", show_permid_short( permid) ## Create metadata_request according to protocol version try: metadata_request = bencode(infohash) self.overlay_bridge.send(permid, GET_METADATA + metadata_request, self.get_metadata_send_callback) self.requested_torrents.add(infohash) except: print_exc() elif DEBUG: print >> sys.stderr, "metadata: GET_METADATA: error connecting to", show_permid_short( permid)
def handle_info_crawler_reply(self, permid, selversion, channel_id, error, message, request_callback): """ <<Crawler-side>> Received a CRAWLER_VIDEOPLAYBACK_INFO_QUERY reply. @param permid The Crawler permid @param selversion The overlay protocol version @param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair @param error The error value. 0 indicates success. @param message The message payload @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload) """ if error: if DEBUG: print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message self._file.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), " INFO REPLY", show_permid(permid), str(error), message, "\n"))) self._file.flush() else: if DEBUG: print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short( permid), cPickle.loads(message) info = cPickle.loads(message) self._file.write("; ".join( (strftime("%Y/%m/%d %H:%M:%S"), " INFO REPLY", show_permid(permid), str(error), str(info), "\n"))) self._file.flush() i = 0 for key, timestamp, piece_size, num_pieces, bitrate, nat in info: i += 1 # do not remove the first item. the list is ordered # DESC so the first item is the last that is added to # the database and we can't affored to remove it, as # it may cause exceptions in the running playback. if i == 1: sql = """ SELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50; DELETE FROM playback_event WHERE key = '%s'; """ % (key, key) else: sql = """ SELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50; DELETE FROM playback_event WHERE key = '%s'; DELETE FROM playback_info WHERE key = '%s'; """ % (key, key, key) # todo: optimize to not select key for each row request_callback(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, sql, channel_data=key, callback=self._after_event_request_callback, frequency=0)
def query_initiator(self, permid, selversion, request_callback): """ Established a new connection. Send a CRAWLER_REPEX_QUERY request. @param permid The Tribler peer permid @param selversion The overlay protocol version @param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload) """ if DEBUG: print >>sys.stderr, "repexcrawler: query_initiator", show_permid_short(permid) request_callback(CRAWLER_REPEX_QUERY, '', callback=self._after_request_callback)
def olbridge_connect_dns_callback(cexc, cdns, cpermid, cselver): # Called by network thread if DEBUG: print >> sys.stderr, "olbridge: connect_dns_callback", cexc, cdns, show_permid_short( cpermid), cselver olbridge_connect_dns_callback_lambda = lambda: callback( cexc, cdns, cpermid, cselver) self.add_task(olbridge_connect_dns_callback_lambda, 0)
def olthread_stop_help_connect_callback(self, exc, dns, permid, selversion): if exc is None: ## Create message according to protocol version stop_request = self.infohash self.overlay_bridge.send(permid, STOP_DOWNLOAD_HELP + stop_request, self.olthread_stop_help_send_callback) elif DEBUG: print >> sys.stderr, "dlhelp: STOP_DOWNLOAD_HELP: error connecting to", show_permid_short( permid), exc
def handleMessage(self, permid, selversion, message): t = message[0] if t == GET_METADATA: # the other peer requests a torrent if DEBUG: print >> sys.stderr, "metadata: Got GET_METADATA", len( message), show_permid_short(permid) return self.send_metadata(permid, message, selversion) elif t == METADATA: # the other peer sends me a torrent if DEBUG: print >> sys.stderr, "metadata: Got METADATA", len( message), show_permid_short( permid), selversion, currentThread().getName() return self.got_metadata(permid, message, selversion) else: if DEBUG: print >> sys.stderr, "metadata: UNKNOWN OVERLAY MESSAGE", ord( t) return False
def _after_send_request(exc, permid): if DEBUG: if exc: print >> sys.stderr, "crawler: could not send request to", show_permid_short( permid), exc if exc: self._release_channel_id(permid, channel_id) # call the optional callback supplied with send_request if callback: callback(exc, permid)
def olbridge_connect_callback(cexc, cdns, cpermid, cselver): # Called by network thread if DEBUG: print >> sys.stderr, "olbridge: connect_callback", cexc, cdns, show_permid_short( cpermid), cselver, callback, currentThread().getName() olbridge_connect_callback_lambda = lambda: callback( cexc, cdns, cpermid, cselver) # Jie: postpone to call this callback to schedule it after the peer has been added to buddycast connection list # Arno, 2008-09-15: No-no-no self.add_task(olbridge_connect_callback_lambda, 0)
def olthread_reserve_pieces_connect_callback(self, exc, dns, permid, selversion, pieces, all_or_nothing): if exc is None: ## Create message according to protocol version if all_or_nothing: all_or_nothing = chr(1) else: all_or_nothing = chr(0) payload = self.torrent_hash + all_or_nothing + bencode(pieces) if DEBUG: print >> sys.stderr, "helper: RESERVE_PIECES: Sending!!!!!!!!!!!!!", show_permid_short( permid) self.overlay_bridge.send( permid, RESERVE_PIECES + payload, self.olthread_reserve_pieces_send_callback) elif DEBUG: print >> sys.stderr, "helper: RESERVE_PIECES: error connecting to", show_permid_short( permid), exc