def query_initiator(self, permid, selversion, request_callback):
"""
<<Crawler-side>>
Established a new connection. Send a CRAWLER_VIDEOPLAYBACK_INFO_QUERY request.
@param permid The Tribler peer permid
@param selversion The oberlay protocol version
@param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
"""
if selversion >= OLPROTO_VER_TENTH:
if DEBUG: print >>sys.stderr, "videoplaybackcrawler: query_initiator", show_permid_short(permid), "version", selversion
# Overlay version 10 provided a simplification in the VOD
# stats collecting. We now have only one database table:
# playback_event that has only 3 columns: key, timestamp,
# and event.
request_callback(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, "SELECT key, timestamp, event FROM playback_event; DELETE FROM playback_event;", callback=self._after_event_request_callback)
elif selversion >= OLPROTO_VER_EIGHTH:
if DEBUG: print >>sys.stderr, "videoplaybackcrawler: query_initiator", show_permid_short(permid), "version", selversion
# boudewijn: order the result DESC! From the resulting
# list we will not remove the first entries from the
# database because this (being the last item added) may
# still be actively used.
request_callback(CRAWLER_VIDEOPLAYBACK_INFO_QUERY, "SELECT key, timestamp, piece_size, num_pieces, bitrate, nat FROM playback_info ORDER BY timestamp DESC LIMIT 50", callback=self._after_info_request_callback)
else:
if DEBUG: print >>sys.stderr, "videoplaybackcrawler: query_info_initiator", show_permid_short(permid), "unsupported overlay version"
def receivedSubsRequest(self, permid, request, selversion):
"""
Reads a received GET_SUBS message and possibly sends a response.
@param permid: the permid of the sender of the GET_SUBS message
@param request: a tuple made of channel_id, infohash, language code
@param selversion: the protocol version of the requesting peer
@return: False if the message had something wrong. (a return value
of False makes the caller close the connection).
Otherwise True
"""
assert self.registered, SUBS_LOG_PREFIX + "Handler not yet registered"
channel_id, infohash, languages = request #happily unpacking
#diction {lang : Subtitle}
allSubtitles = self.subtitlesDb.getAllSubtitles(channel_id,
infohash)
contentsList = {} #{langCode : path}
#for each requested language check if the corresponding subtitle
#is available
for lang in sorted(languages):
if lang in allSubtitles.keys():
if allSubtitles[lang].subtitleExists():
content = self._readSubContent(allSubtitles[lang].path)
if content is not None:
contentsList[lang] = content
else:
if DEBUG:
print >> sys.stderr, time.asctime(),'-', SUBS_LOG_PREFIX + "File not available for " + \
"channel %s, infohash %s, lang %s" % \
(show_permid_short(channel_id), bin2str(infohash),
lang)
self.subtitlesDb.updateSubtitlePath(channel_id,infohash,lang,None)
else:
if DEBUG:
print >> sys.stderr, time.asctime(),'-', SUBS_LOG_PREFIX + "Subtitle not available for " + \
"channel %s, infohash %s, lang %s" % \
(show_permid_short(channel_id), bin2str(infohash),
lang)
if len(contentsList) == 0: #pathlist is empty
if DEBUG:
print >> sys.stderr, time.asctime(),'-', SUBS_LOG_PREFIX + "None of the requested subtitles " + \
" was available. No answer will be sent to %s" % \
show_permid_short(permid)
return True
return self._subsMsgHndlr.sendSubtitleResponse(permid,
(channel_id,infohash,contentsList),
selversion)
def _handleGETSUBS(self,permid, message, selversion):
if selversion < OLPROTO_VER_FOURTEENTH:
if DEBUG:
print >> sys.stderr, "The peer that sent the GET_SUBS request has an old" \
"protcol version: this is strange. Dropping the msg"
return False
decoded = self._decodeGETSUBSMessage(message)
if decoded is None:
if DEBUG:
print >> sys.stderr, "Error decoding a GET_SUBS message from %s" %\
utilities.show_permid_short(permid)
return False
if DEBUG:
channel_id, infohash, languages = decoded
bitmask = self._languagesUtility.langCodesToMask(languages)
print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % ("RG", show_permid_short(permid),
show_permid_short(channel_id),
bin2str(infohash), bitmask, len(message))
# no synch on _listenersList since both this method
# and the registerListener method are called by
# the OLThread
for listener in self._listenersList:
listener.receivedSubsRequest(permid, decoded, selversion)
return True
def handleRMetadata(self, sender_permid, channelCastMessage, fromQuery = False):
'''
Handles the reception of rich metadata.
Called when an "erniched" channelCastMessage (v14) is received.
@param sender_permid: the PermId of the peer who sent the message
@param channelCastMessage: the received message
@return: None
'''
metadataDTOs, sizeList = \
self._splitChannelcastAndRichMetadataContents(channelCastMessage)
if DEBUG:
print >> sys.stderr, "Handling rich metadata from %s..." % show_permid_short(sender_permid)
i=0
for md_and_have in metadataDTOs:
md = md_and_have[0]
havemask = md_and_have[1]
vote = self.votecastDB.getVote(bin2str(md.channel),
bin2str(self.my_permid))
# the next if may seem useless, but since sizeList is defined only when
# logging is enabled for debug, I get an error without this conditional statement
# because the argument for the debug() call getsEvaluated before the logging
# system understands that debug is disabled
#if announceStatsLog.isEnabledFor(logging.INFO):
if DEBUG:
id = "RQ" if fromQuery else "R"
print >> sys.stderr, "%c, %s, %s, %s, %d, %d" % \
(id, md.channel, md.infohash, \
show_permid_short(sender_permid), md.timestamp,
sizeList[i])
#format "R|S (R: received - S: sent), channel, infohash, sender|destination,metadataCreationTimestamp"
# 30-06-2010: "RQ" as received from query
i += 1
# check if the record belongs to a channel
# who we have "reported spam" (negative vote)
if vote == -1:
# if so, ignore the incoming record
continue
isUpdate =self.rmdDb.insertMetadata(md)
self.peerHaveManager.newHaveReceived(md.channel,md.infohash,sender_permid,havemask)
if isUpdate is not None:
#retrieve the metadataDTO from the database in the case it is an update
md = self.rmdDb.getMetadata(md.channel,md.infohash)
self._notifyRichMetadata(md, isUpdate)
# if I am a subscriber send immediately a GET_SUBS to the
# sender
if vote == 2:
if DEBUG:
print >> sys.stderr, "Subscribed to channel %s, trying to retrieve" \
"all subtitle contents" % (show_permid_short(md.channel),)
self._getAllSubtitles(md)
def handle_connection(self, exc, permid, selversion, locally_initiated):
"""
Called when overlay received a connection. Note that this
method is only registered with OverlayApps when the command
line option 'crawl' is used.
"""
if exc:
# connection lost
if DEBUG: print >>sys.stderr, "crawler: overlay connection lost", show_permid_short(permid), exc
elif selversion >= OLPROTO_VER_SEVENTH:
# verify that we do not already have deadlines for this permid
already_known = False
for tup in self._initiator_deadlines:
if tup[4] == permid:
already_known = True
break
if not already_known:
if DEBUG: print >>sys.stderr, "crawler: new overlay connection", show_permid_short(permid)
for initiator_callback, frequency, accept_frequency in self._crawl_initiators:
self._initiator_deadlines.append([0, frequency, accept_frequency, initiator_callback, permid, selversion, 0])
self._initiator_deadlines.sort()
# Start sending crawler requests
self._check_deadlines(False)
else:
if DEBUG: print >>sys.stderr, "crawler: new overlay connection (can not use version %d)" % selversion, show_permid_short(permid)
def tryHolePunching(self):
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: first element in peerlist", self.peerlist[
len(self.peerlist) - 1]
print >> sys.stderr, "NatCheckMsgHandler: second element in peerlist", self.peerlist[
len(self.peerlist) - 2]
holePunchingPort = random.randrange(3200, 4200, 1)
holePunchingAddr = (self.holePunchingIP, holePunchingPort)
peer1 = self.peerlist[len(self.peerlist) - 1]
peer2 = self.peerlist[len(self.peerlist) - 2]
request_id = str(
show_permid_short(peer1[0]) + show_permid_short(peer2[0]) +
str(random.randrange(0, 1000, 1)))
self.udpConnect(peer1[0], request_id, holePunchingAddr)
self.udpConnect(peer2[0], request_id, holePunchingAddr)
# Register peerinfo on file
self._file2.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), "REQUEST", request_id,
show_permid(peer1[0]), str(peer1[1]), str(peer1[2]),
str(self._secure_overlay.get_dns_from_peerdb(peer1[0])),
show_permid(peer2[0]), str(peer2[1]), str(peer2[2]),
str(self._secure_overlay.get_dns_from_peerdb(peer2[0])), "\n")))
self._file2.flush()
self.trav[request_id] = (None, None)
thread.start_new_thread(coordinateHolePunching,
(peer1, peer2, holePunchingAddr))
def tryHolePunching(self):
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: first element in peerlist", self.peerlist[len(self.peerlist)-1]
print >> sys.stderr, "NatCheckMsgHandler: second element in peerlist", self.peerlist[len(self.peerlist)-2]
holePunchingPort = random.randrange(3200, 4200, 1)
holePunchingAddr = (self.holePunchingIP, holePunchingPort)
peer1 = self.peerlist[len(self.peerlist)-1]
peer2 = self.peerlist[len(self.peerlist)-2]
request_id = str(show_permid_short(peer1[0]) + show_permid_short(peer2[0]) + str(random.randrange(0, 1000, 1)))
self.udpConnect(peer1[0], request_id, holePunchingAddr)
self.udpConnect(peer2[0], request_id, holePunchingAddr)
# Register peerinfo on file
self._file2.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"),
"REQUEST",
request_id,
show_permid(peer1[0]),
str(peer1[1]),
str(peer1[2]),
str(self._secure_overlay.get_dns_from_peerdb(peer1[0])),
show_permid(peer2[0]),
str(peer2[1]),
str(peer2[2]),
str(self._secure_overlay.get_dns_from_peerdb(peer2[0])),
"\n")))
self._file2.flush()
self.trav[request_id] = (None, None)
thread.start_new_thread(coordinateHolePunching, (peer1, peer2, holePunchingAddr))
def setUp(self):
self.config_path = tempfile.mkdtemp()
config = {}
config['state_dir'] = self.config_path
config['install_dir'] = os.path.join('..','..')
config['torrent_collecting_dir'] = self.config_path
config['peer_icon_path'] = os.path.join(self.config_path,'peer_icons')
config['superpeer'] = False
sqlitecachedb.init(config, self.rawserver_fatalerrorfunc)
secover1 = SecureOverlay.getInstance()
secover1.resetSingleton()
secover2 = SecureOverlay.getInstance()
secover2.resetSingleton()
self.peer1 = Peer(self,1234,secover1)
self.peer2 = Peer(self,5678,secover2)
self.peer1.start()
self.peer2.start()
self.wanted = False
self.wanted2 = False
self.got = False
self.got2 = False
self.first = True
print >>sys.stderr,"test: setUp: peer1 permid is",show_permid_short(self.peer1.my_permid)
print >>sys.stderr,"test: setUp: peer2 permid is",show_permid_short(self.peer2.my_permid)
sleep(2) # let server threads start
def setUp(self):
print >>sys.stderr,"test: TestOverlayThreadingBridge.setUp()"
secover1 = SecureOverlay.getInstance()
secover1.resetSingleton()
secover2 = SecureOverlay.getInstance()
secover2.resetSingleton()
overbridge1 = OverlayThreadingBridge()
overbridge1.register_bridge(secover1,None)
overbridge2 = OverlayThreadingBridge()
overbridge2.register_bridge(secover2,None)
self.peer1 = Peer(self,1234,overbridge1)
self.peer2 = Peer(self,5678,overbridge2)
self.peer1.start()
self.peer2.start()
self.wanted = False
self.wanted2 = False
self.got = False
self.got2 = False
self.first = True
print >>sys.stderr,"test: setUp: peer1 permid is",show_permid_short(self.peer1.my_permid)
print >>sys.stderr,"test: setUp: peer2 permid is",show_permid_short(self.peer2.my_permid)
time.sleep(2) # let server threads start
def handle_event_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback):
"""
<<Crawler-side>>
Received a CRAWLER_VIDEOPLAYBACK_EVENT_QUERY reply.
@param permid The Crawler permid
@param selversion The overlay protocol version
@param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
@param channel_data Data associated with the request
@param error The error value. 0 indicates success.
@param message The message payload
@param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
"""
if error:
if DEBUG:
print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message
self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), message, "\n")))
self._file.flush()
elif selversion >= OLPROTO_VER_TENTH:
# Overlay version 10 sends the reply pickled and zipped
if DEBUG:
print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(permid), len(message), "bytes zipped"
info = cPickle.loads(zlib.decompress(message))
self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), str(info), "\n")))
self._file.flush()
elif selversion >= OLPROTO_VER_EIGHTH:
if DEBUG:
print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(permid), cPickle.loads(message)
info = cPickle.loads(message)
self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), str(info), "\n")))
self._file.flush()
def olthread_pieces_reserved_send_callback(self,exc,permid):
if exc is not None:
if DEBUG:
print >> sys.stderr,"dlhelp: PIECES_RESERVED: error sending to",show_permid_short(permid),exc
else:
if DEBUG:
print >> sys.stderr,"dlhelp: PIECES_RESERVED: Successfully sent to",show_permid_short(permid)
def setUp(self):
self.config_path = tempfile.mkdtemp()
config = {}
config['state_dir'] = self.config_path
config['install_dir'] = os.path.join('..', '..')
config['peer_icon_path'] = os.path.join(self.config_path, 'peer_icons')
config['superpeer'] = False
sqlitecachedb.init(config, self.rawserver_fatalerrorfunc)
secover1 = SecureOverlay.getInstance()
secover1.resetSingleton()
secover2 = SecureOverlay.getInstance()
secover2.resetSingleton()
self.peer1 = Peer(self, 1234, secover1)
self.peer2 = Peer(self, 5678, secover2)
self.peer1.start()
self.peer2.start()
self.wanted = False
self.wanted2 = False
self.got = False
self.got2 = False
self.first = True
print >> sys.stderr, "test: setUp: peer1 permid is", show_permid_short(
self.peer1.my_permid)
print >> sys.stderr, "test: setUp: peer2 permid is", show_permid_short(
self.peer2.my_permid)
sleep(2) # let server threads start
def _handleSUBS(self, permid, message, selversion):
if selversion < OLPROTO_VER_FOURTEENTH:
if DEBUG:
print >> sys.stderr, "The peer that sent the SUBS request has an old" \
"protcol version: this is strange. Dropping the msg"
return False
decoded = self._decodeSUBSMessage(message)
if decoded is None:
if DEBUG:
print >> sys.stderr, "Error decoding a SUBS message from %s" %\
utilities.show_permid_short(permid)
return False
channel_id, infohash, bitmask,contents = decoded
#if no subtitle was requested drop the whole message
if DEBUG:
print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % ("RS", show_permid_short(permid),
show_permid_short(channel_id),
bin2str(infohash), bitmask, len(message))
requestedSubs = self._checkRequestedSubtitles(channel_id,infohash,bitmask)
if requestedSubs == 0:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + "Received a SUBS message that was not"\
" requested. Dropping"
return False
requestedSubsCodes = self._languagesUtility.maskToLangCodes(requestedSubs)
#drop from the contents subtitles that where not requested
for lang in contents.keys():
if lang not in requestedSubsCodes:
del contents[lang]
#remove the received subtitles from the requested
callbacks = \
self._removeFromRequestedSubtitles(channel_id, infohash, bitmask)
#the receiver does not need the bitmask
tuple = channel_id, infohash, contents
# no synch on _listenersList since both this method
# and the registerListener method are called by
# the OLThread
for listener in self._listenersList:
listener.receivedSubsResponse(permid, tuple, callbacks, selversion)
return True
def _subs_send_callback(self, exc, permid):
'''
Called by the OLThread when a SUBS message is succesfully sent
'''
if exc is not None:
print >> sys.stderr, "Warning: Sending of SUBS message to %s failed: %s" % \
(show_permid_short(permid), str(exc))
else:
if DEBUG:
print >> sys.stderr, "SUBS message succesfully sent to %s" % show_permid_short(permid)
def olthread_request_help_connect_callback(self,exc,dns,permid,selversion):
if exc is None:
if DEBUG:
print >> sys.stderr,"dlhelp: Coordinator sending to",show_permid_short(permid)
## Create message according to protocol version
dlhelp_request = self.infohash
self.overlay_bridge.send(permid, DOWNLOAD_HELP + dlhelp_request,self.olthread_request_help_send_callback)
else:
if DEBUG:
print >> sys.stderr,"dlhelp: DOWNLOAD_HELP: error connecting to",show_permid_short(permid),exc
self.olthread_remove_unreachable_helper(permid)
def _get_subs_connect_callback(self, exception, dns, permid, selversion,
channel_id, infohash, bitmask, msgSentCallback, usrCallback):
"""
Called by the Overlay Thread when a connection with permid is established.
Performs the actual action of sending a GET_SUBS request to the peer
identified by permid. It is called by the OLThread when a connection
with that peer is established.
"""
if exception is not None:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + \
"GET_SUBS not sent. Unable to connect to " + \
utilities.show_permid_short(permid)
else:
if (selversion > 0 and selversion < OLPROTO_VER_FOURTEENTH):
msg = "GET_SUBS not send, the other peers had an old protocol version: %d" %\
selversion
if DEBUG:
print >> sys.stderr, msg
raise SubtitleMsgHandlerException(msg)
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + "sending GET_SUBS to " + \
utilities.show_permid_short(permid)
try :
message = self._createGETSUBSMessage(channel_id, infohash,
bitmask)
if DEBUG:
# Format:
# SS|SG, destination, channel, infohash, bitmask, size
print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % ("SG",show_permid_short(permid),
show_permid_short(channel_id),
bin2str(infohash),bitmask,len(message))
self._overlay_bridge.send(permid, message,
lambda exc, permid: \
self._sent_callback(exc,permid,
channel_id,
infohash,
bitmask,
msgSentCallback,
usrCallback))
except Exception,e:
print_exc()
msg = "GET_SUBS not sent: %s" % str(e)
raise SubtitleMsgHandlerException(e)
def get_metadata_connect_callback(self,exc,dns,permid,selversion,infohash):
if exc is None:
if DEBUG:
print >> sys.stderr,"metadata: Sending GET_METADATA to",show_permid_short(permid)
## Create metadata_request according to protocol version
try:
metadata_request = bencode(infohash)
self.overlay_bridge.send(permid, GET_METADATA + metadata_request,self.get_metadata_send_callback)
self.requested_torrents.add(infohash)
except:
print_exc()
elif DEBUG:
print >> sys.stderr,"metadata: GET_METADATA: error connecting to",show_permid_short(permid)
def setUpMyListenSocket(self):
self.dest_keypair = EC.gen_params(EC.NID_sect233k1)
self.dest_keypair.gen_key()
self.destpermid = str(self.dest_keypair.pub().get_der())
self.destport = 4810
# Start our server side, to with Tribler will try to connect
self.destss = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.destss.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.destss.bind(('', self.destport))
self.destss.listen(1)
print >>sys.stderr,"test: my permid",show_permid_short(self.mypermid)
print >>sys.stderr,"test: his permid",show_permid_short(self.hispermid)
print >>sys.stderr,"test: dest permid",show_permid_short(self.destpermid)
def _sent_callback(self,exc,permid,channel_id,infohash,bitmask, msgSentCallback, usrCallback):
"""
Called by the OverlayThread after a GET_SUBS request has been sent.
"""
if exc is not None:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + "Unable to send GET_SUBS to: " + \
utilities.show_permid_short(permid) + ": " + exc
else:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + "GET_SUBS sent to %s" % \
(utilities.show_permid_short(permid))
self._addToRequestedSubtitles(channel_id, infohash, bitmask, usrCallback)
if msgSentCallback is not None:
msgSentCallback(exc,permid,channel_id,infohash,bitmask)
def olthread_request_help_connect_callback(self, exc, dns, permid,
selversion):
if exc is None:
if DEBUG:
print >> sys.stderr, "dlhelp: Coordinator sending to", show_permid_short(
permid)
## Create message according to protocol version
dlhelp_request = self.infohash
self.overlay_bridge.send(permid, DOWNLOAD_HELP + dlhelp_request,
self.olthread_request_help_send_callback)
else:
if DEBUG:
print >> sys.stderr, "dlhelp: DOWNLOAD_HELP: error connecting to", show_permid_short(
permid), exc
self.olthread_remove_unreachable_helper(permid)
def setUpMyListenSocket(self):
self.dest_keypair = EC.gen_params(EC.NID_sect233k1)
self.dest_keypair.gen_key()
self.destpermid = str(self.dest_keypair.pub().get_der())
self.destport = 4810
# Start our server side, to with Tribler will try to connect
self.destss = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.destss.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.destss.bind(('', self.destport))
self.destss.listen(1)
print >>sys.stderr,time.asctime(),'-', "test: my permid",show_permid_short(self.mypermid)
print >>sys.stderr,time.asctime(),'-', "test: his permid",show_permid_short(self.hispermid)
print >>sys.stderr,time.asctime(),'-', "test: dest permid",show_permid_short(self.destpermid)
def olthread_reserve_pieces_connect_callback(self,exc,dns,permid,selversion,pieces,all_or_nothing):
if exc is None:
## Create message according to protocol version
if all_or_nothing:
all_or_nothing = chr(1)
else:
all_or_nothing = chr(0)
payload = self.torrent_hash + all_or_nothing + bencode(pieces)
if DEBUG:
print >> sys.stderr,"helper: RESERVE_PIECES: Sending!!!!!!!!!!!!!",show_permid_short(permid)
self.overlay_bridge.send(permid, RESERVE_PIECES + payload,self.olthread_reserve_pieces_send_callback)
elif DEBUG:
print >> sys.stderr,"helper: RESERVE_PIECES: error connecting to",show_permid_short(permid),exc
def handle_event_crawler_reply(self, permid, selversion, channel_id,
channel_data, error, message,
request_callback):
"""
<<Crawler-side>>
Received a CRAWLER_VIDEOPLAYBACK_EVENT_QUERY reply.
@param permid The Crawler permid
@param selversion The overlay protocol version
@param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
@param channel_data Data associated with the request
@param error The error value. 0 indicates success.
@param message The message payload
@param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
"""
if error:
if DEBUG:
print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message
self._file.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY",
show_permid(permid), str(error), str(channel_data), message,
"\n")))
self._file.flush()
elif selversion >= OLPROTO_VER_TENTH:
# Overlay version 10 sends the reply pickled and zipped
if DEBUG:
print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(
permid), len(message), "bytes zipped"
info = cPickle.loads(zlib.decompress(message))
self._file.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY",
show_permid(permid), str(error), str(channel_data), str(info),
"\n")))
self._file.flush()
elif selversion >= OLPROTO_VER_EIGHTH:
if DEBUG:
print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(
permid), cPickle.loads(message)
info = cPickle.loads(message)
self._file.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY",
show_permid(permid), str(error), str(channel_data), str(info),
"\n")))
self._file.flush()
def send_metadata_request(self, permid, infohash, selversion=-1, caller="BC"):
if DEBUG:
print >> sys.stderr,"metadata: Connect to send GET_METADATA to",show_permid_short(permid)
if not isValidInfohash(infohash):
return False
filename,metadata = self.torrent_exists(infohash)
if filename is not None: # torrent already exists on disk
if DEBUG:
print >> sys.stderr,"metadata: send_meta_req: Already on disk??!"
self.notify_torrent_is_in(infohash, metadata, filename)
return True
if caller == "dlhelp":
self.requested_torrents.add(infohash)
if self.min_free_space != 0 and (self.free_space - self.avg_torrent_size < self.min_free_space): # no space to collect
self.free_space = self.get_free_space()
if self.free_space - self.avg_torrent_size < self.min_free_space:
self.warn_disk_full()
return True
try:
# Optimization: don't connect if we're connected, although it won't
# do any harm.
if selversion == -1: # not currently connected
self.overlay_bridge.connect(permid,lambda e,d,p,s:self.get_metadata_connect_callback(e,d,p,s,infohash))
else:
self.get_metadata_connect_callback(None,None,permid,selversion,infohash)
except:
print_exc()
return False
return True
def handleConnection(self,exc,permid,selversion,locally_initiated,hisdns):
""" Called by NetworkThread """
# called by SecureOverlay.got_auth_connection() or cleanup_admin_and_callbacks()
if DEBUG:
print >>sys.stderr,"olbridge: handleConnection",exc,show_permid_short(permid),selversion,locally_initiated,hisdns,currentThread().getName()
def olbridge_handle_conn_func():
# Called by OverlayThread
if DEBUG:
print >>sys.stderr,"olbridge: handle_conn_func",exc,show_permid_short(permid),selversion,locally_initiated,hisdns,currentThread().getName()
try:
if hisdns:
self.secover.add_peer_to_db(permid,hisdns,selversion)
if self.olappsconnhandler is not None: # self.olappsconnhandler = OverlayApps.handleConnection
self.olappsconnhandler(exc,permid,selversion,locally_initiated)
except:
print_exc()
if isinstance(exc,CloseException):
self.secover.update_peer_status(permid,exc.was_auth_done())
self.tqueue.add_task(olbridge_handle_conn_func,0)
def handleMessage(self,permid,selversion,message):
""" Called by NetworkThread """
#ProxyService_
#
# DEBUG
#print "### olbridge: handleMessage", show_permid_short(permid), selversion, getMessageName(message[0]), currentThread().getName()
#
#_ProxyService
if DEBUG:
print >>sys.stderr,"olbridge: handleMessage",show_permid_short(permid),selversion,getMessageName(message[0]),currentThread().getName()
def olbridge_handle_msg_func():
# Called by OverlayThread
if DEBUG:
print >>sys.stderr,"olbridge: handle_msg_func",show_permid_short(permid),selversion,getMessageName(message[0]),currentThread().getName()
try:
if self.olappsmsghandler is None:
ret = True
else:
ret = self.olappsmsghandler(permid,selversion,message)
except:
print_exc()
ret = False
if ret == False:
if DEBUG:
print >>sys.stderr,"olbridge: olbridge_handle_msg_func closing!",show_permid_short(permid),selversion,getMessageName(message[0]),currentThread().getName()
self.close(permid)
self.tqueue.add_task(olbridge_handle_msg_func,0)
return True
def natCheckReplySendCallback(self, exc, permid):
if DEBUG:
print >> sys.stderr, "NATCHECK_REPLY was sent to", show_permid_short(
permid), exc
if exc is not None:
return False
return True
def handle_crawler_reply(self, permid, selversion, channel_id,
channel_data, error, message, request_callback):
"""
Received a CRAWLER_DATABASE_QUERY reply.
@param permid The Crawler permid
@param selversion The overlay protocol version
@param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
@param error The error value. 0 indicates success.
@param message The message payload
@param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
"""
if error:
if DEBUG:
print >> sys.stderr, "databasecrawler: handle_crawler_reply", error, message
self._file.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid),
str(error), message, "\n")))
self._file.flush()
else:
if DEBUG:
print >> sys.stderr, "databasecrawler: handle_crawler_reply", show_permid_short(
permid), cPickle.loads(message)
self._file.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid),
str(error), str(cPickle.loads(message)), "\n")))
self._file.flush()
def handleMessage(self,permid,selversion,message):
t = message[0]
if t == GET_METADATA: # the other peer requests a torrent
if DEBUG:
print >> sys.stderr,"metadata: Got GET_METADATA",len(message),show_permid_short(permid)
return self.send_metadata(permid, message, selversion)
elif t == METADATA: # the other peer sends me a torrent
if DEBUG:
print >> sys.stderr,"metadata: Got METADATA",len(message),show_permid_short(permid),selversion, currentThread().getName()
return self.got_metadata(permid, message, selversion)
else:
if DEBUG:
print >> sys.stderr,"metadata: UNKNOWN OVERLAY MESSAGE", ord(t)
return False
def handle_crawler_request(self, permid, selversion, channel_id, message, reply_callback):
"""
Received a CRAWLER_UDPUNCTURE_QUERY request.
@param permid The Crawler permid
@param selversion The overlay protocol version
@param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
@param message The message payload
@param reply_callback Call this function once to send the reply: reply_callback(payload [, error=123])
"""
if DEBUG:
print >> sys.stderr, "puncturecrawler: handle_crawler_request", show_permid_short(permid), message
SimpleFileReporter.lock.acquire()
try:
if not self.reporter.file:
try:
self.reporter.file = open(self.reporter.path, 'a+b')
except Exception, e:
reply_callback(str(e), error=1)
return
file = self.reporter.file
try:
file.seek(0)
result = ("%.2f CRAWL\n" % time.time()) + file.read()
result = zlib.compress(result)
reply_callback(result)
file.truncate(0)
except Exception, e:
reply_callback(str(e), error=1)
def handleConnection(self, exc, permid, selversion, locally_initiated,
hisdns):
""" Called by NetworkThread """
# called by SecureOverlay.got_auth_connection() or cleanup_admin_and_callbacks()
if DEBUG:
print >> sys.stderr, "olbridge: handleConnection", exc, show_permid_short(
permid), selversion, locally_initiated, hisdns, currentThread(
).getName()
def olbridge_handle_conn_func():
# Called by OverlayThread
if DEBUG:
print >> sys.stderr, "olbridge: handle_conn_func", exc, show_permid_short(
permid
), selversion, locally_initiated, hisdns, currentThread(
).getName()
try:
if hisdns:
self.secover.add_peer_to_db(permid, hisdns, selversion)
if self.olappsconnhandler is not None: # self.olappsconnhandler = OverlayApps.handleConnection
self.olappsconnhandler(exc, permid, selversion,
locally_initiated)
except:
print_exc()
if isinstance(exc, CloseException):
self.secover.update_peer_status(permid, exc.was_auth_done())
self.tqueue.add_task(olbridge_handle_conn_func, 0)
def udpConnectReplySendCallback(self, exc, permid):
if DEBUG:
print >> sys.stderr, "NATTRAVERSAL_REPLY was sent to", show_permid_short(permid), exc
if exc is not None:
return False
return True
def process_query_reply(self,permid,query,usercallback,d):
if DEBUG:
print >>sys.stderr,"rquery: process_query_reply:",show_permid_short(permid),query,d
if len(d['a']) > 0:
self.unidecode_hits(query,d)
if query.startswith("CHANNEL"):
# 13-04-2010 Andrea: The gotRemoteHits in SearchGridManager is too slow.
# Since it is run by the GUIThread when there are too many hits the GUI
# gets freezed.
# dropping some random results if they are too many.
# It is just an hack, a better method to improve performance should be found.
if len(d['a']) > self.max_channel_query_results:
if DEBUG:
print >> sys.stderr, "DROPPING some answers: they where %d" % len(d['a'])
newAnswers = {}
newKeys = d['a'].keys()[:self.max_channel_query_results]
for key in newKeys:
newAnswers[key] = d['a'][key]
d['a'] = newAnswers
# Andrea 05-06-2010: updates the database through channelcast. Before this was
# done by the GUIThread in SearchGridManager
self.bc_fac.channelcast_core.updateChannel(permid,query,d['a'])
# Inform user of remote channel hits
remote_query_usercallback_lambda = lambda:usercallback(permid,query,d['a'])
else:
remote_query_usercallback_lambda = lambda:usercallback(permid,query,d['a'])
self.session.uch.perform_usercallback(remote_query_usercallback_lambda)
elif DEBUG:
print >>sys.stderr,"rquery: QUERY_REPLY: no results found"
def handleMessage(self, permid, selversion, message):
""" Called by NetworkThread """
if DEBUG:
print >> sys.stderr, "olbridge: handleMessage", show_permid_short(
permid), selversion, getMessageName(
message[0]), currentThread().getName()
def olbridge_handle_msg_func():
# Called by OverlayThread
if DEBUG:
print >> sys.stderr, "olbridge: handle_msg_func", show_permid_short(
permid), selversion, getMessageName(
message[0]), currentThread().getName()
try:
if self.olappsmsghandler is None:
ret = True
else:
ret = self.olappsmsghandler(permid, selversion, message)
except:
print_exc()
ret = False
if ret == False:
self.close(permid)
self.tqueue.add_task(olbridge_handle_msg_func, 0)
return True
def got_conn_outgoing_conns_callback(self,exc,permid,selversion,locally_initiated,hisdns=None):
print >> sys.stderr,"test: got_conn_outgoing_conns_callback",exc,show_permid_short(permid)
self.assert_(exc is None)
self.assert_(permid == self.peer2.my_permid)
self.assert_(selversion == OLPROTO_VER_CURRENT)
self.assert_(locally_initiated)
self.got = True
def get_metadata_send_callback(self,exc,permid):
if exc is not None:
if DEBUG:
print >> sys.stderr,"metadata: GET_METADATA: error sending to",show_permid_short(permid),exc
pass
else:
pass
def do_help(self, infohash, torrent_data, permid):
basename = binascii.hexlify(
infohash) + '.torrent' # ignore .tribe stuff, not vital
torrentfilename = os.path.join(self.helpdir, basename)
tfile = open(torrentfilename, "wb")
tfile.write(torrent_data)
tfile.close()
if DEBUG:
print >> sys.stderr, "helpmsg: Got metadata required for helping", show_permid_short(
permid)
print >> sys.stderr, "helpmsg: torrent: ", torrentfilename
tdef = TorrentDef.load(torrentfilename)
if self.dlconfig is None:
dscfg = DownloadStartupConfig()
else:
dscfg = DownloadStartupConfig(self.dlconfig)
dscfg.set_coopdl_coordinator_permid(permid)
dscfg.set_dest_dir(self.helpdir)
# Start new download
self.session.start_download(tdef, dscfg)
def olthread_stop_help_connect_callback(self,exc,dns,permid,selversion):
if exc is None:
## Create message according to protocol version
stop_request = self.infohash
self.overlay_bridge.send(permid,STOP_DOWNLOAD_HELP + stop_request,self.olthread_stop_help_send_callback)
elif DEBUG:
print >> sys.stderr,"dlhelp: STOP_DOWNLOAD_HELP: error connecting to",show_permid_short(permid),exc
def olthread_send_request_help(self, permidlist):
for permid in permidlist:
if DEBUG:
print >> sys.stderr, "dlhelp: Coordinator connecting to", show_permid_short(
permid), "for help"
self.overlay_bridge.connect(
permid, self.olthread_request_help_connect_callback)
def query_initiator(self, permid, selversion, request_callback):
"""
Established a new connection. Send a CRAWLER_DATABASE_QUERY request.
@param permid The Tribler peer permid
@param selversion The oberlay protocol version
@param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
"""
if DEBUG: print >>sys.stderr, "databasecrawler: query_initiator", show_permid_short(permid)
sql = []
if selversion >= OLPROTO_VER_SEVENTH:
sql.extend(("SELECT 'peer_count', count(*) FROM Peer",
"SELECT 'torrent_count', count(*) FROM Torrent"))
if selversion >= OLPROTO_VER_ELEVENTH:
sql.extend(("SELECT 'my_subscriptions', count(*) FROM VoteCast where voter_id='" + show_permid(permid) + "' and vote=2",
"SELECT 'my_negative_votes', count(*) FROM VoteCast where voter_id='" + show_permid(permid) + "' and vote=-1",
"SELECT 'my_channel_files', count(*) FROM ChannelCast where publisher_id='" + show_permid(permid) + "'",
"SELECT 'all_subscriptions', count(*) FROM VoteCast where vote=2",
"SELECT 'all_negative_votes', count(*) FROM VoteCast where vote=-1"))
# if OLPROTO_VER_EIGHTH <= selversion <= 11:
# sql.extend(("SELECT 'moderations_count', count(*) FROM ModerationCast"))
# if selversion >= OLPROTO_VER_EIGHTH:
# sql.extend(("SELECT 'positive_votes_count', count(*) FROM Moderators where status=1",
# "SELECT 'negative_votes_count', count(*) FROM Moderators where status=-1"))
request_callback(CRAWLER_DATABASE_QUERY, ";".join(sql), callback=self._after_request_callback)
def _after_send_reply(exc, permid):
if DEBUG:
if exc:
print >> sys.stderr, "crawler: could not send request", show_permid_short(permid), exc
# call the optional callback supplied with send_request
if callback:
callback(exc, permid)
def udpConnect(self, permid, request_id, holePunchingAddr):
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: request UDP connection"
mh_data = request_id + ":" + holePunchingAddr[0] + ":" + str(
holePunchingAddr[1])
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: udpConnect message is", mh_data
try:
mh_msg = bencode(mh_data)
except:
print_exc()
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: error mh_data:", mh_data
return False
# send the message
self.crawler.send_request(permid,
CRAWLER_NATTRAVERSAL,
mh_msg,
frequency=0,
callback=self.udpConnectCallback)
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: request for", show_permid_short(
permid), "sent to crawler"
def get_metadata_send_callback(self, exc, permid):
if exc is not None:
if DEBUG:
print >> sys.stderr, "metadata: GET_METADATA: error sending to", show_permid_short(
permid), exc
pass
else:
pass
def _after_send_reply(exc, permid):
if DEBUG:
if exc:
print >> sys.stderr, "crawler: could not send request", show_permid_short(
permid), exc
# call the optional callback supplied with send_request
if callback:
callback(exc, permid)
def udpConnectReplySendCallback(self, exc, permid):
if DEBUG:
print >> sys.stderr, "NATTRAVERSAL_REPLY was sent to", show_permid_short(
permid), exc
if exc is not None:
return False
return True
def got_conn_outgoing_conns_callback(self, exc, permid, selversion,
locally_initiated, hisdns):
print >> sys.stderr, "test: got_conn_outgoing_conns_callback", exc, show_permid_short(
permid)
self.assert_(exc is None)
self.assert_(permid == self.peer2.my_permid)
self.assert_(selversion == OLPROTO_VER_CURRENT)
self.assert_(locally_initiated)
self.got = True
def olbridge_send_callback(cexc, cpermid):
# Called by network thread
if DEBUG:
print >> sys.stderr, "olbridge: send_callback", cexc, show_permid_short(
cpermid)
olbridge_send_callback_lambda = lambda: callback(cexc, cpermid)
self.add_task(olbridge_send_callback_lambda, 0)
def udpConnectCallback(self, exc, permid):
if exc is not None:
if DEBUG:
print >> sys.stderr, "NATTRAVERSAL_REQUEST failed to", show_permid_short(
permid), exc
# Register peerinfo on file
self._file2.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), "REQUEST FAILED",
show_permid(permid),
str(self._secure_overlay.get_dns_from_peerdb(permid)), "\n")))
return False
if DEBUG:
print >> sys.stderr, "NATTRAVERSAL_REQUEST was sent to", show_permid_short(
permid), exc
return True
def _after_request_callback(self, exc, permid):
"""
Called by the Crawler with the result of the request_callback
call in the query_initiator method.
"""
if not exc:
if DEBUG: print >>sys.stderr, "repexcrawler: request sent to", show_permid_short(permid)
self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "REQUEST", show_permid(permid), "\n")))
self._file.flush()
def olthread_pieces_reserved_send_callback(self, exc, permid):
if exc is not None:
if DEBUG:
print >> sys.stderr, "dlhelp: PIECES_RESERVED: error sending to", show_permid_short(
permid), exc
else:
if DEBUG:
print >> sys.stderr, "dlhelp: PIECES_RESERVED: Successfully sent to", show_permid_short(
permid)
def get_metadata_connect_callback(self, exc, dns, permid, selversion,
infohash):
if exc is None:
if DEBUG:
print >> sys.stderr, "metadata: Sending GET_METADATA to", show_permid_short(
permid)
## Create metadata_request according to protocol version
try:
metadata_request = bencode(infohash)
self.overlay_bridge.send(permid,
GET_METADATA + metadata_request,
self.get_metadata_send_callback)
self.requested_torrents.add(infohash)
except:
print_exc()
elif DEBUG:
print >> sys.stderr, "metadata: GET_METADATA: error connecting to", show_permid_short(
permid)
def handle_info_crawler_reply(self, permid, selversion, channel_id, error,
message, request_callback):
"""
<<Crawler-side>>
Received a CRAWLER_VIDEOPLAYBACK_INFO_QUERY reply.
@param permid The Crawler permid
@param selversion The overlay protocol version
@param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
@param error The error value. 0 indicates success.
@param message The message payload
@param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
"""
if error:
if DEBUG:
print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message
self._file.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), " INFO REPLY",
show_permid(permid), str(error), message, "\n")))
self._file.flush()
else:
if DEBUG:
print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(
permid), cPickle.loads(message)
info = cPickle.loads(message)
self._file.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), " INFO REPLY",
show_permid(permid), str(error), str(info), "\n")))
self._file.flush()
i = 0
for key, timestamp, piece_size, num_pieces, bitrate, nat in info:
i += 1
# do not remove the first item. the list is ordered
# DESC so the first item is the last that is added to
# the database and we can't affored to remove it, as
# it may cause exceptions in the running playback.
if i == 1:
sql = """
SELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;
DELETE FROM playback_event WHERE key = '%s';
""" % (key, key)
else:
sql = """
SELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;
DELETE FROM playback_event WHERE key = '%s';
DELETE FROM playback_info WHERE key = '%s';
""" % (key, key, key)
# todo: optimize to not select key for each row
request_callback(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY,
sql,
channel_data=key,
callback=self._after_event_request_callback,
frequency=0)
def query_initiator(self, permid, selversion, request_callback):
"""
Established a new connection. Send a CRAWLER_REPEX_QUERY request.
@param permid The Tribler peer permid
@param selversion The overlay protocol version
@param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
"""
if DEBUG: print >>sys.stderr, "repexcrawler: query_initiator", show_permid_short(permid)
request_callback(CRAWLER_REPEX_QUERY, '', callback=self._after_request_callback)
def olbridge_connect_dns_callback(cexc, cdns, cpermid, cselver):
# Called by network thread
if DEBUG:
print >> sys.stderr, "olbridge: connect_dns_callback", cexc, cdns, show_permid_short(
cpermid), cselver
olbridge_connect_dns_callback_lambda = lambda: callback(
cexc, cdns, cpermid, cselver)
self.add_task(olbridge_connect_dns_callback_lambda, 0)
def olthread_stop_help_connect_callback(self, exc, dns, permid,
selversion):
if exc is None:
## Create message according to protocol version
stop_request = self.infohash
self.overlay_bridge.send(permid, STOP_DOWNLOAD_HELP + stop_request,
self.olthread_stop_help_send_callback)
elif DEBUG:
print >> sys.stderr, "dlhelp: STOP_DOWNLOAD_HELP: error connecting to", show_permid_short(
permid), exc
def handleMessage(self, permid, selversion, message):
t = message[0]
if t == GET_METADATA: # the other peer requests a torrent
if DEBUG:
print >> sys.stderr, "metadata: Got GET_METADATA", len(
message), show_permid_short(permid)
return self.send_metadata(permid, message, selversion)
elif t == METADATA: # the other peer sends me a torrent
if DEBUG:
print >> sys.stderr, "metadata: Got METADATA", len(
message), show_permid_short(
permid), selversion, currentThread().getName()
return self.got_metadata(permid, message, selversion)
else:
if DEBUG:
print >> sys.stderr, "metadata: UNKNOWN OVERLAY MESSAGE", ord(
t)
return False
def _after_send_request(exc, permid):
if DEBUG:
if exc:
print >> sys.stderr, "crawler: could not send request to", show_permid_short(
permid), exc
if exc:
self._release_channel_id(permid, channel_id)
# call the optional callback supplied with send_request
if callback:
callback(exc, permid)
def olbridge_connect_callback(cexc, cdns, cpermid, cselver):
# Called by network thread
if DEBUG:
print >> sys.stderr, "olbridge: connect_callback", cexc, cdns, show_permid_short(
cpermid), cselver, callback, currentThread().getName()
olbridge_connect_callback_lambda = lambda: callback(
cexc, cdns, cpermid, cselver)
# Jie: postpone to call this callback to schedule it after the peer has been added to buddycast connection list
# Arno, 2008-09-15: No-no-no
self.add_task(olbridge_connect_callback_lambda, 0)
def olthread_reserve_pieces_connect_callback(self, exc, dns, permid,
selversion, pieces,
all_or_nothing):
if exc is None:
## Create message according to protocol version
if all_or_nothing:
all_or_nothing = chr(1)
else:
all_or_nothing = chr(0)
payload = self.torrent_hash + all_or_nothing + bencode(pieces)
if DEBUG:
print >> sys.stderr, "helper: RESERVE_PIECES: Sending!!!!!!!!!!!!!", show_permid_short(
permid)
self.overlay_bridge.send(
permid, RESERVE_PIECES + payload,
self.olthread_reserve_pieces_send_callback)
elif DEBUG:
print >> sys.stderr, "helper: RESERVE_PIECES: error connecting to", show_permid_short(
permid), exc
