def receivedSubsRequest(self, permid, request, selversion):
channel_id, infohash, languages = request
allSubtitles = self.subtitlesDb.getAllSubtitles(channel_id, infohash)
contentsList = {}
for lang in sorted(languages):
if lang in allSubtitles.keys():
if allSubtitles[lang].subtitleExists():
content = self._readSubContent(allSubtitles[lang].path)
if content is not None:
contentsList[lang] = content
else:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'File not available for channel %s, infohash %s, lang %s' % (
show_permid_short(channel_id), bin2str(infohash),
lang)
self.subtitlesDb.updateSubtitlePath(
channel_id, infohash, lang, None)
elif DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'Subtitle not available for channel %s, infohash %s, lang %s' % (
show_permid_short(channel_id), bin2str(infohash), lang)
if len(contentsList) == 0:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'None of the requested subtitles were available. No answer will be sent to %s' % show_permid_short(
permid)
return True
return self._subsMsgHndlr.sendSubtitleResponse(
permid, (channel_id, infohash, contentsList), selversion)
def handle_event_crawler_reply(self, permid, selversion, channel_id,
channel_data, error, message,
request_callback):
if error:
if DEBUG:
print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', error, message
self._file.write('; '.join(
(strftime('%Y/%m/%d %H:%M:%S'), ' EVENT REPLY',
show_permid(permid), str(error), str(channel_data), message,
'\n')))
self._file.flush()
elif selversion >= OLPROTO_VER_TENTH:
if DEBUG:
print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', show_permid_short(
permid), len(message), 'bytes zipped'
info = cPickle.loads(zlib.decompress(message))
self._file.write('; '.join(
(strftime('%Y/%m/%d %H:%M:%S'), ' EVENT REPLY',
show_permid(permid), str(error), str(channel_data), str(info),
'\n')))
self._file.flush()
elif selversion >= OLPROTO_VER_EIGHTH:
if DEBUG:
print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', show_permid_short(
permid), cPickle.loads(message)
info = cPickle.loads(message)
self._file.write('; '.join(
(strftime('%Y/%m/%d %H:%M:%S'), ' EVENT REPLY',
show_permid(permid), str(error), str(channel_data), str(info),
'\n')))
self._file.flush()
def handleRMetadata(self,
sender_permid,
channelCastMessage,
fromQuery=False):
metadataDTOs = self._splitChannelcastAndRichMetadataContents(
channelCastMessage)
if DEBUG:
print >> sys.stderr, 'Handling rich metadata from %s...' % show_permid_short(
sender_permid)
for md_and_have in metadataDTOs:
md = md_and_have[0]
havemask = md_and_have[1]
vote = self.votecastDB.getVote(bin2str(md.channel),
bin2str(self.my_permid))
if DEBUG:
id = 'RQ' if fromQuery else 'R'
print >> sys.stderr, '%s, %s, %s, %s, %d' % (
id, md.channel, md.infohash,
show_permid_short(sender_permid), md.timestamp)
if vote == -1:
continue
isUpdate = self.rmdDb.insertMetadata(md)
self.peerHaveManager.newHaveReceived(md.channel, md.infohash,
sender_permid, havemask)
if isUpdate is not None:
md = self.rmdDb.getMetadata(md.channel, md.infohash)
self._notifyRichMetadata(md, isUpdate)
if vote == 2:
if DEBUG:
print >> sys.stderr, 'Subscribed to channel %s, trying to retrieveall subtitle contents' % (
show_permid_short(md.channel), )
self._getAllSubtitles(md)
def _get_subs_connect_callback(self, exception, dns, permid, selversion, channel_id, infohash, bitmask, msgSentCallback, usrCallback):
if exception is not None:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'GET_SUBS not sent. Unable to connect to ' + utilities.show_permid_short(permid)
else:
if selversion > 0 and selversion < OLPROTO_VER_FOURTEENTH:
msg = 'GET_SUBS not send, the other peers had an old protocol version: %d' % selversion
if DEBUG:
print >> sys.stderr, msg
raise SubtitleMsgHandlerException(msg)
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'sending GET_SUBS to ' + utilities.show_permid_short(permid)
try:
message = self._createGETSUBSMessage(channel_id, infohash, bitmask)
if DEBUG:
print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ('SG',
show_permid_short(permid),
show_permid_short(channel_id),
bin2str(infohash),
bitmask,
len(message))
self._overlay_bridge.send(permid, message, lambda exc, permid: self._sent_callback(exc, permid, channel_id, infohash, bitmask, msgSentCallback, usrCallback))
except Exception as e:
print_exc()
msg = 'GET_SUBS not sent: %s' % str(e)
raise SubtitleMsgHandlerException(e)
def handleRMetadata(self, sender_permid, channelCastMessage, fromQuery = False):
metadataDTOs = self._splitChannelcastAndRichMetadataContents(channelCastMessage)
if DEBUG:
print >> sys.stderr, 'Handling rich metadata from %s...' % show_permid_short(sender_permid)
for md_and_have in metadataDTOs:
md = md_and_have[0]
havemask = md_and_have[1]
vote = self.votecastDB.getVote(bin2str(md.channel), bin2str(self.my_permid))
if DEBUG:
id = 'RQ' if fromQuery else 'R'
print >> sys.stderr, '%s, %s, %s, %s, %d' % (id,
md.channel,
md.infohash,
show_permid_short(sender_permid),
md.timestamp)
if vote == -1:
continue
isUpdate = self.rmdDb.insertMetadata(md)
self.peerHaveManager.newHaveReceived(md.channel, md.infohash, sender_permid, havemask)
if isUpdate is not None:
md = self.rmdDb.getMetadata(md.channel, md.infohash)
self._notifyRichMetadata(md, isUpdate)
if vote == 2:
if DEBUG:
print >> sys.stderr, 'Subscribed to channel %s, trying to retrieveall subtitle contents' % (show_permid_short(md.channel),)
self._getAllSubtitles(md)
def tryHolePunching(self):
if DEBUG:
print >> sys.stderr, 'NatCheckMsgHandler: first element in peerlist', self.peerlist[len(self.peerlist) - 1]
print >> sys.stderr, 'NatCheckMsgHandler: second element in peerlist', self.peerlist[len(self.peerlist) - 2]
holePunchingPort = random.randrange(3200, 4200, 1)
holePunchingAddr = (self.holePunchingIP, holePunchingPort)
peer1 = self.peerlist[len(self.peerlist) - 1]
peer2 = self.peerlist[len(self.peerlist) - 2]
request_id = str(show_permid_short(peer1[0]) + show_permid_short(peer2[0]) + str(random.randrange(0, 1000, 1)))
self.udpConnect(peer1[0], request_id, holePunchingAddr)
self.udpConnect(peer2[0], request_id, holePunchingAddr)
self._file2.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'),
'REQUEST',
request_id,
show_permid(peer1[0]),
str(peer1[1]),
str(peer1[2]),
str(self._secure_overlay.get_dns_from_peerdb(peer1[0])),
show_permid(peer2[0]),
str(peer2[1]),
str(peer2[2]),
str(self._secure_overlay.get_dns_from_peerdb(peer2[0])),
'\n')))
self._file2.flush()
self.trav[request_id] = (None, None)
thread.start_new_thread(coordinateHolePunching, (peer1, peer2, holePunchingAddr))
def handle_connection(self, exc, permid, selversion, locally_initiated):
if exc:
if DEBUG:
print >> sys.stderr, 'crawler: overlay connection lost', show_permid_short(
permid), exc
print >> sys.stderr, repr(permid)
elif selversion >= OLPROTO_VER_SEVENTH:
already_known = False
for tup in self._initiator_deadlines:
if tup[4] == permid:
already_known = True
break
if not already_known:
if DEBUG:
print >> sys.stderr, 'crawler: new overlay connection', show_permid_short(
permid)
print >> sys.stderr, repr(permid)
for initiator_callback, frequency, accept_frequency in self._crawl_initiators:
self._initiator_deadlines.append([
0, frequency, accept_frequency, initiator_callback,
permid, selversion, 0
])
self._initiator_deadlines.sort()
self._check_deadlines(False)
elif DEBUG:
print >> sys.stderr, 'crawler: new overlay connection (can not use version %d)' % selversion, show_permid_short(
permid)
print >> sys.stderr, repr(permid)
def tryHolePunching(self):
if DEBUG:
print >> sys.stderr, 'NatCheckMsgHandler: first element in peerlist', self.peerlist[
len(self.peerlist) - 1]
print >> sys.stderr, 'NatCheckMsgHandler: second element in peerlist', self.peerlist[
len(self.peerlist) - 2]
holePunchingPort = random.randrange(3200, 4200, 1)
holePunchingAddr = (self.holePunchingIP, holePunchingPort)
peer1 = self.peerlist[len(self.peerlist) - 1]
peer2 = self.peerlist[len(self.peerlist) - 2]
request_id = str(
show_permid_short(peer1[0]) + show_permid_short(peer2[0]) +
str(random.randrange(0, 1000, 1)))
self.udpConnect(peer1[0], request_id, holePunchingAddr)
self.udpConnect(peer2[0], request_id, holePunchingAddr)
self._file2.write('; '.join(
(strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST', request_id,
show_permid(peer1[0]), str(peer1[1]), str(peer1[2]),
str(self._secure_overlay.get_dns_from_peerdb(peer1[0])),
show_permid(peer2[0]), str(peer2[1]), str(peer2[2]),
str(self._secure_overlay.get_dns_from_peerdb(peer2[0])), '\n')))
self._file2.flush()
self.trav[request_id] = (None, None)
thread.start_new_thread(coordinateHolePunching,
(peer1, peer2, holePunchingAddr))
def _handleSUBS(self, permid, message, selversion):
if selversion < OLPROTO_VER_FOURTEENTH:
if DEBUG:
print >> sys.stderr, 'The peer that sent the SUBS request has an oldprotcol version: this is strange. Dropping the msg'
return False
decoded = self._decodeSUBSMessage(message)
if decoded is None:
if DEBUG:
print >> sys.stderr, 'Error decoding a SUBS message from %s' % utilities.show_permid_short(permid)
return False
channel_id, infohash, bitmask, contents = decoded
if DEBUG:
print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ('RS',
show_permid_short(permid),
show_permid_short(channel_id),
bin2str(infohash),
bitmask,
len(message))
requestedSubs = self._checkRequestedSubtitles(channel_id, infohash, bitmask)
if requestedSubs == 0:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'Received a SUBS message that was not requested. Dropping'
return False
requestedSubsCodes = self._languagesUtility.maskToLangCodes(requestedSubs)
for lang in contents.keys():
if lang not in requestedSubsCodes:
del contents[lang]
callbacks = self._removeFromRequestedSubtitles(channel_id, infohash, bitmask)
tuple = (channel_id, infohash, contents)
for listener in self._listenersList:
listener.receivedSubsResponse(permid, tuple, callbacks, selversion)
return True
def handle_connection(self, exc, permid, selversion, locally_initiated):
if exc:
if DEBUG:
print >> sys.stderr, 'crawler: overlay connection lost', show_permid_short(permid), exc
print >> sys.stderr, repr(permid)
elif selversion >= OLPROTO_VER_SEVENTH:
already_known = False
for tup in self._initiator_deadlines:
if tup[4] == permid:
already_known = True
break
if not already_known:
if DEBUG:
print >> sys.stderr, 'crawler: new overlay connection', show_permid_short(permid)
print >> sys.stderr, repr(permid)
for initiator_callback, frequency, accept_frequency in self._crawl_initiators:
self._initiator_deadlines.append([0,
frequency,
accept_frequency,
initiator_callback,
permid,
selversion,
0])
self._initiator_deadlines.sort()
self._check_deadlines(False)
elif DEBUG:
print >> sys.stderr, 'crawler: new overlay connection (can not use version %d)' % selversion, show_permid_short(permid)
print >> sys.stderr, repr(permid)
def connect(self, permid, callback):
if DEBUG:
print >> sys.stderr, 'secover: connect', show_permid_short(permid), currentThread().getName()
dns = self.get_dns_from_peerdb(permid)
task = Task(self._connect, permid, dns, callback)
if DEBUG:
print >> sys.stderr, 'secover: connect', show_permid_short(permid), 'currently at', dns
self.rawserver.add_task(task.start, 0)
def query_initiator(self, permid, selversion, request_callback):
if selversion >= OLPROTO_VER_TENTH:
if DEBUG:
print >> sys.stderr, 'videoplaybackcrawler: query_initiator', show_permid_short(permid), 'version', selversion
request_callback(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, 'SELECT key, timestamp, event FROM playback_event; DELETE FROM playback_event;', callback=self._after_event_request_callback)
elif selversion >= OLPROTO_VER_EIGHTH:
if DEBUG:
print >> sys.stderr, 'videoplaybackcrawler: query_initiator', show_permid_short(permid), 'version', selversion
request_callback(CRAWLER_VIDEOPLAYBACK_INFO_QUERY, 'SELECT key, timestamp, piece_size, num_pieces, bitrate, nat FROM playback_info ORDER BY timestamp DESC LIMIT 50', callback=self._after_info_request_callback)
elif DEBUG:
print >> sys.stderr, 'videoplaybackcrawler: query_info_initiator', show_permid_short(permid), 'unsupported overlay version'
def get_metadata_connect_callback(self, exc, dns, permid, selversion, infohash):
if exc is None:
if DEBUG:
print >> sys.stderr, 'metadata: Sending GET_METADATA to', show_permid_short(permid)
try:
metadata_request = bencode(infohash)
self.overlay_bridge.send(permid, GET_METADATA + metadata_request, self.get_metadata_send_callback)
self.requested_torrents.add(infohash)
except:
print_exc()
elif DEBUG:
print >> sys.stderr, 'metadata: GET_METADATA: error connecting to', show_permid_short(permid)
def udpConnectCallback(self, exc, permid):
if exc is not None:
if DEBUG:
print >> sys.stderr, 'NATTRAVERSAL_REQUEST failed to', show_permid_short(permid), exc
self._file2.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'),
'REQUEST FAILED',
show_permid(permid),
str(self._secure_overlay.get_dns_from_peerdb(permid)),
'\n')))
return False
if DEBUG:
print >> sys.stderr, 'NATTRAVERSAL_REQUEST was sent to', show_permid_short(permid), exc
return True
def handleMessage(self, permid, selversion, message):
t = message[0]
if t == GET_METADATA:
if DEBUG:
print >> sys.stderr, 'metadata: Got GET_METADATA', len(message), show_permid_short(permid)
return self.send_metadata(permid, message, selversion)
elif t == METADATA:
if DEBUG:
print >> sys.stderr, 'metadata: Got METADATA', len(message), show_permid_short(permid), selversion, currentThread().getName()
return self.got_metadata(permid, message, selversion)
else:
if DEBUG:
print >> sys.stderr, 'metadata: UNKNOWN OVERLAY MESSAGE', ord(t)
return False
def udpConnectCallback(self, exc, permid):
if exc is not None:
if DEBUG:
print >> sys.stderr, 'NATTRAVERSAL_REQUEST failed to', show_permid_short(
permid), exc
self._file2.write('; '.join(
(strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST FAILED',
show_permid(permid),
str(self._secure_overlay.get_dns_from_peerdb(permid)), '\n')))
return False
if DEBUG:
print >> sys.stderr, 'NATTRAVERSAL_REQUEST was sent to', show_permid_short(
permid), exc
return True
def handle_crawler_request(self, permid, selversion, channel_id, message,
reply_callback):
if DEBUG:
print >> sys.stderr, 'puncturecrawler: handle_crawler_request', show_permid_short(
permid), message
SimpleFileReporter.lock.acquire()
try:
if not self.reporter.file:
try:
self.reporter.file = open(self.reporter.path, 'a+b')
except Exception as e:
reply_callback(str(e), error=1)
return
file = self.reporter.file
try:
file.seek(0)
result = '%.2f CRAWL\n' % time.time() + file.read()
result = zlib.compress(result)
reply_callback(result)
file.truncate(0)
except Exception as e:
reply_callback(str(e), error=1)
try:
file.seek(0, os.SEEK_END)
except:
pass
finally:
SimpleFileReporter.lock.release()
def handle_crawler_reply(self, permid, selversion, channel_id,
channel_data, error, message, request_callback):
try:
if error:
if DEBUG:
print >> sys.stderr, 'puncturecrawler: handle_crawler_reply', error, message
self._file.write(
'ERROR %s %.2f %d %s\n' %
(show_permid(permid), time.time(), error, message))
self._file.flush()
else:
if DEBUG:
print >> sys.stderr, 'puncturecrawler: handle_crawler_reply', show_permid_short(
permid)
data = zlib.decompress(message)
filtered = filter(lambda char: char != '\x00', data)
self._file.write('REPLY %s %.2f\n' %
(show_permid(permid), time.time()))
self._file.write(
'# reply sizes: on-the-wire=%d, decompressed=%d, filtered=%d\n'
% (len(message), len(data), len(filtered)))
self._file.write(filtered)
self._file.flush()
except:
if DEBUG:
print >> sys.stderr, 'puncturecrawler: error writing to file'
def natCheckReplySendCallback(self, exc, permid):
if DEBUG:
print >> sys.stderr, 'NATCHECK_REPLY was sent to', show_permid_short(
permid), exc
if exc is not None:
return False
return True
def query_initiator(self, permid, selversion, request_callback):
if DEBUG:
print >> sys.stderr, 'channelcrawler: query_initiator', show_permid_short(permid)
sql = []
if selversion >= OLPROTO_VER_THIRTEENTH:
sql.extend(("SELECT 'channel_files', publisher_id, count(*) FROM ChannelCast group by publisher_id", "SELECT 'my_votes', mod_id, voter_id, vote, time_stamp FROM VoteCast where voter_id='" + show_permid(permid) + "' order by time_stamp desc limit 100", "SELECT 'other_votes', mod_id, voter_id, vote, time_stamp FROM VoteCast where voter_id<>'" + show_permid(permid) + "' order by time_stamp desc limit 100"))
request_callback(CRAWLER_CHANNEL_QUERY, ';'.join(sql), callback=self._after_request_callback)
def udpConnectReplySendCallback(self, exc, permid):
if DEBUG:
print >> sys.stderr, 'NATTRAVERSAL_REPLY was sent to', show_permid_short(
permid), exc
if exc is not None:
return False
return True
def query_initiator(self, permid, selversion, request_callback):
if DEBUG:
print >> sys.stderr, 'repexcrawler: query_initiator', show_permid_short(
permid)
request_callback(CRAWLER_REPEX_QUERY,
'',
callback=self._after_request_callback)
def handle_info_crawler_reply(self, permid, selversion, channel_id,
channel_data, error, message,
request_callback):
if error:
if DEBUG:
print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', error, message
self._file.write('; '.join(
(strftime('%Y/%m/%d %H:%M:%S'), ' INFO REPLY',
show_permid(permid), str(error), message, '\n')))
self._file.flush()
else:
if DEBUG:
print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', show_permid_short(
permid), cPickle.loads(message)
info = cPickle.loads(message)
self._file.write('; '.join(
(strftime('%Y/%m/%d %H:%M:%S'), ' INFO REPLY',
show_permid(permid), str(error), str(info), '\n')))
self._file.flush()
i = 0
for key, timestamp, piece_size, num_pieces, bitrate, nat in info:
i += 1
if i == 1:
sql = "\nSELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;\nDELETE FROM playback_event WHERE key = '%s';\n" % (
key, key)
else:
sql = "\nSELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;\nDELETE FROM playback_event WHERE key = '%s';\nDELETE FROM playback_info WHERE key = '%s';\n" % (
key, key, key)
request_callback(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY,
sql,
channel_data=key,
callback=self._after_event_request_callback,
frequency=0)
def send_metadata_request(self, permid, infohash, selversion = -1, caller = 'BC'):
if DEBUG:
print >> sys.stderr, 'metadata: Connect to send GET_METADATA to', show_permid_short(permid)
if not isValidInfohash(infohash):
return False
filename, metadata = self.torrent_exists(infohash)
if filename is not None:
if DEBUG:
print >> sys.stderr, 'metadata: send_meta_req: Already on disk??!'
self.notify_torrent_is_in(infohash, metadata, filename)
return True
if caller == 'dlhelp':
self.requested_torrents.add(infohash)
if self.min_free_space != 0 and self.free_space - self.avg_torrent_size < self.min_free_space:
self.free_space = self.get_free_space()
if self.free_space - self.avg_torrent_size < self.min_free_space:
self.warn_disk_full()
return True
try:
if selversion == -1:
self.overlay_bridge.connect(permid, lambda e, d, p, s: self.get_metadata_connect_callback(e, d, p, s, infohash))
else:
self.get_metadata_connect_callback(None, None, permid, selversion, infohash)
except:
print_exc()
return False
return True
def gotUdpConnectRequest(self, permid, selversion, channel_id, mh_msg, reply_callback):
if DEBUG:
print >> sys.stderr, 'NatCheckMsgHandler: gotUdpConnectRequest from', show_permid_short(permid)
try:
mh_data = bdecode(mh_msg)
except:
print_exc()
print >> sys.stderr, 'NatCheckMsgHandler: bad encoded data:', mh_msg
return False
if DEBUG:
print >> sys.stderr, 'NatCheckMsgHandler: gotUdpConnectRequest is', mh_data
try:
request_id, host, port = mh_data.split(':')
except:
print_exc()
print >> sys.stderr, 'NatCheckMsgHandler: error in received data:', mh_data
return False
coordinator = (host, int(port))
if DEBUG:
print >> sys.stderr, 'NatCheckMsgHandler: coordinator address is', coordinator
mhr_data = request_id + ':' + tryConnect(coordinator)
try:
mhr_msg = bencode(mhr_data)
except:
print_exc()
print >> sys.stderr, 'NatCheckMsgHandler: error in encoding data:', mhr_data
return False
reply_callback(mhr_msg, callback=self.udpConnectReplySendCallback)
def send_join_helpers(self, permid):
if DEBUG:
print 'helper: send_join_helpers: sending a join_helpers message to', show_permid_short(
permid)
olthread_send_join_helpers_lambda = lambda: self.olthread_send_join_helpers(
)
self.overlay_bridge.add_task(olthread_send_join_helpers_lambda, 0)
def gotUdpConnectRequest(self, permid, selversion, channel_id, mh_msg,
reply_callback):
if DEBUG:
print >> sys.stderr, 'NatCheckMsgHandler: gotUdpConnectRequest from', show_permid_short(
permid)
try:
mh_data = bdecode(mh_msg)
except:
print_exc()
print >> sys.stderr, 'NatCheckMsgHandler: bad encoded data:', mh_msg
return False
if DEBUG:
print >> sys.stderr, 'NatCheckMsgHandler: gotUdpConnectRequest is', mh_data
try:
request_id, host, port = mh_data.split(':')
except:
print_exc()
print >> sys.stderr, 'NatCheckMsgHandler: error in received data:', mh_data
return False
coordinator = (host, int(port))
if DEBUG:
print >> sys.stderr, 'NatCheckMsgHandler: coordinator address is', coordinator
mhr_data = request_id + ':' + tryConnect(coordinator)
try:
mhr_msg = bencode(mhr_data)
except:
print_exc()
print >> sys.stderr, 'NatCheckMsgHandler: error in encoding data:', mhr_data
return False
reply_callback(mhr_msg, callback=self.udpConnectReplySendCallback)
def _after_request_callback(self, exc, permid):
if not exc:
if DEBUG:
print >> sys.stderr, 'usereventlogcrawler: request send to', show_permid_short(
permid)
msg = '; '.join(['REQUEST', show_permid(permid)])
self.__log(msg)
def handle_crawler_request(self, permid, selversion, channel_id, message, reply_callback):
if DEBUG:
print >> sys.stderr, 'puncturecrawler: handle_crawler_request', show_permid_short(permid), message
SimpleFileReporter.lock.acquire()
try:
if not self.reporter.file:
try:
self.reporter.file = open(self.reporter.path, 'a+b')
except Exception as e:
reply_callback(str(e), error=1)
return
file = self.reporter.file
try:
file.seek(0)
result = '%.2f CRAWL\n' % time.time() + file.read()
result = zlib.compress(result)
reply_callback(result)
file.truncate(0)
except Exception as e:
reply_callback(str(e), error=1)
try:
file.seek(0, os.SEEK_END)
except:
pass
finally:
SimpleFileReporter.lock.release()
def handle_info_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback):
if error:
if DEBUG:
print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', error, message
self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'),
' INFO REPLY',
show_permid(permid),
str(error),
message,
'\n')))
self._file.flush()
else:
if DEBUG:
print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', show_permid_short(permid), cPickle.loads(message)
info = cPickle.loads(message)
self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'),
' INFO REPLY',
show_permid(permid),
str(error),
str(info),
'\n')))
self._file.flush()
i = 0
for key, timestamp, piece_size, num_pieces, bitrate, nat in info:
i += 1
if i == 1:
sql = "\nSELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;\nDELETE FROM playback_event WHERE key = '%s';\n" % (key, key)
else:
sql = "\nSELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;\nDELETE FROM playback_event WHERE key = '%s';\nDELETE FROM playback_info WHERE key = '%s';\n" % (key, key, key)
request_callback(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, sql, channel_data=key, callback=self._after_event_request_callback, frequency=0)
def _send_request(self,
permid,
message_id,
channel_id,
payload,
frequency=3600,
callback=None):
def _after_send_request(exc, permid):
if DEBUG:
if exc:
print >> sys.stderr, 'crawler: could not send request to', show_permid_short(
permid), exc
if exc:
self._release_channel_id(permid, channel_id)
if callback:
callback(exc, permid)
if DEBUG:
print >> sys.stderr, 'crawler: sending', getMessageName(
CRAWLER_REQUEST + message_id), 'with', len(
payload), 'bytes payload to', show_permid_short(permid)
self._overlay_bridge.send(
permid,
''.join((CRAWLER_REQUEST, message_id, chr(channel_id & 255),
chr(frequency >> 8 & 255) + chr(frequency & 255),
str(payload))), _after_send_request)
return channel_id
def _after_send_reply(exc, permid):
if DEBUG:
if exc:
print >> sys.stderr, 'crawler: could not send request', show_permid_short(
permid), exc
if callback:
callback(exc, permid)
def _after_send_reply(exc, permid):
if DEBUG:
print >> sys.stderr, 'crawler: _after_send_reply', show_permid_short(permid), exc
if not exc:
self.send_reply(permid, message_id, channel_id, remaining_payload, error=error)
if callback:
callback(exc, permid)
def got_auth_connection(self, oc):
if DEBUG:
print >> sys.stderr, 'secover: got_auth_connection', show_permid_short(oc.get_auth_permid()), oc.get_ip(), oc.get_auth_listen_port(), currentThread().getName()
if oc.is_locally_initiated() and oc.get_port() != oc.get_auth_listen_port():
if DEBUG:
print >> sys.stderr, 'secover: got_auth_connection: closing because auth', 'listen port not as expected', oc.get_port(), oc.get_auth_listen_port()
self.cleanup_admin_and_callbacks(oc, Exception('closing because auth listen port not as expected'))
return False
ret = True
iplport = ip_and_port2str(oc.get_ip(), oc.get_auth_listen_port())
known = iplport in self.iplport2oc
if not known:
self.iplport2oc[iplport] = oc
elif known and not oc.is_locally_initiated():
if DEBUG:
print >> sys.stderr, 'secover: got_auth_connection:', 'closing because we already have a connection to', iplport
self.cleanup_admin_and_callbacks(oc, Exception('closing because we already have a connection to peer'))
ret = False
if ret:
if oc.is_auth_done():
hisdns = (oc.get_ip(), oc.get_auth_listen_port())
else:
hisdns = None
if self.userconnhandler is not None:
try:
self.userconnhandler(None, oc.get_auth_permid(), oc.get_sel_proto_ver(), oc.is_locally_initiated(), hisdns)
except:
print_exc()
oc.dequeue_callbacks()
return ret
def _after_request_callback(self, exc, permid):
if not exc:
if DEBUG:
print >> sys.stderr, 'repexcrawler: request sent to', show_permid_short(
permid)
self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'),
'REQUEST', show_permid(permid), '\n')))
self._file.flush()
def send_proxy_have(self, aggregated_haves):
if DEBUG:
print 'helper: send_proxy_have: sending a proxy_have message to', show_permid_short(
self.coordinator_permid)
aggregated_string = aggregated_haves.tostring()
olthread_send_proxy_have_lambda = lambda: self.olthread_send_proxy_have(
aggregated_string)
self.overlay_bridge.add_task(olthread_send_proxy_have_lambda, 0)
def _after_request_callback(self, exc, permid):
if not exc:
if DEBUG:
print >> sys.stderr, 'puncturecrawler: request sent to', show_permid_short(
permid)
self._file.write('REQUEST %s %.2f\n' %
(show_permid(permid), time.time()))
self._file.flush()
def _after_send_request(exc, permid):
if DEBUG:
if exc:
print >> sys.stderr, 'crawler: could not send request to', show_permid_short(permid), exc
if exc:
self._release_channel_id(permid, channel_id)
if callback:
callback(exc, permid)
def process_query_reply(self, permid, query, usercallback, d):
if DEBUG:
print >> sys.stderr, 'rquery: process_query_reply:', show_permid_short(permid), query, d
if len(d['a']) > 0:
self.unidecode_hits(query, d)
remote_query_usercallback_lambda = lambda : usercallback(permid, query, d['a'])
self.session.uch.perform_usercallback(remote_query_usercallback_lambda)
elif DEBUG:
print >> sys.stderr, 'rquery: QUERY_REPLY: no results found'
def _after_send_request(exc, permid):
if DEBUG:
if exc:
print >> sys.stderr, 'crawler: could not send request to', show_permid_short(
permid), exc
if exc:
self._release_channel_id(permid, channel_id)
if callback:
callback(exc, permid)
def handle_crawler_request(self, permid, selversion, channel_id, message, reply_callback):
if DEBUG:
print >> sys.stderr, 'repexcrawler: handle_crawler_request', show_permid_short(permid), message
try:
repexhistory = self._repexlog.getHistoryAndCleanup()
except Exception as e:
reply_callback(str(e), error=1)
else:
reply_callback(cPickle.dumps(repexhistory, 2))
def _after_connect(exc, dns, permid, selversion):
self._post_connection_attempt(permid, not exc)
if exc:
if DEBUG:
print >> sys.stderr, 'crawler: could not connect', dns, show_permid_short(permid), exc
if callback:
callback(exc, permid)
else:
self._send_reply(permid, message_id, channel_id, payload, error=error, callback=callback)
def handleMessage(self, permid, selversion, message):
t = message[0]
if t == GET_METADATA:
if DEBUG:
print >> sys.stderr, 'metadata: Got GET_METADATA', len(
message), show_permid_short(permid)
return self.send_metadata(permid, message, selversion)
elif t == METADATA:
if DEBUG:
print >> sys.stderr, 'metadata: Got METADATA', len(
message), show_permid_short(
permid), selversion, currentThread().getName()
return self.got_metadata(permid, message, selversion)
else:
if DEBUG:
print >> sys.stderr, 'metadata: UNKNOWN OVERLAY MESSAGE', ord(
t)
return False
def _after_request_callback(self, exc, permid):
if not exc:
if DEBUG:
print >> sys.stderr, 'channelcrawler: request send to', show_permid_short(permid)
self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'),
'REQUEST',
show_permid(permid),
'\n')))
self._file.flush()
def get_metadata_connect_callback(self, exc, dns, permid, selversion,
infohash):
if exc is None:
if DEBUG:
print >> sys.stderr, 'metadata: Sending GET_METADATA to', show_permid_short(
permid)
try:
metadata_request = bencode(infohash)
self.overlay_bridge.send(permid,
GET_METADATA + metadata_request,
self.get_metadata_send_callback)
self.requested_torrents.add(infohash)
except:
print_exc()
elif DEBUG:
print >> sys.stderr, 'metadata: GET_METADATA: error connecting to', show_permid_short(
permid)
def query_initiator(self, permid, selversion, request_callback):
if DEBUG:
print >> sys.stderr, 'usereventlogcrawler: query_initiator', show_permid_short(
permid)
if selversion >= OLPROTO_VER_FIFTEENTH:
sql = 'SELECT * FROM UserEventLog; DELETE FROM UserEventLog;'
request_callback(CRAWLER_USEREVENTLOG_QUERY,
sql,
callback=self._after_request_callback)
def _sent_callback(self, exc, permid, channel_id, infohash, bitmask, msgSentCallback, usrCallback):
if exc is not None:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'Unable to send GET_SUBS to: ' + utilities.show_permid_short(permid) + ': ' + exc
else:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'GET_SUBS sent to %s' % utilities.show_permid_short(permid)
self._addToRequestedSubtitles(channel_id, infohash, bitmask, usrCallback)
if msgSentCallback is not None:
msgSentCallback(exc, permid, channel_id, infohash, bitmask)
def doNatCheckCallback(self, exc, permid):
if exc is not None:
return False
if DEBUG:
print >> sys.stderr, 'NATCHECK_REQUEST was sent to', show_permid_short(
permid), exc
self._file.write('; '.join(
(strftime('%Y/%m/%d %H:%M:%S'), 'REQUEST', show_permid(permid),
str(self._secure_overlay.get_dns_from_peerdb(permid)), '\n')))
self._file.flush()
return True
def send(self, permid, msg, callback):
if DEBUG:
print >> sys.stderr, 'olbridge: send', show_permid_short(permid), len(msg)
def olbridge_send_callback(cexc, cpermid):
if DEBUG:
print >> sys.stderr, 'olbridge: send_callback', cexc, show_permid_short(cpermid)
olbridge_send_callback_lambda = lambda : callback(cexc, cpermid)
self.add_task(olbridge_send_callback_lambda, 0)
self.secover.send(permid, msg, olbridge_send_callback)
def handle_crawler_request(self, permid, selversion, channel_id, message,
reply_callback):
if DEBUG:
print >> sys.stderr, 'repexcrawler: handle_crawler_request', show_permid_short(
permid), message
try:
repexhistory = self._repexlog.getHistoryAndCleanup()
except Exception as e:
reply_callback(str(e), error=1)
else:
reply_callback(cPickle.dumps(repexhistory, 2))
def connect(self, permid, callback):
if DEBUG:
print >> sys.stderr, 'olbridge: connect', show_permid_short(permid), currentThread().getName()
def olbridge_connect_callback(cexc, cdns, cpermid, cselver):
if DEBUG:
print >> sys.stderr, 'olbridge: connect_callback', cexc, cdns, show_permid_short(cpermid), cselver, callback, currentThread().getName()
olbridge_connect_callback_lambda = lambda : callback(cexc, cdns, cpermid, cselver)
self.add_task(olbridge_connect_callback_lambda, 0)
self.secover.connect(permid, olbridge_connect_callback)
def handle_crawler_request(self, permid, selversion, channel_id, message, reply_callback):
if DEBUG:
print >> sys.stderr, 'channelcrawler: handle_crawler_request', show_permid_short(permid), message
try:
cursor = self._sqlite_cache_db.execute_read(message)
except Exception as e:
reply_callback(str(e), error=1)
else:
if cursor:
reply_callback(cPickle.dumps(list(cursor), 2))
else:
reply_callback('error', error=2)
def doNatCheckCallback(self, exc, permid):
if exc is not None:
return False
if DEBUG:
print >> sys.stderr, 'NATCHECK_REQUEST was sent to', show_permid_short(permid), exc
self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'),
'REQUEST',
show_permid(permid),
str(self._secure_overlay.get_dns_from_peerdb(permid)),
'\n')))
self._file.flush()
return True
def _close(self, permid):
if DEBUG:
print >> sys.stderr, 'secover: actual close', show_permid_short(permid)
try:
oc = self.get_oc_by_permid(permid)
if not oc:
if DEBUG:
print >> sys.stderr, 'secover: error - actual close, but no connection to peer in admin'
else:
oc.close()
except Exception as e:
print_exc()
def _checkingUploadQueue(self):
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'Checking the upload queue...'
if not self._tokenBucket.upload_rate > 0:
return
if not len(self._uploadQueue) > 0:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'Upload queue is empty.'
while len(self._uploadQueue) > 0:
responseData = self._uploadQueue[0]
encodedMsg = self._createSingleResponseMessage(responseData)
if encodedMsg is None:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'Nothing to send'
del self._uploadQueue[0]
continue
msgSize = len(encodedMsg) / 1024.0
if msgSize > self._tokenBucket.capacity:
print >> sys.stderr, 'Warning:' + SUBS_LOG_PREFIX + 'SUBS message too big. Discarded!'
del self._uploadQueue[0]
continue
if self._tokenBucket.consume(msgSize):
if DEBUG:
keys = responseData['subtitles'].keys()
bitmask = self._languagesUtility.langCodesToMask(keys)
print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ('SS',
show_permid_short(responseData['permid']),
show_permid_short(responseData['channel_id']),
bin2str(responseData['infohash']),
bitmask,
int(msgSize * 1024))
self._doSendSubtitles(responseData['permid'], encodedMsg, responseData['selversion'])
del self._uploadQueue[0]
else:
neededCapacity = max(0, msgSize - self._tokenBucket.tokens)
delay = neededCapacity / self._tokenBucket.upload_rate
self._nextUploadTime = time() + delay
self.overlay_bridge.add_task(self._checkingUploadQueue, delay)
return
def olbridge_handle_conn_func():
if DEBUG:
print >> sys.stderr, 'olbridge: handle_conn_func', exc, show_permid_short(permid), selversion, locally_initiated, hisdns, currentThread().getName()
try:
if hisdns:
self.secover.add_peer_to_db(permid, hisdns, selversion)
if self.olappsconnhandler is not None:
self.olappsconnhandler(exc, permid, selversion, locally_initiated)
except:
print_exc()
if isinstance(exc, CloseException):
self.secover.update_peer_status(permid, exc.was_auth_done())
