def handle_event_crawler_reply(self, permid, selversion, channel_id,
channel_data, error, message,
request_callback):
if error:
if DEBUG:
print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', error, message
self._file.write('; '.join(
(strftime('%Y/%m/%d %H:%M:%S'), ' EVENT REPLY',
show_permid(permid), str(error), str(channel_data), message,
'\n')))
self._file.flush()
elif selversion >= OLPROTO_VER_TENTH:
if DEBUG:
print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', show_permid_short(
permid), len(message), 'bytes zipped'
info = cPickle.loads(zlib.decompress(message))
self._file.write('; '.join(
(strftime('%Y/%m/%d %H:%M:%S'), ' EVENT REPLY',
show_permid(permid), str(error), str(channel_data), str(info),
'\n')))
self._file.flush()
elif selversion >= OLPROTO_VER_EIGHTH:
if DEBUG:
print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', show_permid_short(
permid), cPickle.loads(message)
info = cPickle.loads(message)
self._file.write('; '.join(
(strftime('%Y/%m/%d %H:%M:%S'), ' EVENT REPLY',
show_permid(permid), str(error), str(channel_data), str(info),
'\n')))
self._file.flush()
def process_query(self, permid, d, selversion):
hits = None
p = None
sendtorrents = False
netwq = d['q']
if netwq.startswith('SIMPLE'):
if netwq.startswith('SIMPLE+METADATA'):
q = d['q'][len('SIMPLE+METADATA '):]
sendtorrents = True
else:
q = d['q'][len('SIMPLE '):]
uq = self.clean_netwq(q)
kws = split_into_keywords(uq)
hits = self.search_torrents(kws,
maxhits=MAX_RESULTS,
sendtorrents=sendtorrents)
p = self.create_remote_query_reply(d['id'], hits, selversion)
if self.logfile:
self.log(permid, q)
if p:
m = QUERY_REPLY + p
if self.overlay_log:
nqueries = self.get_peer_nqueries(permid)
self.overlay_log('RECV_QRY', show_permid(permid), selversion,
nqueries, repr(d))
self.overlay_log('RPLY_QRY', show_permid(permid), len(hits),
repr(p))
self.overlay_bridge.send(permid, m, self.send_callback)
self.inc_peer_nqueries(permid)
def handle_info_crawler_reply(self, permid, selversion, channel_id,
channel_data, error, message,
request_callback):
if error:
if DEBUG:
print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', error, message
self._file.write('; '.join(
(strftime('%Y/%m/%d %H:%M:%S'), ' INFO REPLY',
show_permid(permid), str(error), message, '\n')))
self._file.flush()
else:
if DEBUG:
print >> sys.stderr, 'videoplaybackcrawler: handle_crawler_reply', show_permid_short(
permid), cPickle.loads(message)
info = cPickle.loads(message)
self._file.write('; '.join(
(strftime('%Y/%m/%d %H:%M:%S'), ' INFO REPLY',
show_permid(permid), str(error), str(info), '\n')))
self._file.flush()
i = 0
for key, timestamp, piece_size, num_pieces, bitrate, nat in info:
i += 1
if i == 1:
sql = "\nSELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;\nDELETE FROM playback_event WHERE key = '%s';\n" % (
key, key)
else:
sql = "\nSELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;\nDELETE FROM playback_event WHERE key = '%s';\nDELETE FROM playback_info WHERE key = '%s';\n" % (
key, key, key)
request_callback(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY,
sql,
channel_data=key,
callback=self._after_event_request_callback,
frequency=0)
def process_query(self, permid, d, selversion):
hits = None
p = None
sendtorrents = False
netwq = d['q']
if netwq.startswith('SIMPLE'):
if netwq.startswith('SIMPLE+METADATA'):
q = d['q'][len('SIMPLE+METADATA '):]
sendtorrents = True
else:
q = d['q'][len('SIMPLE '):]
uq = self.clean_netwq(q)
kws = split_into_keywords(uq)
hits = self.search_torrents(kws, maxhits=MAX_RESULTS, sendtorrents=sendtorrents)
p = self.create_remote_query_reply(d['id'], hits, selversion)
if self.logfile:
self.log(permid, q)
if p:
m = QUERY_REPLY + p
if self.overlay_log:
nqueries = self.get_peer_nqueries(permid)
self.overlay_log('RECV_QRY', show_permid(permid), selversion, nqueries, repr(d))
self.overlay_log('RPLY_QRY', show_permid(permid), len(hits), repr(p))
self.overlay_bridge.send(permid, m, self.send_callback)
self.inc_peer_nqueries(permid)
def tryHolePunching(self):
if DEBUG:
print >> sys.stderr, 'NatCheckMsgHandler: first element in peerlist', self.peerlist[len(self.peerlist) - 1]
print >> sys.stderr, 'NatCheckMsgHandler: second element in peerlist', self.peerlist[len(self.peerlist) - 2]
holePunchingPort = random.randrange(3200, 4200, 1)
holePunchingAddr = (self.holePunchingIP, holePunchingPort)
peer1 = self.peerlist[len(self.peerlist) - 1]
peer2 = self.peerlist[len(self.peerlist) - 2]
request_id = str(show_permid_short(peer1[0]) + show_permid_short(peer2[0]) + str(random.randrange(0, 1000, 1)))
self.udpConnect(peer1[0], request_id, holePunchingAddr)
self.udpConnect(peer2[0], request_id, holePunchingAddr)
self._file2.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'),
'REQUEST',
request_id,
show_permid(peer1[0]),
str(peer1[1]),
str(peer1[2]),
str(self._secure_overlay.get_dns_from_peerdb(peer1[0])),
show_permid(peer2[0]),
str(peer2[1]),
str(peer2[2]),
str(self._secure_overlay.get_dns_from_peerdb(peer2[0])),
'\n')))
self._file2.flush()
self.trav[request_id] = (None, None)
thread.start_new_thread(coordinateHolePunching, (peer1, peer2, holePunchingAddr))
def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback):
if error:
if DEBUG:
print >>sys.stderr, "databasecrawler: handle_crawler_reply", error, message
self._file.write(
"; ".join((strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(error), message, "\n"))
)
self._file.flush()
else:
if DEBUG:
print >>sys.stderr, "databasecrawler: handle_crawler_reply", show_permid_short(permid), cPickle.loads(
message
)
self._file.write(
"; ".join(
(
strftime("%Y/%m/%d %H:%M:%S"),
" REPLY",
show_permid(permid),
str(error),
str(cPickle.loads(message)),
"\n",
)
)
)
self._file.flush()
def handle_event_crawler_reply(
self, permid, selversion, channel_id, channel_data, error, message, request_callback
):
if error:
if DEBUG:
print >>sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message
self._file.write(
"; ".join(
(
strftime("%Y/%m/%d %H:%M:%S"),
" EVENT REPLY",
show_permid(permid),
str(error),
str(channel_data),
message,
"\n",
)
)
)
self._file.flush()
elif selversion >= OLPROTO_VER_TENTH:
if DEBUG:
print >>sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(permid), len(
message
), "bytes zipped"
info = cPickle.loads(zlib.decompress(message))
self._file.write(
"; ".join(
(
strftime("%Y/%m/%d %H:%M:%S"),
" EVENT REPLY",
show_permid(permid),
str(error),
str(channel_data),
str(info),
"\n",
)
)
)
self._file.flush()
elif selversion >= OLPROTO_VER_EIGHTH:
if DEBUG:
print >>sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(
permid
), cPickle.loads(message)
info = cPickle.loads(message)
self._file.write(
"; ".join(
(
strftime("%Y/%m/%d %H:%M:%S"),
" EVENT REPLY",
show_permid(permid),
str(error),
str(channel_data),
str(info),
"\n",
)
)
)
self._file.flush()
def handle_crawler_reply(self, permid, selversion, channel_id,
channel_data, error, message, request_callback):
try:
if error:
if DEBUG:
print >> sys.stderr, 'puncturecrawler: handle_crawler_reply', error, message
self._file.write(
'ERROR %s %.2f %d %s\n' %
(show_permid(permid), time.time(), error, message))
self._file.flush()
else:
if DEBUG:
print >> sys.stderr, 'puncturecrawler: handle_crawler_reply', show_permid_short(
permid)
data = zlib.decompress(message)
filtered = filter(lambda char: char != '\x00', data)
self._file.write('REPLY %s %.2f\n' %
(show_permid(permid), time.time()))
self._file.write(
'# reply sizes: on-the-wire=%d, decompressed=%d, filtered=%d\n'
% (len(message), len(data), len(filtered)))
self._file.write(filtered)
self._file.flush()
except:
if DEBUG:
print >> sys.stderr, 'puncturecrawler: error writing to file'
def gotNatCheckReplyMessage(self, permid, selversion, channel_id, channel_data, error, payload, request_callback):
if error:
if DEBUG:
print >> sys.stderr, 'NatCheckMsgHandler: gotNatCheckReplyMessage'
print >> sys.stderr, 'NatCheckMsgHandler: error', error
self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'),
' REPLY',
show_permid(permid),
str(self._secure_overlay.get_dns_from_peerdb(permid)),
'ERROR(%d)' % error,
payload,
'\n')))
self._file.flush()
else:
try:
recv_data = bdecode(payload)
except:
print_exc()
print >> sys.stderr, 'bad encoded data:', payload
return False
try:
self.validNatCheckReplyMsg(recv_data)
except RuntimeError as e:
print >> sys.stderr, e
return False
if DEBUG:
print >> sys.stderr, 'NatCheckMsgHandler: received NAT_CHECK_REPLY message: ', recv_data
self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'),
' REPLY',
show_permid(permid),
str(self._secure_overlay.get_dns_from_peerdb(permid)),
':'.join([ str(x) for x in recv_data ]),
'\n')))
self._file.flush()
if selversion < OLPROTO_VER_NINETH:
if DEBUG:
print >> sys.stderr, 'NatCheckMsgHandler: ACEStream version too old for NATTRAVERSAL: do nothing'
return True
if DEBUG:
print >> sys.stderr, 'NatCheckMsgHandler: do NATTRAVERSAL'
if len(self.peerlist) == PEERLIST_LEN:
del self.peerlist[0]
self.peerlist.append([permid, recv_data[1], recv_data[2]])
if DEBUG:
print >> sys.stderr, 'NatCheckMsgHandler: peerlist length is: ', len(self.peerlist)
if len(self.peerlist) >= 2:
self.tryHolePunching()
return True
def _after_request_callback(self, exc, permid):
if not exc:
if DEBUG:
print >> sys.stderr, 'puncturecrawler: request sent to', show_permid_short(
permid)
self._file.write('REQUEST %s %.2f\n' %
(show_permid(permid), time.time()))
self._file.flush()
def _after_request_callback(self, exc, permid):
if not exc:
if DEBUG:
print >> sys.stderr, 'databasecrawler: request send to', show_permid_short(
permid)
self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'),
'REQUEST', show_permid(permid), '\n')))
self._file.flush()
def handle_crawler_reply(self, permid, selversion, channel_id,
channel_data, error, message, request_callback):
if error:
if DEBUG:
print >> sys.stderr, 'databasecrawler: handle_crawler_reply', error, message
self._file.write('; '.join(
(strftime('%Y/%m/%d %H:%M:%S'), ' REPLY', show_permid(permid),
str(error), message, '\n')))
self._file.flush()
else:
if DEBUG:
print >> sys.stderr, 'databasecrawler: handle_crawler_reply', show_permid_short(
permid), cPickle.loads(message)
self._file.write('; '.join(
(strftime('%Y/%m/%d %H:%M:%S'), ' REPLY', show_permid(permid),
str(error), str(cPickle.loads(message)), '\n')))
self._file.flush()
def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback):
if error:
if DEBUG:
print >> sys.stderr, 'usereventlogcrawler: handle_crawler_reply', error, message
msg = '; '.join(['REPLY',
show_permid(permid),
str(error),
str(message)])
self.__log(msg)
else:
if DEBUG:
print >> sys.stderr, 'usereventlogcrawler: handle_crawler_reply', show_permid_short(permid), cPickle.loads(message)
msg = '; '.join(['REPLY',
show_permid(permid),
str(error),
str(cPickle.loads(message))])
self.__log(msg)
def _after_request_callback(self, exc, permid):
if not exc:
if DEBUG:
print >> sys.stderr, 'repexcrawler: request sent to', show_permid_short(permid)
self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'),
'REQUEST',
show_permid(permid),
'\n')))
self._file.flush()
def gotUdpConnectReply(self, permid, selversion, channel_id, channel_data, error, mhr_msg, request_callback):
if DEBUG:
print >> sys.stderr, 'NatCheckMsgHandler: gotMakeHoleReplyMessage'
try:
mhr_data = bdecode(mhr_msg)
except:
print_exc()
print >> sys.stderr, 'NatCheckMsgHandler: bad encoded data:', mhr_msg
return False
if DEBUG:
print >> sys.stderr, 'NatCheckMsgHandler: message is', mhr_data
try:
request_id, reply = mhr_data.split(':')
except:
print_exc()
print >> sys.stderr, 'NatCheckMsgHandler: error in received data:', mhr_data
return False
if DEBUG:
print >> sys.stderr, 'NatCheckMsgHandler: request_id is', request_id
if request_id in self.trav:
if DEBUG:
print >> sys.stderr, 'NatCheckMsgHandler: request_id is in the list'
peer, value = self.trav[request_id]
if peer == None:
if DEBUG:
print >> sys.stderr, 'NatCheckMsgHandler: first peer reply'
self.trav[request_id] = ((permid, self._secure_overlay.get_dns_from_peerdb(permid)), reply)
elif type(peer) == TupleType:
if DEBUG:
print >> sys.stderr, 'NatCheckMsgHandler: second peer reply'
self._file2.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'),
' REPLY',
request_id,
show_permid(peer[0]),
str(peer[1]),
value,
show_permid(permid),
str(self._secure_overlay.get_dns_from_peerdb(permid)),
reply,
'\n')))
del self.trav[request_id]
self._file2.flush()
def handle_info_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback):
if error:
if DEBUG:
print >>sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message
self._file.write(
"; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), " INFO REPLY", show_permid(permid), str(error), message, "\n")
)
)
self._file.flush()
else:
if DEBUG:
print >>sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(
permid
), cPickle.loads(message)
info = cPickle.loads(message)
self._file.write(
"; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), " INFO REPLY", show_permid(permid), str(error), str(info), "\n")
)
)
self._file.flush()
i = 0
for key, timestamp, piece_size, num_pieces, bitrate, nat in info:
i += 1
if i == 1:
sql = (
"\nSELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;\nDELETE FROM playback_event WHERE key = '%s';\n"
% (key, key)
)
else:
sql = (
"\nSELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;\nDELETE FROM playback_event WHERE key = '%s';\nDELETE FROM playback_info WHERE key = '%s';\n"
% (key, key, key)
)
request_callback(
CRAWLER_VIDEOPLAYBACK_EVENT_QUERY,
sql,
channel_data=key,
callback=self._after_event_request_callback,
frequency=0,
)
def doNatCheckCallback(self, exc, permid):
if exc is not None:
return False
if DEBUG:
print >> sys.stderr, 'NATCHECK_REQUEST was sent to', show_permid_short(permid), exc
self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'),
'REQUEST',
show_permid(permid),
str(self._secure_overlay.get_dns_from_peerdb(permid)),
'\n')))
self._file.flush()
return True
def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback):
if error:
if DEBUG:
print >> sys.stderr, 'repexcrawler: handle_crawler_reply', error, message
self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'),
' REPLY',
show_permid(permid),
str(error),
message,
'\n')))
self._file.flush()
else:
if DEBUG:
print >> sys.stderr, 'repexcrawler: handle_crawler_reply', show_permid_short(permid), cPickle.loads(message)
self._file.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'),
' REPLY',
show_permid(permid),
str(error),
base64.b64encode(message),
'\n')))
self._file.flush()
def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback):
try:
if error:
if DEBUG:
print >> sys.stderr, 'puncturecrawler: handle_crawler_reply', error, message
self._file.write('ERROR %s %.2f %d %s\n' % (show_permid(permid),
time.time(),
error,
message))
self._file.flush()
else:
if DEBUG:
print >> sys.stderr, 'puncturecrawler: handle_crawler_reply', show_permid_short(permid)
data = zlib.decompress(message)
filtered = filter(lambda char: char != '\x00', data)
self._file.write('REPLY %s %.2f\n' % (show_permid(permid), time.time()))
self._file.write('# reply sizes: on-the-wire=%d, decompressed=%d, filtered=%d\n' % (len(message), len(data), len(filtered)))
self._file.write(filtered)
self._file.flush()
except:
if DEBUG:
print >> sys.stderr, 'puncturecrawler: error writing to file'
def udpConnectCallback(self, exc, permid):
if exc is not None:
if DEBUG:
print >> sys.stderr, 'NATTRAVERSAL_REQUEST failed to', show_permid_short(permid), exc
self._file2.write('; '.join((strftime('%Y/%m/%d %H:%M:%S'),
'REQUEST FAILED',
show_permid(permid),
str(self._secure_overlay.get_dns_from_peerdb(permid)),
'\n')))
return False
if DEBUG:
print >> sys.stderr, 'NATTRAVERSAL_REQUEST was sent to', show_permid_short(permid), exc
return True
def query_initiator(self, permid, selversion, request_callback):
if DEBUG:
print >> sys.stderr, 'databasecrawler: query_initiator', show_permid_short(
permid)
sql = []
if selversion >= OLPROTO_VER_SEVENTH:
sql.extend(("SELECT 'peer_count', count(*) FROM Peer",
"SELECT 'torrent_count', count(*) FROM Torrent"))
if selversion >= OLPROTO_VER_ELEVENTH:
sql.extend((
"SELECT 'my_subscriptions', count(*) FROM VoteCast where voter_id='"
+ show_permid(permid) + "' and vote=2",
"SELECT 'my_negative_votes', count(*) FROM VoteCast where voter_id='"
+ show_permid(permid) + "' and vote=-1",
"SELECT 'my_channel_files', count(*) FROM ChannelCast where publisher_id='"
+ show_permid(permid) + "'",
"SELECT 'all_subscriptions', count(*) FROM VoteCast where vote=2",
"SELECT 'all_negative_votes', count(*) FROM VoteCast where vote=-1"
))
request_callback(CRAWLER_DATABASE_QUERY,
';'.join(sql),
callback=self._after_request_callback)
def query_initiator(self, permid, selversion, request_callback):
if DEBUG:
print >>sys.stderr, "databasecrawler: query_initiator", show_permid_short(permid)
sql = []
if selversion >= OLPROTO_VER_SEVENTH:
sql.extend(("SELECT 'peer_count', count(*) FROM Peer", "SELECT 'torrent_count', count(*) FROM Torrent"))
if selversion >= OLPROTO_VER_ELEVENTH:
sql.extend(
(
"SELECT 'my_subscriptions', count(*) FROM VoteCast where voter_id='"
+ show_permid(permid)
+ "' and vote=2",
"SELECT 'my_negative_votes', count(*) FROM VoteCast where voter_id='"
+ show_permid(permid)
+ "' and vote=-1",
"SELECT 'my_channel_files', count(*) FROM ChannelCast where publisher_id='"
+ show_permid(permid)
+ "'",
"SELECT 'all_subscriptions', count(*) FROM VoteCast where vote=2",
"SELECT 'all_negative_votes', count(*) FROM VoteCast where vote=-1",
)
)
request_callback(CRAWLER_DATABASE_QUERY, ";".join(sql), callback=self._after_request_callback)
def _after_info_request_callback(self, exc, permid):
if not exc:
if DEBUG:
print >>sys.stderr, "videoplaybackcrawler: request send to", show_permid_short(permid)
self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "INFO REQUEST", show_permid(permid), "\n")))
self._file.flush()
def _after_request_callback(self, exc, permid):
if not exc:
if DEBUG:
print >> sys.stderr, 'usereventlogcrawler: request send to', show_permid_short(permid)
msg = '; '.join(['REQUEST', show_permid(permid)])
self.__log(msg)
def query_initiator(self, permid, selversion, request_callback):
if DEBUG:
print >> sys.stderr, 'channelcrawler: query_initiator', show_permid_short(permid)
sql = []
if selversion >= OLPROTO_VER_THIRTEENTH:
sql.extend(("SELECT 'channel_files', publisher_id, count(*) FROM ChannelCast group by publisher_id", "SELECT 'my_votes', mod_id, voter_id, vote, time_stamp FROM VoteCast where voter_id='" + show_permid(permid) + "' order by time_stamp desc limit 100", "SELECT 'other_votes', mod_id, voter_id, vote, time_stamp FROM VoteCast where voter_id<>'" + show_permid(permid) + "' order by time_stamp desc limit 100"))
request_callback(CRAWLER_CHANNEL_QUERY, ';'.join(sql), callback=self._after_request_callback)
def _after_request_callback(self, exc, permid):
if not exc:
if DEBUG:
print >> sys.stderr, 'puncturecrawler: request sent to', show_permid_short(permid)
self._file.write('REQUEST %s %.2f\n' % (show_permid(permid), time.time()))
self._file.flush()