def handle_crawler_reply(self, permid, selversion, channel_id,
channel_data, error, message, request_callback):
"""
Received a CRAWLER_DATABASE_QUERY reply.
@param permid The Crawler permid
@param selversion The overlay protocol version
@param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
@param error The error value. 0 indicates success.
@param message The message payload
@param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
"""
if error:
if DEBUG:
print >> sys.stderr, "databasecrawler: handle_crawler_reply", error, message
self._file.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid),
str(error), message, "\n")))
self._file.flush()
else:
if DEBUG:
print >> sys.stderr, "databasecrawler: handle_crawler_reply", show_permid_short(
permid), cPickle.loads(message)
self._file.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid),
str(error), str(cPickle.loads(message)), "\n")))
self._file.flush()
def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback):
"""
Received a CRAWLER_REPEX_QUERY reply.
@param permid The Crawler permid
@param selversion The overlay protocol version
@param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
@param error The error value. 0 indicates success.
@param message The message payload
@param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
"""
if error:
if DEBUG:
print >> sys.stderr, time.asctime(),'-', "repexcrawler: handle_crawler_reply", error, message
self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(error), message, "\n")))
self._file.flush()
else:
if DEBUG:
print >> sys.stderr, time.asctime(),'-', "repexcrawler: handle_crawler_reply", show_permid_short(permid), cPickle.loads(message)
# The message is pickled, which we will just write to file.
# To make later parsing easier, we base64 encode it
self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(error), base64.b64encode(message), "\n")))
self._file.flush()
def tryHolePunching(self):
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: first element in peerlist", self.peerlist[len(self.peerlist)-1]
print >> sys.stderr, "NatCheckMsgHandler: second element in peerlist", self.peerlist[len(self.peerlist)-2]
holePunchingPort = random.randrange(3200, 4200, 1)
holePunchingAddr = (self.holePunchingIP, holePunchingPort)
peer1 = self.peerlist[len(self.peerlist)-1]
peer2 = self.peerlist[len(self.peerlist)-2]
request_id = str(show_permid_short(peer1[0]) + show_permid_short(peer2[0]) + str(random.randrange(0, 1000, 1)))
self.udpConnect(peer1[0], request_id, holePunchingAddr)
self.udpConnect(peer2[0], request_id, holePunchingAddr)
# Register peerinfo on file
self._file2.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"),
"REQUEST",
request_id,
show_permid(peer1[0]),
str(peer1[1]),
str(peer1[2]),
str(self._secure_overlay.get_dns_from_peerdb(peer1[0])),
show_permid(peer2[0]),
str(peer2[1]),
str(peer2[2]),
str(self._secure_overlay.get_dns_from_peerdb(peer2[0])),
"\n")))
self._file2.flush()
self.trav[request_id] = (None, None)
thread.start_new_thread(coordinateHolePunching, (peer1, peer2, holePunchingAddr))
def tryHolePunching(self):
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: first element in peerlist", self.peerlist[
len(self.peerlist) - 1]
print >> sys.stderr, "NatCheckMsgHandler: second element in peerlist", self.peerlist[
len(self.peerlist) - 2]
holePunchingPort = random.randrange(3200, 4200, 1)
holePunchingAddr = (self.holePunchingIP, holePunchingPort)
peer1 = self.peerlist[len(self.peerlist) - 1]
peer2 = self.peerlist[len(self.peerlist) - 2]
request_id = str(
show_permid_short(peer1[0]) + show_permid_short(peer2[0]) +
str(random.randrange(0, 1000, 1)))
self.udpConnect(peer1[0], request_id, holePunchingAddr)
self.udpConnect(peer2[0], request_id, holePunchingAddr)
# Register peerinfo on file
self._file2.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), "REQUEST", request_id,
show_permid(peer1[0]), str(peer1[1]), str(peer1[2]),
str(self._secure_overlay.get_dns_from_peerdb(peer1[0])),
show_permid(peer2[0]), str(peer2[1]), str(peer2[2]),
str(self._secure_overlay.get_dns_from_peerdb(peer2[0])), "\n")))
self._file2.flush()
self.trav[request_id] = (None, None)
thread.start_new_thread(coordinateHolePunching,
(peer1, peer2, holePunchingAddr))
def handle_info_crawler_reply(self, permid, selversion, channel_id, error,
message, request_callback):
"""
<<Crawler-side>>
Received a CRAWLER_VIDEOPLAYBACK_INFO_QUERY reply.
@param permid The Crawler permid
@param selversion The overlay protocol version
@param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
@param error The error value. 0 indicates success.
@param message The message payload
@param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
"""
if error:
if DEBUG:
print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message
self._file.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), " INFO REPLY",
show_permid(permid), str(error), message, "\n")))
self._file.flush()
else:
if DEBUG:
print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(
permid), cPickle.loads(message)
info = cPickle.loads(message)
self._file.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), " INFO REPLY",
show_permid(permid), str(error), str(info), "\n")))
self._file.flush()
i = 0
for key, timestamp, piece_size, num_pieces, bitrate, nat in info:
i += 1
# do not remove the first item. the list is ordered
# DESC so the first item is the last that is added to
# the database and we can't affored to remove it, as
# it may cause exceptions in the running playback.
if i == 1:
sql = """
SELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;
DELETE FROM playback_event WHERE key = '%s';
""" % (key, key)
else:
sql = """
SELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;
DELETE FROM playback_event WHERE key = '%s';
DELETE FROM playback_info WHERE key = '%s';
""" % (key, key, key)
# todo: optimize to not select key for each row
request_callback(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY,
sql,
channel_data=key,
callback=self._after_event_request_callback,
frequency=0)
def process_query(self, permid, d, selversion):
hits = None
p = None
sendtorrents = False
netwq = d['q']
if netwq.startswith("SIMPLE"): # remote query
# Format: 'SIMPLE '+string of space separated keywords or
# 'SIMPLE+METADATA' +string of space separated keywords
#
# In the future we could support full SQL queries:
# SELECT infohash,torrent_name FROM torrent_db WHERE status = ALIVE
if netwq.startswith('SIMPLE+METADATA'):
q = d['q'][len('SIMPLE+METADATA '):]
sendtorrents = True
else:
q = d['q'][len('SIMPLE '):]
q = self.clean_netwq(q)
q = dunno2unicode(q)
kws = re.split(r'\W+', q.lower())
hits = self.search_torrents(kws,
maxhits=MAX_RESULTS,
sendtorrents=sendtorrents)
p = self.create_remote_query_reply(d['id'], hits, selversion)
elif netwq.startswith("CHANNEL"): # channel query
q = d['q'][len('CHANNEL '):]
q = self.clean_netwq(q)
q = dunno2unicode(q)
hits = self.channelcast_db.searchChannels(q)
p = self.create_channel_query_reply(d['id'], hits, selversion)
# log incoming query, if logfile is set
if self.logfile:
self.log(permid, q)
m = QUERY_REPLY + p
if self.overlay_log:
nqueries = self.get_peer_nqueries(permid)
# RECV_MSG PERMID OVERSION NUM_QUERIES MSG
self.overlay_log('RECV_QRY', show_permid(permid), selversion,
nqueries, repr(d))
# RPLY_QRY PERMID NUM_HITS MSG
self.overlay_log('RPLY_QRY', show_permid(permid), len(hits),
repr(p))
self.overlay_bridge.send(permid, m, self.send_callback)
self.inc_peer_nqueries(permid)
def gotUdpConnectReply(self, permid, selversion, channel_id, channel_data, error, mhr_msg, request_callback):
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: gotMakeHoleReplyMessage"
try:
mhr_data = bdecode(mhr_msg)
except:
print_exc()
print >> sys.stderr, "NatCheckMsgHandler: bad encoded data:", mhr_msg
return False
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: message is", mhr_data
try:
request_id, reply = mhr_data.split(":")
except:
print_exc()
print >> sys.stderr, "NatCheckMsgHandler: error in received data:", mhr_data
return False
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: request_id is", request_id
if request_id in self.trav:
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: request_id is in the list"
peer, value = self.trav[request_id]
if peer == None: # first peer reply
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: first peer reply"
self.trav[request_id] = ( (permid, self._secure_overlay.get_dns_from_peerdb(permid)), reply )
elif type(peer) == TupleType: # second peer reply
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: second peer reply"
# Register peerinfo on file
self._file2.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"),
" REPLY",
request_id,
show_permid(peer[0]),
str(peer[1]),
value,
show_permid(permid),
str(self._secure_overlay.get_dns_from_peerdb(permid)),
reply,
"\n")))
del self.trav[request_id]
self._file2.flush()
def gotUdpConnectReply(self, permid, selversion, channel_id, channel_data,
error, mhr_msg, request_callback):
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: gotMakeHoleReplyMessage"
try:
mhr_data = bdecode(mhr_msg)
except:
print_exc()
print >> sys.stderr, "NatCheckMsgHandler: bad encoded data:", mhr_msg
return False
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: message is", mhr_data
try:
request_id, reply = mhr_data.split(":")
except:
print_exc()
print >> sys.stderr, "NatCheckMsgHandler: error in received data:", mhr_data
return False
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: request_id is", request_id
if request_id in self.trav:
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: request_id is in the list"
peer, value = self.trav[request_id]
if peer == None: # first peer reply
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: first peer reply"
self.trav[request_id] = ((
permid, self._secure_overlay.get_dns_from_peerdb(permid)),
reply)
elif type(peer) == TupleType: # second peer reply
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: second peer reply"
# Register peerinfo on file
self._file2.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), " REPLY", request_id,
show_permid(peer[0]), str(peer[1]), value,
show_permid(permid),
str(self._secure_overlay.get_dns_from_peerdb(permid)),
reply, "\n")))
del self.trav[request_id]
self._file2.flush()
def process_query(self, permid, d, selversion):
hits = None
p = None
sendtorrents = False
netwq = d['q']
if netwq.startswith("SIMPLE"): # remote query
# Format: 'SIMPLE '+string of space separated keywords or
# 'SIMPLE+METADATA' +string of space separated keywords
#
# In the future we could support full SQL queries:
# SELECT infohash,torrent_name FROM torrent_db WHERE status = ALIVE
if netwq.startswith('SIMPLE+METADATA'):
q = d['q'][len('SIMPLE+METADATA '):]
sendtorrents = True
else:
q = d['q'][len('SIMPLE '):]
q = self.clean_netwq(q)
q = dunno2unicode(q)
kws = re.split(r'\W+', q.lower())
hits = self.search_torrents(kws, maxhits=MAX_RESULTS,sendtorrents=sendtorrents)
p = self.create_remote_query_reply(d['id'],hits,selversion)
elif netwq.startswith("CHANNEL"): # channel query
q = d['q'][len('CHANNEL '):]
q = self.clean_netwq(q)
q = dunno2unicode(q)
hits = self.channelcast_db.searchChannels(q)
p = self.create_channel_query_reply(d['id'],hits,selversion)
# log incoming query, if logfile is set
if self.logfile:
self.log(permid, q)
m = QUERY_REPLY+p
if self.overlay_log:
nqueries = self.get_peer_nqueries(permid)
# RECV_MSG PERMID OVERSION NUM_QUERIES MSG
self.overlay_log('RECV_QRY', show_permid(permid), selversion, nqueries, repr(d))
# RPLY_QRY PERMID NUM_HITS MSG
self.overlay_log('RPLY_QRY', show_permid(permid), len(hits), repr(p))
self.overlay_bridge.send(permid, m, self.send_callback)
self.inc_peer_nqueries(permid)
def handle_event_crawler_reply(self, permid, selversion, channel_id,
channel_data, error, message,
request_callback):
"""
<<Crawler-side>>
Received a CRAWLER_VIDEOPLAYBACK_EVENT_QUERY reply.
@param permid The Crawler permid
@param selversion The overlay protocol version
@param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
@param channel_data Data associated with the request
@param error The error value. 0 indicates success.
@param message The message payload
@param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
"""
if error:
if DEBUG:
print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message
self._file.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY",
show_permid(permid), str(error), str(channel_data), message,
"\n")))
self._file.flush()
elif selversion >= OLPROTO_VER_TENTH:
# Overlay version 10 sends the reply pickled and zipped
if DEBUG:
print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(
permid), len(message), "bytes zipped"
info = cPickle.loads(zlib.decompress(message))
self._file.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY",
show_permid(permid), str(error), str(channel_data), str(info),
"\n")))
self._file.flush()
elif selversion >= OLPROTO_VER_EIGHTH:
if DEBUG:
print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(
permid), cPickle.loads(message)
info = cPickle.loads(message)
self._file.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY",
show_permid(permid), str(error), str(channel_data), str(info),
"\n")))
self._file.flush()
def formatForText(self, friends):
lines = []
for friend in friends:
permid = show_permid(friend['permid'])
line = ', '.join([friend['name'], permid, friend['ip'], str(friend['port'])])
line += '\n'
lines.append(line)
return lines
def _after_request_callback(self, exc, permid):
"""
Called by the Crawler with the result of the request_callback
call in the query_initiator method.
"""
if not exc:
if DEBUG: print >>sys.stderr, time.asctime(),'-', "databasecrawler: request send to", show_permid_short(permid)
self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "REQUEST", show_permid(permid), "\n")))
self._file.flush()
def _after_request_callback(self, exc, permid):
"""
Called by the Crawler with the result of the request_callback
call in the query_initiator method.
"""
if not exc:
if DEBUG: print >>sys.stderr, "repexcrawler: request sent to", show_permid_short(permid)
self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "REQUEST", show_permid(permid), "\n")))
self._file.flush()
def _after_request_callback(self, exc, permid):
"""
Called by the Crawler with the result of the request_callback
call in the query_initiator method.
"""
if not exc:
if DEBUG: print >>sys.stderr, "puncturecrawler: request sent to", show_permid_short(permid)
self._file.write("REQUEST %s %.2f\n" % (show_permid(permid), time.time()))
self._file.flush()
def setUpPreSession(self):
""" override TestAsServer """
print >> sys.stderr,"test: Pre Tribler Init"
TestAsServer.setUpPreSession(self)
print >> sys.stderr,"test: Pre Tribler Init: config_path",self.config_path
# Enable dialback support
self.config.set_dialback(True)
self.config.set_buddycast(True) # make sure overlay connections are being made
self.config.set_start_recommender(True)
# Write superpeers.txt
self.install_path = tempfile.mkdtemp()
spdir = os.path.join(self.install_path, LIBRARYNAME, 'Core')
os.makedirs(spdir)
statsdir = os.path.join(self.install_path, LIBRARYNAME, 'Core', 'Statistics')
os.makedirs(statsdir)
superpeerfilename = os.path.join(spdir, 'superpeer.txt')
print >> sys.stderr,"test: writing",self.NLISTENERS,"superpeers to",superpeerfilename
f = open(superpeerfilename, "w")
self.mylistenport = []
self.myss = []
self.mykeypairs = []
self.mypermids = []
for i in range(self.NLISTENERS):
# Start our server side, to with Tribler will try to connect
self.mylistenport.append(4810+i)
self.myss.append(socket.socket(socket.AF_INET, socket.SOCK_STREAM))
self.myss[i].setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.myss[i].bind(('', self.mylistenport[i]))
self.myss[i].listen(1)
self.mykeypairs.append(EC.gen_params(EC.NID_sect233k1))
self.mykeypairs[i].gen_key()
self.mypermids.append(str(self.mykeypairs[i].pub().get_der()))
content = '127.0.0.1, '+str(self.mylistenport[i])+', '+show_permid(self.mypermids[i])+', FakeSuperPeer\n'
f.write(content)
f.close()
self.config.set_install_dir(self.install_path)
srcfiles = []
srcfiles.append(os.path.join(LIBRARYNAME,"schema_sdb_v"+str(CURRENT_MAIN_DB_VERSION)+".sql"))
srcfiles.append(os.path.join(LIBRARYNAME,"Core","Statistics","tribler_seedingstats_sdb.sql"))
srcfiles.append(os.path.join(LIBRARYNAME,"Core","Statistics","tribler_friendship_stats_sdb.sql"))
srcfiles.append(os.path.join(LIBRARYNAME,"Core","Statistics","tribler_videoplayback_stats.sql"))
for srcfile in srcfiles:
sfn = os.path.join('..','..',srcfile)
dfn = os.path.join(self.install_path,srcfile)
print >>sys.stderr,"test: copying",sfn,dfn
shutil.copyfile(sfn,dfn)
"""
def formatForText(self, friends):
lines = []
for friend in friends:
permid = show_permid(friend['permid'])
line = ', '.join(
[friend['name'], permid, friend['ip'],
str(friend['port'])])
line += '\n'
lines.append(line)
return lines
def _after_event_request_callback(self, exc, permid):
"""
<<Crawler-side>>
Called by the Crawler with the result of the request_callback
call in the handle_crawler_reply method.
"""
if not exc:
if DEBUG:
print >> sys.stderr, "videoplaybackcrawler: request send to", show_permid_short(
permid)
self._file.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), " INFO REQUEST",
show_permid(permid), "\n")))
self._file.flush()
def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback):
"""
Received a CRAWLER_CHANNEL_QUERY reply.
@param permid The Crawler permid
@param selversion The overlay protocol version
@param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
@param error The error value. 0 indicates success.
@param message The message payload
@param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
"""
if error:
if DEBUG:
print >> sys.stderr, "channelcrawler: handle_crawler_reply", error, message
self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(error), message, "\n")))
self._file.flush()
else:
if DEBUG:
print >> sys.stderr, "channelcrawler: handle_crawler_reply", show_permid_short(permid), cPickle.loads(message)
self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid), str(error), str(cPickle.loads(message)), "\n")))
self._file.flush()
def doNatCheckCallback(self, exc, permid):
if exc is not None:
return False
if DEBUG:
print >> sys.stderr, "NATCHECK_REQUEST was sent to", show_permid_short(
permid), exc
# Register peerinfo on file
self._file.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), "REQUEST", show_permid(permid),
str(self._secure_overlay.get_dns_from_peerdb(permid)), "\n")))
self._file.flush()
return True
def query_initiator(self, permid, selversion, request_callback):
"""
Established a new connection. Send a CRAWLER_DATABASE_QUERY request.
@param permid The Tribler peer permid
@param selversion The oberlay protocol version
@param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
"""
if DEBUG:
print >> sys.stderr, "databasecrawler: query_initiator", show_permid_short(
permid)
sql = []
if selversion >= OLPROTO_VER_SEVENTH:
sql.extend(("SELECT 'peer_count', count(*) FROM Peer",
"SELECT 'torrent_count', count(*) FROM Torrent"))
if selversion >= OLPROTO_VER_ELEVENTH:
sql.extend((
"SELECT 'my_subscriptions', count(*) FROM VoteCast where voter_id='"
+ show_permid(permid) + "' and vote=2",
"SELECT 'my_negative_votes', count(*) FROM VoteCast where voter_id='"
+ show_permid(permid) + "' and vote=-1",
"SELECT 'my_channel_files', count(*) FROM ChannelCast where publisher_id='"
+ show_permid(permid) + "'",
"SELECT 'all_subscriptions', count(*) FROM VoteCast where vote=2",
"SELECT 'all_negative_votes', count(*) FROM VoteCast where vote=-1"
))
# if OLPROTO_VER_EIGHTH <= selversion <= 11:
# sql.extend(("SELECT 'moderations_count', count(*) FROM ModerationCast"))
# if selversion >= OLPROTO_VER_EIGHTH:
# sql.extend(("SELECT 'positive_votes_count', count(*) FROM Moderators where status=1",
# "SELECT 'negative_votes_count', count(*) FROM Moderators where status=-1"))
request_callback(CRAWLER_DATABASE_QUERY,
";".join(sql),
callback=self._after_request_callback)
def handle_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback):
"""
Received a CRAWLER_UDPUNCTURE_QUERY reply.
@param permid The Crawler permid
@param selversion The overlay protocol version
@param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
@param error The error value. 0 indicates success.
@param message The message payload
@param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
"""
try:
if error:
if DEBUG:
print >> sys.stderr, "puncturecrawler: handle_crawler_reply", error, message
self._file.write("ERROR %s %.2f %d %s\n" % (show_permid(permid), time.time(), error, message))
self._file.flush()
else:
if DEBUG:
print >> sys.stderr, "puncturecrawler: handle_crawler_reply", show_permid_short(permid)
# 25/05/10 Boudewijn: We found that, for unknown
# reasons, the decompressed(message) contains many
# gigabytes worth of \0 characters. For now we filter
# them out until Gert Jan can determine the actual
# cause.
data = zlib.decompress(message)
filtered = filter(lambda char: char != "\0", data)
self._file.write("REPLY %s %.2f\n" % (show_permid(permid), time.time()))
self._file.write("# reply sizes: on-the-wire=%d, decompressed=%d, filtered=%d\n" % (len(message), len(data), len(filtered)))
self._file.write(filtered)
self._file.flush()
except:
if DEBUG:
print >>sys.stderr, "puncturecrawler: error writing to file"
def doNatCheckCallback(self, exc, permid):
if exc is not None:
return False
if DEBUG:
print >> sys.stderr, "NATCHECK_REQUEST was sent to", show_permid_short(permid), exc
# Register peerinfo on file
self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"),
"REQUEST",
show_permid(permid),
str(self._secure_overlay.get_dns_from_peerdb(permid)),
"\n")))
self._file.flush()
return True
def udpConnectCallback(self, exc, permid):
if exc is not None:
if DEBUG:
print >> sys.stderr, "NATTRAVERSAL_REQUEST failed to", show_permid_short(permid), exc
# Register peerinfo on file
self._file2.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"),
"REQUEST FAILED",
show_permid(permid),
str(self._secure_overlay.get_dns_from_peerdb(permid)),
"\n")))
return False
if DEBUG:
print >> sys.stderr, "NATTRAVERSAL_REQUEST was sent to", show_permid_short(permid), exc
return True
def udpConnectCallback(self, exc, permid):
if exc is not None:
if DEBUG:
print >> sys.stderr, "NATTRAVERSAL_REQUEST failed to", show_permid_short(
permid), exc
# Register peerinfo on file
self._file2.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), "REQUEST FAILED",
show_permid(permid),
str(self._secure_overlay.get_dns_from_peerdb(permid)), "\n")))
return False
if DEBUG:
print >> sys.stderr, "NATTRAVERSAL_REQUEST was sent to", show_permid_short(
permid), exc
return True
def setUpPreSession(self):
""" override TestAsServer """
print >> sys.stderr, "test: Pre Tribler Init"
TestAsServer.setUpPreSession(self)
print >> sys.stderr, "test: Pre Tribler Init: config_path", self.config_path
# Enable dialback support
self.config.set_dialback(True)
self.config.set_buddycast(
True) # make sure overlay connections are being made
self.config.set_start_recommender(True)
# Write superpeers.txt
self.install_path = tempfile.mkdtemp()
spdir = os.path.join(self.install_path, LIBRARYNAME, 'Core')
os.makedirs(spdir)
statsdir = os.path.join(self.install_path, LIBRARYNAME, 'Core',
'Statistics')
os.makedirs(statsdir)
superpeerfilename = os.path.join(spdir, 'superpeer.txt')
print >> sys.stderr, "test: writing", self.NLISTENERS, "superpeers to", superpeerfilename
f = open(superpeerfilename, "w")
self.mylistenport = []
self.myss = []
self.mykeypairs = []
self.mypermids = []
for i in range(self.NLISTENERS):
# Start our server side, to with Tribler will try to connect
self.mylistenport.append(4810 + i)
self.myss.append(socket.socket(socket.AF_INET, socket.SOCK_STREAM))
self.myss[i].setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.myss[i].bind(('', self.mylistenport[i]))
self.myss[i].listen(1)
self.mykeypairs.append(EC.gen_params(EC.NID_sect233k1))
self.mykeypairs[i].gen_key()
self.mypermids.append(str(self.mykeypairs[i].pub().get_der()))
content = '127.0.0.1, ' + str(
self.mylistenport[i]) + ', ' + show_permid(
self.mypermids[i]) + ', FakeSuperPeer\n'
f.write(content)
f.close()
self.config.set_install_dir(self.install_path)
srcfiles = []
srcfiles.append(os.path.join(LIBRARYNAME, "schema_sdb_v4.sql"))
srcfiles.append(
os.path.join(LIBRARYNAME, "Core", "Statistics",
"tribler_seedingstats_sdb.sql"))
srcfiles.append(
os.path.join(LIBRARYNAME, "Core", "Statistics",
"tribler_friendship_stats_sdb.sql"))
srcfiles.append(
os.path.join(LIBRARYNAME, "Core", "Statistics",
"tribler_videoplayback_stats.sql"))
for srcfile in srcfiles:
sfn = os.path.join('..', '..', srcfile)
dfn = os.path.join(self.install_path, srcfile)
print >> sys.stderr, "test: copying", sfn, dfn
shutil.copyfile(sfn, dfn)
"""
def _after_info_request_callback(self, exc, permid):
"""
<<Crawler-side>>
Called by the Crawler with the result of the request_callback
call in the query_initiator method.
"""
if not exc:
if DEBUG: print >>sys.stderr, "videoplaybackcrawler: request send to", show_permid_short(permid)
self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), "INFO REQUEST", show_permid(permid), "\n")))
self._file.flush()
def gotNatCheckReplyMessage(self, permid, selversion, channel_id,
channel_data, error, payload,
request_callback):
"""
The handle-reply callback
"""
if error:
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: gotNatCheckReplyMessage"
print >> sys.stderr, "NatCheckMsgHandler: error", error
# generic error: another crawler already obtained these results
self._file.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid),
str(self._secure_overlay.get_dns_from_peerdb(permid)),
"ERROR(%d)" % error, payload, "\n")))
self._file.flush()
else:
try:
recv_data = bdecode(payload)
except:
print_exc()
print >> sys.stderr, "bad encoded data:", payload
return False
try: # check natCheckReply message
self.validNatCheckReplyMsg(recv_data)
except RuntimeError, e:
print >> sys.stderr, e
return False
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: received NAT_CHECK_REPLY message: ", recv_data
# Register peerinfo on file
self._file.write("; ".join(
(strftime("%Y/%m/%d %H:%M:%S"), " REPLY", show_permid(permid),
str(self._secure_overlay.get_dns_from_peerdb(permid)),
":".join([str(x) for x in recv_data]), "\n")))
self._file.flush()
# for Tribler versions < 5.0 : do nothing
if selversion < OLPROTO_VER_NINETH:
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: Tribler version too old for NATTRAVERSAL: do nothing"
return True
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: do NATTRAVERSAL"
# Save peer in peerlist
if len(self.peerlist) == PEERLIST_LEN:
del self.peerlist[0]
self.peerlist.append([permid, recv_data[1], recv_data[2]])
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: peerlist length is: ", len(
self.peerlist)
# Try to perform hole punching
if len(self.peerlist) >= 2:
self.tryHolePunching()
def handle_info_crawler_reply(self, permid, selversion, channel_id, error, message, request_callback):
"""
<<Crawler-side>>
Received a CRAWLER_VIDEOPLAYBACK_INFO_QUERY reply.
@param permid The Crawler permid
@param selversion The overlay protocol version
@param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
@param error The error value. 0 indicates success.
@param message The message payload
@param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
"""
if error:
if DEBUG:
print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message
self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " INFO REPLY", show_permid(permid), str(error), message, "\n")))
self._file.flush()
else:
if DEBUG:
print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(permid), cPickle.loads(message)
info = cPickle.loads(message)
self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " INFO REPLY", show_permid(permid), str(error), str(info), "\n")))
self._file.flush()
i = 0
for key, timestamp, piece_size, num_pieces, bitrate, nat in info:
i += 1
# do not remove the first item. the list is ordered
# DESC so the first item is the last that is added to
# the database and we can't affored to remove it, as
# it may cause exceptions in the running playback.
if i == 1:
sql = """
SELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;
DELETE FROM playback_event WHERE key = '%s';
""" % (key, key)
else:
sql = """
SELECT timestamp, origin, event FROM playback_event WHERE key = '%s' ORDER BY timestamp ASC LIMIT 50;
DELETE FROM playback_event WHERE key = '%s';
DELETE FROM playback_info WHERE key = '%s';
""" % (key, key, key)
# todo: optimize to not select key for each row
request_callback(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, sql, channel_data=key, callback=self._after_event_request_callback, frequency=0)
def gotNatCheckReplyMessage(self, permid, selversion, channel_id, channel_data, error, payload, request_callback):
"""
The handle-reply callback
"""
if error:
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: gotNatCheckReplyMessage"
print >> sys.stderr, "NatCheckMsgHandler: error", error
# generic error: another crawler already obtained these results
self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"),
" REPLY",
show_permid(permid),
str(self._secure_overlay.get_dns_from_peerdb(permid)),
"ERROR(%d)" % error,
payload,
"\n")))
self._file.flush()
else:
try:
recv_data = bdecode(payload)
except:
print_exc()
print >> sys.stderr, "bad encoded data:", payload
return False
try: # check natCheckReply message
self.validNatCheckReplyMsg(recv_data)
except RuntimeError, e:
print >> sys.stderr, e
return False
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: received NAT_CHECK_REPLY message: ", recv_data
# Register peerinfo on file
self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"),
" REPLY",
show_permid(permid),
str(self._secure_overlay.get_dns_from_peerdb(permid)),
":".join([str(x) for x in recv_data]),
"\n")))
self._file.flush()
# for Tribler versions < 5.0 : do nothing
if selversion < OLPROTO_VER_NINETH:
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: Tribler version too old for NATTRAVERSAL: do nothing"
return True
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: do NATTRAVERSAL"
# Save peer in peerlist
if len(self.peerlist) == PEERLIST_LEN:
del self.peerlist[0]
self.peerlist.append([permid,recv_data[1],recv_data[2]])
if DEBUG:
print >> sys.stderr, "NatCheckMsgHandler: peerlist length is: ", len(self.peerlist)
# Try to perform hole punching
if len(self.peerlist) >= 2:
self.tryHolePunching()
def query_initiator(self, permid, selversion, request_callback):
"""
Established a new connection. Send a CRAWLER_DATABASE_QUERY request.
@param permid The Tribler peer permid
@param selversion The oberlay protocol version
@param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
"""
if DEBUG: print >>sys.stderr, time.asctime(),'-', "databasecrawler: query_initiator", show_permid_short(permid)
sql = []
if selversion >= OLPROTO_VER_SEVENTH:
sql.extend(("SELECT 'peer_count', count(*) FROM Peer",
"SELECT 'torrent_count', count(*) FROM Torrent"))
if selversion >= OLPROTO_VER_ELEVENTH:
sql.extend(("SELECT 'my_subscriptions', count(*) FROM VoteCast where voter_id='" + show_permid(permid) + "' and vote=2",
"SELECT 'my_negative_votes', count(*) FROM VoteCast where voter_id='" + show_permid(permid) + "' and vote=-1",
"SELECT 'my_channel_files', count(*) FROM ChannelCast where publisher_id='" + show_permid(permid) + "'",
"SELECT 'all_subscriptions', count(*) FROM VoteCast where vote=2",
"SELECT 'all_negative_votes', count(*) FROM VoteCast where vote=-1"))
# if OLPROTO_VER_EIGHTH <= selversion <= 11:
# sql.extend(("SELECT 'moderations_count', count(*) FROM ModerationCast"))
# if selversion >= OLPROTO_VER_EIGHTH:
# sql.extend(("SELECT 'positive_votes_count', count(*) FROM Moderators where status=1",
# "SELECT 'negative_votes_count', count(*) FROM Moderators where status=-1"))
request_callback(CRAWLER_DATABASE_QUERY, ";".join(sql), callback=self._after_request_callback)
def handle_event_crawler_reply(self, permid, selversion, channel_id, channel_data, error, message, request_callback):
"""
<<Crawler-side>>
Received a CRAWLER_VIDEOPLAYBACK_EVENT_QUERY reply.
@param permid The Crawler permid
@param selversion The overlay protocol version
@param channel_id Identifies a CRAWLER_REQUEST/CRAWLER_REPLY pair
@param channel_data Data associated with the request
@param error The error value. 0 indicates success.
@param message The message payload
@param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
"""
if error:
if DEBUG:
print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", error, message
self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), message, "\n")))
self._file.flush()
elif selversion >= OLPROTO_VER_TENTH:
# Overlay version 10 sends the reply pickled and zipped
if DEBUG:
print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(permid), len(message), "bytes zipped"
info = cPickle.loads(zlib.decompress(message))
self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), str(info), "\n")))
self._file.flush()
elif selversion >= OLPROTO_VER_EIGHTH:
if DEBUG:
print >> sys.stderr, "videoplaybackcrawler: handle_crawler_reply", show_permid_short(permid), cPickle.loads(message)
info = cPickle.loads(message)
self._file.write("; ".join((strftime("%Y/%m/%d %H:%M:%S"), " EVENT REPLY", show_permid(permid), str(error), str(channel_data), str(info), "\n")))
self._file.flush()
def query_initiator(self, permid, selversion, request_callback):
"""
Established a new connection. Send a CRAWLER_CHANNEL_QUERY request.
@param permid The Tribler peer permid
@param selversion The oberlay protocol version
@param request_callback Call this function one or more times to send the requests: request_callback(message_id, payload)
"""
if DEBUG: print >>sys.stderr, "channelcrawler: query_initiator", show_permid_short(permid)
sql = []
if selversion >= OLPROTO_VER_THIRTEENTH:
sql.extend(("SELECT 'channel_files', publisher_id, count(*) FROM ChannelCast group by publisher_id",
"SELECT 'my_votes', mod_id, voter_id, vote, time_stamp FROM VoteCast where voter_id='" + show_permid(permid) + "' order by time_stamp desc limit 100",
"SELECT 'other_votes', mod_id, voter_id, vote, time_stamp FROM VoteCast where voter_id<>'" + show_permid(permid) + "' order by time_stamp desc limit 100"))
request_callback(CRAWLER_CHANNEL_QUERY, ";".join(sql), callback=self._after_request_callback)
