def subtest_good_friendship_stats(self):
"""
Send a valid message-id from a registered crawler peer
"""
print >>sys.stderr, "-"*80, "\ntest: good friendship stats"
s = OLConnection(self.my_keypair, "localhost", self.hisport)
t = time.time() - 100.0
msg_dict = {'current time':int(t)}
payload = bencode(msg_dict)
self.send_crawler_request(s, CRAWLER_FRIENDSHIP_STATS, 0, 0, payload)
error, payload = self.receive_crawler_reply(s, CRAWLER_FRIENDSHIP_STATS, 0)
assert error == 0
d = bdecode(payload)
if DEBUG:
print >>sys.stderr, "test: Got FRIENDSHIPSTATISTICS",`d`
stats = d['stats']
self.assert_(len(stats) == 1)
record = d['stats'][0]
self.assert_(record[0] == bin2str(self.his_permid)) # source_permid
self.assert_(record[1] == bin2str(self.some_permid)) # target_permid
self.assert_(record[2] == 0) # isForwarder
time.sleep(1)
s.close()
def testInsertNewMetadataNoSubs(self):
metadataDTO = MockMetadataDTO([])
metadataDTO.sign(metadataDTO._keypair)
self.underTest.insertMetadata(metadataDTO)
testquery = "SELECT * FROM Metadata WHERE publisher_id=?" \
+ " AND infohash=?;"
channel = bin2str(metadataDTO.channel)
infohash = bin2str(metadataDTO.infohash)
results = self.db.fetchall(testquery, (channel,infohash))
self.assertTrue(len(results) == 1)
tupl = results[0]
self.assertTrue(tupl[0] is not None and isinstance(tupl[0], int))
self.assertEquals(channel,tupl[1])
self.assertEquals(infohash,tupl[2])
self.assertEquals(metadataDTO.description, tupl[3])
self.assertEquals(metadataDTO.timestamp, tupl[4])
self.assertEquals(bin2str(metadataDTO.signature), tupl[5])
subtitlesQuery = "SELECT * FROM Subtitles WHERE metadata_id_fk=?;"
subtitles = self.db.fetchall(subtitlesQuery, (tupl[0],))
self.assertEquals(0,len(subtitles))
def fmsg_send_callback(self,exc,permid,msgid):
# If an exception arises
if exc is None:
self.delete_msg(permid,msgid)
else:
if DEBUG:
print >> sys.stderr, 'friendship: Could not send to ',show_permid_short(permid)
print_exc()
mypermid = self.session.get_permid()
no_of_attempts = 0
no_of_helpers = 10
isForwarder = False
if permid in self.currmsgs:
msgid2rec = self.currmsgs[permid]
for msgid in msgid2rec:
msgrec = msgid2rec[msgid]
no_of_attempts = msgrec['attempt']
if msgrec['forwarded'] == True:
isForwarder = 1
self.friendshipStatistics_db.insertOrUpdateFriendshipStatistics( bin2str(mypermid),
bin2str(permid),
int(time()),
isForwarder,
no_of_attempts ,
no_of_helpers)
def olthread_download_torrent_callback(self,permid,infohash,usercallback, prio = 1):
""" The user has selected a torrent referred to by a peer in a query
reply or channelcast has discovered a new torrent. Try to obtain the actual .torrent file from the peer
"""
assert isinstance(infohash, str), "INFOHASH has invalid type: %s" % type(infohash)
assert len(infohash) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(infohash)
if usercallback:
self.callbacks.setdefault(infohash, set()).add(usercallback)
requester = None
#look for lowest prio requester, which already has this infohash scheduled
for i in range(prio):
if i in self.requesters and self.requesters[i].is_being_requested(infohash):
requester = self.requesters[i]
break
#if not found, then used/create this requester
if not requester:
if prio not in self.requesters:
self.requesters[prio] = TorrentRequester(self, self.metadatahandler, self.overlay_bridge, self.session, prio)
requester = self.requesters[prio]
#make request
requester.add_source(infohash, permid)
if DEBUG:
print >>sys.stderr,'rtorrent: adding request:', bin2str(infohash), bin2str(permid), prio
def test_getPermIDByIP(self):
db = PeerDBHandler.getInstance()
fake_permid_x = 'fake_permid_x' + '0R0\x10\x00\x07*\x86H\xce=\x02\x01\x06\x05+\x81\x04\x00\x1a\x03>\x00\x04'
peer_x = {
'permid': fake_permid_x,
'ip': '1.2.3.4',
'port': 234,
'name': 'fake peer x',
'last_seen': 12345,
'connected_times': 3
}
oldsize = db.size()
p = db.getPeer(fake_permid_x)
assert p == None, p
db.addPeer(fake_permid_x, peer_x)
assert db.hasPeer(fake_permid_x)
assert db.size() == oldsize + 1, (db.size(), oldsize + 1)
permid = db.getPermIDByIP('1.2.3.4')
assert bin2str(permid) == bin2str(fake_permid_x)
db.deletePeer(fake_permid_x, force=True)
assert not db.hasPeer(fake_permid_x)
assert db.size() == oldsize
def testInsertNewMetadataNoSubs(self):
metadataDTO = MockMetadataDTO([])
metadataDTO.sign(metadataDTO._keypair)
self.underTest.insertMetadata(metadataDTO)
testquery = "SELECT * FROM Metadata WHERE publisher_id=?" \
+ " AND infohash=?;"
channel = bin2str(metadataDTO.channel)
infohash = bin2str(metadataDTO.infohash)
results = self.db.fetchall(testquery, (channel,infohash))
self.assertTrue(len(results) == 1)
tupl = results[0]
self.assertTrue(tupl[0] is not None and isinstance(tupl[0], int))
self.assertEquals(channel,tupl[1])
self.assertEquals(infohash,tupl[2])
self.assertEquals(metadataDTO.description, tupl[3])
self.assertEquals(metadataDTO.timestamp, tupl[4])
self.assertEquals(bin2str(metadataDTO.signature), tupl[5])
subtitlesQuery = "SELECT * FROM Subtitles WHERE metadata_id_fk=?;"
subtitles = self.db.fetchall(subtitlesQuery, (tupl[0],))
self.assertEquals(0,len(subtitles))
def fmsg_send_callback(self,exc,permid,msgid):
# If an exception arises
if exc is None:
self.delete_msg(permid,msgid)
else:
if DEBUG:
print >> sys.stderr, time.asctime(),'-', 'friendship: Could not send to ',show_permid_short(permid)
print_exc()
mypermid = self.session.get_permid()
no_of_attempts = 0
no_of_helpers = 10
isForwarder = False
if permid in self.currmsgs:
msgid2rec = self.currmsgs[permid]
for msgid in msgid2rec:
msgrec = msgid2rec[msgid]
no_of_attempts = msgrec['attempt']
if msgrec['forwarded'] == True:
isForwarder = 1
self.friendshipStatistics_db.insertOrUpdateFriendshipStatistics( bin2str(mypermid),
bin2str(permid),
int(time()),
isForwarder,
no_of_attempts ,
no_of_helpers)
def DrawHoverAndInfo(self, gc, dc, circuit_points):
gc.SetBrush(wx.TRANSPARENT_BRUSH)
if self.hop_hover_evt:
self.hop_hover = self.PositionToCircuit(self.hop_hover_evt,
circuit_points)
self.hop_hover_evt = None
if self.hop_hover and self.hop_hover[0] in circuit_points:
circuit, hop_index = self.hop_hover
x, y = circuit_points[circuit][hop_index]
pen = wx.Pen(wx.Colour(229, 229, 229), 1, wx.USER_DASH)
pen.SetDashes([8, 4])
gc.SetPen(pen)
gc.DrawEllipse(x - self.radius, y - self.radius, self.radius * 2,
self.radius * 2)
if self.hop_active_evt:
self.hop_active = self.PositionToCircuit(self.hop_active_evt,
circuit_points)
self.hop_active_evt = None
if self.hop_active and self.hop_active[0] in circuit_points and \
(not self.hop_active[0] or self.hop_active[1] <= len(self.hop_active[0].hops)):
circuit, hop_index = self.hop_active
hop = circuit.hops[hop_index -
1] if hop_index and circuit else None
x, y = circuit_points[circuit][hop_index]
# Draw cicle around node
pen = wx.Pen(self.hop_to_colour.get(self.hop_active, wx.BLACK), 1,
wx.USER_DASH)
pen.SetDashes([8, 4])
gc.SetPen(pen)
gc.DrawEllipse(x - self.radius, y - self.radius, self.radius * 2,
self.radius * 2)
# Determine text
dc.SetFont(self.font_small)
if not hop:
text = 'You\nPERMID ' + bin2str(
self.tunnel_community.my_member.public_key)[:10]
else:
text = 'PERMID ' + bin2str(
self.dispersy.crypto.key_to_hash(hop.public_key))[:10]
if 'UNKNOWN HOST' not in hop.host:
text = 'IP %s:%s\n' % (hop.host, hop.port) + text
# Draw info box + text
box_width, box_height = self.GetTextExtent(dc, text)
box_width += 10
box_height += 10
x = x - box_width - 1.1 * self.radius if x > self.graph_panel.GetSize(
)[0] / 2 else x + 1.1 * self.radius
y = y - box_height - 1.1 * self.radius if y > self.graph_panel.GetSize(
)[1] / 2 else y + 1.1 * self.radius
gc.SetBrush(wx.Brush(wx.Colour(216, 237, 255, 50)))
gc.SetPen(wx.Pen(LIST_BLUE))
gc.DrawRectangle(x, y, box_width, box_height)
self.DrawText(dc, text, x + 5, y + 5)
def test_updateTimes(self):
fake_permid_x = 'fake_permid_x' + '0R0\x10\x00\x07*\x86H\xce=\x02\x01\x06\x05+\x81\x04\x00\x1a\x03>\x00\x04'
peer_x = {'permid': fake_permid_x, 'ip': '1.2.3.4', 'port': 234,
'name': 'fake peer x', 'last_seen': 12345, 'connected_times': 3}
oldsize = self.pdb.size()
p = self.pdb.getPeer(fake_permid_x)
assert p == None, p
self.pdb.addPeer(fake_permid_x, peer_x)
assert self.pdb.hasPeer(fake_permid_x)
assert self.pdb.size() == oldsize + 1, (self.pdb.size(), oldsize + 1)
self.pdb.updateTimes(fake_permid_x, 'connected_times')
sql = 'select connected_times from Peer where permid=' + repr(bin2str(fake_permid_x))
ct = self.pdb._db.fetchone(sql)
assert ct == 4, ct
self.pdb.updateTimes(fake_permid_x, 'buddycast_times')
sql = 'select buddycast_times from Peer where permid=' + repr(bin2str(fake_permid_x))
ct = self.pdb._db.fetchone(sql)
assert ct == 1, ct
self.pdb.updateTimes(fake_permid_x, 'buddycast_times', 3)
sql = 'select buddycast_times from Peer where permid=' + repr(bin2str(fake_permid_x))
ct = self.pdb._db.fetchone(sql)
assert ct == 4, ct
self.pdb.deletePeer(fake_permid_x, force=True)
assert not self.pdb.hasPeer(fake_permid_x)
def doRequest(self):
try:
#request new infohash from queue
while True:
infohash = self.queue.get_nowait()
if infohash in self.sources: #check if still needed
break
else:
self.queue.task_done()
try:
#~load balance sources
permid = choice(list(self.sources[infohash]))
self.sources[infohash].remove(permid)
if len(self.sources[infohash]) < 1:
del self.sources[infohash]
self.nr_times_requested[
infohash] = self.nr_times_requested.get(infohash, 0) + 1
if permid:
if DEBUG:
print >> sys.stderr, "rtorrent: requesting", bin2str(
infohash), bin2str(permid)
#metadatahandler will only do actual request if torrentfile is not on disk
#self.metadatahandler.send_metadata_request(permid, infohash, caller="rquery")
else:
if DEBUG:
print >> sys.stderr, "rtorrent: requesting", bin2str(
infohash), "from dht only"
#schedule a magnet lookup after X seconds
if self.prio <= 1 or (infohash not in self.sources
and infohash in self.nr_times_requested
and self.nr_times_requested[infohash] >
self.MAGNET_THRESHOLD):
self.overlay_bridge.add_task(
lambda infohash=infohash: self.magnet_requester.
add_request(self.prio, infohash),
self.MAGNET_TIMEOUT * (self.prio + 1),
infohash)
#Make sure exceptions wont crash this requesting loop
except:
if DEBUG:
print_exc()
self.queue.task_done()
self.overlay_bridge.add_task(self.doRequest,
self.REQUEST_INTERVAL * self.prio,
self)
except Queue.Empty:
pass
def _updateChannelcastDB(self, query_permid, query, hits, listOfAdditions):
if DEBUG:
print >> sys.stderr, "channelcast: updating channelcastdb", query, len(hits)
publisher_ids = Set()
infohashes = Set()
for hit in listOfAdditions:
publisher_ids.add(hit[0])
infohashes.add(str2bin(hit[2]))
if query and query.startswith('CHANNEL p') and len(publisher_ids) == 1:
publisher_id = publisher_ids.pop()
publisher_ids.add(publisher_id)
nr_torrents = self.channelcastdb.getNrTorrentsInChannel(publisher_id)
if len(infohashes) > nr_torrents:
if len(infohashes) > 50 and len(infohashes) > nr_torrents +1: #peer not behaving according to spec, ignoring
if DEBUG:
print >> sys.stderr, "channelcast: peer not behaving according to spec, ignoring",len(infohashes), show_permid(query_permid)
return
#if my channel, never remove all currently received
if bin2str(self.session.get_permid()) != publisher_id:
self.channelcastdb.deleteTorrentsFromPublisherId(str2bin(publisher_id))
if DEBUG:
print >> sys.stderr, 'Received channelcast message with %d hashes'%len(infohashes), show_permid(query_permid)
else:
#ignore all my favorites, randomness will cause problems with timeframe
my_favorites = self.votecastdb.getPublishersWithPosVote(bin2str(self.session.get_permid()))
#filter listOfAdditions
listOfAdditions = [hit for hit in listOfAdditions if hit[0] not in my_favorites]
#request channeltimeframes for subscribed channels
for publisher_id in my_favorites:
if publisher_id in publisher_ids:
self.updateAChannel(publisher_id, [query_permid])
publisher_ids.remove(publisher_id) #filter publisher_ids
#08/04/10: Andrea: processing rich metadata part.
self.richMetadataInterceptor.handleRMetadata(query_permid, hits, fromQuery = query is not None)
self.channelcastdb.addTorrents(listOfAdditions)
missing_infohashes = {}
for publisher_id in publisher_ids:
for infohash in self.channelcastdb.selectTorrentsToCollect(publisher_id):
missing_infohashes[str2bin(infohash[0])] = publisher_id
def notify(publisher_id):
self.notifier.notify(NTFY_CHANNELCAST, NTFY_UPDATE, publisher_id)
for infohash, publisher_id in missing_infohashes.iteritems():
if infohash in infohashes:
self.rtorrent_handler.download_torrent(query_permid, infohash, lambda infohash, metadata, filename: notify(publisher_id) ,2)
else:
self.rtorrent_handler.download_torrent(query_permid, infohash, lambda infohash, metadata, filename: notify(publisher_id) ,3)
def doRequest(self):
try:
# request new infohash from queue
while True:
infohash = self.queue.get_nowait()
if infohash in self.sources: # check if still needed
break
else:
self.queue.task_done()
try:
# ~load balance sources
permid = choice(list(self.sources[infohash]))
self.sources[infohash].remove(permid)
if len(self.sources[infohash]) < 1:
del self.sources[infohash]
self.nr_times_requested[infohash] = self.nr_times_requested.get(infohash, 0) + 1
if permid:
if DEBUG:
print >>sys.stderr, "rtorrent: requesting", bin2str(infohash), bin2str(permid)
# metadatahandler will only do actual request if torrentfile is not on disk
# self.metadatahandler.send_metadata_request(permid, infohash, caller="rquery")
else:
if DEBUG:
print >>sys.stderr, "rtorrent: requesting", bin2str(infohash), "from dht only"
# schedule a magnet lookup after X seconds
if self.prio <= 1 or (
infohash not in self.sources
and infohash in self.nr_times_requested
and self.nr_times_requested[infohash] > self.MAGNET_THRESHOLD
):
self.overlay_bridge.add_task(
lambda infohash=infohash: self.magnet_requester.add_request(self.prio, infohash),
self.MAGNET_TIMEOUT * (self.prio + 1),
infohash,
)
# Make sure exceptions wont crash this requesting loop
except:
if DEBUG:
print_exc()
self.queue.task_done()
self.overlay_bridge.add_task(self.doRequest, self.REQUEST_INTERVAL * self.prio, self)
except Queue.Empty:
pass
def test_getRecentLivePrefList(self):
pl = self.mdb.getRecentLivePrefList()
assert len(pl) == 11, (len(pl), pl)
infohash_str_126 = 'ByJho7yj9mWY1ORWgCZykLbU1Xc='
assert bin2str(pl[0]) == infohash_str_126
infohash_str_1279 = 'R+grUhp884MnFkt6NuLnnauZFsc='
assert bin2str(pl[1]) == infohash_str_1279
pl = self.mdb.getRecentLivePrefList(8)
assert len(pl) == 8, (len(pl), pl)
assert bin2str(pl[0]) == infohash_str_126
assert bin2str(pl[1]) == infohash_str_1279
def test_getRecentLivePrefList(self):
db = MyPreferenceDBHandler.getInstance()
pl = db.getRecentLivePrefList()
assert len(pl) == 11, (len(pl), pl)
infohash_str_126 = 'ByJho7yj9mWY1ORWgCZykLbU1Xc='
assert bin2str(pl[0]) == infohash_str_126
infohash_str_1279 = 'R+grUhp884MnFkt6NuLnnauZFsc='
assert bin2str(pl[1]) == infohash_str_1279
pl = db.getRecentLivePrefList(8)
assert len(pl) == 8
assert bin2str(pl[0]) == infohash_str_126
assert bin2str(pl[1]) == infohash_str_1279
def test_getRecentLivePrefList(self):
db = MyPreferenceDBHandler.getInstance()
pl = db.getRecentLivePrefList()
assert len(pl) == 11, (len(pl), pl)
infohash_str_126 = 'ByJho7yj9mWY1ORWgCZykLbU1Xc='
assert bin2str(pl[0]) == infohash_str_126
infohash_str_1279 = 'R+grUhp884MnFkt6NuLnnauZFsc='
assert bin2str(pl[1]) == infohash_str_1279
pl = db.getRecentLivePrefList(8)
assert len(pl) == 8
assert bin2str(pl[0]) == infohash_str_126
assert bin2str(pl[1]) == infohash_str_1279
def DrawHoverAndInfo(self, gc, dc, circuit_points):
gc.SetBrush(wx.TRANSPARENT_BRUSH)
if self.hop_hover_evt:
self.hop_hover = self.PositionToCircuit(self.hop_hover_evt, circuit_points)
self.hop_hover_evt = None
if self.hop_hover and self.hop_hover[0] in circuit_points:
circuit, hop_index = self.hop_hover
x, y = circuit_points[circuit][hop_index]
pen = wx.Pen(wx.Colour(229, 229, 229), 1, wx.USER_DASH)
pen.SetDashes([8, 4])
gc.SetPen(pen)
gc.DrawEllipse(x - self.radius, y - self.radius, self.radius * 2, self.radius * 2)
if self.hop_active_evt:
self.hop_active = self.PositionToCircuit(self.hop_active_evt, circuit_points)
self.hop_active_evt = None
if self.hop_active and self.hop_active[0] in circuit_points and \
(not self.hop_active[0] or self.hop_active[1] <= len(self.hop_active[0].hops)):
circuit, hop_index = self.hop_active
hop = circuit.hops[hop_index - 1] if hop_index and circuit else None
x, y = circuit_points[circuit][hop_index]
# Draw cicle around node
pen = wx.Pen(self.hop_to_colour.get(self.hop_active, wx.BLACK), 1, wx.USER_DASH)
pen.SetDashes([8, 4])
gc.SetPen(pen)
gc.DrawEllipse(x - self.radius, y - self.radius, self.radius * 2, self.radius * 2)
# Determine text
dc.SetFont(self.font_small)
if not hop:
text = 'You\nPERMID ' + bin2str(self.tunnel_community.my_member.public_key)[:10]
else:
text = 'PERMID ' + bin2str(self.dispersy.crypto.key_to_hash(hop.public_key))[:10]
if 'UNKNOWN HOST' not in hop.host:
text = 'IP %s:%s\n' % (hop.host, hop.port) + text
# Draw info box + text
box_width, box_height = self.GetTextExtent(dc, text)
box_width += 10
box_height += 10
x = x - box_width - 1.1 * self.radius if x > self.graph_panel.GetSize()[0] / 2 else x + 1.1 * self.radius
y = y - box_height - 1.1 * self.radius if y > self.graph_panel.GetSize()[1] / 2 else y + 1.1 * self.radius
gc.SetBrush(wx.Brush(wx.Colour(216, 237, 255, 50)))
gc.SetPen(wx.Pen(LIST_BLUE))
gc.DrawRectangle(x, y, box_width, box_height)
self.DrawText(dc, text, x + 5, y + 5)
def setUpPostSession(self):
""" override TestAsServer """
TestCrawler.setUpPostSession(self)
self.some_keypair = EC.gen_params(EC.NID_sect233k1)
self.some_keypair.gen_key()
self.some_permid = str(self.some_keypair.pub().get_der())
self.friendshipStatistics_db = FriendshipStatisticsDBHandler.getInstance()
self.friendshipStatistics_db.insertFriendshipStatistics( bin2str(self.his_permid), bin2str(self.some_permid), int(time.time()), 0, commit=True)
self.friendshipStatistics_db.insertFriendshipStatistics( bin2str(self.my_permid), bin2str(self.some_permid), int(time.time()), 0, commit=True)
# make sure that the OLConnection IS in the crawler_db
crawler_db = CrawlerDBHandler.getInstance()
crawler_db.temporarilyAddCrawler(self.my_permid)
def testUpdateExistingWithNewerNewSubs(self):
metadataDTO = MockMetadataDTO(["nld", "ita"])
metadataDTO.sign(metadataDTO._keypair)
self.underTest.insertMetadata(metadataDTO)
newerMetadataDTO = MockMetadataDTO(["nld","ita","eng"])
newerMetadataDTO.channel = metadataDTO.channel
newerMetadataDTO.infohash = metadataDTO.infohash
newerMetadataDTO._keypair = metadataDTO._keypair
newerMetadataDTO.timestamp = metadataDTO.timestamp +1 #newer
newerMetadataDTO.sign(newerMetadataDTO._keypair)
self.underTest.insertMetadata(newerMetadataDTO)
#assert the the older has been replaced
testquery = "SELECT * FROM Metadata WHERE publisher_id=?" \
+ " AND infohash=?;"
channel = bin2str(metadataDTO.channel)
infohash = bin2str(metadataDTO.infohash)
results = self.db.fetchall(testquery, (channel,infohash))
self.assertTrue(len(results) == 1)
tupl = results[0]
self.assertTrue(tupl[0] is not None and isinstance(tupl[0], int))
self.assertEquals(channel,tupl[1])
self.assertEquals(infohash,tupl[2])
self.assertEquals(newerMetadataDTO.description, tupl[3])
self.assertEquals(newerMetadataDTO.timestamp, tupl[4])
self.assertEquals(bin2str(newerMetadataDTO.signature), tupl[5])
subtitlesQuery = "SELECT * FROM Subtitles WHERE metadata_id_fk=?;"
subtitles = self.db.fetchall(subtitlesQuery, (tupl[0],))
self.assertEquals(3,len(subtitles))
for lang in ("ita", "nld","eng"):
found = False
foundSub = None
for subtuple in subtitles:
if subtuple[1] == lang:
found = True
foundSub = subtuple
break
self.assertTrue(found)
self.assertEquals(bin2str(newerMetadataDTO.getSubtitle(lang).checksum), foundSub[3])
def testUpdateExistingWithNewerNewSubs(self):
metadataDTO = MockMetadataDTO(["nld", "ita"])
metadataDTO.sign(metadataDTO._keypair)
self.underTest.insertMetadata(metadataDTO)
newerMetadataDTO = MockMetadataDTO(["nld","ita","eng"])
newerMetadataDTO.channel = metadataDTO.channel
newerMetadataDTO.infohash = metadataDTO.infohash
newerMetadataDTO._keypair = metadataDTO._keypair
newerMetadataDTO.timestamp = metadataDTO.timestamp +1 #newer
newerMetadataDTO.sign(newerMetadataDTO._keypair)
self.underTest.insertMetadata(newerMetadataDTO)
#assert the the older has been replaced
testquery = "SELECT * FROM Metadata WHERE publisher_id=?" \
+ " AND infohash=?;"
channel = bin2str(metadataDTO.channel)
infohash = bin2str(metadataDTO.infohash)
results = self.db.fetchall(testquery, (channel,infohash))
self.assertTrue(len(results) == 1)
tupl = results[0]
self.assertTrue(tupl[0] is not None and isinstance(tupl[0], int))
self.assertEquals(channel,tupl[1])
self.assertEquals(infohash,tupl[2])
self.assertEquals(newerMetadataDTO.description, tupl[3])
self.assertEquals(newerMetadataDTO.timestamp, tupl[4])
self.assertEquals(bin2str(newerMetadataDTO.signature), tupl[5])
subtitlesQuery = "SELECT * FROM Subtitles WHERE metadata_id_fk=?;"
subtitles = self.db.fetchall(subtitlesQuery, (tupl[0],))
self.assertEquals(3,len(subtitles))
for lang in ("ita", "nld","eng"):
found = False
foundSub = None
for subtuple in subtitles:
if subtuple[1] == lang:
found = True
foundSub = subtuple
break
self.assertTrue(found)
self.assertEquals(bin2str(newerMetadataDTO.getSubtitle(lang).checksum), foundSub[3])
def check_progress(self, ds, infohash, roothash, didMagnet):
d = ds.get_download()
cdef = d.get_def()
if ds.get_progress() == 0 or ds.get_status(
) == DLSTATUS_STOPPED_ON_ERROR or time() - getattr(
d, 'started_downloading', time()) > 45:
remove_lambda = lambda d=d: self._remove_download(d)
self.scheduletask(remove_lambda)
if not didMagnet:
if DEBUG:
print >> sys.stderr, "rtorrent: switching to magnet for", cdef.get_name(
), bin2str(infohash)
self.magnet_requester.add_request(infohash,
None,
timeout=SWIFTFAILED_TIMEOUT)
return (0, False)
elif ds.get_progress() == 1:
remove_lambda = lambda d=d: self._remove_download(d, False)
self.scheduletask(remove_lambda)
if DEBUG:
print >> sys.stderr, "rtorrent: swift finished for", cdef.get_name(
)
self.remote_th.notify_possible_torrent_roothash(roothash)
self.requests_success += 1
return (0, False)
return (5.0, True)
def _updateChannelInternal(self, query_permid, query, hits):
dictOfAdditions = dict()
if len(hits) > 0:
# a single read from the db is more efficient
all_spam_channels = self.votecastdb.getChannelsWithNegVote(None)
permid_channel_id = self.channelcastdb.getPermChannelIdDict(binary = True)
for k,v in hits.items():
#create new channel if not found
if v['publisher_id'] not in permid_channel_id:
permid_channel_id[v['publisher_id']] = self.channelcastdb.on_channel_from_channelcast(v['publisher_id'], v['publisher_name'])
#add local channel_id to all messages
v['channel_id'] = permid_channel_id[v['publisher_id']]
#check if the record belongs to a channel who we have "reported spam" (negative vote)
if bin2str(v['publisher_id']) in all_spam_channels:
# if so, ignore the incoming record
continue
# make everything into "string" format, if "binary"
hit = (v['channel_id'],v['publisher_name'],v['infohash'],v['torrentname'],v['time_stamp'])
dictOfAdditions[k] = hit
self._updateChannelcastDB(query_permid, query, hits, dictOfAdditions.values())
return dictOfAdditions
def round():
global session, mygames, myinvite, rounds
gc = GameCast.getInstance()
gc_db = session.open_dbhandler(NTFY_GAMECAST)
if not myinvite and len(mygames) < MAXBOTS:
# Send out a new random game/invite
game = {}
game['game_id'] = gc_db.getNextGameID(0)
game['owner_id'] = 0
game['winner_permid'] = ''
game['moves'] = bencode([])
mycolour = random.choice(['black','white'])
mypermid = bin2str(session.get_permid())
players = {mypermid:mycolour}
game['players'] = bencode(players)
game['gamename'] = 'chess'
game['time'] = random.choice([10,20,30])
game['inc'] = random.choice([60,120,180])
game['is_finished'] = 0
game['lastmove_time'] = 0
game['creation_time'] = 0
gc_db.addGame(game)
invite = {}
invite['target_id'] = -1
invite['game_id'] = game['game_id']
invite['min_rating'] = 0
invite['max_rating'] = 9999
invite['time'] = game['time']
invite['inc'] = game['inc']
invite['gamename'] = 'chess'
if mycolour == 'black':
invite['colour'] = 'white'
else:
invite['colour'] = 'black'
if DEBUG:
print >> sys.stderr, 'chessbot: sending out invite for game (%d,%d)' % (0, game['game_id'])
myinvite = gc._executeSeekOrMatch(invite)
elif myinvite:
# Check if the game related to our invite has started yet
owner_id = myinvite['owner_id']
game_id = myinvite['game_id']
games = gc_db.getGames(game_id = game_id, owner_id = owner_id)
if games and len(games) == 1:
game = games[0]
if game['creation_time']:
# Game has started
myinvite = None
mygames.append((owner_id, game_id))
if DEBUG:
print >> sys.stderr, 'chessbot: starting chessbot for game (%d,%d)' % (owner_id, game_id)
func = lambda:launchchessbot(game)
overlay_bridge.add_task(func, 0)
else:
# Game has not yet started, resend the invite
if int(time()) > myinvite['creation_time']+INV_EXPIRE_TIME:
myinvite = None
elif (rounds % 5) == 0:
gc.resendSeek(myinvite)
else:
myinvite = None
def _updateChannelInternal(self, query_permid, query, hits):
dictOfAdditions = dict()
if len(hits) > 0:
# a single read from the db is more efficient
all_spam_channels = self.votecastdb.getChannelsWithNegVote(None)
permid_channel_id = self.channelcastdb.getPermChannelIdDict(
binary=True)
for k, v in hits.items():
#create new channel if not found
if v['publisher_id'] not in permid_channel_id:
permid_channel_id[v[
'publisher_id']] = self.channelcastdb.on_channel_from_channelcast(
v['publisher_id'], v['publisher_name'])
#add local channel_id to all messages
v['channel_id'] = permid_channel_id[v['publisher_id']]
#check if the record belongs to a channel who we have "reported spam" (negative vote)
if bin2str(v['publisher_id']) in all_spam_channels:
# if so, ignore the incoming record
continue
# make everything into "string" format, if "binary"
hit = (v['channel_id'], v['publisher_name'], v['infohash'],
v['torrentname'], v['time_stamp'])
dictOfAdditions[k] = hit
self._updateChannelcastDB(query_permid, query, hits,
dictOfAdditions.values())
return dictOfAdditions
def on_torrent_callback(rss_url, infohash, torrent_data):
"""
A torrent file is discovered through rss. Create a new
moderation.
"""
if "info" in torrent_data and "name" in torrent_data["info"]:
print >>sys.stderr, time.asctime(),'-', "Creating moderation for %s" % torrent_data["info"]["name"]
else:
print >>sys.stderr, time.asctime(),'-', "Creating moderation"
moderation = {}
moderation['infohash'] = bin2str(infohash)
torrenthash = sha.sha(bencode(data)).digest()
moderation['torrenthash'] = bin2str(torrenthash)
moderation_cast_db.addOwnModeration(moderation)
def check_progress(self, ds, infohash, roothash, didMagnet):
d = ds.get_download()
cdef = d.get_def()
if ds.get_progress() == 0:
remove_lambda = lambda d=d: self._remove_donwload(d)
self.scheduletask(remove_lambda)
if not didMagnet:
if DEBUG:
print >> sys.stderr, "rtorrent: switching to magnet for", cdef.get_name(
), bin2str(infohash)
self.magnet_requester.add_request(infohash, None)
return (0, False)
elif ds.get_progress() == 1:
remove_lambda = lambda d=d: self._remove_donwload(d, False)
self.scheduletask(remove_lambda)
if DEBUG:
print >> sys.stderr, "rtorrent: swift finished for", cdef.get_name(
)
self.remote_th.notify_possible_torrent_roothash(roothash)
return (0, False)
return (5.0, True)
def __torrentdef_retrieved(self, tdef):
infohash = tdef.get_infohash()
if DEBUG:
print >>sys.stderr, "magnetrequester: received torrent", bin2str(infohash)
# remove from requested list
if infohash in self.requestedInfohashes:
self.requestedInfohashes.remove(infohash)
# save torrent
torrent = self.torrent_db.getTorrent(infohash, ["torrent_file_name"], include_mypref=False)
if torrent and torrent.get("torrent_file_name", False) and not os.path.isabs(torrent["torrent_file_name"]):
torrent_filename = os.path.join(self.metadatahandler.torrent_dir, torrent["torrent_file_name"])
else:
torrent_filename = os.path.join(
self.metadatahandler.torrent_dir, get_collected_torrent_filename(infohash)
)
tdef.save(torrent_filename)
# calculate root-hash
sdef = SwiftDef()
sdef.add_content(torrent_filename)
sdef.finalize(self.session.get_swift_path())
# add this new torrent to db
self.torrent_db.addExternalTorrent(tdef, extra_info={"swift_torrent_hash": bin2str(sdef.get_roothash())})
# notify all
self.remoteTorrentHandler.metadatahandler_got_torrent(infohash, tdef, torrent_filename)
self.overlay_bridge.add_task(self.__requestMagnet, self.REQUEST_INTERVAL)
def _doFetch(self, filename, infohash, candidates):
if filename:
if infohash in self.requestedInfohashes:
self.requestedInfohashes.remove(infohash)
self.remote_th.notify_possible_torrent_infohash(infohash, True)
self.requests_on_disk += 1
else:
# try magnet link
magnetlink = "magnet:?xt=urn:btih:" + hexlify(infohash)
if self.remote_th.torrent_db:
# see if we know any trackers for this magnet
trackers = self.remote_th.torrent_db.getTracker(infohash)
for tracker, _ in trackers:
magnetlink += "&tr=" + urllib.quote_plus(tracker)
if DEBUG:
print >> sys.stderr, long(time()), 'rtorrent: requesting magnet', bin2str(infohash), self.prio, magnetlink, len(self.requestedInfohashes)
TorrentDef.retrieve_from_magnet(magnetlink, self.__torrentdef_retrieved, self.MAGNET_RETRIEVE_TIMEOUT, max_connections=30 if self.prio == 0 else 10)
failed_lambda = lambda infohash = infohash: self.__torrentdef_failed(infohash)
self.scheduletask(failed_lambda, t=self.MAGNET_RETRIEVE_TIMEOUT)
return True
def handleVoteCastMsg(self, sender_permid, data):
""" Handles VoteCast message """
if DEBUG:
print >>sys.stderr, "votecast: Processing VOTECAST msg from: ", show_permid_short(
sender_permid
), "; data: ", repr(data)
mod_ids = Set()
for key, value in data.items():
vote = {}
vote["mod_id"] = bin2str(key)
vote["voter_id"] = permid_for_user(sender_permid)
vote["vote"] = value["vote"]
vote["time_stamp"] = value["time_stamp"]
self.votecastdb.addVote(vote)
mod_ids.add(vote["mod_id"])
# Arno, 2010-02-24: Generate event
for mod_id in mod_ids:
try:
self.notifier.notify(NTFY_VOTECAST, NTFY_UPDATE, mod_id)
except:
print_exc()
if DEBUG:
print >>sys.stderr, "votecast: Processing VOTECAST msg from: ", show_permid_short(
sender_permid
), "DONE; data:"
def test_getPeerList(self):
db = PeerDBHandler.getInstance()
peerlist = db.getPeerList()
assert len(peerlist) == 3995
peerlist.sort()
assert bin2str(
peerlist[345]
) == 'MFIwEAYHKoZIzj0CAQYFK4EEABoDPgAEACxVRvG/Gr19EAPJru2Z5gjctEzv973/PJCQIua2ATMP6euq+Kf4gYpdKbsB/PWqJnfY/wSKPHHfIByV'
def __torrentdef_retrieved(self, tdef):
infohash = tdef.get_infohash()
if DEBUG:
print >> sys.stderr, 'rtorrent: received torrent using magnet', bin2str(infohash)
self.remote_th.save_torrent(tdef)
if infohash in self.requestedInfohashes:
self.requestedInfohashes.remove(infohash)
def _download_torrent(self, candidate, infohash, roothash, usercallback,
prio, timeout):
if self.registered:
assert infohash or roothash, "We need either the info or roothash"
doSwiftCollect = candidate and roothash
if doSwiftCollect:
requesters = self.trequesters
hash = (infohash, roothash)
elif infohash:
requesters = self.drequesters
hash = infohash
#fix prio levels to 1 and 0
prio = min(prio, 1)
else:
return
if usercallback:
self.callbacks.setdefault(hash, set()).add(usercallback)
#look for lowest prio requester, which already has this infohash scheduled
requester = None
for i in range(prio, prio + 1):
if i in requesters and requesters[i].is_being_requested(hash):
requester = requesters[i]
break
#if not found, then used/create this requester
if not requester:
if prio not in requesters:
if doSwiftCollect:
requesters[prio] = TorrentRequester(
self, self.drequesters[1], self.session, prio)
else:
requesters[prio] = MagnetRequester(self, prio)
requester = requesters[prio]
#make request
requester.add_request(hash, candidate, timeout)
if DEBUG:
print >> sys.stderr, 'rtorrent: adding torrent request:', bin2str(
infohash or ''), bin2str(roothash or ''), candidate, prio
def test_getPermIDByIP(self):
fake_permid_x = 'fake_permid_x' + '0R0\x10\x00\x07*\x86H\xce=\x02\x01\x06\x05+\x81\x04\x00\x1a\x03>\x00\x04'
peer_x = {'permid': fake_permid_x, 'ip': '1.2.3.4', 'port': 234,
'name': 'fake peer x', 'last_seen': 12345, 'connected_times': 3}
oldsize = self.pdb.size()
p = self.pdb.getPeer(fake_permid_x)
assert p == None, p
self.pdb.addPeer(fake_permid_x, peer_x)
assert self.pdb.hasPeer(fake_permid_x)
assert self.pdb.size() == oldsize + 1, (self.pdb.size(), oldsize + 1)
permid = self.pdb.getPermIDByIP('1.2.3.4')
assert bin2str(permid) == bin2str(fake_permid_x)
self.pdb.deletePeer(fake_permid_x, force=True)
assert not self.pdb.hasPeer(fake_permid_x)
assert self.pdb.size() == oldsize
def __torrentdef_retrieved(self, tdef):
infohash = tdef.get_infohash()
if DEBUG:
print >> sys.stderr, 'rtorrent: received torrent using magnet', bin2str(
infohash)
self.remote_th.save_torrent(tdef)
if infohash in self.requestedInfohashes:
self.requestedInfohashes.remove(infohash)
def __requestMagnet(self):
try:
if len(self.requestedInfohashes) < self.MAX_CONCURRENT:
#request new infohash from queue
while True:
if len(self.list) == 0:
return
prio, infohash = self.list.pop(0)
if infohash in self.requestedInfohashes:
if DEBUG:
print >> sys.stderr, 'magnetrequester: magnet already requested', bin2str(
infohash)
continue
torrent = self.torrent_db.getTorrent(infohash,
['torrent_file_name'],
include_mypref=False)
torrent_alt_filename = os.path.join(
self.metadatahandler.torrent_dir,
get_collected_torrent_filename(infohash))
if torrent and torrent.get('torrent_file_name', False):
torrent_filename = os.path.join(
self.metadatahandler.torrent_dir,
torrent['torrent_file_name'])
else:
torrent_filename = torrent_alt_filename
if os.path.isfile(torrent_filename) or os.path.isfile(
torrent_alt_filename):
if DEBUG:
print >> sys.stderr, 'magnetrequester: magnet already on disk', bin2str(
infohash)
else:
break #do request
else: #requesting max_concurrent
return
except:
print_exc()
#try magnet link
magnetlink = "magnet:?xt=urn:btih:" + hexlify(infohash)
if DEBUG:
print >> sys.stderr, 'magnetrequester: requesting magnet', bin2str(
infohash), prio, magnetlink
self.requestedInfohashes.add(infohash)
TorrentDef.retrieve_from_magnet(magnetlink,
self.__torrentdef_retrieved,
self.MAGNET_RETRIEVE_TIMEOUT)
self.overlay_bridge.add_task(
lambda: self.__torrentdef_failed(infohash),
self.MAGNET_RETRIEVE_TIMEOUT, infohash)
if len(self.requestedInfohashes) < self.MAX_CONCURRENT:
self.overlay_bridge.add_task(self.__requestMagnet,
self.REQUEST_INTERVAL)
def __torrentdef_retrieved(self, tdef):
infohash = tdef.get_infohash()
if DEBUG:
print >> sys.stderr, 'magnetrequester: received torrent', bin2str(
infohash)
#remove from requested list
if infohash in self.requestedInfohashes:
self.requestedInfohashes.remove(infohash)
#save torrent
torrent = self.torrent_db.getTorrent(infohash,
['torrent_file_name'],
include_mypref=False)
if torrent and torrent.get(
'torrent_file_name',
False) and not os.path.isabs(torrent['torrent_file_name']):
torrent_filename = os.path.join(
self.metadatahandler.torrent_dir,
torrent['torrent_file_name'])
else:
torrent_filename = os.path.join(
self.metadatahandler.torrent_dir,
get_collected_torrent_filename(infohash))
tdef.save(torrent_filename)
#calculate root-hash
sdef = SwiftDef()
sdef.add_content(torrent_filename)
sdef.finalize(self.session.get_swift_path())
#add this new torrent to db
self.torrent_db.addExternalTorrent(tdef,
extra_info={
'swift_torrent_hash':
bin2str(sdef.get_roothash())
})
#notify all
self.remoteTorrentHandler.metadatahandler_got_torrent(
infohash, tdef, torrent_filename)
self.overlay_bridge.add_task(self.__requestMagnet,
self.REQUEST_INTERVAL)
def updateChannel(self,query_permid, query, hits):
"""
This function is called when there is a reply from remote peer regarding updating of a channel
@param query_permid: the peer who returned the results
@param query: the query string (None if this is not the results of a query)
@param hits: details of all matching results related to the query
"""
if DEBUG:
print >> sys.stderr, "channelcast: sending message to", bin2str(query_permid), query, len(hits)
return self._updateChannelInternal(query_permid, query, hits)
def log(self, permid, decoded_message):
lt = T.localtime(T.time())
timestamp = "%04d-%02d-%02d %02d:%02d:%02d" % (lt[0], lt[1], lt[2], lt[3], lt[4], lt[5])
ip = self.peer_db.getPeer(permid, "ip")
#ip = "x.y.z.1"
s = "%s\t%s\t%s\t%s\n"% (timestamp, bin2str(permid), ip, decoded_message)
print dunno2unicode(s)
self.logfile.write(dunno2unicode(s)) # bin2str(
self.logfile.flush()
def log(self, loggertype, *args, **kwargs):
if loggertype == GCLOG:
logger = self.gc_logger
elif loggertype == GCGLOG:
logger = self.gcg_logger
else:
return
if logger and args:
d = {'event_type' : args[0],
'at_peer_short' : '%s (%s:%s)' % (showPermid(self.mypermid), self.showIP(self.myip), self.showPort(self.myport)),
'at_peer' : '%s (%s:%s)' % (bin2str(self.mypermid), self.showIP(self.myip), self.showPort(self.myport)),
'tf_peer_short' : '%s (%s:%s)' % (showPermid(args[3]), self.showIP(args[1]), self.showPort(args[2])) if len(args) > 1 else '',
'tf_peer' : '%s (%s:%s)' % (bin2str(args[3]), self.showIP(args[1]), self.showPort(args[2])) if len(args) > 1 else ''}
msg = ''
iterator = iter(sorted(kwargs.iteritems()))
for key, value in iterator:
msg += "%s = %s ; " % (key, value)
if msg:
msg = msg[:-3]
logger.info(msg, extra=d)
def download_torrent(self,permid,infohash,usercallback, prio = 1):
if self.registered:
""" The user has selected a torrent referred to by a peer in a query
reply. Try to obtain the actual .torrent file from the peer and then
start the actual download.
"""
assert isinstance(infohash, str), "INFOHASH has invalid type: %s" % type(infohash)
assert len(infohash) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(infohash)
self.callbacks.setdefault(infohash, set()).add(usercallback)
if prio not in self.requesters:
self.requesters[prio] = TorrentRequester(self, self.metadatahandler, self.overlay_bridge, self.session, prio)
self.requesters[prio].add_source(infohash, permid)
if DEBUG:
if permid:
print >>sys.stderr,'rtorrent: adding request:', bin2str(infohash), bin2str(permid), prio
else:
print >>sys.stderr,'rtorrent: adding request:', bin2str(infohash), prio
def subtest_votecast(self):
print >> sys.stderr, "test: votecast-----------------------------"
s = OLConnection(self.my_keypair, 'localhost', self.hisport)
vcast = VoteCastCore(None, s, self.session, None, log='', dnsindb=None)
#Send Good VoteCast message
vdata = {self.hispermid: {'vote': -1, 'time_stamp': 12345345}}
print >> sys.stderr, "Test Good VoteCast", ` vdata `
msg = VOTECAST + bencode(vdata)
s.send(msg)
resp = s.recv()
#print >> sys.stderr, "printing resp", resp
if len(resp) > 0:
print >> sys.stderr, "test: votecast: got", getMessageName(resp[0])
self.assert_(resp[0] == VOTECAST)
print >> sys.stderr, "test: votecast: got msg", ` bdecode(resp[1:]) `
vdata_rcvd = bdecode(resp[1:])
self.assert_(validVoteCastMsg(vdata_rcvd) == True)
s.close()
#Now, send a bad ChannelCast messages
# The other side should close the connection
#Bad time_stamp: it can only int
vdata = {bin2str(self.hispermid): {'vote': -1, 'time_stamp': 'halo'}}
self.subtest_bad_votecast(vdata)
#Bad Vote: Vote can only -1 or 2
vdata = {
bin2str(self.hispermid): {
'vote': -15,
'time_stamp': 12345345
}
}
self.subtest_bad_votecast(vdata)
# Bad Message format ... Correct format is 'time_stamp'
vdata = {bin2str(self.hispermid): {'vote': -15, 'timestamp': 12345345}}
self.subtest_bad_votecast(vdata)
print >> sys.stderr, "End of votecast test"
def saveFriendshipStatistics(self, permid, currentTime, stats):
if stats:
# 20/10/08. Boudewijn: A mistake in the code results in
# only 7 items in the list instead of 8. We add one here
# to get things working.
for stat in stats:
if len(stat) == 7:
stat.append(0)
if len(stat) == 7 or len(stat) == 8:
stat.append(bin2str(permid))
self.friendshipStatistics_db.saveFriendshipStatisticData(stats)
def saveFriendshipStatistics(self,permid,currentTime,stats):
if stats:
# 20/10/08. Boudewijn: A mistake in the code results in
# only 7 items in the list instead of 8. We add one here
# to get things working.
for stat in stats:
if len(stat) == 7:
stat.append(0)
if len(stat) == 7 or len(stat) == 8:
stat.append(bin2str(permid))
self.friendshipStatistics_db.saveFriendshipStatisticData(stats)
def testInsertNewMetadataSubs(self):
metadataDTO = MockMetadataDTO(["nld","ita"])
metadataDTO.sign(metadataDTO._keypair)
self.underTest.insertMetadata(metadataDTO)
testquery = "SELECT * FROM Metadata WHERE publisher_id=?" \
+ " AND infohash=?;"
results = self.db.fetchall(testquery, (bin2str(metadataDTO.channel),bin2str(metadataDTO.infohash)))
self.assertTrue(len(results) == 1)
tupl = results[0]
self.assertTrue(tupl[0] is not None and isinstance(tupl[0], int))
self.assertEquals(bin2str(metadataDTO.channel),tupl[1])
self.assertEquals(bin2str(metadataDTO.infohash),tupl[2])
self.assertEquals(metadataDTO.description, tupl[3])
self.assertEquals(metadataDTO.timestamp, tupl[4])
self.assertEquals(bin2str(metadataDTO.signature), tupl[5])
subtitlesQuery = "SELECT * FROM Subtitles WHERE metadata_id_fk=?;"
subtitles = self.db.fetchall(subtitlesQuery, (tupl[0],))
self.assertEquals(2,len(subtitles))
for lang in ("ita", "nld"):
found = False
foundSub = None
for subtuple in subtitles:
if subtuple[1] == lang:
found = True
foundSub = subtuple
break
self.assertTrue(found)
self.assertEquals(bin2str(metadataDTO.getSubtitle(lang).checksum), foundSub[3])
def testInsertNewMetadataSubs(self):
metadataDTO = MockMetadataDTO(["nld","ita"])
metadataDTO.sign(metadataDTO._keypair)
self.underTest.insertMetadata(metadataDTO)
testquery = "SELECT * FROM Metadata WHERE publisher_id=?" \
+ " AND infohash=?;"
results = self.db.fetchall(testquery, (bin2str(metadataDTO.channel),bin2str(metadataDTO.infohash)))
self.assertTrue(len(results) == 1)
tupl = results[0]
self.assertTrue(tupl[0] is not None and isinstance(tupl[0], int))
self.assertEquals(bin2str(metadataDTO.channel),tupl[1])
self.assertEquals(bin2str(metadataDTO.infohash),tupl[2])
self.assertEquals(metadataDTO.description, tupl[3])
self.assertEquals(metadataDTO.timestamp, tupl[4])
self.assertEquals(bin2str(metadataDTO.signature), tupl[5])
subtitlesQuery = "SELECT * FROM Subtitles WHERE metadata_id_fk=?;"
subtitles = self.db.fetchall(subtitlesQuery, (tupl[0],))
self.assertEquals(2,len(subtitles))
for lang in ("ita", "nld"):
found = False
foundSub = None
for subtuple in subtitles:
if subtuple[1] == lang:
found = True
foundSub = subtuple
break
self.assertTrue(found)
self.assertEquals(bin2str(metadataDTO.getSubtitle(lang).checksum), foundSub[3])
def olthread_download_torrent_callback(self,
permid,
infohash,
usercallback,
prio=1):
""" The user has selected a torrent referred to by a peer in a query
reply or channelcast has discovered a new torrent. Try to obtain the actual .torrent file from the peer
"""
assert isinstance(
infohash, str), "INFOHASH has invalid type: %s" % type(infohash)
assert len(
infohash
) == INFOHASH_LENGTH, "INFOHASH has invalid length: %d" % len(infohash)
if usercallback:
self.callbacks.setdefault(infohash, set()).add(usercallback)
requester = None
#look for lowest prio requester, which already has this infohash scheduled
for i in range(prio):
if i in self.requesters and self.requesters[i].is_being_requested(
infohash):
requester = self.requesters[i]
break
#if not found, then used/create this requester
if not requester:
if prio not in self.requesters:
self.requesters[prio] = TorrentRequester(
self, self.metadatahandler, self.overlay_bridge,
self.session, prio)
requester = self.requesters[prio]
#make request
requester.add_source(infohash, permid)
if DEBUG:
print >> sys.stderr, 'rtorrent: adding request:', bin2str(
infohash), bin2str(permid), prio
def process_response(self,permid,d):
mypermid = self.session.get_permid()
self.friendshipStatistics_db.updateFriendshipResponseTime( bin2str(mypermid),
bin2str(permid),
int(time()))
fs = self.frienddb.getFriendState(permid)
# If the request to add has been approved
if d['response'] == 1:
if fs == FS_I_INVITED:
self.frienddb.setFriendState(permid, commit=True, state = FS_MUTUAL)
elif fs != FS_MUTUAL:
# Unsollicited response, consider this an invite, if not already friend
self.frienddb.setFriendState(permid, commit=True, state = FS_HE_INVITED)
else:
# He denied our friendship
self.frienddb.setFriendState(permid, commit=True, state = FS_HE_DENIED)
def process_response(self,permid,d):
mypermid = self.session.get_permid()
self.friendshipStatistics_db.updateFriendshipResponseTime( bin2str(mypermid),
bin2str(permid),
int(time()))
fs = self.frienddb.getFriendState(permid)
# If the request to add has been approved
if d['response'] == 1:
if fs == FS_I_INVITED:
self.frienddb.setFriendState(permid, commit=True, state = FS_MUTUAL)
elif fs != FS_MUTUAL:
# Unsollicited response, consider this an invite, if not already friend
self.frienddb.setFriendState(permid, commit=True, state = FS_HE_INVITED)
else:
# He denied our friendship
self.frienddb.setFriendState(permid, commit=True, state = FS_HE_DENIED)
def _updateChannelInternal(self, query_permid, query, hits):
dictOfAdditions = dict()
if len(hits) > 0:
# a single read from the db is more efficient
all_spam_channels = self.votecastdb.getPublishersWithNegVote(bin2str(self.session.get_permid()))
for k,v in hits.items():
#check if the record belongs to a channel who we have "reported spam" (negative vote)
if bin2str(v['publisher_id']) in all_spam_channels:
# if so, ignore the incoming record
continue
# make everything into "string" format, if "binary"
hit = (bin2str(v['publisher_id']),v['publisher_name'],bin2str(v['infohash']),bin2str(v['torrenthash']),v['torrentname'],v['time_stamp'],bin2str(k))
# 29/06/11 boudewijn: note that k contains the signature (whatever that is) and NOT
# the infohash. this makes this result incompatible with
# SearchGridManager.getRemoteHits(). Hence these hits are NOT propagated there
# anymore.
dictOfAdditions[k] = hit
# Arno, 2010-06-11: We're on the OverlayThread
self._updateChannelcastDB(query_permid, query, hits, dictOfAdditions.values())
return dictOfAdditions
def subtest_votecast(self):
print >>sys.stderr,"test: votecast-----------------------------"
s = OLConnection(self.my_keypair,'localhost',self.hisport)
vcast = VoteCastCore(None, s, self.session, None, log = '', dnsindb = None)
#Send Good VoteCast message
vdata = {self.hispermid:{'vote':-1,'time_stamp':12345345}}
print >> sys.stderr, "Test Good VoteCast", `vdata`
msg = VOTECAST+bencode(vdata)
s.send(msg)
resp = s.recv()
#print >> sys.stderr, "printing resp", resp
if len(resp) > 0:
print >>sys.stderr,"test: votecast: got",getMessageName(resp[0])
self.assert_(resp[0]==VOTECAST)
print >>sys.stderr, "test: votecast: got msg", `bdecode(resp[1:])`
vdata_rcvd = bdecode(resp[1:])
self.assert_(validVoteCastMsg(vdata_rcvd)==True)
s.close()
#Now, send a bad ChannelCast messages
# The other side should close the connection
#Bad time_stamp: it can only int
vdata = {bin2str(self.hispermid):{'vote':-1,'time_stamp':'halo'}}
self.subtest_bad_votecast(vdata)
#Bad Vote: Vote can only -1 or 2
vdata = {bin2str(self.hispermid):{'vote':-15,'time_stamp':12345345}}
self.subtest_bad_votecast(vdata)
# Bad Message format ... Correct format is 'time_stamp'
vdata = {bin2str(self.hispermid):{'vote':-15,'timestamp':12345345}}
self.subtest_bad_votecast(vdata)
print>>sys.stderr, "End of votecast test"
def showChannel(self, channelname, channel_permid):
self.frame.top_bg.selectTab('channels')
if channel_permid == bin2str(self.utility.session.get_permid()):
self.ShowPage('mychannel')
else:
self.frame.selectedchannellist.SetTitle(channelname)
description_list = ["Marking a channel as your favorite will help to distribute it.", "If many Tribler users mark a channel as their favorite, it is considered popular."]
self.frame.channelcategories.Quicktip(random.choice(description_list))
self.ShowPage('selectedchannel')
manager = self.frame.selectedchannellist.GetManager()
manager.refresh(channel_permid)
def on_torrent_collect_response(self, messages, verifyRequest=True):
logger.debug("%d messages received", len(messages))
toInsert = {}
toCollect = {}
toPopularity = {}
for message in messages:
if verifyRequest:
pong_request = self._dispersy.request_cache.pop(
message.payload.identifier,
SearchCommunity.PingRequestCache)
logger.debug(
"pop %s", pong_request.helper_candidate
if pong_request else "unknown")
else:
logger.debug("no-pop")
pong_request = True
if pong_request and message.payload.hashtype == SWIFT_INFOHASHES:
for roothash, infohash, seeders, leechers, ago in message.payload.torrents:
toInsert[infohash] = [infohash, roothash]
toPopularity[infohash] = [
seeders, leechers,
time() - (ago * 60)
]
toCollect.setdefault(infohash,
[]).append(message.candidate)
if len(toInsert) > 0:
self._torrent_db.on_torrent_collect_response(toInsert.values())
hashes = [hash_ for hash_ in toCollect.keys() if hash_]
if hashes:
hashesToCollect = self._torrent_db.selectSwiftTorrentsToCollect(
hashes)
for infohash, roothash in hashesToCollect[:5]:
for candidate in toCollect[infohash]:
if DEBUG:
from Tribler.Core.CacheDB.sqlitecachedb import bin2str
print >> sys.stderr, "SearchCommunity: requesting .torrent after receiving ping/pong ", candidate, bin2str(
infohash), bin2str(roothash)
self._rtorrent_handler.download_torrent(
candidate,
infohash,
roothash,
prio=LOW_PRIO_COLLECTING,
timeout=CANDIDATE_WALK_LIFETIME)
def subtest_channel_permid_query(self, nickname):
print >> sys.stderr, "test: chquery permid-----------------------------"
s = OLConnection(self.my_keypair, 'localhost', self.hisport)
data = {}
uq = u'CHANNEL p ' + bin2str(self.hispermid)
data['q'] = uq.encode("UTF-8")
data['id'] = 'b' * 20
msg = QUERY + bencode(data)
s.send(msg)
resp = s.recv()
#print >> sys.stderr, "printing resp", resp
if len(resp) > 0:
print >> sys.stderr, "test: chquery: got", getMessageName(resp[0])
self.assert_(resp[0] == QUERY_REPLY)
self.check_chquery_reply(resp[1:], nickname)
print >> sys.stderr, "test:", ` bdecode(resp[1:]) `
s.close()
def on_torrent_collect_response(self, messages, verifyRequest=True):
if __debug__: dprint(len(messages))
toInsert = {}
toCollect = {}
toPopularity = {}
for message in messages:
if verifyRequest:
pong_request = self._dispersy.request_cache.pop(
message.payload.identifier,
SearchCommunity.PingRequestCache)
if __debug__: dprint("pop", pong_request.helper_candidate)
else:
if __debug__: dprint("no-pop")
pong_request = True
if pong_request and message.payload.hashtype == SWIFT_INFOHASHES:
for roothash, infohash, seeders, leechers, ago in message.payload.torrents:
toInsert[infohash] = [infohash, roothash]
toPopularity[infohash] = [
seeders, leechers,
time() - (ago * 60)
]
toCollect.setdefault(infohash,
[]).append(message.candidate)
self._torrent_db.on_torrent_collect_response(toInsert.values())
hashesToCollect = self._torrent_db.selectSwiftTorrentsToCollect(
toCollect.keys())
for infohash, roothash in hashesToCollect[:5]:
for candidate in toCollect[infohash]:
if DEBUG:
from Tribler.Core.CacheDB.sqlitecachedb import bin2str
print >> sys.stderr, "SearchCommunity: requesting .torrent after receiving ping/pong ", candidate, bin2str(
infohash), bin2str(roothash)
self._rtorrent_handler.download_torrent(candidate,
infohash,
roothash,
prio=2)
def fmsg_connect_callback(self,exc,dns,permid,selversion, type = None):
""" Callback function for the overlay connect function """
# Called by OverlayThread
if exc is None:
if selversion < OLPROTO_VER_SEVENTH:
self.remove_msgs_for_ltv7_peer(permid)
return
# Reached him
sendlist = self.get_msgs_as_sendlist(targetpermid=permid)
if DEBUG:
print >> sys.stderr, 'friendship: fmsg_connect_callback: sendlist len',len(sendlist)
#print_stack()
for i in range(0,len(sendlist)):
tuple = sendlist[i]
permid,msgid,msg = tuple
send_callback = lambda exc,permid:self.fmsg_send_callback(exc,permid,msgid)
if DEBUG:
print >>sys.stderr,"friendship: fmsg_connect_callback: Sending",`msg`,msgid
mypermid = self.session.get_permid()
commit = (i == len(sendlist)-1)
isForwarder = 0
no_of_helpers = 0
# if type == F_REQUEST_MSG:
# print
# elif type == F_RESPONSE_MSG:
# print
#Set forwarder to True and also no of helpers to 10
if type == F_FORWARD_MSG:
isForwarder = 1
no_of_helpers = 10
no_of_attempts = 0
if permid in self.currmsgs:
msgid2rec = self.currmsgs[permid]
if msgid in msgid2rec:
msgrec = msgid2rec[msgid]
no_of_attempts = msgrec['attempt']
# insertFriendshipStatistics(self, my_permid, target_permid, current_time, isForwarder = 0, no_of_attempts = 0, no_of_helpers = 0, commit = True):
self.friendshipStatistics_db.insertOrUpdateFriendshipStatistics( bin2str(mypermid),
bin2str(permid),
int(time()),
isForwarder,
no_of_attempts ,
no_of_helpers,
commit=commit)
self.overlay_bridge.send(permid, FRIENDSHIP + bencode(msg), send_callback)
else:
if DEBUG:
peer = self.peerdb.getPeer(permid)
if peer is None:
print >>sys.stderr, 'friendship: Could not connect to peer', show_permid_short(permid),peer
else:
print >>sys.stderr, 'friendship: Could not connect to peer', show_permid_short(permid),peer['name']
print >>sys.stderr,exc
mypermid = self.session.get_permid()
isForwarder = 0
no_of_helpers = 0
if type == F_FORWARD_MSG:
isForwarder = 1
no_of_helpers = 10
no_of_attempts = 0
if permid in self.currmsgs:
msgid2rec = self.currmsgs[permid]
for msgid in msgid2rec:
msgrec = msgid2rec[msgid]
no_of_attempts = msgrec['attempt']
self.friendshipStatistics_db.insertOrUpdateFriendshipStatistics( bin2str(mypermid),
bin2str(permid),
int(time()),
isForwarder,
no_of_attempts ,
no_of_helpers)
def _doFetch(self, filename, hash, candidates):
infohash, roothash = hash
attempting_download = False
if filename:
self.remote_th.notify_possible_torrent_infohash(infohash, True)
self.remote_th.notify_possible_torrent_infohash(hash, True)
self.requests_on_disk += 1
elif candidates:
candidate = candidates[0]
candidates = candidates[1:]
ip, port = candidate.sock_addr
if not candidate.tunnel:
port = 7758
if DEBUG:
print >> sys.stderr, "rtorrent: requesting torrent", hash, ip, port
doMagnet = self.prio <= 1
download = None
sdef = SwiftDef(roothash, tracker="%s:%d" % (ip, port))
dcfg = self.dscfg.copy()
try:
# hide download from gui
download = self.session.start_download(sdef, dcfg, hidden=True)
state_lambda = lambda ds, infohash = infohash, roothash = roothash, doMagnet = doMagnet: self.check_progress(ds, infohash, roothash, doMagnet)
download.set_state_callback(state_lambda, delay=self.REQUEST_INTERVAL * (self.prio + 1))
download.started_downloading = time()
except DuplicateDownloadException:
download = self.session.get_download(roothash)
download.add_peer((ip, port))
except OperationNotEnabledByConfigurationException:
doMagnet = True
else:
if DEBUG:
print >> sys.stderr, "rtorrent: start swift download for", bin2str(roothash), ip, port
attempting_download = True
if download and candidates:
try:
for candidate in candidates:
ip, port = candidate.sock_addr
if not candidate.tunnel:
port = 7758
download.add_peer((ip, port))
except:
print_exc()
# schedule a magnet lookup after X seconds
if doMagnet and self.magnet_requester:
magnet_lambda = lambda infohash = infohash: self.magnet_requester.add_request(infohash, None)
self.scheduletask(magnet_lambda, t=self.MAGNET_TIMEOUT * (self.prio))
return attempting_download
def check_progress(self, ds, infohash, roothash, didMagnet):
d = ds.get_download()
cdef = d.get_def()
if ds.get_progress() == 1:
remove_lambda = lambda d = d: self._remove_download(d, False)
self.scheduletask(remove_lambda)
if DEBUG:
print >> sys.stderr, "rtorrent: swift finished for", cdef.get_name(), bin2str(infohash)
self.remote_th.notify_possible_torrent_roothash(roothash)
self.requests_success += 1
return (0, False)
else:
diff = time() - getattr(d, 'started_downloading', time())
if (diff > self.SWIFT_CANCEL and ds.get_progress() == 0) or diff > 45 or ds.get_status() == DLSTATUS_STOPPED_ON_ERROR:
remove_lambda = lambda d = d: self._remove_download(d)
self.scheduletask(remove_lambda)
if DEBUG:
print >> sys.stderr, "rtorrent: swift failed download for", cdef.get_name(), bin2str(infohash)
if not didMagnet and self.magnet_requester:
if DEBUG:
print >> sys.stderr, "rtorrent: switching to magnet for", cdef.get_name(), bin2str(infohash)
self.magnet_requester.add_request(infohash, None, timeout=SWIFTFAILED_TIMEOUT)
self.requests_fail += 1
return (0, False)
return (self.REQUEST_INTERVAL * (self.prio + 1), True)
def subtest_voting(self):
self.votecast_db.unsubscribe(bin2str(self.mypermid))
self.assertEqual(
self.votecast_db.getVote(bin2str(self.mypermid),
bin2str(self.hispermid)), None)
#print >> sys.stderr, self.votecast_db.getAll()
self.votecast_db.spam(bin2str(self.mypermid))
self.assertEqual(
self.votecast_db.getVote(bin2str(self.mypermid),
bin2str(self.hispermid)), -1)
#print >> sys.stderr, self.votecast_db.getAll()
self.votecast_db.subscribe(bin2str(self.mypermid))
self.assertEqual(
self.votecast_db.getVote(bin2str(self.mypermid),
bin2str(self.hispermid)), 2)
#print >> sys.stderr, self.votecast_db.getAll()
self.votecast_db.unsubscribe(bin2str(self.mypermid))
self.assertEqual(
self.votecast_db.getVote(bin2str(self.mypermid),
bin2str(self.hispermid)), None)
#print >> sys.stderr, self.votecast_db.getAll()
self.votecast_db.spam(bin2str(self.mypermid))
self.assertEqual(
self.votecast_db.getVote(bin2str(self.mypermid),
bin2str(self.hispermid)), -1)
