def __init__(self, data_handler, overlay_bridge, session, buddycast_interval_function, log = '', dnsindb = None):
""" Returns an instance of this class """
if ChannelCastCore.__single:
raise RuntimeError, "ChannelCastCore is singleton"
ChannelCastCore.__single = self
#Keep reference to interval-function of BuddycastFactory
self.interval = buddycast_interval_function
self.data_handler = data_handler
self.dnsindb = dnsindb
self.log = log
self.overlay_bridge = overlay_bridge
self.channelcastdb = ChannelCastDBHandler.getInstance()
self.votecastdb = VoteCastDBHandler.getInstance()
self.rtorrent_handler = RemoteTorrentHandler.getInstance()
self.my_permid = self.channelcastdb.my_permid
self.session = session
self.network_delay = 30
#Reference to buddycast-core, set by the buddycast-core (as it is created by the
#buddycast-factory after calling this constructor).
self.buddycast_core = None
#Extend logging with ChannelCast-messages and status
if self.log:
self.overlay_log = OverlayLogger.getInstance(self.log)
self.dnsindb = self.data_handler.get_dns_from_peerdb
self.notifier = Notifier.getInstance()
self.metadataDbHandler = MetadataDBHandler.getInstance()
#subtitlesHandler = SubtitlesHandler.getInstance()
subtitleSupport = SubtitlesSupport.getInstance()
# better if an instance of RMDInterceptor was provided from the
# outside
self.peersHaveManger = PeersHaveManager.getInstance()
if not self.peersHaveManger.isRegistered():
self.peersHaveManger.register(self.metadataDbHandler, self.overlay_bridge)
self.richMetadataInterceptor = RichMetadataInterceptor(self.metadataDbHandler,self.votecastdb,
self.my_permid, subtitleSupport, self.peersHaveManger,
self.notifier)
def setUp(self):
self.metadataDBHandler = MockMetadataDBHandler()
self.voteCastDBHandler = MockVoteCastHandler()
self.subSupp = MockSubtitlesHandler()
self.my_permid_and_keypair = generatePermIds(1)[0]
self.advertisedChannelIds = generatePermIds(CHANNELCAST_NUM_OF_ENTRIES)
self.advertisedInfohash = generateInfohashes(CHANNELCAST_NUM_OF_ENTRIES)
self.peersHaveMngr = MockPeersHaveMngr()
self.channelcastMsg = dict()
for i in range(CHANNELCAST_NUM_OF_ENTRIES):
signature, msg = generateChannelCastEntry(self.advertisedChannelIds[i][0],
self.advertisedInfohash[i],
self.advertisedChannelIds[i][1])
self.channelcastMsg[signature] = msg
self.undertest = RichMetadataInterceptor(self.metadataDBHandler, self.voteCastDBHandler,
self.my_permid_and_keypair[0], self.subSupp,
self.peersHaveMngr)
self.metadataDBHandler.nextKeypair = self.advertisedChannelIds[0][1]
class ChannelCastCore:
__single = None
TESTASSERVER = False # for unit testing
def __init__(self, data_handler, overlay_bridge, session, buddycast_interval_function, log = '', dnsindb = None):
""" Returns an instance of this class """
if ChannelCastCore.__single:
raise RuntimeError, "ChannelCastCore is singleton"
ChannelCastCore.__single = self
#Keep reference to interval-function of BuddycastFactory
self.interval = buddycast_interval_function
self.data_handler = data_handler
self.dnsindb = dnsindb
self.log = log
self.overlay_bridge = overlay_bridge
self.channelcastdb = ChannelCastDBHandler.getInstance()
self.votecastdb = VoteCastDBHandler.getInstance()
self.rtorrent_handler = RemoteTorrentHandler.getInstance()
self.my_permid = self.channelcastdb.my_permid
self.session = session
self.network_delay = 30
#Reference to buddycast-core, set by the buddycast-core (as it is created by the
#buddycast-factory after calling this constructor).
self.buddycast_core = None
#Extend logging with ChannelCast-messages and status
if self.log:
self.overlay_log = OverlayLogger.getInstance(self.log)
self.dnsindb = self.data_handler.get_dns_from_peerdb
self.notifier = Notifier.getInstance()
self.metadataDbHandler = MetadataDBHandler.getInstance()
#subtitlesHandler = SubtitlesHandler.getInstance()
subtitleSupport = SubtitlesSupport.getInstance()
# better if an instance of RMDInterceptor was provided from the
# outside
self.peersHaveManger = PeersHaveManager.getInstance()
if not self.peersHaveManger.isRegistered():
self.peersHaveManger.register(self.metadataDbHandler, self.overlay_bridge)
self.richMetadataInterceptor = RichMetadataInterceptor(self.metadataDbHandler,self.votecastdb,
self.my_permid, subtitleSupport, self.peersHaveManger,
self.notifier)
def initialized(self):
return self.buddycast_core is not None
def getInstance(*args, **kw):
if ChannelCastCore.__single is None:
ChannelCastCore(*args, **kw)
return ChannelCastCore.__single
getInstance = staticmethod(getInstance)
def createAndSendChannelCastMessage(self, target_permid, selversion):
""" Create and send a ChannelCast Message """
# ChannelCast feature starts from eleventh version; hence, do not send to lower version peers
# Arno, 2010-02-05: v12 uses a different on-the-wire format, ignore those.
# Andrea, 2010-04-08: sending the "old-style" channelcast message to older
# peers, and enriched channelcast messages to new versions, for full backward
# compatibility
if selversion < OLPROTO_VER_THIRTEENTH:
if DEBUG:
print >> sys.stderr, "channelcast: Do not send to lower version peer:", selversion
return
# 3/5/2010 Andrea: adding the destination parameters to createChannelCastMessage for
# logging reasons only. When logging will be disabled, that parameter will
# become useless
channelcast_data = self.createChannelCastMessage(selversion, target_permid)
if channelcast_data is None or len(channelcast_data)==0:
if DEBUG:
print >>sys.stderr, "channelcast: No channels there.. hence we do not send"
return
channelcast_msg = bencode(channelcast_data)
if self.log:
dns = self.dnsindb(target_permid)
if dns:
ip,port = dns
MSG_ID = "CHANNELCAST"
msg = repr(channelcast_data)
self.overlay_log('SEND_MSG', ip, port, show_permid(target_permid), selversion, MSG_ID, msg)
data = CHANNELCAST + channelcast_msg
self.overlay_bridge.send(target_permid, data, self.channelCastSendCallback)
#if DEBUG: print >> sys.stderr, "channelcast: Sent channelcastmsg",repr(channelcast_data)
def createChannelCastMessage(self, selversion, dest_permid=None):
"""
Create a ChannelCast Message
@param selversion: the protocol version of the destination
@param dest_permid: the destination of the message. Actually this parameter is not really needed. If
not none, it is used for logging purposes only
@return a channelcast message, possibly enrich with rich metadata content in the
case selversion is sufficiently high
"""
# 09-04-2010 Andrea: I addedd the selversion param, to intercept and modify
# the ChannelCast message contents if the protocol version allows rich metadata
# enrichment
if DEBUG:
print >> sys.stderr, "channelcast: Creating channelcastmsg..."
hits = self.channelcastdb.getRecentAndRandomTorrents(NUM_OWN_RECENT_TORRENTS,NUM_OWN_RANDOM_TORRENTS,NUM_OTHERS_RECENT_TORRENTS,NUM_OTHERS_RECENT_TORRENTS)
# 3/5/2010 Andrea:
# hits is of the form: [(mod_id, mod_name, infohash, torrenthash, torrent_name, time_stamp, signature)]
# adding the destination parameter to buildChannelcastMessageFrom Hits for
# logging reasons only. When logging will be disabled, that parameter will
# become useless
d = self.buildChannelcastMessageFromHits(hits, selversion, dest_permid)
# #assert validChannelCastMsg(d)
return d
def channelCastSendCallback(self, exc, target_permid, other=0):
if DEBUG:
if exc is None:
print >> sys.stderr,"channelcast: *** msg was sent successfully to peer", show_permid_short(target_permid)
else:
print >> sys.stderr, "channelcast: *** warning - error in sending msg to", show_permid_short(target_permid), exc
def gotChannelCastMessage(self, recv_msg, sender_permid, selversion):
""" Receive and handle a ChannelCast message """
# ChannelCast feature starts from eleventh version; hence, do not receive from lower version peers
# Arno, 2010-02-05: v12 uses a different on-the-wire format, ignore those.
# Andrea: 2010-04-08: v14 can still receive v13 channelcast messages
if selversion < OLPROTO_VER_THIRTEENTH:
if DEBUG:
print >> sys.stderr, "channelcast: Do not receive from lower version peer:", selversion
return True
if DEBUG:
print >> sys.stderr,'channelcast: Received a msg from ', show_permid_short(sender_permid)
print >> sys.stderr,"channelcast: my_permid=", show_permid_short(self.my_permid)
if not sender_permid or sender_permid == self.my_permid:
if DEBUG:
print >> sys.stderr, "channelcast: warning - got channelcastMsg from a None/Self peer", \
show_permid_short(sender_permid), recv_msg
return False
#if len(recv_msg) > self.max_length:
# if DEBUG:
# print >> sys.stderr, "channelcast: warning - got large channelCastHaveMsg", len(recv_msg)
# return False
channelcast_data = {}
try:
channelcast_data = bdecode(recv_msg)
except:
print >> sys.stderr, "channelcast: warning, invalid bencoded data"
return False
# check message-structure
if not validChannelCastMsg(channelcast_data):
print >> sys.stderr, "channelcast: invalid channelcast_message"
return False
# 19/02/10 Boudewijn: validChannelCastMsg passes when
# PUBLISHER_NAME and TORRENTNAME are either string or
# unicode-string. However, all further code requires that
# these are unicode!
for ch in channelcast_data.values():
if isinstance(ch["publisher_name"], str):
ch["publisher_name"] = str2unicode(ch["publisher_name"])
if isinstance(ch["torrentname"], str):
ch["torrentname"] = str2unicode(ch["torrentname"])
self.handleChannelCastMsg(sender_permid, channelcast_data)
#Log RECV_MSG of uncompressed message
if self.log:
dns = self.dnsindb(sender_permid)
if dns:
ip,port = dns
MSG_ID = "CHANNELCAST"
# 08/04/10 Andrea: representing the whole channelcast + metadata message
msg = repr(channelcast_data)
self.overlay_log('RECV_MSG', ip, port, show_permid(sender_permid), selversion, MSG_ID, msg)
if self.TESTASSERVER:
self.createAndSendChannelCastMessage(sender_permid, selversion)
return True
def handleChannelCastMsg(self, sender_permid, data):
self._updateChannelInternal(sender_permid, None, data)
def updateChannel(self,query_permid, query, hits):
"""
This function is called when there is a reply from remote peer regarding updating of a channel
@param query_permid: the peer who returned the results
@param query: the query string (None if this is not the results of a query)
@param hits: details of all matching results related to the query
"""
if DEBUG:
print >> sys.stderr, "channelcast: sending message to", bin2str(query_permid), query, len(hits)
return self._updateChannelInternal(query_permid, query, hits)
def _updateChannelInternal(self, query_permid, query, hits):
dictOfAdditions = dict()
if len(hits) > 0:
# a single read from the db is more efficient
all_spam_channels = self.votecastdb.getPublishersWithNegVote(bin2str(self.session.get_permid()))
for k,v in hits.items():
#check if the record belongs to a channel who we have "reported spam" (negative vote)
if bin2str(v['publisher_id']) in all_spam_channels:
# if so, ignore the incoming record
continue
# make everything into "string" format, if "binary"
hit = (bin2str(v['publisher_id']),v['publisher_name'],bin2str(v['infohash']),bin2str(v['torrenthash']),v['torrentname'],v['time_stamp'],bin2str(k))
# 29/06/11 boudewijn: note that k contains the signature (whatever that is) and NOT
# the infohash. this makes this result incompatible with
# SearchGridManager.getRemoteHits(). Hence these hits are NOT propagated there
# anymore.
dictOfAdditions[k] = hit
# Arno, 2010-06-11: We're on the OverlayThread
self._updateChannelcastDB(query_permid, query, hits, dictOfAdditions.values())
return dictOfAdditions
def _updateChannelcastDB(self, query_permid, query, hits, listOfAdditions):
if DEBUG:
print >> sys.stderr, "channelcast: updating channelcastdb", query, len(hits)
publisher_ids = Set()
infohashes = Set()
for hit in listOfAdditions:
publisher_ids.add(hit[0])
infohashes.add(str2bin(hit[2]))
if query and query.startswith('CHANNEL p') and len(publisher_ids) == 1:
publisher_id = publisher_ids.pop()
publisher_ids.add(publisher_id)
nr_torrents = self.channelcastdb.getNrTorrentsInChannel(publisher_id)
if len(infohashes) > nr_torrents:
if len(infohashes) > 50 and len(infohashes) > nr_torrents +1: #peer not behaving according to spec, ignoring
if DEBUG:
print >> sys.stderr, "channelcast: peer not behaving according to spec, ignoring",len(infohashes), show_permid(query_permid)
return
#if my channel, never remove all currently received
if bin2str(self.session.get_permid()) != publisher_id:
self.channelcastdb.deleteTorrentsFromPublisherId(str2bin(publisher_id))
if DEBUG:
print >> sys.stderr, 'Received channelcast message with %d hashes'%len(infohashes), show_permid(query_permid)
else:
#ignore all my favorites, randomness will cause problems with timeframe
my_favorites = self.votecastdb.getPublishersWithPosVote(bin2str(self.session.get_permid()))
#filter listOfAdditions
listOfAdditions = [hit for hit in listOfAdditions if hit[0] not in my_favorites]
#request channeltimeframes for subscribed channels
for publisher_id in my_favorites:
if publisher_id in publisher_ids:
self.updateAChannel(publisher_id, [query_permid])
publisher_ids.remove(publisher_id) #filter publisher_ids
#08/04/10: Andrea: processing rich metadata part.
self.richMetadataInterceptor.handleRMetadata(query_permid, hits, fromQuery = query is not None)
self.channelcastdb.addTorrents(listOfAdditions)
missing_infohashes = {}
for publisher_id in publisher_ids:
for infohash in self.channelcastdb.selectTorrentsToCollect(publisher_id):
missing_infohashes[str2bin(infohash[0])] = publisher_id
def notify(publisher_id):
self.notifier.notify(NTFY_CHANNELCAST, NTFY_UPDATE, publisher_id)
for infohash, publisher_id in missing_infohashes.iteritems():
if infohash in infohashes:
self.rtorrent_handler.download_torrent(query_permid, infohash, lambda infohash, metadata, filename: notify(publisher_id) ,2)
else:
self.rtorrent_handler.download_torrent(query_permid, infohash, lambda infohash, metadata, filename: notify(publisher_id) ,3)
def updateMySubscribedChannels(self):
def update(permids):
permid = permids.pop()
self.updateAChannel(permid)
if len(permids) > 0:
self.overlay_bridge.add_task(lambda: update(permids), 20)
subscribed_channels = self.channelcastdb.getMySubscribedChannels()
permids = [values[0] for values in subscribed_channels]
if len(permids) > 0:
update(permids)
self.overlay_bridge.add_task(self.updateMySubscribedChannels, RELOAD_FREQUENCY)
def updateAChannel(self, publisher_id, peers = None, timeframe = None):
if peers == None:
peers = RemoteQueryMsgHandler.getInstance().get_connected_peers(OLPROTO_VER_FOURTEENTH)
else:
#use the specified peers list, small problem we dont have the selversion
#use oversion 14, eventually RemoteQueryMsgHandler will convert the query for oversion13 peers
peers = [(permid, OLPROTO_VER_FOURTEENTH) for permid in peers]
shuffle(peers)
# Create separate thread which does all the requesting
self.overlay_bridge.add_task(lambda: self._sequentialQueryPeers(publisher_id, peers, timeframe))
def _sequentialQueryPeers(self, publisher_id, peers, timeframe = None):
def seqtimeout(permid):
if peers and permid == peers[0][0]:
peers.pop(0)
dorequest()
def seqcallback(query_permid, query, hits):
self.updateChannel(query_permid, query, hits)
if peers and query_permid == peers[0][0]:
peers.pop(0)
dorequest()
def dorequest():
if peers:
permid, selversion = peers[0]
q = "CHANNEL p "+publisher_id
if timeframe:
record = timeframe
else:
record = self.channelcastdb.getTimeframeForChannel(publisher_id)
if record:
q+= " "+" ".join(map(str,record))
self.session.query_peers(q,[permid],usercallback = seqcallback)
self.overlay_bridge.add_task(lambda: seqtimeout(permid), 30)
peers = peers[:]
dorequest()
def buildChannelcastMessageFromHits(self, hits, selversion, dest_permid=None, fromQuery=False):
'''
Creates a channelcast message from database hits.
This method is used to create channel results both when a channelcast message
is created in the "normal" buddycast epidemic protocol, and when a remote
query for channels arrives and is processed. It substitutes a lot of duplicated
code in the old versions.
@param hits: a tuple (publisher_id, publisher_name, infohash,
torrenthash, torrentname, time_stamp, signature) representing
a channelcast entry in the db
@param selversion: the protocol version of the destination
@param dest_permid: the permid of the destination of the message. Actually this parameter
is used for logging purposes only, when not None. If None, nothing
bad happens.
'''
# 09-04-2010 Andrea : I introduced this separate method because this code was
# duplicated in RemoteQueryMessageHandler
enrichWithMetadata = False
if selversion >= OLPROTO_VER_FOURTEENTH:
enrichWithMetadata = True
if DEBUG:
print >> sys.stderr, "channelcast: creating enriched messages"\
"since peer has version: ", selversion
d = {}
for hit in hits:
# ARNOUNICODE: temp fixes until data is sent not Base64-encoded
# 08/04/10 Andrea: I substituted the keys with constnats, otherwise a change here
# would break my code in the RichMetadataInterceptor
r = {}
r['publisher_id'] = str(hit[0]) # ARNOUNICODE: must be str
r['publisher_name'] = hit[1].encode("UTF-8") # ARNOUNICODE: must be explicitly UTF-8 encoded
r['infohash'] = str(hit[2]) # ARNOUNICODE: must be str
r['torrenthash'] = str(hit[3]) # ARNOUNICODE: must be str
r['torrentname'] = hit[4].encode("UTF-8") # ARNOUNICODE: must be explicitly UTF-8 encoded
r['time_stamp'] = int(hit[5])
# hit[6]: signature, which is unique for any torrent published by a user
signature = hit[6]
d[signature] = r
# 08/04/10 Andrea: intercepting a channelcast message and enriching it with
# subtitles information
# 3/5/2010 Andrea: adding the destination parameter to addRichMetadataContent for
# logging reasons only. When logging will be disabled, that parameter will
# become useless
if enrichWithMetadata:
d = self.richMetadataInterceptor.addRichMetadataContent(d, dest_permid, fromQuery)
return d
class TestRichMetadataInterceptor(unittest.TestCase):
def setUp(self):
self.metadataDBHandler = MockMetadataDBHandler()
self.voteCastDBHandler = MockVoteCastHandler()
self.subSupp = MockSubtitlesHandler()
self.my_permid_and_keypair = generatePermIds(1)[0]
self.advertisedChannelIds = generatePermIds(CHANNELCAST_NUM_OF_ENTRIES)
self.advertisedInfohash = generateInfohashes(CHANNELCAST_NUM_OF_ENTRIES)
self.peersHaveMngr = MockPeersHaveMngr()
self.channelcastMsg = dict()
for i in range(CHANNELCAST_NUM_OF_ENTRIES):
signature, msg = generateChannelCastEntry(self.advertisedChannelIds[i][0],
self.advertisedInfohash[i],
self.advertisedChannelIds[i][1])
self.channelcastMsg[signature] = msg
self.undertest = RichMetadataInterceptor(self.metadataDBHandler, self.voteCastDBHandler,
self.my_permid_and_keypair[0], self.subSupp,
self.peersHaveMngr)
self.metadataDBHandler.nextKeypair = self.advertisedChannelIds[0][1]
def testAddRMDContentNoContent(self):
self.metadataDBHandler.returnMetadata = False
newMessage = self.undertest.addRichMetadataContent(self.channelcastMsg)
#message should be left untouched
self.assertEquals(self.channelcastMsg,newMessage)
#check if the the db handler is called
self.assertEquals(CHANNELCAST_NUM_OF_ENTRIES,self.metadataDBHandler.getMetadataCount)
for i in range(CHANNELCAST_NUM_OF_ENTRIES):
self.assertTrue(
(self.advertisedChannelIds[i][0], self.advertisedInfohash[i])
in self.metadataDBHandler.getMetadataParametesHistory)
def testAddRichMetadataContentSomeContent(self):
self.metadataDBHandler.returnMetadata = True
newMessage = self.undertest.addRichMetadataContent(self.channelcastMsg)
#message should have been changed
self.assertNotEquals(self.channelcastMsg,newMessage)
for item in newMessage.itervalues():
#check the contents of the modified message
self.assertTrue('rich_metadata' in item.keys())
#description,bitmask,timestamp,listofchecksums, signature
self.assertEquals(6, len(item['rich_metadata']))
def test_splitChnAndRmdNoContent(self):
self.metadataDBHandler.returnMetadata = False
newMessage = self.undertest.addRichMetadataContent(self.channelcastMsg)
listOfmetadata = \
self.undertest._splitChannelcastAndRichMetadataContents(newMessage)
self.assertEquals(([],[]),listOfmetadata)
def test_splitChnAndRmdSomeContent(self):
self.metadataDBHandler.returnMetadata = True
newMessage = self.undertest.addRichMetadataContent(self.channelcastMsg)
listOfmetadata = \
self.undertest._splitChannelcastAndRichMetadataContents(newMessage)
listOfmetadata = listOfmetadata[0]
self.assertEquals(2,len(listOfmetadata))
for dto in listOfmetadata:
self.assertTrue(isinstance(dto[0], MetadataDTO))
def testHandleRMetadata(self):
self.metadataDBHandler.returnMetadata = True
newMessage = self.undertest.addRichMetadataContent(self.channelcastMsg)
#it will result that i am a subscriber for any channel
self.voteCastDBHandler.nextVoteValue = 2
self.undertest.handleRMetadata(self.advertisedChannelIds[0][0],
newMessage)
self.assertEquals(2, self.metadataDBHandler.insertMetadataCount)
self.assertEquals(2, self.subSupp.retrieveMultipleCount)
pass