def subtest_receptionOfSUBSTwoRequestsOneAvailable(self):
"""
Asking for two subtitles while the recipent of the request has only one.
The response should contain only the one available subtitle content,
plus a bitmask that reflects the contents of the response.
"""
print >> sys.stderr, "test: test_subtitles_msgs_2_1 -----------------------"
ol_conn = OLConnection(self.my_keypair, 'localhost', self.hisport)
bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(
['nld', 'eng'])
binmask = utilities.uintToBinaryString(bitmask, length=4)
request = GET_SUBS + \
bencode((
self.anotherpermid,
self.testInfohash,
binmask
))
subshandler = SubtitlesHandler()
subshandler.register(ol_conn, self.richMetadata_db, self.session)
ol_conn.send(request)
subs_data = ol_conn.recv()
self.assertEquals(SUBS, subs_data[0])
data = bdecode(subs_data[1:])
print >> sys.stderr, "test: subtitles_messages : received SUBS repsonse: ", data
#check on the format of the response
self.assertTrue(isinstance(data, list))
self.assertEquals(4, len(data)) # for fields
self.assertEquals(self.mdto.channel, data[0])
self.assertEquals(self.mdto.infohash, data[1])
#the receiver had only one of the two requested subtitles
# so I expect a different bitmask
bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(
['nld'])
expectedBinarymask = utilities.uintToBinaryString(bitmask, length=4)
self.assertEquals(expectedBinarymask, data[2])
self.assertTrue(isinstance(data[3], list))
self.assertEquals(1, len(data[3]))
with codecs.open(self.sub1, "rb", "utf-8") as sub:
expectedContents = sub.read()
self.assertEquals(expectedContents, data[3][0])
ol_conn.close()
print >> sys.stderr, "test: subtitles_messages: received content is valid."
print >> sys.stderr, "End of test_subtitles_msgs_2_1 test --------------------"
def subtest_receptionOfSUBSTwoRequestsOneAvailable(self):
"""
Asking for two subtitles while the recipent of the request has only one.
The response should contain only the one available subtitle content,
plus a bitmask that reflects the contents of the response.
"""
print >> sys.stderr, "test: test_subtitles_msgs_2_1 -----------------------"
ol_conn = OLConnection(self.my_keypair,'localhost',self.hisport)
bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(['nld','eng'])
binmask = utilities.uintToBinaryString(bitmask, length=4)
request = GET_SUBS + \
bencode((
self.anotherpermid,
self.testInfohash,
binmask
))
subshandler = SubtitlesHandler()
subshandler.register(ol_conn, self.richMetadata_db, self.session)
ol_conn.send(request)
subs_data = ol_conn.recv()
self.assertEquals(SUBS, subs_data[0])
data = bdecode(subs_data[1:])
print >> sys.stderr, "test: subtitles_messages : received SUBS repsonse: ", data
#check on the format of the response
self.assertTrue(isinstance(data,list))
self.assertEquals(4, len(data)) # for fields
self.assertEquals(self.mdto.channel,data[0])
self.assertEquals(self.mdto.infohash, data[1])
#the receiver had only one of the two requested subtitles
# so I expect a different bitmask
bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(['nld'])
expectedBinarymask = utilities.uintToBinaryString(bitmask, length=4)
self.assertEquals(expectedBinarymask, data[2])
self.assertTrue(isinstance(data[3],list))
self.assertEquals(1, len(data[3]))
with codecs.open(self.sub1, "rb", "utf-8") as sub:
expectedContents = sub.read()
self.assertEquals(expectedContents, data[3][0])
ol_conn.close()
print >> sys.stderr, "test: subtitles_messages: received content is valid."
print >> sys.stderr, "End of test_subtitles_msgs_2_1 test --------------------"
def subtest_receptionOfSUBS(self):
'''
Asking for the single available subtitle. The response should be
a valid SUBS message containing its contents
'''
print >> sys.stderr, "test: test_subtitles_msgs_1_1 -----------------------"
ol_conn = OLConnection(self.my_keypair, 'localhost', self.hisport)
bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(
['nld'])
binmask = utilities.uintToBinaryString(bitmask, length=4)
request = GET_SUBS + \
bencode((
self.anotherpermid,
self.testInfohash,
binmask
))
subshandler = SubtitlesHandler()
subshandler.register(ol_conn, self.richMetadata_db, self.session)
ol_conn.send(request)
subs_data = ol_conn.recv()
print >> sys.stderr, "test: subtitles_messages : received SUBS response: len", len(
subs_data)
self.assertEquals(SUBS, subs_data[0])
data = bdecode(subs_data[1:])
print >> sys.stderr, "test: subtitles_messages : received SUBS response: ", data
#check on the format of the response
self.assertTrue(isinstance(data, list))
self.assertEquals(4, len(data)) # for fields
self.assertEquals(self.mdto.channel, data[0])
self.assertEquals(self.mdto.infohash, data[1])
self.assertEquals(binmask, data[2])
self.assertTrue(isinstance(data[3], list))
self.assertEquals(1, len(data[3]))
with codecs.open(self.sub1, "rb", "utf-8") as sub:
expectedContents = sub.read()
self.assertEquals(expectedContents, data[3][0])
ol_conn.close()
print >> sys.stderr, "test: subtitles_messages: received content is valid."
print >> sys.stderr, "End of test_subtitles_msgs_1_1 test --------------------"
def subtest_receptionOfSUBS(self):
'''
Asking for the single available subtitle. The response should be
a valid SUBS message containing its contents
'''
print >> sys.stderr, "test: test_subtitles_msgs_1_1 -----------------------"
ol_conn = OLConnection(self.my_keypair,'localhost',self.hisport)
bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(['nld'])
binmask = utilities.uintToBinaryString(bitmask, length=4)
request = GET_SUBS + \
bencode((
self.anotherpermid,
self.testInfohash,
binmask
))
subshandler = SubtitlesHandler()
subshandler.register(ol_conn, self.richMetadata_db, self.session)
ol_conn.send(request)
subs_data = ol_conn.recv()
print >> sys.stderr, "test: subtitles_messages : received SUBS response: len",len(subs_data)
self.assertEquals(SUBS, subs_data[0])
data = bdecode(subs_data[1:])
print >> sys.stderr, "test: subtitles_messages : received SUBS response: ", data
#check on the format of the response
self.assertTrue(isinstance(data,list))
self.assertEquals(4, len(data)) # for fields
self.assertEquals(self.mdto.channel,data[0])
self.assertEquals(self.mdto.infohash, data[1])
self.assertEquals(binmask, data[2])
self.assertTrue(isinstance(data[3],list))
self.assertEquals(1, len(data[3]))
with codecs.open(self.sub1, "rb", "utf-8") as sub:
expectedContents = sub.read()
self.assertEquals(expectedContents, data[3][0])
ol_conn.close()
print >> sys.stderr, "test: subtitles_messages: received content is valid."
print >> sys.stderr, "End of test_subtitles_msgs_1_1 test --------------------"
def set_subtitles_upload_rate(self, value):
with self.sesslock:
SubtitlesHandler.getInstance().setUploadRate(value)
SessionConfigInterface.set_subtitles_uploade_rate(self, value)
def setUp(self):
self._session = MockSession()
self.ol_bridge = MockOverlayBridge()
self.rmdDBHandler = MockMetadataDBHandler()
self.underTest = SubtitlesHandler()
class TestSubtitlesHandler(unittest.TestCase):
def setUp(self):
self._session = MockSession()
self.ol_bridge = MockOverlayBridge()
self.rmdDBHandler = MockMetadataDBHandler()
self.underTest = SubtitlesHandler()
def tearDown(self):
self.ol_bridge = None
#cleanup the mess in collected dir
if self.underTest.subs_dir is not None:
toDelete = [ os.path.join(self.underTest.subs_dir, entry) for entry in os.listdir(self.underTest.subs_dir)\
if entry.endswith(".srt")]
for xfile in toDelete:
if os.path.isfile(xfile):
os.remove(xfile)
def testRegisterStuff(self):
self.underTest.register(self.ol_bridge, self.rmdDBHandler,
self._session)
self.assertTrue(self.underTest.registered)
self.assertTrue(self.ol_bridge is self.underTest.overlay_bridge)
self.assertTrue(self.rmdDBHandler is self.underTest.subtitlesDb)
expectedPath = os.path.join(os.path.abspath(os.path.curdir),
self._session.get_state_dir(),
"subtitles_collecting_dir")
self.assertEqual(os.path.normpath(expectedPath),
self.underTest.subs_dir)
# preaparing the mock msg handler
self.mockMsgHandler = MockSubsMsgHander()
self.underTest._subsMsgHndlr = self.mockMsgHandler
def testGetSubtitlesFileRelativeName(self):
#subtitles filenames are build from the sha1 hash
#of the triple (channel_id, infohash, langCode)
name = getSubtitleFileRelativeName(testChannelId, testInfohash, "rus")
hasher = sha()
for value in (testChannelId, testInfohash, "rus"):
hasher.update(value)
self.assertEquals(hasher.hexdigest() + ".srt", name)
def testSendSubtitleRequestSimple(self):
self.testRegisterStuff()
self.underTest.sendSubtitleRequest(testDestPermId, testChannelId,
testInfohash, ["zho", "rus"], None,
OLPROTO_VER_FOURTEENTH)
self.assertEquals(1, self.mockMsgHandler.sendReqCount)
def testReceibedGETSUBSNoSubs(self):
self.testRegisterStuff()
self.underTest.receivedSubsRequest(
testDestPermId, (testDestPermId, testChannelId, ["ita", "nld"]),
OLPROTO_VER_FOURTEENTH)
self.assertEquals(0, self.mockMsgHandler.sendResCount)
def testReceivedGETSUBSTwoSubs(self):
self.testRegisterStuff()
self.underTest.receivedSubsRequest(
testDestPermId, (testChannelId, testInfohash, ["eng", "rus"]),
OLPROTO_VER_FOURTEENTH)
eng = u"this is a fake srt\n\nonly needed for testing\n\ncheers :)\n\n"
rus = eng + \
u"asfasgb sagas gba\n\nasfasfas 24214 a \nThe checksum is different yuppy!\n\n"
self.assertEquals(1, self.mockMsgHandler.sendResCount)
destination, response, selversion = self.mockMsgHandler.sendResParams[
0]
self.assertEquals(testDestPermId, destination)
self.assertEquals(OLPROTO_VER_FOURTEENTH, selversion)
channelId, infohash, contentsList = response
self.assertEquals(testChannelId, channelId)
self.assertEquals(testInfohash, infohash)
self.assertEquals(contentsList, {"eng": eng, "rus": rus})
def testReceivedSUBSMessage(self):
self.testRegisterStuff()
languages = ["eng", "rus"]
zho = u"Subtitle Content 1"
kor = u"Subtitle Content 2"
contentsDict = {"eng": zho, "rus": kor}
msg = (testChannelId, testInfohash, contentsDict)
simpleCallback = lambda x: x
bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(
["eng", "rus"])
self.underTest.receivedSubsResponse(testDestPermId, msg,
[(simpleCallback, bitmask)],
OLPROTO_VER_FOURTEENTH)
#self.assertEquals(languages,callbackParams)
expectedFilename1 = getSubtitleFileRelativeName(
testChannelId, testInfohash, "eng")
expectedFilename2 = getSubtitleFileRelativeName(
testChannelId, testInfohash, "rus")
expectedPath1 = os.path.join(self._session.get_state_dir(),
self.underTest.subs_dir,
expectedFilename1)
expectedPath2 = os.path.join(self._session.get_state_dir(),
self.underTest.subs_dir,
expectedFilename2)
self.assertTrue(os.path.isfile(expectedPath1))
self.assertTrue(os.path.isfile(expectedPath2))
with codecs.open(expectedPath1, "rb", "utf-8") as file1:
content1 = file1.read()
self.assertEquals(zho, content1)
with codecs.open(expectedPath2, "rb", "utf-8") as file2:
content2 = file2.read()
self.assertEquals(kor, content2)
self.assertEquals(1, self.ol_bridge.add_task_count)
params = self.ol_bridge.add_taskParametersHistory[0]
#calling the lambda
val = params[0]()
self.assertEquals(languages, val)
def test_saveSubsOnDisk(self):
self.testRegisterStuff()
subContent = u"Test Content\nFor a pseudo subtitle file\n\nYo!\n"
self.underTest._saveSubOnDisk(testChannelId, testInfohash, "eng",
subContent)
expectedFilename = getSubtitleFileRelativeName(testChannelId,
testInfohash, "eng")
expectedPath = os.path.join(self.underTest.subs_dir, expectedFilename)
self.assertTrue(os.path.isfile(expectedPath))
#check the contents
with codecs.open(expectedPath, "rb", "utf-8") as file:
cont = file.read()
self.assertEquals(subContent, cont)
##now the file exists. If a new subtitle is saved for the same
# channel, infohash, lang but with a different content
# the old one should be overwritten
newContent = u"I'm the new content! I shall win over the old one!"
self.underTest._saveSubOnDisk(testChannelId, testInfohash, "eng",
newContent)
self.assertTrue(os.path.isfile(expectedPath))
#check the contents
with codecs.open(expectedPath, "rb", "utf-8") as file:
cont = file.read()
self.assertEquals(newContent, cont)
def register(self, overlay_bridge, session, launchmany, config, requestPolicy):
self.overlay_bridge = overlay_bridge
self.launchmany = launchmany
self.requestPolicy = requestPolicy
self.text_mode = config.has_key('text_mode')
# OverlayApps gets all messages, and demultiplexes
overlay_bridge.register_recv_callback(self.handleMessage)
overlay_bridge.register_conns_callback(self.handleConnection)
# Arno, 2010-01-28: Start with crawler support, other mods depend on
# that, e.g. BuddyCast
i_am_crawler = False
if config['crawler']:
crawler = Crawler.get_instance(session)
self.register_msg_handler([CRAWLER_REQUEST], crawler.handle_request)
database_crawler = DatabaseCrawler.get_instance()
crawler.register_message_handler(CRAWLER_DATABASE_QUERY, database_crawler.handle_crawler_request, database_crawler.handle_crawler_reply)
# seeding_stats_crawler = SeedingStatsCrawler.get_instance()
# crawler.register_message_handler(CRAWLER_SEEDINGSTATS_QUERY, seeding_stats_crawler.handle_crawler_request, seeding_stats_crawler.handle_crawler_reply)
friendship_crawler = FriendshipCrawler.get_instance(session)
crawler.register_message_handler(CRAWLER_FRIENDSHIP_STATS, friendship_crawler.handle_crawler_request, friendship_crawler.handle_crawler_reply)
natcheck_handler = NatCheckMsgHandler.getInstance()
natcheck_handler.register(launchmany)
crawler.register_message_handler(CRAWLER_NATCHECK, natcheck_handler.gotDoNatCheckMessage, natcheck_handler.gotNatCheckReplyMessage)
crawler.register_message_handler(CRAWLER_NATTRAVERSAL, natcheck_handler.gotUdpConnectRequest, natcheck_handler.gotUdpConnectReply)
# videoplayback_crawler = VideoPlaybackCrawler.get_instance()
# crawler.register_message_handler(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, videoplayback_crawler.handle_event_crawler_request, videoplayback_crawler.handle_event_crawler_reply)
# crawler.register_message_handler(CRAWLER_VIDEOPLAYBACK_INFO_QUERY, videoplayback_crawler.handle_info_crawler_request, videoplayback_crawler.handle_info_crawler_reply)
repex_crawler = RepexCrawler.get_instance(session)
crawler.register_message_handler(CRAWLER_REPEX_QUERY, repex_crawler.handle_crawler_request, repex_crawler.handle_crawler_reply)
puncture_crawler = PunctureCrawler.get_instance()
crawler.register_message_handler(CRAWLER_PUNCTURE_QUERY, puncture_crawler.handle_crawler_request, puncture_crawler.handle_crawler_reply)
channel_crawler = ChannelCrawler.get_instance()
crawler.register_message_handler(CRAWLER_CHANNEL_QUERY, channel_crawler.handle_crawler_request, channel_crawler.handle_crawler_reply)
usereventlog_crawler = UserEventLogCrawler.get_instance()
crawler.register_message_handler(CRAWLER_USEREVENTLOG_QUERY, usereventlog_crawler.handle_crawler_request, usereventlog_crawler.handle_crawler_reply)
if crawler.am_crawler():
i_am_crawler = True
# we will only accept CRAWLER_REPLY messages when we are actully a crawler
self.register_msg_handler([CRAWLER_REPLY], crawler.handle_reply)
self.register_connection_handler(crawler.handle_connection)
if "database" in sys.argv:
# allows access to tribler database (boudewijn)
crawler.register_crawl_initiator(database_crawler.query_initiator)
# if "videoplayback" in sys.argv:
# # allows access to video-playback statistics (boudewijn)
# crawler.register_crawl_initiator(videoplayback_crawler.query_initiator)
# if "seedingstats" in sys.argv:
# # allows access to seeding statistics (Boxun)
# crawler.register_crawl_initiator(seeding_stats_crawler.query_initiator, frequency=60*30)
if "friendship" in sys.argv:
# allows access to friendship statistics (Ali)
crawler.register_crawl_initiator(friendship_crawler.query_initiator)
if "natcheck" in sys.argv:
# allows access to nat-check statistics (Lucia)
crawler.register_crawl_initiator(natcheck_handler.doNatCheck, 3600)
if "repex" in sys.argv:
# allows access to RePEX log statistics (Raynor Vliegendhart)
crawler.register_crawl_initiator(repex_crawler.query_initiator)
if "puncture" in sys.argv:
# allows access to UDPPuncture log statistics (Gertjan)
crawler.register_crawl_initiator(puncture_crawler.query_initiator)
if "channel" in sys.argv:
# allows access to tribler channels' database (nitin)
crawler.register_crawl_initiator(channel_crawler.query_initiator)
if "usereventlog" in sys.argv:
# allows access to usereventlog database (Niels)
crawler.register_crawl_initiator(usereventlog_crawler.query_initiator)
else:
self.register_msg_handler([CRAWLER_REQUEST, CRAWLER_REPLY], self.handleDisabledMessage)
# ProxyService_
#
# Create handler for metadata messages in two parts, as
# ProxyService needs to know the metadata_handler and we need
# to know the proxy message handler.
# Part 1:
self.metadata_handler = MetadataHandler.getInstance()
# Create handler for messages to doe
self.doe_handler = DoeMessageHandler(launchmany)
self.register_msg_handler(ProxyMessages, self.doe_handler.handleMessage)
# Create handler for messages to proxy
self.proxy_handler = ProxyMessageHandler()
self.proxy_handler.register(session, self.metadata_handler, config['proxyservice_dir'], config.get('proxy_default_dlcfg', False))
self.register_msg_handler(DoeMessages, self.proxy_handler.handleMessage)
# Part 2:
# Register the ProxyHandler to be called when a .torrent is in
self.metadata_handler.register(overlay_bridge, self.proxy_handler, launchmany, config)
self.register_msg_handler(MetadataMessages, self.metadata_handler.handleMessage)
# Create the ProxyPeerManager
self.proxy_peer_manager = ProxyPeerManager(launchmany)
#
# _ProxyService
# 13-04-2010 Andrea: subtitles collecting
if not config['subtitles_collecting'] :
self.subtitles_handler = None
else:
self.subtitles_handler = SubtitlesHandler.getInstance()
self.subtitles_handler.register(self.overlay_bridge, self.launchmany.richmetadataDbHandler, self.launchmany.session)
self.peersHaveManger = PeersHaveManager.getInstance()
if not self.peersHaveManger.isRegistered():
self.peersHaveManger.register(self.launchmany.richmetadataDbHandler, self.overlay_bridge)
# I'm not sure if this is the best place to init this
self.subtitle_support = SubtitlesSupport.getInstance()
keypair = self.launchmany.session.keypair
permid = self.launchmany.session.get_permid()
self.subtitle_support._register(self.launchmany.richmetadataDbHandler,
self.subtitles_handler,
self.launchmany.channelcast_db, permid,
keypair, self.peersHaveManger,
self.overlay_bridge)
# cleanup the subtitles database at the first launch
self.subtitle_support.runDBConsinstencyRoutine()
if not config['torrent_collecting']:
self.torrent_collecting_solution = 0
else:
self.torrent_collecting_solution = config['buddycast_collecting_solution']
if config['buddycast']:
# Create handler for Buddycast messages
self.buddycast = BuddyCastFactory.getInstance(superpeer=config['superpeer'], log=config['overlay_log'])
# Using buddycast to handle torrent collecting since they are dependent
self.buddycast.register(overlay_bridge, launchmany,
launchmany.rawserver_fatalerrorfunc,
self.metadata_handler,
self.torrent_collecting_solution,
config['start_recommender'],config['buddycast_max_peers'],i_am_crawler)
self.register_msg_handler(BuddyCastMessages, self.buddycast.handleMessage)
self.register_connection_handler(self.buddycast.handleConnection)
if config['dialback']:
self.dialback_handler = DialbackMsgHandler.getInstance()
# The Dialback mechanism needs the real rawserver, not the overlay_bridge
self.dialback_handler.register(overlay_bridge, launchmany, launchmany.rawserver, config)
self.register_msg_handler([DIALBACK_REQUEST],
self.dialback_handler.olthread_handleSecOverlayMessage)
self.register_connection_handler(self.dialback_handler.olthread_handleSecOverlayConnection)
else:
self.register_msg_handler([DIALBACK_REQUEST], self.handleDisabledMessage)
if config['socnet']:
self.socnet_handler = SocialNetworkMsgHandler.getInstance()
self.socnet_handler.register(overlay_bridge, launchmany, config)
self.register_msg_handler(SocialNetworkMessages,self.socnet_handler.handleMessage)
self.register_connection_handler(self.socnet_handler.handleConnection)
self.friendship_handler = FriendshipMsgHandler.getInstance()
self.friendship_handler.register(overlay_bridge, launchmany.session)
self.register_msg_handler(FriendshipMessages,self.friendship_handler.handleMessage)
self.register_connection_handler(self.friendship_handler.handleConnection)
if config['rquery']:
self.rquery_handler = RemoteQueryMsgHandler.getInstance()
self.rquery_handler.register(overlay_bridge,launchmany,config,self.buddycast,log=config['overlay_log'])
self.register_msg_handler(RemoteQueryMessages,self.rquery_handler.handleMessage)
self.register_connection_handler(self.rquery_handler.handleConnection)
if config['subtitles_collecting']:
hndl = self.subtitles_handler.getMessageHandler()
self.register_msg_handler(SubtitleMessages, hndl)
if config['torrent_collecting']:
self.rtorrent_handler = RemoteTorrentHandler.getInstance()
self.rtorrent_handler.register(overlay_bridge,self.metadata_handler,session)
self.metadata_handler.register2(self.rtorrent_handler)
# Add notifier as connection handler
self.register_connection_handler(self.notifier_handles_connection)
if config['buddycast']:
# Arno: to prevent concurrency between mainthread and overlay
# thread where BuddyCast schedules tasks
self.buddycast.register2()
# ProxyService_
#
# Register the ProxyPeerManager with BuddyCast
# The ProxyPeerManager has to register after the BuddyCastCore object is created by the BuddyCastFactory.
# The BuddyCastCore object is created in a method scheduled by the overlay_bridge.
# If the ProxyPeerManager.register is also scheduled by the overlay_bridge, it will be executed after the
# BuddyCastCore is created (overlay_bridge executes the scheduled tasks in the order of their execution)
if self.proxy_peer_manager:
self.overlay_bridge.add_task(self.proxy_peer_manager.register, 0)
def set_subtitles_upload_rate(self, value):
with self.sesslock:
SubtitlesHandler.getInstance().setUploadRate(value)
SessionConfigInterface.set_subtitles_uploade_rate(self, value)
def register(self, overlay_bridge, session, launchmany, config, requestPolicy):
self.overlay_bridge = overlay_bridge
self.launchmany = launchmany
self.requestPolicy = requestPolicy
self.text_mode = config.has_key('text_mode')
# OverlayApps gets all messages, and demultiplexes
overlay_bridge.register_recv_callback(self.handleMessage)
overlay_bridge.register_conns_callback(self.handleConnection)
# Arno, 2010-01-28: Start with crawler support, other mods depend on
# that, e.g. BuddyCast
i_am_crawler = False
if config['crawler']:
crawler = Crawler.get_instance(session)
self.register_msg_handler([CRAWLER_REQUEST], crawler.handle_request)
database_crawler = DatabaseCrawler.get_instance()
crawler.register_message_handler(CRAWLER_DATABASE_QUERY, database_crawler.handle_crawler_request, database_crawler.handle_crawler_reply)
# seeding_stats_crawler = SeedingStatsCrawler.get_instance()
# crawler.register_message_handler(CRAWLER_SEEDINGSTATS_QUERY, seeding_stats_crawler.handle_crawler_request, seeding_stats_crawler.handle_crawler_reply)
friendship_crawler = FriendshipCrawler.get_instance(session)
crawler.register_message_handler(CRAWLER_FRIENDSHIP_STATS, friendship_crawler.handle_crawler_request, friendship_crawler.handle_crawler_reply)
natcheck_handler = NatCheckMsgHandler.getInstance()
natcheck_handler.register(launchmany)
crawler.register_message_handler(CRAWLER_NATCHECK, natcheck_handler.gotDoNatCheckMessage, natcheck_handler.gotNatCheckReplyMessage)
crawler.register_message_handler(CRAWLER_NATTRAVERSAL, natcheck_handler.gotUdpConnectRequest, natcheck_handler.gotUdpConnectReply)
# videoplayback_crawler = VideoPlaybackCrawler.get_instance()
# crawler.register_message_handler(CRAWLER_VIDEOPLAYBACK_EVENT_QUERY, videoplayback_crawler.handle_event_crawler_request, videoplayback_crawler.handle_event_crawler_reply)
# crawler.register_message_handler(CRAWLER_VIDEOPLAYBACK_INFO_QUERY, videoplayback_crawler.handle_info_crawler_request, videoplayback_crawler.handle_info_crawler_reply)
repex_crawler = RepexCrawler.get_instance(session)
crawler.register_message_handler(CRAWLER_REPEX_QUERY, repex_crawler.handle_crawler_request, repex_crawler.handle_crawler_reply)
puncture_crawler = PunctureCrawler.get_instance()
crawler.register_message_handler(CRAWLER_PUNCTURE_QUERY, puncture_crawler.handle_crawler_request, puncture_crawler.handle_crawler_reply)
channel_crawler = ChannelCrawler.get_instance()
crawler.register_message_handler(CRAWLER_CHANNEL_QUERY, channel_crawler.handle_crawler_request, channel_crawler.handle_crawler_reply)
usereventlog_crawler = UserEventLogCrawler.get_instance()
crawler.register_message_handler(CRAWLER_USEREVENTLOG_QUERY, usereventlog_crawler.handle_crawler_request, usereventlog_crawler.handle_crawler_reply)
if crawler.am_crawler():
i_am_crawler = True
# we will only accept CRAWLER_REPLY messages when we are actully a crawler
self.register_msg_handler([CRAWLER_REPLY], crawler.handle_reply)
self.register_connection_handler(crawler.handle_connection)
if "database" in sys.argv:
# allows access to tribler database (boudewijn)
crawler.register_crawl_initiator(database_crawler.query_initiator)
# if "videoplayback" in sys.argv:
# # allows access to video-playback statistics (boudewijn)
# crawler.register_crawl_initiator(videoplayback_crawler.query_initiator)
# if "seedingstats" in sys.argv:
# # allows access to seeding statistics (Boxun)
# crawler.register_crawl_initiator(seeding_stats_crawler.query_initiator, frequency=60*30)
if "friendship" in sys.argv:
# allows access to friendship statistics (Ali)
crawler.register_crawl_initiator(friendship_crawler.query_initiator)
if "natcheck" in sys.argv:
# allows access to nat-check statistics (Lucia)
crawler.register_crawl_initiator(natcheck_handler.doNatCheck, 3600)
if "repex" in sys.argv:
# allows access to RePEX log statistics (Raynor Vliegendhart)
crawler.register_crawl_initiator(repex_crawler.query_initiator)
if "puncture" in sys.argv:
# allows access to UDPPuncture log statistics (Gertjan)
crawler.register_crawl_initiator(puncture_crawler.query_initiator)
if "channel" in sys.argv:
# allows access to tribler channels' database (nitin)
crawler.register_crawl_initiator(channel_crawler.query_initiator)
if "usereventlog" in sys.argv:
# allows access to usereventlog database (Niels)
crawler.register_crawl_initiator(usereventlog_crawler.query_initiator)
else:
self.register_msg_handler([CRAWLER_REQUEST, CRAWLER_REPLY], self.handleDisabledMessage)
# ProxyService_
#
# Create handler for metadata messages in two parts, as
# ProxyService needs to know the metadata_handler and we need
# to know the proxy message handler.
# Part 1:
self.metadata_handler = MetadataHandler.getInstance()
# Create handler for messages to doe
self.doe_handler = DoeMessageHandler(launchmany)
self.register_msg_handler(ProxyMessages, self.doe_handler.handleMessage)
# Create handler for messages to proxy
self.proxy_handler = ProxyMessageHandler()
self.proxy_handler.register(session, self.metadata_handler, config['proxyservice_dir'], config.get('proxy_default_dlcfg', False))
self.register_msg_handler(DoeMessages, self.proxy_handler.handleMessage)
# Part 2:
# Register the ProxyHandler to be called when a .torrent is in
self.metadata_handler.register(overlay_bridge, self.proxy_handler, launchmany, config)
self.register_msg_handler(MetadataMessages, self.metadata_handler.handleMessage)
# Create the ProxyPeerManager
self.proxy_peer_manager = ProxyPeerManager(launchmany)
#
# _ProxyService
# 09-02-2011 Niels: disabling subtitles (no more channelcast
config['subtitles_collecting'] = False
# 13-04-2010 Andrea: subtitles collecting
if not config['subtitles_collecting'] :
self.subtitles_handler = None
else:
self.subtitles_handler = SubtitlesHandler.getInstance()
self.subtitles_handler.register(self.overlay_bridge, self.launchmany.richmetadataDbHandler, self.launchmany.session)
self.peersHaveManger = PeersHaveManager.getInstance()
if not self.peersHaveManger.isRegistered():
self.peersHaveManger.register(self.launchmany.richmetadataDbHandler, self.overlay_bridge)
# I'm not sure if this is the best place to init this
self.subtitle_support = SubtitlesSupport.getInstance()
keypair = self.launchmany.session.keypair
permid = self.launchmany.session.get_permid()
self.subtitle_support._register(self.launchmany.richmetadataDbHandler,
self.subtitles_handler,
self.launchmany.channelcast_db, permid,
keypair, self.peersHaveManger,
self.overlay_bridge)
# cleanup the subtitles database at the first launch
self.subtitle_support.runDBConsinstencyRoutine()
if not config['torrent_collecting']:
self.torrent_collecting_solution = 0
else:
self.torrent_collecting_solution = config['buddycast_collecting_solution']
if config['buddycast']:
# Create handler for Buddycast messages
self.buddycast = BuddyCastFactory.getInstance(superpeer=config['superpeer'], log=config['overlay_log'])
# Using buddycast to handle torrent collecting since they are dependent
self.buddycast.register(overlay_bridge, launchmany,
launchmany.rawserver_fatalerrorfunc,
self.metadata_handler,
self.torrent_collecting_solution,
config['start_recommender'],config['buddycast_max_peers'],i_am_crawler)
self.register_msg_handler(BuddyCastMessages, self.buddycast.handleMessage)
self.register_connection_handler(self.buddycast.handleConnection)
if config['dialback']:
self.dialback_handler = DialbackMsgHandler.getInstance()
# The Dialback mechanism needs the real rawserver, not the overlay_bridge
self.dialback_handler.register(overlay_bridge, launchmany, launchmany.rawserver, config)
self.register_msg_handler([DIALBACK_REQUEST],
self.dialback_handler.olthread_handleSecOverlayMessage)
self.register_connection_handler(self.dialback_handler.olthread_handleSecOverlayConnection)
else:
self.register_msg_handler([DIALBACK_REQUEST], self.handleDisabledMessage)
if config['socnet']:
self.socnet_handler = SocialNetworkMsgHandler.getInstance()
self.socnet_handler.register(overlay_bridge, launchmany, config)
self.register_msg_handler(SocialNetworkMessages,self.socnet_handler.handleMessage)
self.register_connection_handler(self.socnet_handler.handleConnection)
self.friendship_handler = FriendshipMsgHandler.getInstance()
self.friendship_handler.register(overlay_bridge, launchmany.session)
self.register_msg_handler(FriendshipMessages,self.friendship_handler.handleMessage)
self.register_connection_handler(self.friendship_handler.handleConnection)
if config['rquery']:
self.rquery_handler = RemoteQueryMsgHandler.getInstance()
self.rquery_handler.register(overlay_bridge,launchmany,config,self.buddycast,log=config['overlay_log'])
self.register_msg_handler(RemoteQueryMessages,self.rquery_handler.handleMessage)
self.register_connection_handler(self.rquery_handler.handleConnection)
if config['subtitles_collecting']:
hndl = self.subtitles_handler.getMessageHandler()
self.register_msg_handler(SubtitleMessages, hndl)
if config['torrent_collecting']:
self.rtorrent_handler = RemoteTorrentHandler.getInstance()
self.rtorrent_handler.register(overlay_bridge,self.metadata_handler,session)
self.metadata_handler.register2(self.rtorrent_handler)
# Add notifier as connection handler
self.register_connection_handler(self.notifier_handles_connection)
if config['buddycast']:
# Arno: to prevent concurrency between mainthread and overlay
# thread where BuddyCast schedules tasks
self.buddycast.register2()
# ProxyService_
#
# Register the ProxyPeerManager with BuddyCast
# The ProxyPeerManager has to register after the BuddyCastCore object is created by the BuddyCastFactory.
# The BuddyCastCore object is created in a method scheduled by the overlay_bridge.
# If the ProxyPeerManager.register is also scheduled by the overlay_bridge, it will be executed after the
# BuddyCastCore is created (overlay_bridge executes the scheduled tasks in the order of their execution)
if self.proxy_peer_manager:
self.overlay_bridge.add_task(self.proxy_peer_manager.register, 0)
def setUp(self):
self._session = MockSession()
self.ol_bridge = MockOverlayBridge()
self.rmdDBHandler = MockMetadataDBHandler()
self.underTest = SubtitlesHandler()
class TestSubtitlesHandler(unittest.TestCase):
def setUp(self):
self._session = MockSession()
self.ol_bridge = MockOverlayBridge()
self.rmdDBHandler = MockMetadataDBHandler()
self.underTest = SubtitlesHandler()
def tearDown(self):
self.ol_bridge = None
#cleanup the mess in collected dir
if self.underTest.subs_dir is not None:
toDelete = [ os.path.join(self.underTest.subs_dir, entry) for entry in os.listdir(self.underTest.subs_dir)\
if entry.endswith(".srt")]
for xfile in toDelete:
if os.path.isfile(xfile) :
os.remove(xfile)
def testRegisterStuff(self):
self.underTest.register(self.ol_bridge, self.rmdDBHandler, self._session)
self.assertTrue(self.underTest.registered)
self.assertTrue(self.ol_bridge is self.underTest.overlay_bridge)
self.assertTrue(self.rmdDBHandler is self.underTest.subtitlesDb)
expectedPath = os.path.join(os.path.abspath(os.path.curdir), self._session.get_state_dir(),
"subtitles_collecting_dir")
self.assertEqual(os.path.normpath(expectedPath), self.underTest.subs_dir)
# preaparing the mock msg handler
self.mockMsgHandler = MockSubsMsgHander()
self.underTest._subsMsgHndlr = self.mockMsgHandler
def testGetSubtitlesFileRelativeName(self):
#subtitles filenames are build from the sha1 hash
#of the triple (channel_id, infohash, langCode)
name = getSubtitleFileRelativeName(testChannelId, testInfohash, "rus")
hasher = sha()
for value in (testChannelId, testInfohash, "rus"):
hasher.update(value)
self.assertEquals(hasher.hexdigest() + ".srt", name)
def testSendSubtitleRequestSimple(self):
self.testRegisterStuff()
self.underTest.sendSubtitleRequest(testDestPermId, testChannelId, testInfohash,
["zho","rus"], None, OLPROTO_VER_FOURTEENTH)
self.assertEquals(1,self.mockMsgHandler.sendReqCount)
def testReceibedGETSUBSNoSubs(self):
self.testRegisterStuff()
self.underTest.receivedSubsRequest(testDestPermId,
(testDestPermId,testChannelId,["ita","nld"]), OLPROTO_VER_FOURTEENTH)
self.assertEquals(0,self.mockMsgHandler.sendResCount)
def testReceivedGETSUBSTwoSubs(self):
self.testRegisterStuff()
self.underTest.receivedSubsRequest(testDestPermId,
(testChannelId,testInfohash,["eng","rus"]), OLPROTO_VER_FOURTEENTH)
eng = u"this is a fake srt\n\nonly needed for testing\n\ncheers :)\n\n"
rus = eng + \
u"asfasgb sagas gba\n\nasfasfas 24214 a \nThe checksum is different yuppy!\n\n"
self.assertEquals(1,self.mockMsgHandler.sendResCount)
destination, response, selversion = self.mockMsgHandler.sendResParams[0]
self.assertEquals(testDestPermId,destination)
self.assertEquals(OLPROTO_VER_FOURTEENTH,selversion)
channelId, infohash, contentsList = response
self.assertEquals(testChannelId,channelId)
self.assertEquals(testInfohash,infohash)
self.assertEquals(contentsList,{"eng":eng,"rus":rus})
def testReceivedSUBSMessage(self):
self.testRegisterStuff()
languages = ["eng","rus"]
zho = u"Subtitle Content 1"
kor = u"Subtitle Content 2"
contentsDict = {"eng":zho, "rus":kor}
msg = (testChannelId,
testInfohash, contentsDict)
simpleCallback = lambda x : x
bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(["eng","rus"])
self.underTest.receivedSubsResponse(testDestPermId, msg, [(simpleCallback,bitmask)], OLPROTO_VER_FOURTEENTH)
#self.assertEquals(languages,callbackParams)
expectedFilename1 = getSubtitleFileRelativeName(testChannelId, testInfohash, "eng")
expectedFilename2 = getSubtitleFileRelativeName(testChannelId, testInfohash, "rus")
expectedPath1 = os.path.join(self._session.get_state_dir(),self.underTest.subs_dir,expectedFilename1)
expectedPath2 = os.path.join(self._session.get_state_dir(),self.underTest.subs_dir,expectedFilename2)
self.assertTrue(os.path.isfile(expectedPath1))
self.assertTrue(os.path.isfile(expectedPath2))
with codecs.open(expectedPath1,"rb","utf-8") as file1:
content1 = file1.read()
self.assertEquals(zho,content1)
with codecs.open(expectedPath2,"rb","utf-8") as file2:
content2 = file2.read()
self.assertEquals(kor,content2)
self.assertEquals(1, self.ol_bridge.add_task_count)
params = self.ol_bridge.add_taskParametersHistory[0]
#calling the lambda
val = params[0]()
self.assertEquals(languages,val)
def test_saveSubsOnDisk(self):
self.testRegisterStuff()
subContent = u"Test Content\nFor a pseudo subtitle file\n\nYo!\n"
self.underTest._saveSubOnDisk(testChannelId, testInfohash,
"eng", subContent)
expectedFilename = getSubtitleFileRelativeName(testChannelId,
testInfohash, "eng")
expectedPath = os.path.join(self.underTest.subs_dir, expectedFilename)
self.assertTrue(os.path.isfile(expectedPath))
#check the contents
with codecs.open(expectedPath, "rb", "utf-8") as file:
cont = file.read()
self.assertEquals(subContent,cont)
##now the file exists. If a new subtitle is saved for the same
# channel, infohash, lang but with a different content
# the old one should be overwritten
newContent = u"I'm the new content! I shall win over the old one!"
self.underTest._saveSubOnDisk(testChannelId, testInfohash,
"eng", newContent)
self.assertTrue(os.path.isfile(expectedPath))
#check the contents
with codecs.open(expectedPath, "rb", "utf-8") as file:
cont = file.read()
self.assertEquals(newContent,cont)