def register(self, overlay_bridge, metadataDBHandler, session): """ Injects the required dependencies on the instance. @param overlay_bridge: a reference to a working instance of OverlayTrheadingBridge @param metadataDBHandler: a reference to the current instance of L{MetadataDBHandler} @param session: a reference to the running session """ self.overlay_bridge = overlay_bridge self.subtitlesDb = metadataDBHandler self.config_dir = os.path.abspath(session.get_state_dir()) subs_path = os.path.join(self.config_dir, session.get_subtitles_collecting_dir()) # George Milescu, 19.11.2010 # I replaced self.subs_dir because os.path.abspath(subs_path) returns an inexistent path for non-standard tribler paths # self.subs_dir = os.path.abspath(subs_path) self.subs_dir = os.path.abspath(session.get_subtitles_collecting_dir()) self._upload_rate = session.get_subtitles_upload_rate() self.max_subs_message_size = MAX_SUBS_MESSAGE_SIZE self._session = session #the upload rate is controlled by a token bucket. #a token corresponds to 1 KB. #The max burst size corresponds to 2 subtitles of the maximum size (2 MBs) tokenBucket = SimpleTokenBucket(self._upload_rate, self.max_subs_message_size) self._subsMsgHndlr = SubsMessageHandler(self.overlay_bridge, tokenBucket, MAX_SUBTITLE_SIZE) self._subsMsgHndlr.registerListener(self) #assure that the directory exists if os.path.isdir(self.config_dir) : if not os.path.isdir(self.subs_dir): try: os.mkdir(self.subs_dir) except: msg = u"Cannot create collecting dir %s " % self.subs_dir print >> sys.stderr, "Error: %s" % msg raise IOError(msg) else: msg = u"Configuration dir %s does not exists" % self.subs_dir print >> sys.stderr, "Error: %s" % msg raise IOError(msg) #event notifier self._notifier = Notifier.getInstance() self.registered = True
def setUp(self): self.ol_bridge = MockOverlayBridge() self.tokenBucket = MockTokenBucket() self.underTest = SubsMessageHandler(self.ol_bridge,self.tokenBucket,1000000)
class TestSubtitlesMsgHandlerIsolation(unittest.TestCase): def setUp(self): self.ol_bridge = MockOverlayBridge() self.tokenBucket = MockTokenBucket() self.underTest = SubsMessageHandler(self.ol_bridge,self.tokenBucket,1000000) def test_addToRequestedSubtitles(self): langUtil = LanguagesProvider.getLanguagesInstance() bitmask1 = langUtil.langCodesToMask(["nld"]) self.underTest._addToRequestedSubtitles(testChannelId, testInfohash, bitmask1) key = "".join((testChannelId, testInfohash)) self.assertEquals(bitmask1, self.underTest.requestedSubtitles[ key ].cumulativeBitmask) bitmask2 = langUtil.langCodesToMask(["jpn", "ita"]) self.underTest._addToRequestedSubtitles(testChannelId, testInfohash, bitmask2) self.assertEquals(bitmask1 | bitmask2, self.underTest.requestedSubtitles[ key ].cumulativeBitmask) removeBitmask = langUtil.langCodesToMask(["nld", "ita"]) self.underTest._removeFromRequestedSubtitles(testChannelId, testInfohash, removeBitmask) codes = langUtil.maskToLangCodes(self.underTest.requestedSubtitles[ key ].cumulativeBitmask) self.assertEquals(["jpn"], codes) def testSendSubtitlesRequestConnected(self): langUtil = LanguagesProvider.getLanguagesInstance() request = {} request['channel_id'] = testChannelId request['infohash'] = testInfohash request['languages'] = ["kor"] self.underTest.sendSubtitleRequest(testDestPermId, request, None, None, OLPROTO_VER_FOURTEENTH) self.assertEquals(0, self.ol_bridge.connect_count) #selversion was 1 self.assertEquals(1, self.ol_bridge.send_count) #send called one time binaryBitmask = pack("!L", langUtil.langCodesToMask(["kor"])) expectedMsg = GET_SUBS + \ bencode(( testChannelId, testInfohash, binaryBitmask )) passedParameters = self.ol_bridge.sendParametersHistory[0] self.assertEquals(testDestPermId, passedParameters[0]) self.assertEquals(expectedMsg, passedParameters[1]) def testSendSubtitlesRequestNotConnected(self): langUtil = LanguagesProvider.getLanguagesInstance() request = {} request['channel_id'] = testChannelId request['infohash'] = testInfohash request['languages'] = ["kor"] self.underTest.sendSubtitleRequest(testDestPermId, request) self.assertEquals(1, self.ol_bridge.connect_count) #selversion was -1 self.assertEquals(1, self.ol_bridge.send_count) #send called one time binaryBitmask = pack("!L", langUtil.langCodesToMask(["kor"])) expectedMsg = GET_SUBS + \ bencode(( testChannelId, testInfohash, binaryBitmask )) passedParameters = self.ol_bridge.sendParametersHistory[0] self.assertEquals(testDestPermId, passedParameters[0]) self.assertEquals(expectedMsg, passedParameters[1]) def test_decodeGETSUBSMessage(self): langUtil = LanguagesProvider.getLanguagesInstance() binaryBitmask = pack("!L", langUtil.langCodesToMask(["kor", "spa"])) bencodedMessage = GET_SUBS + \ bencode(( testChannelId, testInfohash, binaryBitmask )) channel_id, infohash, languages = \ self.underTest._decodeGETSUBSMessage(bencodedMessage) self.assertEquals(testChannelId, channel_id) self.assertEquals(testInfohash, infohash) self.assertEquals(["kor", "spa"], languages) def test_decodeGETSUBSMessageInvalid(self): langUtil = LanguagesProvider.getLanguagesInstance() binaryBitmask = pack("!L", langUtil.langCodesToMask(["kor", "spa"])) invalidTypeMsg = chr(25) + \ bencode(( testChannelId, testInfohash, binaryBitmask )) self.assertRaises(AssertionError, self.underTest._decodeGETSUBSMessage, (invalidTypeMsg,)) invalidMsgField = GET_SUBS + \ bencode(( 42, testChannelId, testInfohash, binaryBitmask )) decoded = \ self.underTest._decodeGETSUBSMessage(invalidMsgField) #when something in the body is wrong returns None self.assertTrue(decoded is None) invalidBitamsk = "\xff\xff\xff\xff\xbb" invalidMsgField = GET_SUBS + \ bencode(( testChannelId, testInfohash, invalidBitamsk #40 bit bitmask!) )) decoded = \ self.underTest._decodeGETSUBSMessage(invalidMsgField) #when something in the body is wrong returns None self.assertTrue(decoded is None) def test_createSingleResponseMessage(self): langUtil = LanguagesProvider.getLanguagesInstance() data = { 'permid' : testDestPermId, 'channel_id' : testChannelId, 'infohash' : testInfohash, 'subtitles' : {"eng" : "This is content 1", "nld": "This is content 2", "ita" : "This is content 3"}, 'selversion' : OLPROTO_VER_FOURTEENTH } langs = data['subtitles'].keys() bitmask = langUtil.langCodesToMask(langs) binaryBitmask = pack("!L", bitmask) expextedMessage = SUBS + \ bencode(( data['channel_id'], data['infohash'], binaryBitmask, [data['subtitles']['eng'], data['subtitles']['ita'], data['subtitles']['nld']] )) msg = self.underTest._createSingleResponseMessage(data) decoded = bdecode(msg[1:]) self.assertEquals(expextedMessage, msg) def test_receivedGETSUBSSimple(self): langUtil = LanguagesProvider.getLanguagesInstance() bitmask = langUtil.langCodesToMask(["eng", "rus"]) binaryBitmask = pack("!L", bitmask) request = GET_SUBS + \ bencode(( testChannelId, testInfohash, binaryBitmask )) list = MockMsgListener() self.underTest.registerListener(list) self.underTest.handleMessage(testDestPermId, OLPROTO_VER_FOURTEENTH, request) self.assertEquals(1,list.receivedCount) self.assertEquals(testDestPermId, list.receivedParams[0][0]) self.assertEquals(OLPROTO_VER_FOURTEENTH,list.receivedParams[0][2]) self.assertEquals((testChannelId,testInfohash,["eng","rus"]),list.receivedParams[0][1]) def test_receivedGETSUBSInvalid1(self): bitmask = -1 request = GET_SUBS + \ bencode(( testChannelId, testInfohash, bitmask )) list = MockMsgListener() self.underTest.registerListener(list) val = self.underTest.handleMessage(testDestPermId, OLPROTO_VER_FOURTEENTH, request) self.assertFalse(val) self.assertEquals(0,list.receivedCount) #the invalid msg has been dropped def test_receivedGETSUBSInvalid2(self): bitmask = -1 request = GET_SUBS + \ bencode(( testChannelId, testInfohash, bitmask )) list = MockMsgListener() self.underTest.registerListener(list) val = self.underTest.handleMessage(testDestPermId, 13,request) self.assertFalse(val) self.assertEquals(0,list.receivedCount) #the invalid msg has been dropped def test_receivedSUBSSimpleNoRequest(self): langUtil = LanguagesProvider.getLanguagesInstance() data = { 'permid' : testDestPermId, 'channel_id' : testChannelId, 'infohash' : testInfohash, 'subtitles' : {"eng" : "This is content 1", "nld": "This is content 2", "ita" : "This is content 3"}, 'selversion' : OLPROTO_VER_FOURTEENTH } langs = data['subtitles'].keys() bitmask = langUtil.langCodesToMask(langs) binaryBitmask = pack("!L", bitmask) expextedMessage = SUBS + \ bencode(( data['channel_id'], data['infohash'], binaryBitmask, [data['subtitles']['eng'], data['subtitles']['ita'], data['subtitles']['nld']] )) list = MockMsgListener() self.underTest.registerListener(list) val = self.underTest.handleMessage(testDestPermId, OLPROTO_VER_FOURTEENTH, expextedMessage) # never had a request for this message should be dropped self.assertFalse(val) self.assertEquals(0,list.subsCount) def test_receivedSUBSOtherRequest(self): langUtil = LanguagesProvider.getLanguagesInstance() data = { 'permid' : testDestPermId, 'channel_id' : testChannelId, 'infohash' : testInfohash, 'subtitles' : {"eng" : "This is content 1", "nld": "This is content 2", "ita" : "This is content 3"}, 'selversion' : OLPROTO_VER_FOURTEENTH } langs = data['subtitles'].keys() bitmask = langUtil.langCodesToMask(langs) binaryBitmask = pack("!L", bitmask) expextedMessage = SUBS + \ bencode(( data['channel_id'], data['infohash'], binaryBitmask, [data['subtitles']['eng'], data['subtitles']['ita'], data['subtitles']['nld']] )) list = MockMsgListener() self.underTest.registerListener(list) #invalid bitmask self.underTest._addToRequestedSubtitles(testChannelId, testInfohash, int(0xFFFFFFFF & ~bitmask), None) val = self.underTest.handleMessage(testDestPermId, OLPROTO_VER_FOURTEENTH, expextedMessage) # never had a request for this message should be dropped self.assertFalse(val) self.assertEquals(0,list.subsCount) def test_receivedSUBSSomeRequest(self): langUtil = LanguagesProvider.getLanguagesInstance() data = { 'permid' : testDestPermId, 'channel_id' : testChannelId, 'infohash' : testInfohash, 'subtitles' : {"eng" : "This is content 1", "nld": "This is content 2", "ita" : "This is content 3"}, 'selversion' : OLPROTO_VER_FOURTEENTH } langs = data['subtitles'].keys() bitmask = langUtil.langCodesToMask(langs) binaryBitmask = pack("!L", bitmask) expextedMessage = SUBS + \ bencode(( data['channel_id'], data['infohash'], binaryBitmask, [data['subtitles']['eng'], data['subtitles']['ita'], data['subtitles']['nld']] )) list = MockMsgListener() self.underTest.registerListener(list) #invalid bitmask self.underTest._addToRequestedSubtitles(testChannelId, testInfohash, langUtil.langCodesToMask(["ita"]), None) val = self.underTest.handleMessage(testDestPermId, OLPROTO_VER_FOURTEENTH, expextedMessage) # never had a request for this message should be dropped self.assertTrue(val) self.assertEquals(1,list.subsCount) params = list.subsParams[0] channel_id, infohash, contentsDictionary = params[1] self.assertEquals(testChannelId,channel_id) self.assertEquals(testInfohash, infohash) contentKeys = contentsDictionary.keys() self.assertEquals(["ita"],contentKeys) def test_cleanSUSRequests(self): self.underTest._requestValidityTime = 0.001 #ds self.underTest._addToRequestedSubtitles(testChannelId, testInfohash, 3, None) self.assertEquals(1,len(self.underTest.requestedSubtitles)) time.sleep(1.2) self.underTest._cleanUpRequestedSubtitles() self.assertEquals(0,len(self.underTest.requestedSubtitles))
class SubtitlesHandler(object): __single = None def __init__(self): # notice that singleton pattern is not enforced. # This is better, since this way the code is more easy # to test. SubtitlesHandler.__single = self self.languagesUtility = LanguagesProvider.getLanguagesInstance() #instance of MetadataDBHandler self.subtitlesDb = None self.registered = False self.subs_dir = None #other useful attributes are injected by the register method @staticmethod def getInstance(*args, **kw): if SubtitlesHandler.__single is None: SubtitlesHandler(*args, **kw) return SubtitlesHandler.__single def register(self, overlay_bridge, metadataDBHandler, session): """ Injects the required dependencies on the instance. @param overlay_bridge: a reference to a working instance of OverlayTrheadingBridge @param metadataDBHandler: a reference to the current instance of L{MetadataDBHandler} @param session: a reference to the running session """ self.overlay_bridge = overlay_bridge self.subtitlesDb = metadataDBHandler self.config_dir = os.path.abspath(session.get_state_dir()) subs_path = os.path.join(self.config_dir, session.get_subtitles_collecting_dir()) # George Milescu, 19.11.2010 # I replaced self.subs_dir because os.path.abspath(subs_path) returns an inexistent path for non-standard tribler paths # self.subs_dir = os.path.abspath(subs_path) self.subs_dir = os.path.abspath(session.get_subtitles_collecting_dir()) self._upload_rate = session.get_subtitles_upload_rate() self.max_subs_message_size = MAX_SUBS_MESSAGE_SIZE self._session = session #the upload rate is controlled by a token bucket. #a token corresponds to 1 KB. #The max burst size corresponds to 2 subtitles of the maximum size (2 MBs) tokenBucket = SimpleTokenBucket(self._upload_rate, self.max_subs_message_size) self._subsMsgHndlr = SubsMessageHandler(self.overlay_bridge, tokenBucket, MAX_SUBTITLE_SIZE) self._subsMsgHndlr.registerListener(self) #assure that the directory exists if os.path.isdir(self.config_dir) : if not os.path.isdir(self.subs_dir): try: os.mkdir(self.subs_dir) except: msg = u"Cannot create collecting dir %s " % self.subs_dir print >> sys.stderr, "Error: %s" % msg raise IOError(msg) else: msg = u"Configuration dir %s does not exists" % self.subs_dir print >> sys.stderr, "Error: %s" % msg raise IOError(msg) #event notifier self._notifier = Notifier.getInstance() self.registered = True def sendSubtitleRequest(self, permid, channel_id, infohash, languages, callback=None, selversion= -1): """ Send a request for subtitle files. Only called by the OLThread Send a GET_SUBS request to the peer identified by permid. The request asks for several subtitles file, for a given channel_id and torrent infohash. The subtitles file to request are specified by the languages parameter that is a list of 3 characters language codes. The contents of a GET_SUBS request are: - channel_id: the identifier of the channel for which the subtitles were added. (a permid). Binary. - infohash: the infohash of the torrent, the subtitles refer to. Binary. - bitmask: a 32 bit bitmask (an integer) which specifies the languages requested @param permid: the destination of the request (binary) @param channel_id: the identifier of the channel for which the subtitle was added (binary) @param infohash: the infohash of a torrent the subtitles refers to (binary). @param languages: a list of 3-characters language codes. It must be on of the supported language codes (see Languages) @param callback: a function that will be called WHENEVER some of the requested subtitles are received. It must have exactly one parameter that will be bound to a list of the languages that were received @param selversion: the protocol version of the peer whe are sending the request to @raise SubtitleMsgHandlerException: if the message failed its attempt to be sent. Notice that also if the method returns without raising any exception it doesn't mean that the message has been sent. """ assert utilities.isValidInfohash(infohash), SUBS_LOG_PREFIX + "Invalid infohash %s" % infohash assert utilities.isValidPermid(permid), SUBS_LOG_PREFIX + "Invlaid destination permid %s" % permid assert self.languagesUtility.isLangListSupported(languages), SUBS_LOG_PREFIX + "Some of the languages where not supported" if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + "preparing to send GET_SUBS to " + utilities.show_permid_short(permid) if len(languages) == 0: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + " no subtitles to request." return requestDetails = dict() requestDetails['channel_id'] = channel_id requestDetails['infohash'] = infohash requestDetails['languages'] = languages self._subsMsgHndlr.sendSubtitleRequest(permid, requestDetails, lambda e,d,c,i,b : \ self._subsRequestSent(e,d,c,i,b), callback, selversion) def _subsRequestSent(self,exception,dest, channel_id, infohash, bitmask ): ''' Gets called when a subtitle request has been successfully sent. ''' pass def receivedSubsRequest(self, permid, request, selversion): """ Reads a received GET_SUBS message and possibly sends a response. @param permid: the permid of the sender of the GET_SUBS message @param request: a tuple made of channel_id, infohash, language code @param selversion: the protocol version of the requesting peer @return: False if the message had something wrong. (a return value of False makes the caller close the connection). Otherwise True """ assert self.registered, SUBS_LOG_PREFIX + "Handler not yet registered" channel_id, infohash, languages = request #happily unpacking #diction {lang : Subtitle} allSubtitles = self.subtitlesDb.getAllSubtitles(channel_id, infohash) contentsList = {} #{langCode : path} #for each requested language check if the corresponding subtitle #is available for lang in sorted(languages): if lang in allSubtitles.keys(): if allSubtitles[lang].subtitleExists(): content = self._readSubContent(allSubtitles[lang].path) if content is not None: contentsList[lang] = content else: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + "File not available for channel %s, infohash %s, lang %s" % \ (show_permid_short(channel_id), bin2str(infohash),lang) self.subtitlesDb.updateSubtitlePath(channel_id,infohash,lang,None) else: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + "Subtitle not available for channel %s, infohash %s, lang %s" % \ (show_permid_short(channel_id), bin2str(infohash),lang) if len(contentsList) == 0: #pathlist is empty if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + "None of the requested subtitles were available. No answer will be sent to %s" % \ show_permid_short(permid) return True return self._subsMsgHndlr.sendSubtitleResponse(permid, (channel_id,infohash,contentsList), selversion) def _readSubContent(self,path): try: # 09/12/10 boudewijn: just before calling this method, the # subtitleExists method is called. This checks that # path.isfile(path) is True. Hence no relpath and or join # is required. Also, we support Python version 2.5, and # this does NOT contain path.relpath. fileName = path # relativeName = os.path.relpath(path, self.subs_dir) # fileName = os.path.join(self.subs_dir, relativeName) # print >> sys.stderr, "SUBTITLES PATH:", path # print >> sys.stderr, "SUBTITLES ISFILE:", os.path.isfile(path) # print >> sys.stderr, "SUBTITLES SUBS_DIR:", self.subs_dir # print >> sys.stderr, "SUBTITLES RELNAME:", relativeName # print >> sys.stderr, "SUBTITLES FILENAME:", fileName # print >> sys.stderr, "SUBTITLES TEST:", os.path.normpath(os.path.join(self.subs_dir, path)) file = open(fileName, 'rb') fileContent = file.read() file.close() except IOError,e: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + "Error reading from subs file %s: %s" % (relativeName, e) fileContent = None if fileContent and len(fileContent) <= MAX_SUBTITLE_SIZE: return fileContent print >> sys.stderr, "Warning: Subtitle %s dropped. Bigger than %d" % (relativeName, MAX_SUBTITLE_SIZE) return None