class SubtitlesHandler(object): __single = None def __init__(self): # notice that singleton pattern is not enforced. # This is better, since this way the code is more easy # to test. SubtitlesHandler.__single = self self.avg_subtitle_size = 100 # 100 KB, experimental avg self.languagesUtility = LanguagesProvider.getLanguagesInstance() # instance of MetadataDBHandler self.subtitlesDb = None self.registered = False self.subs_dir = None # other useful attributes are injected by the register method @staticmethod def getInstance(*args, **kw): if SubtitlesHandler.__single is None: SubtitlesHandler(*args, **kw) return SubtitlesHandler.__single def register(self, overlay_bridge, metadataDBHandler, session): """ Injects the required dependencies on the instance. @param overlay_bridge: a reference to a working instance of OverlayTrheadingBridge @param metadataDBHandler: a reference to the current instance of L{MetadataDBHandler} @param session: a reference to the running session """ self.overlay_bridge = overlay_bridge self.subtitlesDb = metadataDBHandler self.config_dir = os.path.abspath(session.get_state_dir()) subs_path = os.path.join(self.config_dir, session.get_subtitles_collecting_dir()) self.subs_dir = os.path.abspath(subs_path) self.min_free_space = DEFAULT_MIN_FREE_SPACE self._upload_rate = session.get_subtitles_upload_rate() self.max_subs_message_size = MAX_SUBS_MESSAGE_SIZE self._session = session # the upload rate is controlled by a token bucket. # a token corresponds to 1 KB. # The max burst size corresponds to 2 subtitles of the maximum size (2 MBs) tokenBucket = SimpleTokenBucket(self._upload_rate, self.max_subs_message_size) self._subsMsgHndlr = SubsMessageHandler(self.overlay_bridge, tokenBucket, MAX_SUBTITLE_SIZE) self._subsMsgHndlr.registerListener(self) # assure that the directory exists if os.path.isdir(self.config_dir): if not os.path.isdir(self.subs_dir): try: os.mkdir(self.subs_dir) except: msg = u"Cannot create collecting dir %s " % self.subs_dir print >>sys.stderr, "Error: %s" % msg raise IOError(msg) else: msg = u"Configuration dir %s does not exists" % self.subs_dir print >>sys.stderr, "Error: %s" % msg raise IOError(msg) diskManager = DiskManager(self.min_free_space, self.config_dir) self.diskManager = diskManager dmConfig = { "maxDiskUsage": MAX_SUBTITLE_DISK_USAGE, "diskPolicy": DISK_FULL_DELETE_SOME | DELETE_OLDEST_FIRST, "encoding": "utf-8", } self.diskManager.registerDir(self.subs_dir, dmConfig) freeSpace = self.diskManager.getAvailableSpace() if DEBUG: print >>sys.stderr, SUBS_LOG_PREFIX + "Avaialble %d MB for subtitle collecting" % (freeSpace / (2 ** 20)) # event notifier self._notifier = Notifier.getInstance() self.registered = True def sendSubtitleRequest(self, permid, channel_id, infohash, languages, callback=None, selversion=-1): """ Send a request for subtitle files. Only called by the OLThread Send a GET_SUBS request to the peer indentified by permid. The request asks for several subtitles file, for a given channel_id and torrent infohash. The subtitles file to request are specified by the languages parameter that is a list of 3 characters language codes. The contents of a GET_SUBS request are: - channel_id: the identifier of the channel for which the subtitles were added. (a permid). Binary. - infohash: the infohash of the torrent, the subtitles refer to. Binary. - bitmask: a 32 bit bitmask (an integer) which specifies the languages requested @param permid: the destination of the request (binary) @param channel_id: the identifier of the channel for which the subtitle was added (binary) @param infohash: the infohash of a torrent the subtitles refers to (binary). @param languages: a list of 3-characters language codes. It must be on of the supported language codes (see Languages) @param callback: a function that will be called WHENEVER some of the requested subtitles are received. It must have exactly one parameter that will be bound to a list of the languages that were received @param selversion: the protocol version of the peer whe are sending the request to @raise SubtitleMsgHandlerException: if the message failed its attempt to be sent. Notice that also if the method returns without raising any exception it doesn't mean that the message has been sent. """ assert utilities.isValidInfohash(infohash), SUBS_LOG_PREFIX + "Invalid infohash %s" % infohash assert utilities.isValidPermid(permid), SUBS_LOG_PREFIX + "Invlaid destination permid %s" % permid assert self.languagesUtility.isLangListSupported(languages), ( SUBS_LOG_PREFIX + "Some of the languages where not supported" ) if DEBUG: print >>sys.stderr, SUBS_LOG_PREFIX + "preparing to send GET_SUBS to " + utilities.show_permid_short(permid) # Better to leave up to the caller the responsibility to check # if the subtitle is already available and as correct checsum and so on.. # onDisk = [] # for langCode in languages: # # filename = self.diskManager.isFilenOnDisk(self.subs_dir, # getSubtitleFileRelativeName(channel_id, infohash, langCode)) # # # # should I skip this part and just send the request anyway? # # (thus leaving to the caller the responsibility to avoid useless # # requests) # if filename: # log.debug(SUBS_LOG_PREFIX + langCode + # " subtitle already on disk. Skipping it"\ # " in the request") # onDisk.append(langCode) # self._notify_sub_is_in(channel_id, infohash, langCode, filename) # # for deleteme in onDisk: # languages.remove(deleteme) if len(languages) == 0: if DEBUG: print >>sys.stderr, SUBS_LOG_PREFIX + " no subtitles to request." return if not self.diskManager.tryReserveSpace(self.subs_dir, len(languages) * self.avg_subtitle_size): self._warn_disk_full() return False requestDetails = dict() requestDetails["channel_id"] = channel_id requestDetails["infohash"] = infohash requestDetails["languages"] = languages self._subsMsgHndlr.sendSubtitleRequest( permid, requestDetails, lambda e, d, c, i, b: self._subsRequestSent(e, d, c, i, b), callback, selversion ) def _subsRequestSent(self, exception, dest, channel_id, infohash, bitmask): """ Gets called when a subtitle request has been succesfully sent. """ pass def receivedSubsRequest(self, permid, request, selversion): """ Reads a received GET_SUBS message and possibly sends a response. @param permid: the permid of the sender of the GET_SUBS message @param request: a tuple made of channel_id, infohash, language code @param selversion: the protocol version of the requesting peer @return: False if the message had something wrong. (a return value of False makes the caller close the connection). Otherwise True """ assert self.registered, SUBS_LOG_PREFIX + "Handler not yet registered" channel_id, infohash, languages = request # happily unpacking # diction {lang : Subtitle} allSubtitles = self.subtitlesDb.getAllSubtitles(channel_id, infohash) contentsList = {} # {langCode : path} # for each requested language check if the corresponding subtitle # is available for lang in sorted(languages): if lang in allSubtitles.keys(): if allSubtitles[lang].subtitleExists(): content = self._readSubContent(allSubtitles[lang].path) if content is not None: contentsList[lang] = content else: if DEBUG: print >>sys.stderr, SUBS_LOG_PREFIX + "File not available for " + "channel %s, infohash %s, lang %s" % ( show_permid_short(channel_id), bin2str(infohash), lang, ) self.subtitlesDb.updateSubtitlePath(channel_id, infohash, lang, None) else: if DEBUG: print >>sys.stderr, SUBS_LOG_PREFIX + "Subtitle not available for " + "channel %s, infohash %s, lang %s" % ( show_permid_short(channel_id), bin2str(infohash), lang, ) if len(contentsList) == 0: # pathlist is empty if DEBUG: print >>sys.stderr, SUBS_LOG_PREFIX + "None of the requested subtitles " + " was available. No answer will be sent to %s" % show_permid_short( permid ) return True return self._subsMsgHndlr.sendSubtitleResponse(permid, (channel_id, infohash, contentsList), selversion) def _readSubContent(self, path): try: relativeName = os.path.relpath(path, self.subs_dir) fileContent = self.diskManager.readContent(self.subs_dir, relativeName) except IOError, e: if DEBUG: print >>sys.stderr, SUBS_LOG_PREFIX + "Error reading from subs file %s: %s" % (relativeName, e) fileContent = None if fileContent is not None and len(fileContent) <= MAX_SUBTITLE_SIZE: return fileContent else: print >>sys.stderr, "Warning: Subtitle %s dropped. Bigger then %d" % (relativeName, MAX_SUBTITLE_SIZE) return None
class TestSubtitlesMsgHandlerIsolation(unittest.TestCase): def setUp(self): self.ol_bridge = MockOverlayBridge() self.tokenBucket = MockTokenBucket() self.underTest = SubsMessageHandler(self.ol_bridge,self.tokenBucket,1000000) def test_addToRequestedSubtitles(self): langUtil = LanguagesProvider.getLanguagesInstance() bitmask1 = langUtil.langCodesToMask(["nld"]) self.underTest._addToRequestedSubtitles(testChannelId, testInfohash, bitmask1) key = "".join((testChannelId, testInfohash)) self.assertEquals(bitmask1, self.underTest.requestedSubtitles[ key ].cumulativeBitmask) bitmask2 = langUtil.langCodesToMask(["jpn", "ita"]) self.underTest._addToRequestedSubtitles(testChannelId, testInfohash, bitmask2) self.assertEquals(bitmask1 | bitmask2, self.underTest.requestedSubtitles[ key ].cumulativeBitmask) removeBitmask = langUtil.langCodesToMask(["nld", "ita"]) self.underTest._removeFromRequestedSubtitles(testChannelId, testInfohash, removeBitmask) codes = langUtil.maskToLangCodes(self.underTest.requestedSubtitles[ key ].cumulativeBitmask) self.assertEquals(["jpn"], codes) def testSendSubtitlesRequestConnected(self): langUtil = LanguagesProvider.getLanguagesInstance() request = {} request['channel_id'] = testChannelId request['infohash'] = testInfohash request['languages'] = ["kor"] self.underTest.sendSubtitleRequest(testDestPermId, request, None, None, OLPROTO_VER_FOURTEENTH) self.assertEquals(0, self.ol_bridge.connect_count) #selversion was 1 self.assertEquals(1, self.ol_bridge.send_count) #send called one time binaryBitmask = uintToBinaryString(langUtil.langCodesToMask(["kor"])) expectedMsg = GET_SUBS + \ bencode(( testChannelId, testInfohash, binaryBitmask )) passedParameters = self.ol_bridge.sendParametersHistory[0] self.assertEquals(testDestPermId, passedParameters[0]) self.assertEquals(expectedMsg, passedParameters[1]) def testSendSubtitlesRequestNotConnected(self): langUtil = LanguagesProvider.getLanguagesInstance() request = {} request['channel_id'] = testChannelId request['infohash'] = testInfohash request['languages'] = ["kor"] self.underTest.sendSubtitleRequest(testDestPermId, request) self.assertEquals(1, self.ol_bridge.connect_count) #selversion was -1 self.assertEquals(1, self.ol_bridge.send_count) #send called one time binaryBitmask = uintToBinaryString(langUtil.langCodesToMask(["kor"])) expectedMsg = GET_SUBS + \ bencode(( testChannelId, testInfohash, binaryBitmask )) passedParameters = self.ol_bridge.sendParametersHistory[0] self.assertEquals(testDestPermId, passedParameters[0]) self.assertEquals(expectedMsg, passedParameters[1]) def test_decodeGETSUBSMessage(self): langUtil = LanguagesProvider.getLanguagesInstance() binaryBitmask = uintToBinaryString(langUtil.langCodesToMask(["kor", "spa"])) bencodedMessage = GET_SUBS + \ bencode(( testChannelId, testInfohash, binaryBitmask )) channel_id, infohash, languages = \ self.underTest._decodeGETSUBSMessage(bencodedMessage) self.assertEquals(testChannelId, channel_id) self.assertEquals(testInfohash, infohash) self.assertEquals(["kor", "spa"], languages) def test_decodeGETSUBSMessageInvalid(self): langUtil = LanguagesProvider.getLanguagesInstance() binaryBitmask = uintToBinaryString(langUtil.langCodesToMask(["kor", "spa"])) invalidTypeMsg = chr(25) + \ bencode(( testChannelId, testInfohash, binaryBitmask )) self.assertRaises(AssertionError, self.underTest._decodeGETSUBSMessage, (invalidTypeMsg,)) invalidMsgField = GET_SUBS + \ bencode(( 42, testChannelId, testInfohash, binaryBitmask )) decoded = \ self.underTest._decodeGETSUBSMessage(invalidMsgField) #when something in the body is wrong returns None self.assertTrue(decoded is None) invalidBitamsk = uintToBinaryString(0xFFFFFFFF11, 5) invalidMsgField = GET_SUBS + \ bencode(( testChannelId, testInfohash, invalidBitamsk #40 bit bitmask!) )) decoded = \ self.underTest._decodeGETSUBSMessage(invalidMsgField) #when something in the body is wrong returns None self.assertTrue(decoded is None) def test_createSingleResponseMessage(self): langUtil = LanguagesProvider.getLanguagesInstance() data = { 'permid' : testDestPermId, 'channel_id' : testChannelId, 'infohash' : testInfohash, 'subtitles' : {"eng" : "This is content 1", "nld": "This is content 2", "ita" : "This is content 3"}, 'selversion' : OLPROTO_VER_FOURTEENTH } langs = data['subtitles'].keys() bitmask = langUtil.langCodesToMask(langs) binaryBitmask = uintToBinaryString(bitmask, length=4) expextedMessage = SUBS + \ bencode(( data['channel_id'], data['infohash'], binaryBitmask, [data['subtitles']['eng'], data['subtitles']['ita'], data['subtitles']['nld']] )) msg = self.underTest._createSingleResponseMessage(data) decoded = bdecode(msg[1:]) self.assertEquals(expextedMessage, msg) def test_receivedGETSUBSSimple(self): langUtil = LanguagesProvider.getLanguagesInstance() bitmask = langUtil.langCodesToMask(["eng", "rus"]) binaryBitmask = uintToBinaryString(bitmask, length=4) request = GET_SUBS + \ bencode(( testChannelId, testInfohash, binaryBitmask )) list = MockMsgListener() self.underTest.registerListener(list) self.underTest.handleMessage(testDestPermId, OLPROTO_VER_FOURTEENTH, request) self.assertEquals(1,list.receivedCount) self.assertEquals(testDestPermId, list.receivedParams[0][0]) self.assertEquals(OLPROTO_VER_FOURTEENTH,list.receivedParams[0][2]) self.assertEquals((testChannelId,testInfohash,["eng","rus"]),list.receivedParams[0][1]) def test_receivedGETSUBSInvalid1(self): bitmask = -1 request = GET_SUBS + \ bencode(( testChannelId, testInfohash, bitmask )) list = MockMsgListener() self.underTest.registerListener(list) val = self.underTest.handleMessage(testDestPermId, OLPROTO_VER_FOURTEENTH, request) self.assertFalse(val) self.assertEquals(0,list.receivedCount) #the invalid msg has been dropped def test_receivedGETSUBSInvalid2(self): bitmask = -1 request = GET_SUBS + \ bencode(( testChannelId, testInfohash, bitmask )) list = MockMsgListener() self.underTest.registerListener(list) val = self.underTest.handleMessage(testDestPermId, 13,request) self.assertFalse(val) self.assertEquals(0,list.receivedCount) #the invalid msg has been dropped def test_receivedSUBSSimpleNoRequest(self): langUtil = LanguagesProvider.getLanguagesInstance() data = { 'permid' : testDestPermId, 'channel_id' : testChannelId, 'infohash' : testInfohash, 'subtitles' : {"eng" : "This is content 1", "nld": "This is content 2", "ita" : "This is content 3"}, 'selversion' : OLPROTO_VER_FOURTEENTH } langs = data['subtitles'].keys() bitmask = langUtil.langCodesToMask(langs) binaryBitmask = uintToBinaryString(bitmask, length=4) expextedMessage = SUBS + \ bencode(( data['channel_id'], data['infohash'], binaryBitmask, [data['subtitles']['eng'], data['subtitles']['ita'], data['subtitles']['nld']] )) list = MockMsgListener() self.underTest.registerListener(list) val = self.underTest.handleMessage(testDestPermId, OLPROTO_VER_FOURTEENTH, expextedMessage) # never had a request for this message should be dropped self.assertFalse(val) self.assertEquals(0,list.subsCount) def test_receivedSUBSOtherRequest(self): langUtil = LanguagesProvider.getLanguagesInstance() data = { 'permid' : testDestPermId, 'channel_id' : testChannelId, 'infohash' : testInfohash, 'subtitles' : {"eng" : "This is content 1", "nld": "This is content 2", "ita" : "This is content 3"}, 'selversion' : OLPROTO_VER_FOURTEENTH } langs = data['subtitles'].keys() bitmask = langUtil.langCodesToMask(langs) binaryBitmask = uintToBinaryString(bitmask, length=4) expextedMessage = SUBS + \ bencode(( data['channel_id'], data['infohash'], binaryBitmask, [data['subtitles']['eng'], data['subtitles']['ita'], data['subtitles']['nld']] )) list = MockMsgListener() self.underTest.registerListener(list) #invalid bitmask self.underTest._addToRequestedSubtitles(testChannelId, testInfohash, int(0xFFFFFFFF & ~bitmask), None) val = self.underTest.handleMessage(testDestPermId, OLPROTO_VER_FOURTEENTH, expextedMessage) # never had a request for this message should be dropped self.assertFalse(val) self.assertEquals(0,list.subsCount) def test_receivedSUBSSomeRequest(self): langUtil = LanguagesProvider.getLanguagesInstance() data = { 'permid' : testDestPermId, 'channel_id' : testChannelId, 'infohash' : testInfohash, 'subtitles' : {"eng" : "This is content 1", "nld": "This is content 2", "ita" : "This is content 3"}, 'selversion' : OLPROTO_VER_FOURTEENTH } langs = data['subtitles'].keys() bitmask = langUtil.langCodesToMask(langs) binaryBitmask = uintToBinaryString(bitmask, length=4) expextedMessage = SUBS + \ bencode(( data['channel_id'], data['infohash'], binaryBitmask, [data['subtitles']['eng'], data['subtitles']['ita'], data['subtitles']['nld']] )) list = MockMsgListener() self.underTest.registerListener(list) #invalid bitmask self.underTest._addToRequestedSubtitles(testChannelId, testInfohash, langUtil.langCodesToMask(["ita"]), None) val = self.underTest.handleMessage(testDestPermId, OLPROTO_VER_FOURTEENTH, expextedMessage) # never had a request for this message should be dropped self.assertTrue(val) self.assertEquals(1,list.subsCount) params = list.subsParams[0] channel_id, infohash, contentsDictionary = params[1] self.assertEquals(testChannelId,channel_id) self.assertEquals(testInfohash, infohash) contentKeys = contentsDictionary.keys() self.assertEquals(["ita"],contentKeys) def test_cleanSUSRequests(self): self.underTest._requestValidityTime = 0.001 #ds self.underTest._addToRequestedSubtitles(testChannelId, testInfohash, 3, None) self.assertEquals(1,len(self.underTest.requestedSubtitles)) time.sleep(1.2) self.underTest._cleanUpRequestedSubtitles() self.assertEquals(0,len(self.underTest.requestedSubtitles))