def register(self, overlay_bridge, metadataDBHandler, session):
"""
Injects the required dependencies on the instance.
@param overlay_bridge: a reference to a working instance
of OverlayTrheadingBridge
@param metadataDBHandler: a reference to the current instance of
L{MetadataDBHandler}
@param session: a reference to the running session
"""
self.overlay_bridge = overlay_bridge
self.subtitlesDb = metadataDBHandler
self.config_dir = os.path.abspath(session.get_state_dir())
subs_path = os.path.join(self.config_dir, session.get_subtitles_collecting_dir())
self.subs_dir = os.path.abspath(subs_path)
self.min_free_space = DEFAULT_MIN_FREE_SPACE
self._upload_rate = session.get_subtitles_upload_rate()
self.max_subs_message_size = MAX_SUBS_MESSAGE_SIZE
self._session = session
# the upload rate is controlled by a token bucket.
# a token corresponds to 1 KB.
# The max burst size corresponds to 2 subtitles of the maximum size (2 MBs)
tokenBucket = SimpleTokenBucket(self._upload_rate, self.max_subs_message_size)
self._subsMsgHndlr = SubsMessageHandler(self.overlay_bridge, tokenBucket, MAX_SUBTITLE_SIZE)
self._subsMsgHndlr.registerListener(self)
# assure that the directory exists
if os.path.isdir(self.config_dir):
if not os.path.isdir(self.subs_dir):
try:
os.mkdir(self.subs_dir)
except:
msg = u"Cannot create collecting dir %s " % self.subs_dir
print >>sys.stderr, "Error: %s" % msg
raise IOError(msg)
else:
msg = u"Configuration dir %s does not exists" % self.subs_dir
print >>sys.stderr, "Error: %s" % msg
raise IOError(msg)
diskManager = DiskManager(self.min_free_space, self.config_dir)
self.diskManager = diskManager
dmConfig = {
"maxDiskUsage": MAX_SUBTITLE_DISK_USAGE,
"diskPolicy": DISK_FULL_DELETE_SOME | DELETE_OLDEST_FIRST,
"encoding": "utf-8",
}
self.diskManager.registerDir(self.subs_dir, dmConfig)
freeSpace = self.diskManager.getAvailableSpace()
if DEBUG:
print >>sys.stderr, SUBS_LOG_PREFIX + "Avaialble %d MB for subtitle collecting" % (freeSpace / (2 ** 20))
# event notifier
self._notifier = Notifier.getInstance()
self.registered = True
def setUp(self):
self.ol_bridge = MockOverlayBridge()
self.tokenBucket = MockTokenBucket()
self.underTest = SubsMessageHandler(self.ol_bridge,self.tokenBucket,1000000)
class SubtitlesHandler(object):
__single = None
def __init__(self):
# notice that singleton pattern is not enforced.
# This is better, since this way the code is more easy
# to test.
SubtitlesHandler.__single = self
self.avg_subtitle_size = 100 # 100 KB, experimental avg
self.languagesUtility = LanguagesProvider.getLanguagesInstance()
# instance of MetadataDBHandler
self.subtitlesDb = None
self.registered = False
self.subs_dir = None
# other useful attributes are injected by the register method
@staticmethod
def getInstance(*args, **kw):
if SubtitlesHandler.__single is None:
SubtitlesHandler(*args, **kw)
return SubtitlesHandler.__single
def register(self, overlay_bridge, metadataDBHandler, session):
"""
Injects the required dependencies on the instance.
@param overlay_bridge: a reference to a working instance
of OverlayTrheadingBridge
@param metadataDBHandler: a reference to the current instance of
L{MetadataDBHandler}
@param session: a reference to the running session
"""
self.overlay_bridge = overlay_bridge
self.subtitlesDb = metadataDBHandler
self.config_dir = os.path.abspath(session.get_state_dir())
subs_path = os.path.join(self.config_dir, session.get_subtitles_collecting_dir())
self.subs_dir = os.path.abspath(subs_path)
self.min_free_space = DEFAULT_MIN_FREE_SPACE
self._upload_rate = session.get_subtitles_upload_rate()
self.max_subs_message_size = MAX_SUBS_MESSAGE_SIZE
self._session = session
# the upload rate is controlled by a token bucket.
# a token corresponds to 1 KB.
# The max burst size corresponds to 2 subtitles of the maximum size (2 MBs)
tokenBucket = SimpleTokenBucket(self._upload_rate, self.max_subs_message_size)
self._subsMsgHndlr = SubsMessageHandler(self.overlay_bridge, tokenBucket, MAX_SUBTITLE_SIZE)
self._subsMsgHndlr.registerListener(self)
# assure that the directory exists
if os.path.isdir(self.config_dir):
if not os.path.isdir(self.subs_dir):
try:
os.mkdir(self.subs_dir)
except:
msg = u"Cannot create collecting dir %s " % self.subs_dir
print >>sys.stderr, "Error: %s" % msg
raise IOError(msg)
else:
msg = u"Configuration dir %s does not exists" % self.subs_dir
print >>sys.stderr, "Error: %s" % msg
raise IOError(msg)
diskManager = DiskManager(self.min_free_space, self.config_dir)
self.diskManager = diskManager
dmConfig = {
"maxDiskUsage": MAX_SUBTITLE_DISK_USAGE,
"diskPolicy": DISK_FULL_DELETE_SOME | DELETE_OLDEST_FIRST,
"encoding": "utf-8",
}
self.diskManager.registerDir(self.subs_dir, dmConfig)
freeSpace = self.diskManager.getAvailableSpace()
if DEBUG:
print >>sys.stderr, SUBS_LOG_PREFIX + "Avaialble %d MB for subtitle collecting" % (freeSpace / (2 ** 20))
# event notifier
self._notifier = Notifier.getInstance()
self.registered = True
def sendSubtitleRequest(self, permid, channel_id, infohash, languages, callback=None, selversion=-1):
"""
Send a request for subtitle files. Only called by the OLThread
Send a GET_SUBS request to the peer indentified by permid.
The request asks for several subtitles file, for a given channel_id
and torrent infohash. The subtitles file to request are specified
by the languages parameter that is a list of 3 characters language
codes.
The contents of a GET_SUBS request are:
- channel_id: the identifier of the channel for which the subtitles
were added. (a permid). Binary.
- infohash: the infohash of the torrent, the subtitles refer to.
Binary.
- bitmask: a 32 bit bitmask (an integer) which specifies the
languages requested
@param permid: the destination of the request (binary)
@param channel_id: the identifier of the channel for which the subtitle
was added (binary)
@param infohash: the infohash of a torrent the subtitles refers to (binary).
@param languages: a list of 3-characters language codes. It must be
on of the supported language codes (see Languages)
@param callback: a function that will be called WHENEVER some of the
requested subtitles are received. It must have exactly
one parameter that will be bound to a list of
the languages that were received
@param selversion: the protocol version of the peer whe are sending
the request to
@raise SubtitleMsgHandlerException: if the message failed its attempt to be sent.
Notice that also if the method returns without
raising any exception it doesn't mean
that the message has been sent.
"""
assert utilities.isValidInfohash(infohash), SUBS_LOG_PREFIX + "Invalid infohash %s" % infohash
assert utilities.isValidPermid(permid), SUBS_LOG_PREFIX + "Invlaid destination permid %s" % permid
assert self.languagesUtility.isLangListSupported(languages), (
SUBS_LOG_PREFIX + "Some of the languages where not supported"
)
if DEBUG:
print >>sys.stderr, SUBS_LOG_PREFIX + "preparing to send GET_SUBS to " + utilities.show_permid_short(permid)
# Better to leave up to the caller the responsibility to check
# if the subtitle is already available and as correct checsum and so on..
# onDisk = []
# for langCode in languages:
#
# filename = self.diskManager.isFilenOnDisk(self.subs_dir,
# getSubtitleFileRelativeName(channel_id, infohash, langCode))
#
#
# # should I skip this part and just send the request anyway?
# # (thus leaving to the caller the responsibility to avoid useless
# # requests)
# if filename:
# log.debug(SUBS_LOG_PREFIX + langCode +
# " subtitle already on disk. Skipping it"\
# " in the request")
# onDisk.append(langCode)
# self._notify_sub_is_in(channel_id, infohash, langCode, filename)
#
# for deleteme in onDisk:
# languages.remove(deleteme)
if len(languages) == 0:
if DEBUG:
print >>sys.stderr, SUBS_LOG_PREFIX + " no subtitles to request."
return
if not self.diskManager.tryReserveSpace(self.subs_dir, len(languages) * self.avg_subtitle_size):
self._warn_disk_full()
return False
requestDetails = dict()
requestDetails["channel_id"] = channel_id
requestDetails["infohash"] = infohash
requestDetails["languages"] = languages
self._subsMsgHndlr.sendSubtitleRequest(
permid, requestDetails, lambda e, d, c, i, b: self._subsRequestSent(e, d, c, i, b), callback, selversion
)
def _subsRequestSent(self, exception, dest, channel_id, infohash, bitmask):
"""
Gets called when a subtitle request has been succesfully sent.
"""
pass
def receivedSubsRequest(self, permid, request, selversion):
"""
Reads a received GET_SUBS message and possibly sends a response.
@param permid: the permid of the sender of the GET_SUBS message
@param request: a tuple made of channel_id, infohash, language code
@param selversion: the protocol version of the requesting peer
@return: False if the message had something wrong. (a return value
of False makes the caller close the connection).
Otherwise True
"""
assert self.registered, SUBS_LOG_PREFIX + "Handler not yet registered"
channel_id, infohash, languages = request # happily unpacking
# diction {lang : Subtitle}
allSubtitles = self.subtitlesDb.getAllSubtitles(channel_id, infohash)
contentsList = {} # {langCode : path}
# for each requested language check if the corresponding subtitle
# is available
for lang in sorted(languages):
if lang in allSubtitles.keys():
if allSubtitles[lang].subtitleExists():
content = self._readSubContent(allSubtitles[lang].path)
if content is not None:
contentsList[lang] = content
else:
if DEBUG:
print >>sys.stderr, SUBS_LOG_PREFIX + "File not available for " + "channel %s, infohash %s, lang %s" % (
show_permid_short(channel_id),
bin2str(infohash),
lang,
)
self.subtitlesDb.updateSubtitlePath(channel_id, infohash, lang, None)
else:
if DEBUG:
print >>sys.stderr, SUBS_LOG_PREFIX + "Subtitle not available for " + "channel %s, infohash %s, lang %s" % (
show_permid_short(channel_id),
bin2str(infohash),
lang,
)
if len(contentsList) == 0: # pathlist is empty
if DEBUG:
print >>sys.stderr, SUBS_LOG_PREFIX + "None of the requested subtitles " + " was available. No answer will be sent to %s" % show_permid_short(
permid
)
return True
return self._subsMsgHndlr.sendSubtitleResponse(permid, (channel_id, infohash, contentsList), selversion)
def _readSubContent(self, path):
try:
relativeName = os.path.relpath(path, self.subs_dir)
fileContent = self.diskManager.readContent(self.subs_dir, relativeName)
except IOError, e:
if DEBUG:
print >>sys.stderr, SUBS_LOG_PREFIX + "Error reading from subs file %s: %s" % (relativeName, e)
fileContent = None
if fileContent is not None and len(fileContent) <= MAX_SUBTITLE_SIZE:
return fileContent
else:
print >>sys.stderr, "Warning: Subtitle %s dropped. Bigger then %d" % (relativeName, MAX_SUBTITLE_SIZE)
return None
class TestSubtitlesMsgHandlerIsolation(unittest.TestCase):
def setUp(self):
self.ol_bridge = MockOverlayBridge()
self.tokenBucket = MockTokenBucket()
self.underTest = SubsMessageHandler(self.ol_bridge,self.tokenBucket,1000000)
def test_addToRequestedSubtitles(self):
langUtil = LanguagesProvider.getLanguagesInstance()
bitmask1 = langUtil.langCodesToMask(["nld"])
self.underTest._addToRequestedSubtitles(testChannelId,
testInfohash, bitmask1)
key = "".join((testChannelId, testInfohash))
self.assertEquals(bitmask1,
self.underTest.requestedSubtitles[
key
].cumulativeBitmask)
bitmask2 = langUtil.langCodesToMask(["jpn", "ita"])
self.underTest._addToRequestedSubtitles(testChannelId,
testInfohash, bitmask2)
self.assertEquals(bitmask1 | bitmask2,
self.underTest.requestedSubtitles[
key
].cumulativeBitmask)
removeBitmask = langUtil.langCodesToMask(["nld", "ita"])
self.underTest._removeFromRequestedSubtitles(testChannelId,
testInfohash,
removeBitmask)
codes = langUtil.maskToLangCodes(self.underTest.requestedSubtitles[
key
].cumulativeBitmask)
self.assertEquals(["jpn"], codes)
def testSendSubtitlesRequestConnected(self):
langUtil = LanguagesProvider.getLanguagesInstance()
request = {}
request['channel_id'] = testChannelId
request['infohash'] = testInfohash
request['languages'] = ["kor"]
self.underTest.sendSubtitleRequest(testDestPermId, request, None, None, OLPROTO_VER_FOURTEENTH)
self.assertEquals(0, self.ol_bridge.connect_count) #selversion was 1
self.assertEquals(1, self.ol_bridge.send_count) #send called one time
binaryBitmask = uintToBinaryString(langUtil.langCodesToMask(["kor"]))
expectedMsg = GET_SUBS + \
bencode((
testChannelId,
testInfohash,
binaryBitmask
))
passedParameters = self.ol_bridge.sendParametersHistory[0]
self.assertEquals(testDestPermId, passedParameters[0])
self.assertEquals(expectedMsg, passedParameters[1])
def testSendSubtitlesRequestNotConnected(self):
langUtil = LanguagesProvider.getLanguagesInstance()
request = {}
request['channel_id'] = testChannelId
request['infohash'] = testInfohash
request['languages'] = ["kor"]
self.underTest.sendSubtitleRequest(testDestPermId, request)
self.assertEquals(1, self.ol_bridge.connect_count) #selversion was -1
self.assertEquals(1, self.ol_bridge.send_count) #send called one time
binaryBitmask = uintToBinaryString(langUtil.langCodesToMask(["kor"]))
expectedMsg = GET_SUBS + \
bencode((
testChannelId,
testInfohash,
binaryBitmask
))
passedParameters = self.ol_bridge.sendParametersHistory[0]
self.assertEquals(testDestPermId, passedParameters[0])
self.assertEquals(expectedMsg, passedParameters[1])
def test_decodeGETSUBSMessage(self):
langUtil = LanguagesProvider.getLanguagesInstance()
binaryBitmask = uintToBinaryString(langUtil.langCodesToMask(["kor", "spa"]))
bencodedMessage = GET_SUBS + \
bencode((
testChannelId,
testInfohash,
binaryBitmask
))
channel_id, infohash, languages = \
self.underTest._decodeGETSUBSMessage(bencodedMessage)
self.assertEquals(testChannelId, channel_id)
self.assertEquals(testInfohash, infohash)
self.assertEquals(["kor", "spa"], languages)
def test_decodeGETSUBSMessageInvalid(self):
langUtil = LanguagesProvider.getLanguagesInstance()
binaryBitmask = uintToBinaryString(langUtil.langCodesToMask(["kor", "spa"]))
invalidTypeMsg = chr(25) + \
bencode((
testChannelId,
testInfohash,
binaryBitmask
))
self.assertRaises(AssertionError, self.underTest._decodeGETSUBSMessage,
(invalidTypeMsg,))
invalidMsgField = GET_SUBS + \
bencode((
42,
testChannelId,
testInfohash,
binaryBitmask
))
decoded = \
self.underTest._decodeGETSUBSMessage(invalidMsgField)
#when something in the body is wrong returns None
self.assertTrue(decoded is None)
invalidBitamsk = uintToBinaryString(0xFFFFFFFF11, 5)
invalidMsgField = GET_SUBS + \
bencode((
testChannelId,
testInfohash,
invalidBitamsk #40 bit bitmask!)
))
decoded = \
self.underTest._decodeGETSUBSMessage(invalidMsgField)
#when something in the body is wrong returns None
self.assertTrue(decoded is None)
def test_createSingleResponseMessage(self):
langUtil = LanguagesProvider.getLanguagesInstance()
data = {
'permid' : testDestPermId,
'channel_id' : testChannelId,
'infohash' : testInfohash,
'subtitles' : {"eng" : "This is content 1", "nld": "This is content 2",
"ita" : "This is content 3"},
'selversion' : OLPROTO_VER_FOURTEENTH
}
langs = data['subtitles'].keys()
bitmask = langUtil.langCodesToMask(langs)
binaryBitmask = uintToBinaryString(bitmask, length=4)
expextedMessage = SUBS + \
bencode((
data['channel_id'],
data['infohash'],
binaryBitmask,
[data['subtitles']['eng'], data['subtitles']['ita'],
data['subtitles']['nld']]
))
msg = self.underTest._createSingleResponseMessage(data)
decoded = bdecode(msg[1:])
self.assertEquals(expextedMessage, msg)
def test_receivedGETSUBSSimple(self):
langUtil = LanguagesProvider.getLanguagesInstance()
bitmask = langUtil.langCodesToMask(["eng", "rus"])
binaryBitmask = uintToBinaryString(bitmask, length=4)
request = GET_SUBS + \
bencode((
testChannelId,
testInfohash,
binaryBitmask
))
list = MockMsgListener()
self.underTest.registerListener(list)
self.underTest.handleMessage(testDestPermId, OLPROTO_VER_FOURTEENTH, request)
self.assertEquals(1,list.receivedCount)
self.assertEquals(testDestPermId, list.receivedParams[0][0])
self.assertEquals(OLPROTO_VER_FOURTEENTH,list.receivedParams[0][2])
self.assertEquals((testChannelId,testInfohash,["eng","rus"]),list.receivedParams[0][1])
def test_receivedGETSUBSInvalid1(self):
bitmask = -1
request = GET_SUBS + \
bencode((
testChannelId,
testInfohash,
bitmask
))
list = MockMsgListener()
self.underTest.registerListener(list)
val = self.underTest.handleMessage(testDestPermId, OLPROTO_VER_FOURTEENTH, request)
self.assertFalse(val)
self.assertEquals(0,list.receivedCount) #the invalid msg has been dropped
def test_receivedGETSUBSInvalid2(self):
bitmask = -1
request = GET_SUBS + \
bencode((
testChannelId,
testInfohash,
bitmask
))
list = MockMsgListener()
self.underTest.registerListener(list)
val = self.underTest.handleMessage(testDestPermId, 13,request)
self.assertFalse(val)
self.assertEquals(0,list.receivedCount) #the invalid msg has been dropped
def test_receivedSUBSSimpleNoRequest(self):
langUtil = LanguagesProvider.getLanguagesInstance()
data = {
'permid' : testDestPermId,
'channel_id' : testChannelId,
'infohash' : testInfohash,
'subtitles' : {"eng" : "This is content 1", "nld": "This is content 2",
"ita" : "This is content 3"},
'selversion' : OLPROTO_VER_FOURTEENTH
}
langs = data['subtitles'].keys()
bitmask = langUtil.langCodesToMask(langs)
binaryBitmask = uintToBinaryString(bitmask, length=4)
expextedMessage = SUBS + \
bencode((
data['channel_id'],
data['infohash'],
binaryBitmask,
[data['subtitles']['eng'], data['subtitles']['ita'],
data['subtitles']['nld']]
))
list = MockMsgListener()
self.underTest.registerListener(list)
val = self.underTest.handleMessage(testDestPermId, OLPROTO_VER_FOURTEENTH, expextedMessage)
# never had a request for this message should be dropped
self.assertFalse(val)
self.assertEquals(0,list.subsCount)
def test_receivedSUBSOtherRequest(self):
langUtil = LanguagesProvider.getLanguagesInstance()
data = {
'permid' : testDestPermId,
'channel_id' : testChannelId,
'infohash' : testInfohash,
'subtitles' : {"eng" : "This is content 1", "nld": "This is content 2",
"ita" : "This is content 3"},
'selversion' : OLPROTO_VER_FOURTEENTH
}
langs = data['subtitles'].keys()
bitmask = langUtil.langCodesToMask(langs)
binaryBitmask = uintToBinaryString(bitmask, length=4)
expextedMessage = SUBS + \
bencode((
data['channel_id'],
data['infohash'],
binaryBitmask,
[data['subtitles']['eng'], data['subtitles']['ita'],
data['subtitles']['nld']]
))
list = MockMsgListener()
self.underTest.registerListener(list)
#invalid bitmask
self.underTest._addToRequestedSubtitles(testChannelId, testInfohash, int(0xFFFFFFFF & ~bitmask), None)
val = self.underTest.handleMessage(testDestPermId, OLPROTO_VER_FOURTEENTH, expextedMessage)
# never had a request for this message should be dropped
self.assertFalse(val)
self.assertEquals(0,list.subsCount)
def test_receivedSUBSSomeRequest(self):
langUtil = LanguagesProvider.getLanguagesInstance()
data = {
'permid' : testDestPermId,
'channel_id' : testChannelId,
'infohash' : testInfohash,
'subtitles' : {"eng" : "This is content 1", "nld": "This is content 2",
"ita" : "This is content 3"},
'selversion' : OLPROTO_VER_FOURTEENTH
}
langs = data['subtitles'].keys()
bitmask = langUtil.langCodesToMask(langs)
binaryBitmask = uintToBinaryString(bitmask, length=4)
expextedMessage = SUBS + \
bencode((
data['channel_id'],
data['infohash'],
binaryBitmask,
[data['subtitles']['eng'], data['subtitles']['ita'],
data['subtitles']['nld']]
))
list = MockMsgListener()
self.underTest.registerListener(list)
#invalid bitmask
self.underTest._addToRequestedSubtitles(testChannelId, testInfohash, langUtil.langCodesToMask(["ita"]), None)
val = self.underTest.handleMessage(testDestPermId, OLPROTO_VER_FOURTEENTH, expextedMessage)
# never had a request for this message should be dropped
self.assertTrue(val)
self.assertEquals(1,list.subsCount)
params = list.subsParams[0]
channel_id, infohash, contentsDictionary = params[1]
self.assertEquals(testChannelId,channel_id)
self.assertEquals(testInfohash, infohash)
contentKeys = contentsDictionary.keys()
self.assertEquals(["ita"],contentKeys)
def test_cleanSUSRequests(self):
self.underTest._requestValidityTime = 0.001 #ds
self.underTest._addToRequestedSubtitles(testChannelId, testInfohash, 3, None)
self.assertEquals(1,len(self.underTest.requestedSubtitles))
time.sleep(1.2)
self.underTest._cleanUpRequestedSubtitles()
self.assertEquals(0,len(self.underTest.requestedSubtitles))
