def subtest_receptionOfSUBSTwoRequestsOneAvailable(self):
"""
Asking for two subtitles while the recipent of the request has only one.
The response should contain only the one available subtitle content,
plus a bitmask that reflects the contents of the response.
"""
print >> sys.stderr, time.asctime(),'-', "test: test_subtitles_msgs_2_1 -----------------------"
ol_conn = OLConnection(self.my_keypair,'localhost',self.hisport)
bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(['nld','eng'])
binmask = utilities.uintToBinaryString(bitmask, length=4)
request = GET_SUBS + \
bencode((
self.anotherpermid,
self.testInfohash,
binmask
))
subshandler = SubtitlesHandler()
subshandler.register(ol_conn, self.richMetadata_db, self.session)
ol_conn.send(request)
subs_data = ol_conn.recv()
self.assertEquals(SUBS, subs_data[0])
data = bdecode(subs_data[1:])
print >> sys.stderr, time.asctime(),'-', "test: subtitles_messages : received SUBS repsonse: ", data
#check on the format of the response
self.assertTrue(isinstance(data,list))
self.assertEquals(4, len(data)) # for fields
self.assertEquals(self.mdto.channel,data[0])
self.assertEquals(self.mdto.infohash, data[1])
#the receiver had only one of the two requested subtitles
# so I expect a different bitmask
bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(['nld'])
expectedBinarymask = utilities.uintToBinaryString(bitmask, length=4)
self.assertEquals(expectedBinarymask, data[2])
self.assertTrue(isinstance(data[3],list))
self.assertEquals(1, len(data[3]))
with codecs.open(self.sub1, "rb", "utf-8") as sub:
expectedContents = sub.read()
self.assertEquals(expectedContents, data[3][0])
ol_conn.close()
print >> sys.stderr, time.asctime(),'-', "test: subtitles_messages: received content is valid."
print >> sys.stderr, time.asctime(),'-', "End of test_subtitles_msgs_2_1 test --------------------"
def subtest_invalidRequest3(self):
"""
Trying to send an invalid message (valid for everythin except that there is one field more)
The connection should be closed by the receiver
"""
print >> sys.stderr, time.asctime(),'-', "test: test_subtitles_msgs_invalid_request_3 ------------------"
ol_conn = OLConnection(self.my_keypair,'localhost',self.hisport)
bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(['nld','eng'])
binmask = utilities.uintToBinaryString(bitmask, length=4)
request = GET_SUBS + \
bencode((
self.anotherpermid,
self.testInfohash,
binmask,
42
))
ol_conn.send(request)
self.assertEquals(0, len(ol_conn.recv()))
print >> sys.stderr, time.asctime(),'-', "test: test_subtitles_msgs_invalid_request_3: connection closed as expected"
ol_conn.close()
print >> sys.stderr, time.asctime(),'-', "End of test_subtitles_msgs_invalid_request_3 ------------------"
def test_addToRequestedSubtitles(self):
langUtil = LanguagesProvider.getLanguagesInstance()
bitmask1 = langUtil.langCodesToMask(["nld"])
self.underTest._addToRequestedSubtitles(testChannelId,
testInfohash, bitmask1)
key = "".join((testChannelId, testInfohash))
self.assertEquals(bitmask1,
self.underTest.requestedSubtitles[
key
].cumulativeBitmask)
bitmask2 = langUtil.langCodesToMask(["jpn", "ita"])
self.underTest._addToRequestedSubtitles(testChannelId,
testInfohash, bitmask2)
self.assertEquals(bitmask1 | bitmask2,
self.underTest.requestedSubtitles[
key
].cumulativeBitmask)
removeBitmask = langUtil.langCodesToMask(["nld", "ita"])
self.underTest._removeFromRequestedSubtitles(testChannelId,
testInfohash,
removeBitmask)
codes = langUtil.maskToLangCodes(self.underTest.requestedSubtitles[
key
].cumulativeBitmask)
self.assertEquals(["jpn"], codes)
def test_receivedSUBSOtherRequest(self):
langUtil = LanguagesProvider.getLanguagesInstance()
data = {
'permid' : testDestPermId,
'channel_id' : testChannelId,
'infohash' : testInfohash,
'subtitles' : {"eng" : "This is content 1", "nld": "This is content 2",
"ita" : "This is content 3"},
'selversion' : OLPROTO_VER_FOURTEENTH
}
langs = data['subtitles'].keys()
bitmask = langUtil.langCodesToMask(langs)
binaryBitmask = uintToBinaryString(bitmask, length=4)
expextedMessage = SUBS + \
bencode((
data['channel_id'],
data['infohash'],
binaryBitmask,
[data['subtitles']['eng'], data['subtitles']['ita'],
data['subtitles']['nld']]
))
list = MockMsgListener()
self.underTest.registerListener(list)
#invalid bitmask
self.underTest._addToRequestedSubtitles(testChannelId, testInfohash, int(0xFFFFFFFF & ~bitmask), None)
val = self.underTest.handleMessage(testDestPermId, OLPROTO_VER_FOURTEENTH, expextedMessage)
# never had a request for this message should be dropped
self.assertFalse(val)
self.assertEquals(0,list.subsCount)
def test_createSingleResponseMessage(self):
langUtil = LanguagesProvider.getLanguagesInstance()
data = {
'permid' : testDestPermId,
'channel_id' : testChannelId,
'infohash' : testInfohash,
'subtitles' : {"eng" : "This is content 1", "nld": "This is content 2",
"ita" : "This is content 3"},
'selversion' : OLPROTO_VER_FOURTEENTH
}
langs = data['subtitles'].keys()
bitmask = langUtil.langCodesToMask(langs)
binaryBitmask = uintToBinaryString(bitmask, length=4)
expextedMessage = SUBS + \
bencode((
data['channel_id'],
data['infohash'],
binaryBitmask,
[data['subtitles']['eng'], data['subtitles']['ita'],
data['subtitles']['nld']]
))
msg = self.underTest._createSingleResponseMessage(data)
decoded = bdecode(msg[1:])
self.assertEquals(expextedMessage, msg)
def __init__(self):
# notice that singleton pattern is not enforced.
# This is better, since this way the code is more easy
# to test.
SubtitlesHandler.__single = self
self.avg_subtitle_size = 100 # 100 KB, experimental avg
self.languagesUtility = LanguagesProvider.getLanguagesInstance()
# instance of MetadataDBHandler
self.subtitlesDb = None
self.registered = False
self.subs_dir = None
def __init__(self):
#singleton pattern not really enforced if someone just calls
# the normal constructor. But this way I can test the instance easier
try:
SubtitlesSupport._singletonLock.acquire()
SubtitlesSupport.__single = self
finally:
SubtitlesSupport._singletonLock.release()
self.richMetadata_db = None
self.subtitlesHandler = None
self.channelcast_db = None
self.langUtility = LanguagesProvider.getLanguagesInstance()
self._registered = False
def test_decodeGETSUBSMessage(self):
langUtil = LanguagesProvider.getLanguagesInstance()
binaryBitmask = uintToBinaryString(langUtil.langCodesToMask(["kor", "spa"]))
bencodedMessage = GET_SUBS + \
bencode((
testChannelId,
testInfohash,
binaryBitmask
))
channel_id, infohash, languages = \
self.underTest._decodeGETSUBSMessage(bencodedMessage)
self.assertEquals(testChannelId, channel_id)
self.assertEquals(testInfohash, infohash)
self.assertEquals(["kor", "spa"], languages)
def subtest_receptionOfSUBS(self):
'''
Asking for the single available subtitle. The response should be
a valid SUBS message containing its contents
'''
print >> sys.stderr, time.asctime(),'-', "test: test_subtitles_msgs_1_1 -----------------------"
ol_conn = OLConnection(self.my_keypair,'localhost',self.hisport)
bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(['nld'])
binmask = utilities.uintToBinaryString(bitmask, length=4)
request = GET_SUBS + \
bencode((
self.anotherpermid,
self.testInfohash,
binmask
))
subshandler = SubtitlesHandler()
subshandler.register(ol_conn, self.richMetadata_db, self.session)
ol_conn.send(request)
subs_data = ol_conn.recv()
print >> sys.stderr, time.asctime(),'-', "test: subtitles_messages : received SUBS response: len",len(subs_data)
self.assertEquals(SUBS, subs_data[0])
data = bdecode(subs_data[1:])
print >> sys.stderr, time.asctime(),'-', "test: subtitles_messages : received SUBS response: ", data
#check on the format of the response
self.assertTrue(isinstance(data,list))
self.assertEquals(4, len(data)) # for fields
self.assertEquals(self.mdto.channel,data[0])
self.assertEquals(self.mdto.infohash, data[1])
self.assertEquals(binmask, data[2])
self.assertTrue(isinstance(data[3],list))
self.assertEquals(1, len(data[3]))
with codecs.open(self.sub1, "rb", "utf-8") as sub:
expectedContents = sub.read()
self.assertEquals(expectedContents, data[3][0])
ol_conn.close()
print >> sys.stderr, time.asctime(),'-', "test: subtitles_messages: received content is valid."
print >> sys.stderr, time.asctime(),'-', "End of test_subtitles_msgs_1_1 test --------------------"
def getSubtitleFileRelativeName(channel_id, infohash, langCode):
# subtitles filenames are build from the sha1 hash
# of the triple (channel_id, infohash, langCode)
# channel_id and infohash are binary versions
assert utilities.validPermid(channel_id), "Invalid channel_id %s" % utilities.show_permid_short(channel_id)
assert utilities.validInfohash(infohash), "Invalid infohash %s" % bin2str(infohash)
assert LanguagesProvider.getLanguagesInstance().isLangCodeSupported(langCode), (
"Unsupported language code %s" % langCode
)
hasher = sha()
for data in (channel_id, infohash, langCode):
hasher.update(data)
subtitleName = hasher.hexdigest() + SUBS_EXTENSION
return subtitleName
def __init__(self, overlayBridge, tokenBucket, maxSubsSize):
self._languagesUtility = LanguagesProvider.getLanguagesInstance()
self._overlay_bridge = overlayBridge
# handleMessage() is called by the OLThread
# registerListener is called by the OLThread
# no synchronization should be needed for this list :)
self._listenersList = list()
self._tokenBucket = tokenBucket
#controls the interval the uploadQueue gets checked
self._nextUploadTime = 0
#dictionary of type { "".join(channel_id,infohash) : _RequestedSubtitlesEntry}
#bits get cleaned when subtitles are recevied
#when the bitmask is 000 the netry is removed from the dictionary
#also entries older then REQUEST_VALIDITY_TIME get dropped
self.requestedSubtitles = {}
self._requestsLock = threading.RLock()
self._nextCleanUpTime = int(time()) + CLEANUP_PERIOD
#subtitles to send get queueued in this queue
#each subtitle message to send is a dictionary whose keys are:
#permid: destionation of the message
#channel_id: identifier of the channel from wich the subtitles to
# upload are
#infohash: identifier of the torrent for which the subtitles to
# upload are
#subtitles: a dictionary of the form {langCode : path} for the
# subtitles to send
#selversion:
self._uploadQueue = []
self._requestValidityTime = REQUEST_VALIDITY_TIME
self._maxSubSize = maxSubsSize
def testSendSubtitlesRequestConnected(self):
langUtil = LanguagesProvider.getLanguagesInstance()
request = {}
request['channel_id'] = testChannelId
request['infohash'] = testInfohash
request['languages'] = ["kor"]
self.underTest.sendSubtitleRequest(testDestPermId, request, None, None, OLPROTO_VER_FOURTEENTH)
self.assertEquals(0, self.ol_bridge.connect_count) #selversion was 1
self.assertEquals(1, self.ol_bridge.send_count) #send called one time
binaryBitmask = uintToBinaryString(langUtil.langCodesToMask(["kor"]))
expectedMsg = GET_SUBS + \
bencode((
testChannelId,
testInfohash,
binaryBitmask
))
passedParameters = self.ol_bridge.sendParametersHistory[0]
self.assertEquals(testDestPermId, passedParameters[0])
self.assertEquals(expectedMsg, passedParameters[1])
def test_decodeGETSUBSMessageInvalid(self):
langUtil = LanguagesProvider.getLanguagesInstance()
binaryBitmask = uintToBinaryString(langUtil.langCodesToMask(["kor", "spa"]))
invalidTypeMsg = chr(25) + \
bencode((
testChannelId,
testInfohash,
binaryBitmask
))
self.assertRaises(AssertionError, self.underTest._decodeGETSUBSMessage,
(invalidTypeMsg,))
invalidMsgField = GET_SUBS + \
bencode((
42,
testChannelId,
testInfohash,
binaryBitmask
))
decoded = \
self.underTest._decodeGETSUBSMessage(invalidMsgField)
#when something in the body is wrong returns None
self.assertTrue(decoded is None)
invalidBitamsk = uintToBinaryString(0xFFFFFFFF11, 5)
invalidMsgField = GET_SUBS + \
bencode((
testChannelId,
testInfohash,
invalidBitamsk #40 bit bitmask!)
))
decoded = \
self.underTest._decodeGETSUBSMessage(invalidMsgField)
#when something in the body is wrong returns None
self.assertTrue(decoded is None)
def test_receivedGETSUBSSimple(self):
langUtil = LanguagesProvider.getLanguagesInstance()
bitmask = langUtil.langCodesToMask(["eng", "rus"])
binaryBitmask = uintToBinaryString(bitmask, length=4)
request = GET_SUBS + \
bencode((
testChannelId,
testInfohash,
binaryBitmask
))
list = MockMsgListener()
self.underTest.registerListener(list)
self.underTest.handleMessage(testDestPermId, OLPROTO_VER_FOURTEENTH, request)
self.assertEquals(1,list.receivedCount)
self.assertEquals(testDestPermId, list.receivedParams[0][0])
self.assertEquals(OLPROTO_VER_FOURTEENTH,list.receivedParams[0][2])
self.assertEquals((testChannelId,testInfohash,["eng","rus"]),list.receivedParams[0][1])
def __init__(self, lang, path=None, checksum=None):
"""
Create a subtitle instance.
@param lang: an ISO 639-2 language code. Notice that not every language
code described by the standard is supported, but only
a small subset. See the Languages module
@param path: a file system path to the subtitles file
@param checksum: a sha1 checksum of the contents
of the subitles file
"""
self._languages = LanguagesProvider.getLanguagesInstance()
if lang not in self._languages.supportedLanguages.keys():
raise ValueError("Language" + lang + " not supported")
#ISO 639-2 code. See Languages for supported languages
self._lang = lang #final property
#A string representing the path in the filesystme for this subtitle
self._path = path
#sha1 checksum
self._checksum = checksum
def test_packDataWithSubs(self):
badInfohash = str2bin("GEh/o8rtTLB1wZJzFcSZSS4u9qo=")
dto = MetadataDTO(test_perm_id, badInfohash)
subtitles = [SubtitleInfo(lang,path) for lang,path in self._srtSubs.iteritems()]
for sub in subtitles :
sub.computeChecksum()
dto.addSubtitle(sub)
packed = dto._packData()
decoded = bdecode(packed)
self.assertTrue(len(decoded) == 6)
decodedChannelId = decoded[0]
decodedInfohash = decoded[1]
decodedDescription = decoded[2]
decodedTimestamp = decoded[3]
decodedBitmask = decoded[4]
checksums = decoded[5]
expectedMask = \
LanguagesProvider.getLanguagesInstance().langCodesToMask(self._srtSubs.keys())
binaryExpexted = uintToBinaryString(expectedMask)
self.assertEquals(dto.channel, decodedChannelId)
self.assertEquals(dto.infohash, decodedInfohash)
self.assertEquals(dto.description,decodedDescription)
self.assertAlmostEquals(dto.timestamp,decodedTimestamp)
self.assertEquals(binaryExpexted,decodedBitmask)
self.assertEquals(3,len(checksums))
subs = dto.getAllSubtitles()
i=0
for key in sorted(subs.iterkeys()):
self.assertEquals(subs[key].checksum, checksums[i])
i += 1
def testReceivedSUBSMessage(self):
self.testRegisterStuff()
languages = ["eng", "rus"]
zho = u"Subtitle Content 1"
kor = u"Subtitle Content 2"
contentsDict = {"eng": zho, "rus": kor}
msg = (testChannelId, testInfohash, contentsDict)
simpleCallback = lambda x: x
bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(["eng", "rus"])
self.underTest.receivedSubsResponse(testDestPermId, msg, [(simpleCallback, bitmask)], OLPROTO_VER_FOURTEENTH)
# self.assertEquals(languages,callbackParams)
expectedFilename1 = getSubtitleFileRelativeName(testChannelId, testInfohash, "eng")
expectedFilename2 = getSubtitleFileRelativeName(testChannelId, testInfohash, "rus")
expectedPath1 = os.path.join(self._session.get_state_dir(), self.underTest.subs_dir, expectedFilename1)
expectedPath2 = os.path.join(self._session.get_state_dir(), self.underTest.subs_dir, expectedFilename2)
self.assertTrue(os.path.isfile(expectedPath1))
self.assertTrue(os.path.isfile(expectedPath2))
with codecs.open(expectedPath1, "rb", "utf-8") as file1:
content1 = file1.read()
self.assertEquals(zho, content1)
with codecs.open(expectedPath2, "rb", "utf-8") as file2:
content2 = file2.read()
self.assertEquals(kor, content2)
self.assertEquals(1, self.ol_bridge.add_task_count)
params = self.ol_bridge.add_taskParametersHistory[0]
# calling the lambda
val = params[0]()
self.assertEquals(languages, val)
# Written by Andrea Reale
# see LICENSE.txt for license information
from BaseLib.Core.BitTornado.bencode import bencode, bdecode
from BaseLib.Core.Subtitles.MetadataDomainObjects.Languages import LanguagesProvider
from BaseLib.Core.Subtitles.MetadataDomainObjects.MetadataExceptions import SerializationException
from BaseLib.Core.Subtitles.MetadataDomainObjects.SubtitleInfo import SubtitleInfo
from BaseLib.Core.Overlay.permid import sign_data, verify_data
from BaseLib.Core.Utilities.utilities import isValidInfohash, isValidPermid, uintToBinaryString, binaryStringToUint
from math import floor
import sys
import time
DEBUG = False
_languagesUtil = LanguagesProvider.getLanguagesInstance()
class MetadataDTO(object):
"""
Metdata DataTransferObject
"""
def __init__(self, publisher, infohash, timestamp=None, description=u"", subtitles=None, signature=None):
"""
Create a MetataDTO instance.
publisher and infohash are mandatory to be not null
@param publisher: the permid of the owner of the
channel this instance refers to
@param infohash: the infohash of the item in the channel this instance
