def subtest_receptionOfSUBSTwoRequestsOneAvailable(self):
     """
     Asking for two subtitles while the recipent of the request has only one.
     The response should contain only the one available subtitle content,
     plus a bitmask that reflects the contents of the response.
     """
     
     print >> sys.stderr, "test: test_subtitles_msgs_2_1 -----------------------"
     ol_conn = OLConnection(self.my_keypair,'localhost',self.hisport)
     
     bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(['nld','eng'])
     binmask = utilities.uintToBinaryString(bitmask, length=4)
     
     request = GET_SUBS + \
                   bencode((
                           self.anotherpermid,
                           self.testInfohash,
                           binmask
                           ))
                   
     subshandler = SubtitlesHandler()
     subshandler.register(ol_conn, self.richMetadata_db, self.session)
     
     ol_conn.send(request)
     subs_data = ol_conn.recv()
     self.assertEquals(SUBS, subs_data[0])
     data = bdecode(subs_data[1:])
     print >> sys.stderr, "test: subtitles_messages : received SUBS repsonse: ", data
     
     #check on the format of the response
     self.assertTrue(isinstance(data,list))
     self.assertEquals(4, len(data)) # for fields
     self.assertEquals(self.mdto.channel,data[0])
     self.assertEquals(self.mdto.infohash, data[1])
     
     #the receiver had only one of the two requested subtitles
     # so I expect a different bitmask
     bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(['nld'])
     expectedBinarymask = utilities.uintToBinaryString(bitmask, length=4)
     
     self.assertEquals(expectedBinarymask, data[2])
     self.assertTrue(isinstance(data[3],list))
     self.assertEquals(1, len(data[3]))
     with codecs.open(self.sub1, "rb", "utf-8") as sub:
         expectedContents = sub.read()
     self.assertEquals(expectedContents, data[3][0])
     
     ol_conn.close()
     print >> sys.stderr, "test: subtitles_messages: received content is valid."
     print >> sys.stderr, "End of test_subtitles_msgs_2_1 test --------------------"
 def test_addToRequestedSubtitles(self):
     langUtil = LanguagesProvider.getLanguagesInstance()
     bitmask1 = langUtil.langCodesToMask(["nld"])
     
     self.underTest._addToRequestedSubtitles(testChannelId,
                                             testInfohash, bitmask1)
     
     key = "".join((testChannelId, testInfohash))
     self.assertEquals(bitmask1,
                       self.underTest.requestedSubtitles[
                                                         key
                                                         ].cumulativeBitmask)
     
     bitmask2 = langUtil.langCodesToMask(["jpn", "ita"])
     self.underTest._addToRequestedSubtitles(testChannelId,
                                             testInfohash, bitmask2)
     
     self.assertEquals(bitmask1 | bitmask2,
                       self.underTest.requestedSubtitles[
                                                         key
                                                         ].cumulativeBitmask)
     
     removeBitmask = langUtil.langCodesToMask(["nld", "ita"])
     self.underTest._removeFromRequestedSubtitles(testChannelId,
                                                    testInfohash,
                                                    removeBitmask)
     
     codes = langUtil.maskToLangCodes(self.underTest.requestedSubtitles[
                                                         key
                                                         ].cumulativeBitmask)
     
     self.assertEquals(["jpn"], codes)
 def test_receivedSUBSOtherRequest(self):
     langUtil = LanguagesProvider.getLanguagesInstance()
     data = {
             'permid' : testDestPermId,
             'channel_id' : testChannelId,
             'infohash' : testInfohash,
             'subtitles' : {"eng" : "This is content 1", "nld": "This is content  2",
                            "ita" : "This is content 3"},
             'selversion' : OLPROTO_VER_FOURTEENTH
             }
     langs = data['subtitles'].keys()
     
     bitmask = langUtil.langCodesToMask(langs)
     binaryBitmask = pack("!L", bitmask)
     expextedMessage = SUBS + \
                         bencode((
                                 data['channel_id'],
                                 data['infohash'],
                                 binaryBitmask,
                                 [data['subtitles']['eng'], data['subtitles']['ita'],
                                  data['subtitles']['nld']]
                                  ))
     
     list = MockMsgListener()
     self.underTest.registerListener(list) 
     
     #invalid bitmask
     self.underTest._addToRequestedSubtitles(testChannelId, testInfohash, int(0xFFFFFFFF & ~bitmask), None)                   
     
     val = self.underTest.handleMessage(testDestPermId, OLPROTO_VER_FOURTEENTH, expextedMessage)
     # never had a request for this message should be dropped
     self.assertFalse(val)
     self.assertEquals(0,list.subsCount)
 def test_createSingleResponseMessage(self):
     langUtil = LanguagesProvider.getLanguagesInstance()
     data = {
             'permid' : testDestPermId,
             'channel_id' : testChannelId,
             'infohash' : testInfohash,
             'subtitles' : {"eng" : "This is content 1", "nld": "This is content  2",
                            "ita" : "This is content 3"},
             'selversion' : OLPROTO_VER_FOURTEENTH
             }
     langs = data['subtitles'].keys()
     
     bitmask = langUtil.langCodesToMask(langs)
     binaryBitmask = pack("!L", bitmask)
     expextedMessage = SUBS + \
                         bencode((
                                 data['channel_id'],
                                 data['infohash'],
                                 binaryBitmask,
                                 [data['subtitles']['eng'], data['subtitles']['ita'],
                                  data['subtitles']['nld']]
                                  ))
     msg = self.underTest._createSingleResponseMessage(data)
     decoded = bdecode(msg[1:])
     
     self.assertEquals(expextedMessage, msg)
    def subtest_invalidRequest3(self):
        """
        Trying to send an invalid message (valid for everythin except that there is one field more)
        The connection should be closed by the receiver
        """
        print >> sys.stderr, "test: test_subtitles_msgs_invalid_request_3 ------------------"
        ol_conn = OLConnection(self.my_keypair,'localhost',self.hisport)

        bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(['nld','eng'])
        binmask = utilities.uintToBinaryString(bitmask, length=4)
        
        request = GET_SUBS + \
                      bencode((
                              self.anotherpermid,
                              self.testInfohash,
                              binmask,
                              42
                              ))
        
        ol_conn.send(request)
        self.assertEquals(0, len(ol_conn.recv()))
        print >> sys.stderr, "test: test_subtitles_msgs_invalid_request_3: connection closed as expected"
        
        ol_conn.close()
        print >> sys.stderr, "End of test_subtitles_msgs_invalid_request_3 ------------------"
    def __init__(self):
        # notice that singleton pattern is not enforced.
        # This is better, since this way the code is more easy
        # to test.
        
        SubtitlesHandler.__single = self
        self.languagesUtility = LanguagesProvider.getLanguagesInstance()

        #instance of MetadataDBHandler
        self.subtitlesDb = None
        self.registered = False
        self.subs_dir = None
 def __init__(self):
     
     #singleton pattern not really enforced if someone just calls 
     # the normal constructor. But this way I can test the instance easier
     try:
         SubtitlesSupport._singletonLock.acquire()
         SubtitlesSupport.__single = self
     finally:
         SubtitlesSupport._singletonLock.release()
         
     self.richMetadata_db = None
     self.subtitlesHandler = None
     self.channelcast_db = None
     self.langUtility = LanguagesProvider.getLanguagesInstance()
     self._registered = False
def getSubtitleFileRelativeName(channel_id, infohash, langCode):
    # subtitles filenames are build from the sha1 hash
    # of the triple (channel_id, infohash, langCode)
    
    # channel_id and infohash are binary versions
    
    assert utilities.validPermid(channel_id), "Invalid channel_id %s" % utilities.show_permid_short(channel_id)
    assert utilities.validInfohash(infohash), "Invalid infohash %s" % bin2str(infohash)
    assert LanguagesProvider.getLanguagesInstance().isLangCodeSupported(langCode), "Unsupported language code %s" % langCode
        
    hasher = sha()
    for data in (channel_id, infohash, langCode):
        hasher.update(data)
    subtitleName = hasher.hexdigest() + SUBS_EXTENSION
    return subtitleName
 def test_decodeGETSUBSMessage(self):
     langUtil = LanguagesProvider.getLanguagesInstance()
     binaryBitmask = pack("!L", langUtil.langCodesToMask(["kor", "spa"]))
     
     bencodedMessage = GET_SUBS + \
                   bencode((
                           testChannelId,
                           testInfohash,
                           binaryBitmask
                           ))
     channel_id, infohash, languages = \
         self.underTest._decodeGETSUBSMessage(bencodedMessage)
         
     self.assertEquals(testChannelId, channel_id)
     self.assertEquals(testInfohash, infohash)
     self.assertEquals(["kor", "spa"], languages)
 def subtest_receptionOfSUBS(self):
     '''
     Asking for the single available subtitle. The response should be 
     a valid SUBS message containing its contents
     '''
     
     print >> sys.stderr, "test: test_subtitles_msgs_1_1 -----------------------"
     ol_conn = OLConnection(self.my_keypair,'localhost',self.hisport)
     
     bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(['nld'])
     binmask = utilities.uintToBinaryString(bitmask, length=4)
     
     request = GET_SUBS + \
                   bencode((
                           self.anotherpermid,
                           self.testInfohash,
                           binmask
                           ))
                   
     subshandler = SubtitlesHandler()
     subshandler.register(ol_conn, self.richMetadata_db, self.session)
     
     ol_conn.send(request)
     subs_data = ol_conn.recv()
     print >> sys.stderr, "test: subtitles_messages : received SUBS response: len",len(subs_data)
     self.assertEquals(SUBS, subs_data[0])
     data = bdecode(subs_data[1:])
     print >> sys.stderr, "test: subtitles_messages : received SUBS response: ", data
     
     #check on the format of the response
     self.assertTrue(isinstance(data,list))
     self.assertEquals(4, len(data)) # for fields
     self.assertEquals(self.mdto.channel,data[0])
     self.assertEquals(self.mdto.infohash, data[1])
     self.assertEquals(binmask, data[2])
     self.assertTrue(isinstance(data[3],list))
     self.assertEquals(1, len(data[3]))
     with codecs.open(self.sub1, "rb", "utf-8") as sub:
         expectedContents = sub.read()
     self.assertEquals(expectedContents, data[3][0])
     
     ol_conn.close()
     
     print >> sys.stderr, "test: subtitles_messages: received content is valid."
     print >> sys.stderr, "End of test_subtitles_msgs_1_1 test --------------------"
 def testSendSubtitlesRequestConnected(self):
     langUtil = LanguagesProvider.getLanguagesInstance()
     request = {}
     request['channel_id'] = testChannelId
     request['infohash'] = testInfohash
     request['languages'] = ["kor"]
     self.underTest.sendSubtitleRequest(testDestPermId, request, None, None, OLPROTO_VER_FOURTEENTH)
     
     self.assertEquals(0, self.ol_bridge.connect_count) #selversion was 1
     self.assertEquals(1, self.ol_bridge.send_count) #send called one time 
     
     binaryBitmask = pack("!L", langUtil.langCodesToMask(["kor"]))
     expectedMsg = GET_SUBS + \
                   bencode((
                           testChannelId,
                           testInfohash,
                           binaryBitmask
                           ))
     passedParameters = self.ol_bridge.sendParametersHistory[0]
     self.assertEquals(testDestPermId, passedParameters[0])
     self.assertEquals(expectedMsg, passedParameters[1])
Example #12
0
    def __init__(self, lang, path=None, checksum=None):
        """
        Create a subtitle instance.
        
        @param lang: an ISO 639-2 language code. Notice that not every language
                     code described by the standard is supported, but only
                     a small subset. See the Languages module
        @param path: a file system path to the subtitles file
        @param checksum: a sha1 checksum of the contents 
                         of the subitles file
        """
        self._languages = LanguagesProvider.getLanguagesInstance()
        if lang not in self._languages.supportedLanguages.keys():
            raise ValueError("Language" + lang + " not supported")

        # ISO 639-2 code. See Languages for supported languages
        self._lang = lang  # final property
        # A string representing the path in the filesystme for this subtitle
        self._path = path
        # sha1 checksum
        self._checksum = checksum
 def test_decodeGETSUBSMessageInvalid(self):
     langUtil = LanguagesProvider.getLanguagesInstance()
     
     binaryBitmask = pack("!L", langUtil.langCodesToMask(["kor", "spa"]))
     invalidTypeMsg = chr(25) + \
                   bencode((
                           testChannelId,
                           testInfohash,
                           binaryBitmask
                           ))
     
     self.assertRaises(AssertionError, self.underTest._decodeGETSUBSMessage,
                        (invalidTypeMsg,))
     
     invalidMsgField = GET_SUBS + \
                      bencode((
                           42,
                           testChannelId,
                           testInfohash,
                           binaryBitmask
                           ))
           
     decoded = \
         self.underTest._decodeGETSUBSMessage(invalidMsgField)
     #when something in the body is wrong returns None
     self.assertTrue(decoded is None)
     
     invalidBitamsk = "\xff\xff\xff\xff\xbb"
     invalidMsgField = GET_SUBS + \
                      bencode((
                           testChannelId,
                           testInfohash,
                           invalidBitamsk #40 bit bitmask!)
                           ))
     
     decoded = \
         self.underTest._decodeGETSUBSMessage(invalidMsgField)
         
     #when something in the body is wrong returns None
     self.assertTrue(decoded is None)
Example #14
0
    def testReceivedSUBSMessage(self):
        self.testRegisterStuff()
        languages = ["eng","rus"]
        zho = u"Subtitle Content 1"
        kor = u"Subtitle Content 2"
        contentsDict = {"eng":zho, "rus":kor}
        
        msg = (testChannelId,
               testInfohash, contentsDict)
        
        
        simpleCallback = lambda x : x

        
        bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(["eng","rus"])
        self.underTest.receivedSubsResponse(testDestPermId, msg, [(simpleCallback,bitmask)], OLPROTO_VER_FOURTEENTH)
        
        #self.assertEquals(languages,callbackParams)
        expectedFilename1 = getSubtitleFileRelativeName(testChannelId, testInfohash, "eng")
        expectedFilename2 = getSubtitleFileRelativeName(testChannelId, testInfohash, "rus")
        expectedPath1 = os.path.join(self._session.get_state_dir(),self.underTest.subs_dir,expectedFilename1)
        expectedPath2 = os.path.join(self._session.get_state_dir(),self.underTest.subs_dir,expectedFilename2)
        self.assertTrue(os.path.isfile(expectedPath1))
        self.assertTrue(os.path.isfile(expectedPath2))
        
        with codecs.open(expectedPath1,"rb","utf-8") as file1:
            content1 = file1.read()
            
        self.assertEquals(zho,content1)
        
        with codecs.open(expectedPath2,"rb","utf-8") as file2:
            content2 = file2.read()
            
        self.assertEquals(kor,content2)
        
        self.assertEquals(1, self.ol_bridge.add_task_count)
        params = self.ol_bridge.add_taskParametersHistory[0]
        #calling the lambda
        val = params[0]()
        self.assertEquals(languages,val)
 def test_receivedGETSUBSSimple(self):
     langUtil = LanguagesProvider.getLanguagesInstance()
     bitmask = langUtil.langCodesToMask(["eng", "rus"])
     binaryBitmask = pack("!L", bitmask)
     request = GET_SUBS + \
                   bencode((
                           testChannelId,
                           testInfohash,
                           binaryBitmask
                           ))
     
     
     
     list = MockMsgListener()
     
     self.underTest.registerListener(list)
     self.underTest.handleMessage(testDestPermId, OLPROTO_VER_FOURTEENTH, request)
     
     self.assertEquals(1,list.receivedCount)
     self.assertEquals(testDestPermId, list.receivedParams[0][0])
     self.assertEquals(OLPROTO_VER_FOURTEENTH,list.receivedParams[0][2])
     self.assertEquals((testChannelId,testInfohash,["eng","rus"]),list.receivedParams[0][1])
Example #16
0
 def test_packDataWithSubs(self):
     badInfohash = str2bin("GEh/o8rtTLB1wZJzFcSZSS4u9qo=")
     dto = MetadataDTO(test_perm_id, badInfohash)
     
     subtitles = [SubtitleInfo(lang,path) for lang,path in self._srtSubs.iteritems()]
     
     for sub in subtitles :
         sub.computeChecksum()
         dto.addSubtitle(sub)
     
     packed = dto._packData()
     decoded = bdecode(packed)
     
     self.assertTrue(len(decoded) == 6)
     decodedChannelId = decoded[0]
     decodedInfohash = decoded[1]
     decodedDescription = decoded[2]
     decodedTimestamp = decoded[3]
     decodedBitmask = decoded[4]
     checksums = decoded[5]
     
     expectedMask = \
       LanguagesProvider.getLanguagesInstance().langCodesToMask(self._srtSubs.keys())
       
     binaryExpexted = pack("!L", expectedMask)
     
     self.assertEquals(dto.channel, decodedChannelId)
     self.assertEquals(dto.infohash, decodedInfohash)
     self.assertEquals(dto.description,decodedDescription)
     self.assertAlmostEquals(dto.timestamp,decodedTimestamp)
     self.assertEquals(binaryExpexted,decodedBitmask)
     self.assertEquals(3,len(checksums))
     
     subs = dto.getAllSubtitles()
     i=0
     for key in sorted(subs.iterkeys()):
         self.assertEquals(subs[key].checksum, checksums[i])
         i += 1
    def __init__(self, overlayBridge, tokenBucket, maxSubsSize):
        self._languagesUtility = LanguagesProvider.getLanguagesInstance()
        self._overlay_bridge = overlayBridge
        
        # handleMessage() is called by the OLThread
        # registerListener is called by the OLThread
        # no synchronization should be needed for this list :)
        self._listenersList = list()
        
        self._tokenBucket = tokenBucket
        
        #controls the interval the uploadQueue gets checked
        self._nextUploadTime = 0
        
        
        #dictionary of type { "".join(channel_id,infohash) : _RequestedSubtitlesEntry}
        #bits get cleaned when subtitles are received
        #when the bitmask is 000 the entry is removed from the dictionary
        #also entries older then REQUEST_VALIDITY_TIME get dropped
        
        self.requestedSubtitles = {}
        self._requestsLock = threading.RLock()
        
        self._nextCleanUpTime = int(time()) + CLEANUP_PERIOD

        #subtitles to send get queued in this queue
        #each subtitle message to send is a dictionary whose keys are:
        #permid: destination of the message
        #channel_id: identifier of the channel from which the subtitles to upload are
        #infohash: identifier of the torrent for which the subtitles to upload are
        #subtitles: a dictionary of the form {langCode : path} for the subtitles to send
        #selversion: 
    
        self._uploadQueue = []
        self._requestValidityTime = REQUEST_VALIDITY_TIME
        self._maxSubSize = maxSubsSize
# see LICENSE.txt for license information

from Tribler.Core.BitTornado.bencode import bencode, bdecode
from Tribler.Core.Subtitles.MetadataDomainObjects.Languages import LanguagesProvider
from Tribler.Core.Subtitles.MetadataDomainObjects.MetadataExceptions import SerializationException
from Tribler.Core.Subtitles.MetadataDomainObjects.SubtitleInfo import SubtitleInfo
from Tribler.Core.Overlay.permid import sign_data, verify_data
from Tribler.Core.Utilities.utilities import isValidInfohash, isValidPermid
from math import floor
from struct import pack, unpack
import sys
import time

DEBUG = False

_languagesUtil = LanguagesProvider.getLanguagesInstance()

class MetadataDTO(object):
    '''
    Metdata DataTransferObject
    '''


    def __init__(self, publisher,infohash,timestamp = None,
                 description=u"", subtitles=None,signature=None):
        """
        Create a MetataDTO instance.
        
        publisher and infohash are mandatory to be not null
        @param publisher: the permid  of the owner of the 
                          channel this instance refers to