def getSomeMetadata(self, channel_id, infohash): s1 = SubtitleInfo("eng", None) s2 = SubtitleInfo("rus", None) self.content1 = u"Subtitle Content 1" self.content2 = u"Subtitle Content 2" hasher = sha() hasher.update(self.content1) s1.checksum = hasher.digest() hasher = sha() hasher.update(self.content2) s2.checksum = hasher.digest() metadata = MetadataDTO(channel_id, infohash, time.time(), "", {"eng":s1, "rus":s2}) if self.nextKeypair is None: metadata.signature = "fake" else: metadata.sign(self.nextKeypair) return metadata
def _createSubtitlesDict(bitmask, listOfChecksums): langList = _languagesUtil.maskToLangCodes(bitmask) if len(langList) != len(listOfChecksums): raise SerializationException("Unexpected num of checksums") subtitles = {} for i in range(0, len(langList)): sub = SubtitleInfo(langList[i]) sub.checksum = listOfChecksums[i] subtitles[langList[i]] = sub return subtitles
def _createSubtitlesDict(bitmask, listOfChecksums): langList = _languagesUtil.maskToLangCodes(bitmask) if len(langList) != len(listOfChecksums): raise SerializationException("Unexpected num of checksums") subtitles = {} for i in range(0, len(langList)): sub = SubtitleInfo(langList[i]) sub.checksum = listOfChecksums[i] subtitles[langList[i]] = sub return subtitles
def setUpDB(self): try: self.richMetadata_db = self.session.open_dbhandler( NTFY_RICH_METADATA) #add some metadata self.mdto = MetadataDTO(self.anotherpermid, self.testInfohash) subtitle1 = SubtitleInfo("nld", self.sub1) subtitle1.computeChecksum() subtitle2 = SubtitleInfo("eng", os.path.join(RES_DIR, "fake0.srt")) subtitle2.computeChecksum() self.mdto.addSubtitle(subtitle1) self.mdto.addSubtitle(subtitle2) self.mdto.sign(self.another_keypair) self.richMetadata_db.insertMetadata(self.mdto) #hisoermid has the nld subtitle but doesn't have the english one self.richMetadata_db.updateSubtitlePath(self.mdto.channel, self.mdto.infohash, "eng", None) except: print_exc()
def testChecksums(self): path = os.path.join(RES_DIR, PATH_TO_SRT) sub = SubtitleInfo("ita", path) #I know from the outside the the correct sha1 is # eb8ada2a2094675ea047c27207e449fbfce04e85 sha1Hasher = hashlib.sha1() with codecs.open(path, "rb", "utf-8") as subFile: sha1Hasher.update(subFile.read()) expectedChecksum = sha1Hasher.digest() sub.computeChecksum() self.assertEquals(expectedChecksum, sub.checksum) self.assertTrue(sub.verifyChecksum())
def getLocalSubtitles(self, channel, infohash): ''' Returns a dictionary containing all the subtitles pointing to a local pathm for the given channel, infohash @param channel: binary channel_id(permid) @param infohash: binary infohash @rtype: dict @return: a dictionary like this: { ... langCode : SubtitleInfo, ... } The dictionary will be empty if no local subtitle is available. ''' query = QUERIES["SELECT SUBTITLES WITH PATH BY CHN INFO"] channel = bin2str(channel) infohash = bin2str(infohash) res = self._db.fetchall(query, (channel, infohash)) result = {} for entry in res: location = entry[0] language = entry[1] checksum = str2bin(entry[2]) subInfo = SubtitleInfo(language, location, checksum) result[language] = subInfo return result
def getAllLocalSubtitles(self): ''' Returns a structure containing all the subtitleInfos that are pointing to a local path @return a dictionary like this: { ... channel1 : { infohash1 : [ SubtitleInfo1, ...] } ... } ''' query = QUERIES["SELECT SUBTITLES WITH PATH"] res = self._db.fetchall(query) result = {} for entry in res: # fk = entry[0] path = entry[1] lang = entry[2] checksum = str2bin(entry[3]) channel = str2bin(entry[4]) infohash = str2bin(entry[5]) s = SubtitleInfo(lang, path, checksum) if channel not in result: result[channel] = {} if infohash not in result[channel]: result[channel][infohash] = [] result[channel][infohash].append(s) return result
def getSubtitle(self, channel, infohash, lang): """ Get a subtitle for a language for a given item in a given channel. Returns the details reguarding a subtitles in a given language for a given item in a given channel, if it exists. Otherwise it returns None. @param channel: a perm_id identifying the owner of the channel. @param infohash: the infohash of an item, as announced in channelcast messages. @param lang: a 3 characters ISO 639-2 language code, identifying the desired subtitle langugage @return: a SubtitleInfo instance """ query = QUERIES["SELECT SUBS JOIN HASH ONE"] infohash = bin2str(infohash) channel = bin2str(channel) res = self._db.fetchall(query, (infohash, channel, lang)) if len(res) == 0: return None elif len(res) == 1: checksum = str2bin(res[0][3]) return SubtitleInfo(res[0][1], res[0][2], checksum) else: # This should be not possible to database constraints raise MetadataDBException("Metadata DB Constraint violeted!")
def testChecksums(self): path = os.path.join(RES_DIR,PATH_TO_SRT) sub = SubtitleInfo("ita",path) #I know from the outside the the correct sha1 is # eb8ada2a2094675ea047c27207e449fbfce04e85 sha1Hasher = hashlib.sha1() with codecs.open(path, "rb", "utf-8") as subFile: sha1Hasher.update(subFile.read()) expectedChecksum = sha1Hasher.digest() sub.computeChecksum() self.assertEquals(expectedChecksum, sub.checksum) self.assertTrue(sub.verifyChecksum())
def getSomeMetadata(self, channel_id, infohash): s1 = SubtitleInfo("eng", None) s2 = SubtitleInfo("rus", None) self.content1 = u"Subtitle Content 1" self.content2 = u"Subtitle Content 2" hasher = sha() hasher.update(self.content1) s1.checksum = hasher.digest() hasher = sha() hasher.update(self.content2) s2.checksum = hasher.digest() metadata = MetadataDTO(channel_id, infohash, time.time(), "", { "eng": s1, "rus": s2 }) if self.nextKeypair is None: metadata.signature = "fake" else: metadata.sign(self.nextKeypair) return metadata
def __init__(self): self.returnMetadata = True self.getAllSubsCount = 0 self.getAllSubsParametersHistory = list() self.getMetadataCount = 0 self.getMetadataParametesHistory = list() self.updateSubtitleCount = 0 self.updateSubtitleParameterHistory = list() self.commitCount = 0 s1 = SubtitleInfo("eng", os.path.join(RES_DIR,"fake0.srt")) s1.computeChecksum() s2 = SubtitleInfo("rus", os.path.join(RES_DIR,"fake1.srt")) s2.computeChecksum() self.returnSubs = { "eng" : s1, "rus" : s2 } self.insertMetadataCount = 0 self.insertMetadataParameters = list() self.nextKeypair = None
def testUpdateSubtitles(self): sub1path= os.path.join(RES_DIR,"fake0.srt") sub2path=os.path.join(RES_DIR,"fake1.srt") infohash = _generateFakeInfohash() metadataDTO = MockMetadataDTO([], infohash) sub1 = SubtitleInfo("ita", None, _computeSHA1(sub1path)) sub2 = SubtitleInfo("eng",None,_computeSHA1(sub2path)) metadataDTO.addSubtitle(sub1) metadataDTO.addSubtitle(sub2) metadataDTO.sign(metadataDTO._keypair) self.underTest.insertMetadata(metadataDTO) res1 = self.underTest.getSubtitle(metadataDTO.channel, infohash,"ita") self.assertEquals(sub1,res1) res2 = self.underTest.getSubtitle(metadataDTO.channel, infohash, "eng") self.assertEquals(sub2,res2) sub1bis = copy.copy(sub1) sub1bis.path = sub1path sub2bis = copy.copy(sub2) sub2bis.path = sub2path self.underTest.updateSubtitlePath(metadataDTO.channel, infohash, sub1bis.lang, sub1bis.path, False) self.underTest.updateSubtitlePath(metadataDTO.channel, infohash, sub2bis.lang, sub2bis.path , False) self.underTest.commit() #still unchanged since I did not commit res1 = self.underTest.getSubtitle(metadataDTO.channel, infohash,"ita") self.assertTrue(sub1== res1 and sub1.path != res1.path) self.assertTrue(sub1bis == res1 and sub1bis.path == res1.path) res2 = self.underTest.getSubtitle(metadataDTO.channel, infohash, "eng") self.assertTrue(sub2 == res2 and sub2.path != res2.path) self.assertTrue(sub2bis == res2 and sub2bis.path == res2.path)
def testDeserializeWithSubs(self): badInfohash = str2bin("GEh/o8rtTLB1wZJzFcSZSS4u9qo=") dto = MetadataDTO(test_perm_id, badInfohash) subtitles = [ SubtitleInfo(lang, path) for lang, path in self._srtSubs.iteritems() ] for sub in subtitles: sub.computeChecksum() dto.addSubtitle(sub) dto.sign(test_keypair) serial = dto.serialize() newDto = MDUtil.deserialize(serial) self.assertEquals(dto, newDto)
def setupDB(self, nickname): TestChannels.setupDB(self, nickname) try: self.richMetadata_db = self.session.open_dbhandler( NTFY_RICH_METADATA) #add some metadata for torrents (they are defined in TestChannels.setupDB() self.mdto = MetadataDTO(self.hispermid, self.infohash1) subtitle1 = SubtitleInfo("nld", os.path.join(RES_DIR, "fake.srt")) subtitle1.computeChecksum() subtitle2 = SubtitleInfo("eng", os.path.join(RES_DIR, "fake0.srt")) subtitle2.computeChecksum() self.mdto.addSubtitle(subtitle1) self.mdto.addSubtitle(subtitle2) self.mdto.sign(self.his_keypair) self.richMetadata_db.insertMetadata(self.mdto) except: print_exc()
def testSerializeWithSubs(self): badInfohash = str2bin("GEh/o8rtTLB1wZJzFcSZSS4u9qo=") dto = MetadataDTO(test_perm_id, badInfohash) subtitles = [ SubtitleInfo(lang, path) for lang, path in self._srtSubs.iteritems() ] for sub in subtitles: sub.computeChecksum() dto.addSubtitle(sub) dto.sign(test_keypair) serial = dto.serialize() decoded = serial self.assertEquals(7, len(decoded)) signature = decoded[6] self.assertEquals(dto.signature, signature)
def testSignatureOnChecksums(self): badInfohash = str2bin("GEh/o8rtTLB1wZJzFcSZSS4u9qo=") dto = MetadataDTO(test_perm_id, badInfohash) subtitles = [ SubtitleInfo(lang, path) for lang, path in self._srtSubs.iteritems() ] for sub in subtitles: sub.computeChecksum() dto.addSubtitle(sub) dto.sign(test_keypair) self.assertTrue(dto.verifySignature()) dto.getSubtitle("rus").checksum = "ABCDEFGHILMOPQRS" self.assertFalse(dto.verifySignature())
def _getAllSubtitlesByKey(self, metadataKey): ''' Retrieves every subtitles given a Metadata table key Given an instance of the Metadata table artificial key, retrieves every subtitle instance associated to that key @param metadataKey: a value of an artificial key in the Metadata table @return : a dictionary of type {lang : SubtitleInfo}, empty if no results ''' query = QUERIES["SELECT SUBS FK ALL"] results = self._db.fetchall(query, (metadataKey, )) subsDict = {} for entry in results: subsDict[entry[1]] = SubtitleInfo(entry[1], entry[2], str2bin(entry[3])) return subsDict
def test_packDataWithSubs(self): badInfohash = str2bin("GEh/o8rtTLB1wZJzFcSZSS4u9qo=") dto = MetadataDTO(test_perm_id, badInfohash) subtitles = [ SubtitleInfo(lang, path) for lang, path in self._srtSubs.iteritems() ] for sub in subtitles: sub.computeChecksum() dto.addSubtitle(sub) packed = dto._packData() decoded = bdecode(packed) self.assertTrue(len(decoded) == 6) decodedChannelId = decoded[0] decodedInfohash = decoded[1] decodedDescription = decoded[2] decodedTimestamp = decoded[3] decodedBitmask = decoded[4] checksums = decoded[5] expectedMask = \ LanguagesProvider.getLanguagesInstance().langCodesToMask(self._srtSubs.keys()) binaryExpexted = pack("!L", expectedMask) self.assertEquals(dto.channel, decodedChannelId) self.assertEquals(dto.infohash, decodedInfohash) self.assertEquals(dto.description, decodedDescription) self.assertAlmostEquals(dto.timestamp, decodedTimestamp) self.assertEquals(binaryExpexted, decodedBitmask) self.assertEquals(3, len(checksums)) subs = dto.getAllSubtitles() i = 0 for key in sorted(subs.iterkeys()): self.assertEquals(subs[key].checksum, checksums[i]) i += 1
def __init__(self): self.returnMetadata = True self.getAllSubsCount = 0 self.getAllSubsParametersHistory = list() self.getMetadataCount = 0 self.getMetadataParametesHistory = list() self.updateSubtitleCount = 0 self.updateSubtitleParameterHistory = list() self.commitCount = 0 s1 = SubtitleInfo("eng", os.path.join(RES_DIR, "fake0.srt")) s1.computeChecksum() s2 = SubtitleInfo("rus", os.path.join(RES_DIR, "fake1.srt")) s2.computeChecksum() self.returnSubs = {"eng": s1, "rus": s2} self.insertMetadataCount = 0 self.insertMetadataParameters = list() self.nextKeypair = None
def _getSubtitleByKey(self, metadata_fk, lang): """ Return a subtitle in a given language for a key of the Metadata table. Given an instance of the artificial key in the metadata table, retrieves a SubtitleInfo instance for that key and the language passed in. @param metadata_fk: a key in the metadata table @param lang: a language code for the subtitle to be retrieved @return: a SubtitleInfo instance, or None """ query = QUERIES["SELECT SUBS FK ONE"] res = self._db.fetchall(query, (metadata_fk, lang)) if len(res) == 0: return None elif len(res) == 1: checksum = str2bin(res[0][3]) return SubtitleInfo(res[0][1], res[0][2], checksum) else: # This should be not possible to database constraints raise MetadataDBException("Metadata DB Constraint violeted!")
def __init__(self, availableLangs, infohash = None): self._keypair = generate_keypair() self._permId = str(self._keypair.pub().get_der()) if infohash == None : hasher = hashlib.sha1() hasher.update(self._permId + "a") infohash = hasher.digest() self.channel = self._permId self.infohash = infohash self.description = u"" self.resetTimestamp() self._subtitles = {} hasher = hashlib.sha1() #fake checksums for subs for lang in availableLangs: hasher.update(lang + "123") checksum = hasher.digest() self.addSubtitle(SubtitleInfo(lang, None, checksum))
def getAllSubtitles(self, channel, infohash): """ Get all the available subtitles for a channel and infohash. Returns a list representing subtitles that are available for a givenchannel and infohash. @param channel: the perm_id of the channel owner (binary) @param infohash: the infhash of a channel elements as it is announced in ChannelCast (binary) @return: a dictionary of { lang : SubtitleInfo instance} """ query = QUERIES["SELECT SUBS JOIN HASH ALL"] infohash = bin2str(infohash) channel = bin2str(channel) results = self._db.fetchall(query, (infohash, channel)) subsDict = {} for entry in results: subsDict[entry[1]] = SubtitleInfo(entry[1], entry[2], entry[3]) return subsDict
def setupDB(self,nickname): TestChannels.setupDB(self,nickname) try: self.richMetadata_db = self.session.open_dbhandler(NTFY_RICH_METADATA) #add some metadata for torrents (they are defined in TestChannels.setupDB() self.mdto = MetadataDTO(self.hispermid, self.infohash1) subtitle1 = SubtitleInfo("nld", os.path.join(RES_DIR,"fake.srt")) subtitle1.computeChecksum() subtitle2 = SubtitleInfo("eng", os.path.join(RES_DIR, "fake0.srt")) subtitle2.computeChecksum() self.mdto.addSubtitle(subtitle1) self.mdto.addSubtitle(subtitle2) self.mdto.sign(self.his_keypair) self.richMetadata_db.insertMetadata(self.mdto) except: print_exc()
def setUpDB(self): try: self.richMetadata_db = self.session.open_dbhandler(NTFY_RICH_METADATA) #add some metadata self.mdto = MetadataDTO(self.anotherpermid, self.testInfohash) subtitle1 = SubtitleInfo("nld", self.sub1) subtitle1.computeChecksum() subtitle2 = SubtitleInfo("eng", os.path.join(RES_DIR, "fake0.srt")) subtitle2.computeChecksum() self.mdto.addSubtitle(subtitle1) self.mdto.addSubtitle(subtitle2) self.mdto.sign(self.another_keypair) self.richMetadata_db.insertMetadata(self.mdto) #hisoermid has the nld subtitle but doesn't have the english one self.richMetadata_db.updateSubtitlePath(self.mdto.channel,self.mdto.infohash,"eng",None) except: print_exc()
def testSubsExists(self): path = os.path.join(RES_DIR, PATH_TO_SRT) sub = SubtitleInfo("rus", "fakepath") self.assertFalse(sub.subtitleExists()) sub.path = os.path.abspath(path) self.assertTrue(sub.subtitleExists())
def testSubsExists(self): path = os.path.join(RES_DIR,PATH_TO_SRT) sub = SubtitleInfo("rus","fakepath") self.assertFalse(sub.subtitleExists()) sub.path = os.path.abspath(path) self.assertTrue(sub.subtitleExists())
def testInitialization(self): sub = SubtitleInfo("eng","fakepath") self.assertFalse(sub is None) self.assertFalse(sub.subtitleExists()) self.assertRaises(AssertionError, sub.computeChecksum)
def testInitialization(self): sub = SubtitleInfo("eng", "fakepath") self.assertFalse(sub is None) self.assertFalse(sub.subtitleExists()) self.assertRaises(AssertionError, sub.computeChecksum)
class SubtitlesSupport(object): ''' Subtitle dissemination system facade. Acts as the only faced between the subtitle dissemination system and the GUI (or whoever needs to subtitles). Provides methods to query the subtitles database. Allows publishers to add their own subtitles, and if necessary permits to retrieve the subtitle remotely if not available. ''' __single = None _singletonLock = threading.RLock() def __init__(self): #singleton pattern not really enforced if someone just calls # the normal constructor. But this way I can test the instance easier try: SubtitlesSupport._singletonLock.acquire() SubtitlesSupport.__single = self finally: SubtitlesSupport._singletonLock.release() self.richMetadata_db = None self.subtitlesHandler = None self.channelcast_db = None self.langUtility = LanguagesProvider.getLanguagesInstance() self._registered = False @staticmethod def getInstance(*args, **kw): try: SubtitlesSupport._singletonLock.acquire() if SubtitlesSupport.__single == None: SubtitlesSupport(*args, **kw) finally: SubtitlesSupport._singletonLock.release() return SubtitlesSupport.__single def _register(self, richMetadataDBHandler, subtitlesHandler, channelcast_db, my_permid, my_keypair, peersHaveManger, ol_bridge): assert richMetadataDBHandler is not None assert subtitlesHandler is not None assert channelcast_db is not None assert peersHaveManger is not None assert ol_bridge is not None assert isValidPermid(my_permid) self.richMetadata_db = richMetadataDBHandler self.subtitlesHandler = subtitlesHandler self.channelcast_db = channelcast_db self.my_permid = my_permid self.my_keypair = my_keypair self._peersHaveManager = peersHaveManger #used to decouple calls to SubtitleHandler self._ol_bridge = ol_bridge self._registered = True def getSubtileInfosForInfohash(self, infohash): ''' Retrieve available information about subtitles for the given infohash. Given the infohash of a .torrent, retrieves every information about subtitles published for that .torrent that is currently available in the DB. @param infohash: a .torrent infohash (binary) @return: a dictionary. The dictionary looks like this:: { channel_id1 : {langCode : L{SubtitleInfo}, ...} , channel_id2 : {langCode : L{SubtitleInfo}, ... }, ... } Each entry in the dictionary has the following semantics: - channel_id is the permid identifiying the channel (binary). - langCode is an ISO 693-2 three characters language code ''' assert utilities.isValidInfohash(infohash) assert self._registered, "Instance is not registered" returnDictionary = dict() #a metadataDTO corresponds to all metadata for a pair channel, infohash metadataDTOs = self.richMetadata_db.getAllMetadataForInfohash(infohash) for metadataDTO in metadataDTOs: channel = metadataDTO.channel subtitles = metadataDTO.getAllSubtitles() if len(subtitles) > 0: returnDictionary[channel] = subtitles return returnDictionary def getSubtitleInfos(self, channel, infohash): ''' Retrieve subtitles information for the given channel-infohash pair. Searches in the local database for information about subtitles that are currently available. @param channel: the channel_id (perm_id) of a channel (binary) @param infohash: a .torrent infohash (binary) @return: a dictionary of SubtitleInfo instances. The keys are the language codes of the subtitles ''' assert self._registered, "Instance is not registered" metadataDTO = self.richMetadata_db.getMetadata(channel, infohash) if metadataDTO: return metadataDTO.getAllSubtitles() return {} def publishSubtitle(self, infohash, lang, pathToSrtSubtitle): ''' Allows an user to publish an srt subtitle file in his channel. Called by a channel owner this method inserts a new subtitle for a torrent published in his channel. The method assumes that the torrent identified by the infohash parameter is already in the channel, and that the parameter pathToSrtSubtitle points to an existing srt file on the local filesystem. If a subtitle for the same language was already associated to the specified infohash and channel, it will be overwritten. After calling this method the newly inserted subtitle will be disseminated via Channelcast. @param infohash: the infohash of the torrent to associate the subtitle with, binary @param lang: a 3 characters code for the language of the subtitle as specified in ISO 639-2. Currently just 32 language codes will be supported. @param pathToSrtSubtitle: a path in the local filesystem to a subtitle in srt format. @raise RichMetadataException: if something "general" goes wrong while adding new metadata @raise IOError: if disk related problems occur ''' assert utilities.isValidInfohash(infohash), "Invalid Infohash" assert lang is not None and self.langUtility.isLangCodeSupported(lang) assert self._registered, "Instance is not registered" channelid = bin2str(self.my_permid) base64infohash = bin2str(infohash) # consisnstency check: I want to assure that this method is called # for an item that is actually in my channel consinstent = self.channelcast_db.isItemInChannel( channelid, base64infohash) if not consinstent: msg = "Infohash %s not found in my channel. Rejecting subtitle" % base64infohash if DEBUG: print >> sys.stderr, msg raise RichMetadataException(msg) try: filepath = self.subtitlesHandler.copyToSubtitlesFolder( pathToSrtSubtitle, self.my_permid, infohash, lang) except Exception, e: if DEBUG: print >> sys.stderr, "Failed to read and copy subtitle to appropriate folder: %s" % str( e) # retrieve existing metadata from my channel, infohash metadataDTO = self.richMetadata_db.getMetadata(self.my_permid, infohash) # can be none if no metadata was available if metadataDTO is None: metadataDTO = MetadataDTO(self.my_permid, infohash) else: #update the timestamp metadataDTO.resetTimestamp() newSubtitle = SubtitleInfo(lang, filepath) # this check should be redundant, since i should be sure that subtitle # exists at this point if newSubtitle.subtitleExists(): newSubtitle.computeChecksum() else: msg = "Inconsistency found. The subtitle was not published" if DEBUG: print >> sys.stderr, msg raise RichMetadataException(msg) metadataDTO.addSubtitle(newSubtitle) metadataDTO.sign(self.my_keypair) self.richMetadata_db.insertMetadata(metadataDTO)