Пример #1
0
    def getLocalSubtitles(self, channel, infohash):
        '''
        Returns a dictionary containing all the subtitles pointing
        to a local pathm for the given channel, infohash
        @param channel: binary channel_id(permid)
        @param infohash: binary infohash
        
        @rtype: dict
        @return: a dictionary like this:
                {
                 ...
                 langCode : SubtitleInfo,
                 ...
                }
                The dictionary will be empty if no local subtitle
                is available.
        '''
        query = QUERIES["SELECT SUBTITLES WITH PATH BY CHN INFO"]

        channel = bin2str(channel)
        infohash = bin2str(infohash)
        res = self._db.fetchall(query, (channel, infohash))

        result = {}

        for entry in res:
            location = entry[0]
            language = entry[1]
            checksum = str2bin(entry[2])
            subInfo = SubtitleInfo(language, location, checksum)
            result[language] = subInfo

        return result
 def updateHaveMask(self,channel,infohash,peer_id, newMask, timestamp=None):
     '''
     Store a received have mask in the db
     
     (See insertHaveMask for description)
     
     @type channel: str
     @param channel: channel_id (binary)
     
     @type infohash: str
     @param infohash: the infohash of a torrent (binary)
     
     @type peer_id: str
     @param peer_id: peer from whom the infomask was received.(ie its binary permid)
     
     @type havemask: int
     "param havemask: a non-negative integer. It must be smaller
                     then 2**32.
     '''
     channel = bin2str(channel)
     infohash = bin2str(infohash)
     peer_id = bin2str(peer_id)
     
     updateQuery = QUERIES["UPDATE HAVE MASK"]
     if timestamp is None:
         timestamp = int(time.time())
     self._db.execute_write(updateQuery, 
                            (newMask,timestamp,peer_id, channel, infohash))
Пример #3
0
    def getSubtitle(self, channel, infohash, lang):
        """
        Get a subtitle for a language for a given item in a given channel.
        
        Returns the details reguarding a subtitles in a given language for a
        given item in a given channel, if it exists. Otherwise it returns
        None.
        
        @param channel: a perm_id identifying the owner of the channel.
        @param infohash: the infohash of an item, as announced in channelcast
                         messages.
        @param lang: a 3 characters ISO 639-2 language code, identifying
                     the desired subtitle langugage
        @return: a SubtitleInfo instance
        """
        query = QUERIES["SELECT SUBS JOIN HASH ONE"]

        infohash = bin2str(infohash)
        channel = bin2str(channel)

        res = self._db.fetchall(query, (infohash, channel, lang))
        if len(res) == 0:
            return None
        elif len(res) == 1:
            checksum = str2bin(res[0][3])
            return SubtitleInfo(res[0][1], res[0][2], checksum)
        else:
            # This should be not possible to database constraints
            raise MetadataDBException("Metadata DB Constraint violeted!")
Пример #4
0
    def getHaveMask(self, channel, infohash, peer_id):
        '''
        Returns the have mask for a single peer if available.
        
        @type channel: str
        @param channel: channel_id (binary)
        
        @type infohash: str
        @param infohash: the infohash of a torrent (binary)
        
        @type peer_id: str
        @param peer_id: peer from whom the infomask was received.(ie its binary permid)
        
        @rtype: int
        @return: the have mask relative to channel, infohash, and peer.
                 If not available returns None
                 
        @postcondition: the return value is either None or a non-negative
                        integer smaller then 2**32
        '''

        query = QUERIES["GET ONE HAVE MASK"]

        channel = bin2str(channel)
        infohash = bin2str(infohash)
        peer_id = bin2str(peer_id)

        res = self._db.fetchall(query, (channel, infohash, peer_id))

        if len(res) <= 0:
            return None
        elif len(res) > 1:
            raise AssertionError("channel,infohash,peer_id should be unique")
        else:
            return res[0][0]
Пример #5
0
    def updateHaveMask(self,
                       channel,
                       infohash,
                       peer_id,
                       newMask,
                       timestamp=None):
        '''
        Store a received have mask in the db
        
        (See insertHaveMask for description)
        
        @type channel: str
        @param channel: channel_id (binary)
        
        @type infohash: str
        @param infohash: the infohash of a torrent (binary)
        
        @type peer_id: str
        @param peer_id: peer from whom the infomask was received.(ie its binary permid)
        
        @type havemask: int
        "param havemask: a non-negative integer. It must be smaller
                        then 2**32.
        '''
        channel = bin2str(channel)
        infohash = bin2str(infohash)
        peer_id = bin2str(peer_id)

        updateQuery = QUERIES["UPDATE HAVE MASK"]
        if timestamp is None:
            timestamp = int(time.time())
        self._db.execute_write(
            updateQuery, (newMask, timestamp, peer_id, channel, infohash))
Пример #6
0
 def deleteHaveEntry(self, channel, infohash, peer_id):
     '''
     Delete a row from the SubtitlesHave db.
     
     If the row is not in the db nothing happens.
     
     @type channel: str
     @param channel: channel_id (binary)
     
     @type infohash: str
     @param infohash: the infohash of a torrent (binary)
     
     @type peer_id: str
     @param peer_id: peer from whom the infomask was received.(ie its binary permid)
     
     @postcondition: if a row identified by channel, infohash, peer_id
                     was in the database, it will no longer be there
                     at the end of this method call
     
     '''
     channel = bin2str(channel)
     infohash = bin2str(infohash)
     peer_id = bin2str(peer_id)
     deleteQuery = QUERIES["DELETE HAVE"]
     self._db.execute_write(deleteQuery, (peer_id, channel, infohash))
 def getHaveMask(self, channel, infohash, peer_id):
     '''
     Returns the have mask for a single peer if available.
     
     @type channel: str
     @param channel: channel_id (binary)
     
     @type infohash: str
     @param infohash: the infohash of a torrent (binary)
     
     @type peer_id: str
     @param peer_id: peer from whom the infomask was received.(ie its binary permid)
     
     @rtype: int
     @return: the have mask relative to channel, infohash, and peer.
              If not available returns None
              
     @postcondition: the return value is either None or a non-negative
                     integer smaller then 2**32
     '''
     
     query = QUERIES["GET ONE HAVE MASK"]
     
     channel = bin2str(channel)
     infohash = bin2str(infohash)
     peer_id = bin2str(peer_id)
     
     res = self._db.fetchall(query,(channel,infohash,peer_id))
     
     if len(res) <= 0:
         return None
     elif len(res) > 1:
         raise AssertionError("channel,infohash,peer_id should be unique")
     else:
         return res[0][0]
 def getSubtitle(self, channel, infohash, lang):
     """
     Get a subtitle for a language for a given item in a given channel.
     
     Returns the details reguarding a subtitles in a given language for a
     given item in a given channel, if it exists. Otherwise it returns
     None.
     
     @param channel: a perm_id identifying the owner of the channel.
     @param infohash: the infohash of an item, as announced in channelcast
                      messages.
     @param lang: a 3 characters ISO 639-2 language code, identifying
                  the desired subtitle langugage
     @return: a SubtitleInfo instance
     """
     query = QUERIES["SELECT SUBS JOIN HASH ONE"]
     
     infohash = bin2str(infohash)
     channel = bin2str(channel)
       
       
     res = self._db.fetchall(query, (infohash, channel, lang))
     if len(res) == 0 :
         return None
     elif len(res) == 1 :
         checksum = str2bin(res[0][3])
         return SubtitleInfo(res[0][1], res[0][2], checksum)
     else : 
         # This should be not possible to database constraints
         raise MetadataDBException("Metadata DB Constraint violeted!")
 def deleteHaveEntry(self, channel, infohash, peer_id):
     '''
     Delete a row from the SubtitlesHave db.
     
     If the row is not in the db nothing happens.
     
     @type channel: str
     @param channel: channel_id (binary)
     
     @type infohash: str
     @param infohash: the infohash of a torrent (binary)
     
     @type peer_id: str
     @param peer_id: peer from whom the infomask was received.(ie its binary permid)
     
     @postcondition: if a row identified by channel, infohash, peer_id
                     was in the database, it will no longer be there
                     at the end of this method call
     
     '''
     channel = bin2str(channel)
     infohash = bin2str(infohash)
     peer_id = bin2str(peer_id)
     deleteQuery = QUERIES["DELETE HAVE"]
     self._db.execute_write(deleteQuery,
                            (peer_id,channel,infohash))
 def getLocalSubtitles(self, channel, infohash):
     '''
     Returns a dictionary containing all the subtitles pointing
     to a local pathm for the given channel, infohash
     @param channel: binary channel_id(permid)
     @param infohash: binary infohash
     
     @rtype: dict
     @return: a dictionary like this:
             {
              ...
              langCode : SubtitleInfo,
              ...
             }
             The dictionary will be empty if no local subtitle
             is available.
     '''
     query = QUERIES["SELECT SUBTITLES WITH PATH BY CHN INFO"]
     
     channel = bin2str(channel)
     infohash = bin2str(infohash)
     res = self._db.fetchall(query,(channel,infohash))
     
     result  = {}
     
     for entry in res:
         location = entry[0]
         language = entry[1]
         checksum = str2bin(entry[2])
         subInfo = SubtitleInfo(language, location, checksum)
         result[language] = subInfo
     
     return result
Пример #11
0
    def handleRMetadata(self,
                        sender_permid,
                        channelCastMessage,
                        fromQuery=False):
        '''
        Handles the reception of rich metadata.
        
        Called when an "erniched" channelCastMessage (v14) is received.
        @param sender_permid: the PermId of the peer who sent the message
        @param channelCastMessage: the received message
        @return: None
        '''
        metadataDTOs = self._splitChannelcastAndRichMetadataContents(
            channelCastMessage)

        if DEBUG:
            print >> sys.stderr, "Handling rich metadata from %s..." % show_permid_short(
                sender_permid)

        for md_and_have in metadataDTOs:
            md = md_and_have[0]
            havemask = md_and_have[1]

            vote = self.votecastDB.getVote(bin2str(md.channel),
                                           bin2str(self.my_permid))

            if DEBUG:
                id = "RQ" if fromQuery else "R"
                print >> sys.stderr, "%s, %s, %s, %s, %d" % \
                                       (id, md.channel, md.infohash, \
                                        show_permid_short(sender_permid), md.timestamp)
                #format "R|S (R: received - S: sent), channel, infohash, sender|destination,metadataCreationTimestamp"
                # 30-06-2010: "RQ" as received from query

            # check if the record belongs to a channel
            # who we have "reported spam" (negative vote)
            if vote == -1:
                # if so, ignore the incoming record
                continue

            isUpdate = self.rmdDb.insertMetadata(md)
            self.peerHaveManager.newHaveReceived(md.channel, md.infohash,
                                                 sender_permid, havemask)

            if isUpdate is not None:
                #retrieve the metadataDTO from the database in the case it is an update
                md = self.rmdDb.getMetadata(md.channel, md.infohash)
                self._notifyRichMetadata(md, isUpdate)

            # if I am a subscriber send immediately a GET_SUBS to the
            # sender
            if vote == 2:
                if DEBUG:
                    print >> sys.stderr, "Subscribed to channel %s, trying to retrieve" \
                         "all subtitle contents" % (show_permid_short(md.channel),)

                self._getAllSubtitles(md)
Пример #12
0
    def insertHaveMask(self,
                       channel,
                       infohash,
                       peer_id,
                       havemask,
                       timestamp=None):
        '''
        Store a received have mask in the db
        
        Each inserted rows represent a delcaration of subtitle 
        availability from peer_id, for some subtitles for
        a torrent identified by infohash in a channel identified
        by channel.
        
        @type channel: str
        @param channel: channel_id (binary)
        
        @type infohash: str
        @param infohash: the infohash of a torrent (binary)
        
        @type peer_id: str
        @param peer_id: peer from whom the infomask was received.(ie its binary permid)
        
        @type havemask: int
        @param havemask: a non-negative integer. It must be smaller
                        then 2**32.
                        
        @precondition: an entry for (channel, infohash) must already
                       exist in the database
        '''
        query = QUERIES["SELECT METADATA"]

        if timestamp is None:
            timestamp = int(time.time())

        channel = bin2str(channel)
        infohash = bin2str(infohash)
        peer_id = bin2str(peer_id)

        res = self._db.fetchall(query, (infohash, channel))

        if len(res) != 1:
            raise MetadataDBException("No entry in the MetadataDB for %s, %s" %\
                                      (channel[-10:],infohash))

        metadata_fk = res[0][0]

        insertQuery = QUERIES["INSERT HAVE MASK"]

        try:
            self._db.execute_write(insertQuery,
                                   (metadata_fk, peer_id, havemask, timestamp))
        except sqlite3.IntegrityError, e:
            raise MetadataDBException(str(e))
Пример #13
0
    def publishSubtitle(self, infohash, lang, pathToSrtSubtitle):
        '''
        Allows an user to publish an srt subtitle file in his channel.
        
        Called by a channel owner this method inserts a new subtitle for
        a torrent published in his channel. 
        The method assumes that the torrent identified by the infohash
        parameter is already in the channel, and that the parameter 
        pathToSrtSubtitle points to an existing srt file on the local
        filesystem.
        If a subtitle for the same language was already associated to the 
        specified infohash and channel, it will be overwritten.
        After calling this method the newly inserted subtitle will be 
        disseminated via Channelcast.
        
        @param infohash: the infohash of the torrent to associate the subtitle
                         with, binary
        @param lang: a 3 characters code for the language of the subtitle as
                     specified in ISO 639-2. Currently just 32 language codes
                     will be supported.
        @param pathToSrtSubtitle: a path in the local filesystem to a subtitle
                                  in srt format.
        
        @raise RichMetadataException: if something "general" goes wrong while
                                      adding new metadata
        @raise IOError: if disk related problems occur
        '''
        assert utilities.isValidInfohash(infohash), "Invalid Infohash"
        assert lang is not None and self.langUtility.isLangCodeSupported(lang)
        assert self._registered, "Instance is not registered"

        channelid = bin2str(self.my_permid)
        base64infohash = bin2str(infohash)
        # consisnstency check: I want to assure that this method is called
        # for an item that is actually in my channel
        consinstent = self.channelcast_db.isItemInChannel(
            channelid, base64infohash)

        if not consinstent:
            msg = "Infohash %s not found in my channel. Rejecting subtitle" % base64infohash
            if DEBUG:
                print >> sys.stderr, msg
            raise RichMetadataException(msg)
        try:
            filepath = self.subtitlesHandler.copyToSubtitlesFolder(
                pathToSrtSubtitle, self.my_permid, infohash, lang)
        except Exception, e:
            if DEBUG:
                print >> sys.stderr, "Failed to read and copy subtitle to appropriate folder: %s" % str(
                    e)
    def handleRMetadata(self, sender_permid, channelCastMessage, fromQuery = False):
        '''
        Handles the reception of rich metadata.
        
        Called when an "erniched" channelCastMessage (v14) is received.
        @param sender_permid: the PermId of the peer who sent the message
        @param channelCastMessage: the received message
        @return: None
        '''
        metadataDTOs = self._splitChannelcastAndRichMetadataContents(channelCastMessage)
          
        if DEBUG:
            print >> sys.stderr, "Handling rich metadata from %s..." % show_permid_short(sender_permid)

        for md_and_have in metadataDTOs:
            md = md_and_have[0]
            havemask = md_and_have[1]
            
            vote = self.votecastDB.getVote(bin2str(md.channel), bin2str(self.my_permid))
            
            if DEBUG:
                id = "RQ" if fromQuery else "R"
                print >> sys.stderr, "%s, %s, %s, %s, %d" % \
                                       (id, md.channel, md.infohash, \
                                        show_permid_short(sender_permid), md.timestamp)
                #format "R|S (R: received - S: sent), channel, infohash, sender|destination,metadataCreationTimestamp"
                # 30-06-2010: "RQ" as received from query
        
            # check if the record belongs to a channel 
            # who we have "reported spam" (negative vote)
            if  vote == -1:
                # if so, ignore the incoming record
                continue
            
            isUpdate = self.rmdDb.insertMetadata(md)
            self.peerHaveManager.newHaveReceived(md.channel,md.infohash,sender_permid,havemask)
            
            if isUpdate is not None:
                #retrieve the metadataDTO from the database in the case it is an update
                md = self.rmdDb.getMetadata(md.channel,md.infohash)
                self._notifyRichMetadata(md, isUpdate)
            
            # if I am a subscriber send immediately a GET_SUBS to the 
            # sender
            if vote == 2:
                if DEBUG:
                    print >> sys.stderr, "Subscribed to channel %s, trying to retrieve" \
                         "all subtitle contents" % (show_permid_short(md.channel),)
                
                self._getAllSubtitles(md)
   def publishSubtitle(self, infohash, lang, pathToSrtSubtitle):
       '''
       Allows an user to publish an srt subtitle file in his channel.
       
       Called by a channel owner this method inserts a new subtitle for
       a torrent published in his channel. 
       The method assumes that the torrent identified by the infohash
       parameter is already in the channel, and that the parameter 
       pathToSrtSubtitle points to an existing srt file on the local
       filesystem.
       If a subtitle for the same language was already associated to the 
       specified infohash and channel, it will be overwritten.
       After calling this method the newly inserted subtitle will be 
       disseminated via Channelcast.
       
       @param infohash: the infohash of the torrent to associate the subtitle
                        with, binary
       @param lang: a 3 characters code for the language of the subtitle as
                    specified in ISO 639-2. Currently just 32 language codes
                    will be supported.
       @param pathToSrtSubtitle: a path in the local filesystem to a subtitle
                                 in srt format.
       
       @raise RichMetadataException: if something "general" goes wrong while
                                     adding new metadata
       @raise IOError: if disk related problems occur
       '''
       assert utilities.isValidInfohash(infohash), "Invalid Infohash"
       assert lang is not None and self.langUtility.isLangCodeSupported(lang)
       assert self._registered, "Instance is not registered"
 
       channelid = bin2str(self.my_permid)
       base64infohash = bin2str(infohash)
       # consisnstency check: I want to assure that this method is called
       # for an item that is actually in my channel
       consinstent = self.channelcast_db.isItemInChannel(channelid,base64infohash)
       
       if not consinstent:
           msg = "Infohash %s not found in my channel. Rejecting subtitle"% base64infohash
           if DEBUG:
               print >> sys.stderr, msg
           raise RichMetadataException(msg)
       try:
           filepath = self.subtitlesHandler.copyToSubtitlesFolder(pathToSrtSubtitle,
                                                           self.my_permid,infohash,
                                                           lang)
       except Exception,e:
           if DEBUG:
               print >> sys.stderr, "Failed to read and copy subtitle to appropriate folder: %s" % str(e)
 def insertHaveMask(self, channel, infohash, peer_id, havemask, timestamp=None):
     '''
     Store a received have mask in the db
     
     Each inserted rows represent a delcaration of subtitle 
     availability from peer_id, for some subtitles for
     a torrent identified by infohash in a channel identified
     by channel.
     
     @type channel: str
     @param channel: channel_id (binary)
     
     @type infohash: str
     @param infohash: the infohash of a torrent (binary)
     
     @type peer_id: str
     @param peer_id: peer from whom the infomask was received.(ie its binary permid)
     
     @type havemask: int
     @param havemask: a non-negative integer. It must be smaller
                     then 2**32.
                     
     @precondition: an entry for (channel, infohash) must already
                    exist in the database
     '''
     query = QUERIES["SELECT METADATA"]
     
     if timestamp is None:
         timestamp = int(time.time())
         
     channel = bin2str(channel)
     infohash = bin2str(infohash)
     peer_id = bin2str(peer_id)
     
     res = self._db.fetchall(query, (infohash, channel))
     
     if len(res) != 1:
         raise MetadataDBException("No entry in the MetadataDB for %s, %s" %\
                                   (channel[-10:],infohash))
                                   
     metadata_fk = res[0][0]
     
     insertQuery = QUERIES["INSERT HAVE MASK"]
     
     try:
         self._db.execute_write(insertQuery, (metadata_fk, peer_id, havemask, timestamp))
     except sqlite3.IntegrityError,e:
         raise MetadataDBException(str(e))
Пример #17
0
    def updateSubtitlePath(self,
                           channel,
                           infohash,
                           lang,
                           newPath,
                           commitNow=True):
        """
        Updates a subtitle entry in the database if it exists.
        
        Given the channel, the infohash, and a SubtitleInfo instance,
        the entry relative to that subtitle is updated accordingly 
        to the details in the SubtitleInfo instance.
        If an instance for the provided channel, infohash, and language
        does not already exist in the db, nothing is done.
        
        @param channel: the channel id (permid) of the channel for the
                        subtitle (binary)
        @param infohash: the infohash of the item the subtitle refrs to
                        (binary)
        @param lang: the language of the subtitle to update
        @param path: the new path of the subtitle. None to indicate that the
                    subtitle is not available
        @return True if an entry was updated in the db. False if nothing
                got written on the db
                
        @precondition: subtitle.lang is not None
        """
        query = QUERIES["SELECT SUBS JOIN HASH ONE"]

        channel = bin2str(channel)
        infohash = bin2str(infohash)

        res = self._db.fetchall(query, (infohash, channel, lang))

        if len(res) > 1:
            raise MetadataDBException("Metadata DB constraint violated")
        elif len(res) == 0:
            if DEBUG:
                print >> sys.stderr, "Nothing to update for channel %s, infohash %s, lang"\
                        " %s. Doing nothing." % (channel[-10:],\
                                                 infohash, lang)
            return False
        else:
            query = QUERIES["UPDATE SUBTITLES"]
            self._db.execute_write(query,
                                   (newPath, res[0][3], res[0][0], lang),
                                   commitNow)
            return True
Пример #18
0
 def getNrMetadata(self, channel):
     """
     Returns the number of metadata objects inside a channel
     """
     query = QUERIES['SELECT NRMETADATA']
     channel = bin2str(channel)
     return self._db.fetchone(query, (channel, ))
Пример #19
0
    def _handleGETSUBS(self, permid, message, selversion):

        if selversion < OLPROTO_VER_FOURTEENTH:
            if DEBUG:
                print >> sys.stderr, "The peer that sent the GET_SUBS request has an old" \
                     "protcol version: this is strange. Dropping the msg"
            return False
        decoded = self._decodeGETSUBSMessage(message)

        if decoded is None:
            if DEBUG:
                print >> sys.stderr, "Error decoding a GET_SUBS message from %s" %\
                      utilities.show_permid_short(permid)
            return False

        if DEBUG:
            channel_id, infohash, languages = decoded
            bitmask = self._languagesUtility.langCodesToMask(languages)
            print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % (
                "RG", show_permid_short(permid), show_permid_short(channel_id),
                bin2str(infohash), bitmask, len(message))

        # no synch on _listenersList since both this method
        # and the registerListener method are called by
        # the OLThread
        for listener in self._listenersList:
            listener.receivedSubsRequest(permid, decoded, selversion)

        return True
 def getNrMetadata(self, channel):
     """
     Returns the number of metadata objects inside a channel
     """
     query = QUERIES['SELECT NRMETADATA']
     channel = bin2str(channel)
     return self._db.fetchone(query, (channel, ))
 def hasMetadata(self, channel, infohash):
     """
     Checks whether there exists some metadata for an item in a channel.
     
     @param channel: a perm_id identifying the owner of the channel.
     @param infohash: the infohash of an item, as announced in channelcast
                      messages.
     @return boolean
     """
     query = QUERIES["SELECT METADATA"]
     
     infohash = bin2str(infohash)
     channel = bin2str(channel)
     
     res = self._db.fetchall(query, (infohash, channel))
     return len(res) != 0
 def _updateSubtitle(self, metadata_fk, subtitle, commitNow=True):
     """
     Update an entry in the Subtitles database.
     
     If the entry identified by metadata_fk, subtitle.lang does not exist
     in the subtitle database this method does nothing.
     
     @param metadata_fk: foreign key of the metadata table
     @param subtitle: instance of Subitle containing the data to insert
     @param commitNow: if False, this method does not commit the changes to
                       the database
     """
     assert metadata_fk is not None
     assert subtitle is not None
     assert isinstance(subtitle, SubtitleInfo)
              
     toUpdate = self._getSubtitleByKey(metadata_fk, subtitle.lang)
     
     if toUpdate is None:
         return
     
    
     query = QUERIES["UPDATE SUBTITLES"]
     
     checksum = bin2str(subtitle.checksum)
                         
     self._db.execute_write(query, (subtitle.path,
                     checksum, metadata_fk, subtitle.lang),
                     commitNow) 
Пример #23
0
    def _updateSubtitle(self, metadata_fk, subtitle, commitNow=True):
        """
        Update an entry in the Subtitles database.
        
        If the entry identified by metadata_fk, subtitle.lang does not exist
        in the subtitle database this method does nothing.
        
        @param metadata_fk: foreign key of the metadata table
        @param subtitle: instance of Subitle containing the data to insert
        @param commitNow: if False, this method does not commit the changes to
                          the database
        """
        assert metadata_fk is not None
        assert subtitle is not None
        assert isinstance(subtitle, SubtitleInfo)

        toUpdate = self._getSubtitleByKey(metadata_fk, subtitle.lang)

        if toUpdate is None:
            return

        query = QUERIES["UPDATE SUBTITLES"]

        checksum = bin2str(subtitle.checksum)

        self._db.execute_write(
            query, (subtitle.path, checksum, metadata_fk, subtitle.lang),
            commitNow)
 def _handleGETSUBS(self,permid, message, selversion):
     
     if selversion < OLPROTO_VER_FOURTEENTH:
         if DEBUG:
             print >> sys.stderr, "The peer that sent the GET_SUBS request has an old" \
                  "protcol version: this is strange. Dropping the msg"
         return False
     decoded = self._decodeGETSUBSMessage(message)
     
     if decoded is None:
         if DEBUG:
             print >> sys.stderr, "Error decoding a GET_SUBS message from %s" %\
                   utilities.show_permid_short(permid)
         return False
 
     if DEBUG:
         channel_id, infohash, languages = decoded
         bitmask = self._languagesUtility.langCodesToMask(languages)
         print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % ("RG", show_permid_short(permid), 
                                                  show_permid_short(channel_id),
                                                  bin2str(infohash), bitmask, len(message))
     
     # no synch on _listenersList since both this method
     # and the registerListener method are called by
     # the OLThread
     for listener in self._listenersList:
         listener.receivedSubsRequest(permid, decoded, selversion)
     
     return True
Пример #25
0
    def hasMetadata(self, channel, infohash):
        """
        Checks whether there exists some metadata for an item in a channel.
        
        @param channel: a perm_id identifying the owner of the channel.
        @param infohash: the infohash of an item, as announced in channelcast
                         messages.
        @return boolean
        """
        query = QUERIES["SELECT METADATA"]

        infohash = bin2str(infohash)
        channel = bin2str(channel)

        res = self._db.fetchall(query, (infohash, channel))
        return len(res) != 0
 def getHaveEntries(self, channel, infohash):
     '''
     Return a list of have entries for subtitles for a torrent
     in a channel.
     
     This method returns a list of tuple, like:
     [ 
       ...
       (peer_id, haveMask, timestamp),
       ...
     ]
     
     (peer_id) is the perm_id of a Tribler
     Peer, haveMask is an integer value representing a 
     bitmask of subtitles owned by that peer. 
     Timestamp is the timestamp at the time the havemask
     was received. 
     The results are ordered by descending timestamp.
     If there are no
     entris for the givenn channel,infohash pair, the returned
     list will be empty
     
     @type channel: str
     @param channel: channel_id (binary)
     
     @type infohash: str
     @param infohash: the infohash of a torrent (binary)
     
     @rtype: list
     @return: see description
     
     '''
     query = QUERIES["GET ALL HAVE MASK"]
     
     channel = bin2str(channel)
     infohash = bin2str(infohash)
     
     res = self._db.fetchall(query,(channel,infohash))
     returnlist = list()
     
     for entry in res:
         peer_id = str2bin(entry[0])
         haveMask = entry[1]
         timestamp = entry[2]
         returnlist.append((peer_id, haveMask, timestamp))
         
     return returnlist
Пример #27
0
    def getHaveEntries(self, channel, infohash):
        '''
        Return a list of have entries for subtitles for a torrent
        in a channel.
        
        This method returns a list of tuple, like:
        [ 
          ...
          (peer_id, haveMask, timestamp),
          ...
        ]
        
        (peer_id) is the perm_id of a Tribler
        Peer, haveMask is an integer value representing a 
        bitmask of subtitles owned by that peer. 
        Timestamp is the timestamp at the time the havemask
        was received. 
        The results are ordered by descending timestamp.
        If there are no
        entris for the givenn channel,infohash pair, the returned
        list will be empty
        
        @type channel: str
        @param channel: channel_id (binary)
        
        @type infohash: str
        @param infohash: the infohash of a torrent (binary)
        
        @rtype: list
        @return: see description
        
        '''
        query = QUERIES["GET ALL HAVE MASK"]

        channel = bin2str(channel)
        infohash = bin2str(infohash)

        res = self._db.fetchall(query, (channel, infohash))
        returnlist = list()

        for entry in res:
            peer_id = str2bin(entry[0])
            haveMask = entry[1]
            timestamp = entry[2]
            returnlist.append((peer_id, haveMask, timestamp))

        return returnlist
Пример #28
0
    def send_metadata(self, permid, message, selversion):
        try:
            infohash = bdecode(message[1:])
        except:
            print_exc()
            if DEBUG:
                print >> sys.stderr,"metadata: GET_METADATA: error becoding"
            return False
        if not isValidInfohash(infohash):
            if DEBUG:
                print >> sys.stderr,"metadata: GET_METADATA: invalid hash"
            return False

        # TODO:
        res = self.torrent_db.getOne(('torrent_file_name', 'status_id'), infohash=bin2str(infohash))
        if not res:
            if DEBUG:
                print >> sys.stderr,"metadata: GET_METADATA: not in database"
            return True    # don't close connection because I don't have the torrent
        torrent_file_name, status_id = res
        if status_id == self.torrent_db._getStatusID('dead'):
            if DEBUG:
                print >> sys.stderr,"metadata: GET_METADATA: Torrent was dead"
            return True
        if not torrent_file_name:
            return True
        torrent_path = os.path.join(self.torrent_dir, torrent_file_name)
        if not os.path.isfile(torrent_path):
            if DEBUG:
                print >> sys.stderr,"metadata: GET_METADATA: not existing", res, torrent_path
            return True
#        
#        data = self.torrent_db.getTorrent(infohash)
#        if not data or not data['torrent_name']:
#            return True     # don't close connection
#        live = data.get('status', 'unknown')
#        #print "**************** check live before send metadata", live
#        if live == 'dead':
#            return True    # don't send dead torrents around
#        
#        torrent_path = None
#        try:
#            torrent_path = os.path.join(data['torrent_dir'], data['torrent_name'])
#            if not os.path.isfile(torrent_path):
#                torrent_path = None
#        except:
#            print_exc()
#            
#        if not torrent_path:
#            if DEBUG:
#                print >> sys.stderr,"metadata: GET_METADATA: not torrent path"
#            return True
        
        task = {'permid':permid, 'infohash':infohash, 'torrent_path':torrent_path, 'selversion':selversion}
        self.upload_queue.append(task)
        if int(time()) >= self.next_upload_time:
            self.checking_upload_queue()
        
        return True
Пример #29
0
    def _deleteSubtitleByChannel(self, channel, infohash, lang):
        '''
        Remove a subtitle for a channel infohash
        
        @param channel: the channel where the subtitle is (binary)
        @param infohash: the infohash of the torrent referred by the subtitle
                        (binary)
        @param lang: ISO-639-2 language code of the subtitle to remove
        
        '''

        query = QUERIES["DELETE ONE SUBTITLE JOIN"]

        infohash = bin2str(infohash)
        channel = bin2str(channel)

        self._db.execute_write(query, (channel, infohash, lang))
Пример #30
0
 def receivedSubsRequest(self, permid, request, selversion):
     """
     Reads a received GET_SUBS message and possibly sends a response.
     
     @param permid: the permid of the sender of the GET_SUBS message
     @param request: a tuple made of channel_id, infohash, language code
     @param selversion: the protocol version of the requesting peer
     
     @return: False if the message had something wrong. (a return value
              of False makes the caller close the connection).
              Otherwise True
     """
     
     assert self.registered, SUBS_LOG_PREFIX + "Handler not yet registered"
     
     channel_id, infohash, languages = request #happily unpacking
     
     #diction {lang : Subtitle}
     allSubtitles = self.subtitlesDb.getAllSubtitles(channel_id, infohash)
     
     contentsList = {} #{langCode : path}
     #for each requested language check if the corresponding subtitle
     #is available
     for lang in sorted(languages):
         if lang in allSubtitles.keys():
             if allSubtitles[lang].subtitleExists():
                 content = self._readSubContent(allSubtitles[lang].path)
                 if content is not None:
                     contentsList[lang] = content 
             else:
                 if DEBUG:
                     print >> sys.stderr, SUBS_LOG_PREFIX + "File not available for channel %s, infohash %s, lang %s" % \
                           (show_permid_short(channel_id), bin2str(infohash),lang)
                           
                 self.subtitlesDb.updateSubtitlePath(channel_id,infohash,lang,None)
         else:
             if DEBUG:
                 print >> sys.stderr, SUBS_LOG_PREFIX + "Subtitle not available for channel %s, infohash %s, lang %s" % \
                       (show_permid_short(channel_id), bin2str(infohash),lang)
     
     if len(contentsList) == 0: #pathlist is empty
         if DEBUG:
             print >> sys.stderr, SUBS_LOG_PREFIX + "None of the requested subtitles were available. No answer will be sent to %s" % \
                   show_permid_short(permid)
         return True
     return self._subsMsgHndlr.sendSubtitleResponse(permid, (channel_id,infohash,contentsList), selversion)
 def _deleteSubtitleByChannel(self, channel, infohash, lang):
     '''
     Remove a subtitle for a channel infohash
     
     @param channel: the channel where the subtitle is (binary)
     @param infohash: the infohash of the torrent referred by the subtitle
                     (binary)
     @param lang: ISO-639-2 language code of the subtitle to remove
     
     '''
     
     query = QUERIES["DELETE ONE SUBTITLE JOIN"]
     
     infohash = bin2str(infohash)
     channel = bin2str(channel)
     
     self._db.execute_write(query,(channel, infohash, lang))
    def _handleSUBS(self, permid, message, selversion):
        if selversion < OLPROTO_VER_FOURTEENTH:
            if DEBUG:
                print >> sys.stderr, "The peer that sent the SUBS request has an old" \
                     "protcol version: this is strange. Dropping the msg"
            return False
        
        decoded = self._decodeSUBSMessage(message)
        
        if decoded is None:
            if DEBUG:
                print >> sys.stderr, "Error decoding a SUBS message from %s" %\
                      utilities.show_permid_short(permid)
            return False
        
        
        channel_id, infohash, bitmask,contents = decoded
        #if no subtitle was requested drop the whole message
        
        if DEBUG:
            print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % ("RS", show_permid_short(permid), 
                                                     show_permid_short(channel_id),
                                                     bin2str(infohash), bitmask, len(message))
        
        

        requestedSubs = self._checkRequestedSubtitles(channel_id,infohash,bitmask) 
        if requestedSubs == 0:
            if DEBUG:
                print >> sys.stderr, SUBS_LOG_PREFIX + "Received a SUBS message that was not"\
                      " requested. Dropping"
            return False
        
        requestedSubsCodes = self._languagesUtility.maskToLangCodes(requestedSubs)
        #drop from the contents subtitles that where not requested
        
        
        for lang in contents.keys():
            if lang not in requestedSubsCodes:
                del contents[lang]
        
        #remove the received subtitles from the requested 
        callbacks = \
            self._removeFromRequestedSubtitles(channel_id, infohash, bitmask)

        
        
        #the receiver does not need the bitmask
        tuple = channel_id, infohash, contents
        
        # no synch on _listenersList since both this method
        # and the registerListener method are called by
        # the OLThread
        for listener in self._listenersList:
            listener.receivedSubsResponse(permid, tuple, callbacks, selversion)
        
    
        return True
 def getMetadata(self, channel, infohash):
     """
     Returns a MetadataDTO instance for channel/infohash if available in DB
     
     Given a channel/infhash couple returns a MetadataDTO instance, built
     with the values retrieved from the Metadata and Subtitles DB. If
     no result returns None
     
     @param channel: the permid of the channel's owner (binary)
     @param infohash: the infohash of the item the metadata refers to
                      (binary)
     @return: a MetadataDTO instance comprehensive of subtitles if any
              metadata is found in the DB. None otherwise.
     """
     
     query = QUERIES["SELECT METADATA"]
     
     infohash = bin2str(infohash)
     channel = bin2str(channel)
     
     res = self._db.fetchall(query, (infohash, channel))
     
     if len(res) == 0:
         return None
     if len(res) > 1:
         raise MetadataDBException("Metadata DB Constraint violated")
     
     metaTuple = res[0]
     
     subsDictionary = self._getAllSubtitlesByKey(metaTuple[0])
     
     publisher = str2bin(metaTuple[1])
     infohash =  str2bin(metaTuple[2])
     timestamp = int(metaTuple[4])
     description = unicode(metaTuple[3])
     signature = str2bin(metaTuple[5])
     
     toReturn = MetadataDTO(publisher, infohash,
                            timestamp, description, None,
                            signature)
     
     for sub in subsDictionary.itervalues():
         toReturn.addSubtitle(sub)
     
     return toReturn
Пример #34
0
    def getMetadata(self, channel, infohash):
        """
        Returns a MetadataDTO instance for channel/infohash if available in DB
        
        Given a channel/infhash couple returns a MetadataDTO instance, built
        with the values retrieved from the Metadata and Subtitles DB. If
        no result returns None
        
        @param channel: the permid of the channel's owner (binary)
        @param infohash: the infohash of the item the metadata refers to
                         (binary)
        @return: a MetadataDTO instance comprehensive of subtitles if any
                 metadata is found in the DB. None otherwise.
        """

        query = QUERIES["SELECT METADATA"]

        infohash = bin2str(infohash)
        channel = bin2str(channel)

        res = self._db.fetchall(query, (infohash, channel))

        if len(res) == 0:
            return None
        if len(res) > 1:
            raise MetadataDBException("Metadata DB Constraint violated")

        metaTuple = res[0]

        subsDictionary = self._getAllSubtitlesByKey(metaTuple[0])

        publisher = str2bin(metaTuple[1])
        infohash = str2bin(metaTuple[2])
        timestamp = int(metaTuple[4])
        description = unicode(metaTuple[3])
        signature = str2bin(metaTuple[5])

        toReturn = MetadataDTO(publisher, infohash, timestamp, description,
                               None, signature)

        for sub in subsDictionary.itervalues():
            toReturn.addSubtitle(sub)

        return toReturn
 def updateSubtitlePath(self, channel, infohash, lang, newPath, commitNow=True):
     """
     Updates a subtitle entry in the database if it exists.
     
     Given the channel, the infohash, and a SubtitleInfo instance,
     the entry relative to that subtitle is updated accordingly 
     to the details in the SubtitleInfo instance.
     If an instance for the provided channel, infohash, and language
     does not already exist in the db, nothing is done.
     
     @param channel: the channel id (permid) of the channel for the
                     subtitle (binary)
     @param infohash: the infohash of the item the subtitle refrs to
                     (binary)
     @param lang: the language of the subtitle to update
     @param path: the new path of the subtitle. None to indicate that the
                 subtitle is not available
     @return True if an entry was updated in the db. False if nothing
             got written on the db
             
     @precondition: subtitle.lang is not None
     """
     query = QUERIES["SELECT SUBS JOIN HASH ONE"]
     
     channel = bin2str(channel)
     infohash = bin2str(infohash)
     
     res = self._db.fetchall(query, (infohash, channel, lang))
     
     if len(res) > 1 :
         raise MetadataDBException("Metadata DB constraint violated")
     elif len(res) == 0 :
         if DEBUG:
             print >> sys.stderr, "Nothing to update for channel %s, infohash %s, lang"\
                     " %s. Doing nothing." % (channel[-10:],\
                                              infohash, lang)
         return False
     else:
         query = QUERIES["UPDATE SUBTITLES"]
         self._db.execute_write(query, (newPath,
                     res[0][3], res[0][0], lang),
                     commitNow) 
         return True
Пример #36
0
    def _get_subs_connect_callback(self, exception, dns, permid, selversion,
                                   channel_id, infohash, bitmask,
                                   msgSentCallback, usrCallback):
        """
        Called by the Overlay Thread when a connection with permid is established.
        
        Performs the actual action of sending a GET_SUBS request to the peer
        identified by permid. It is called by the OLThread when a connection
        with that peer is established.
    
        """

        if exception is not None:
            if DEBUG:
                print >> sys.stderr, SUBS_LOG_PREFIX + \
                      "GET_SUBS not sent. Unable to connect to " + \
                      utilities.show_permid_short(permid)
        else:

            if (selversion > 0 and selversion < OLPROTO_VER_FOURTEENTH):
                msg = "GET_SUBS not send, the other peers had an old protocol version: %d" %\
                    selversion
                if DEBUG:
                    print >> sys.stderr, msg
                raise SubtitleMsgHandlerException(msg)

            if DEBUG:
                print >> sys.stderr, SUBS_LOG_PREFIX + "sending GET_SUBS to " + \
                      utilities.show_permid_short(permid)
            try:
                message = self._createGETSUBSMessage(channel_id, infohash,
                                                     bitmask)

                if DEBUG:
                    # Format:
                    # SS|SG, destination, channel, infohash, bitmask, size
                    print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % (
                        "SG", show_permid_short(permid),
                        show_permid_short(channel_id), bin2str(infohash),
                        bitmask, len(message))

                self._overlay_bridge.send(permid, message,
                                          lambda exc, permid: \
                                            self._sent_callback(exc,permid,
                                                            channel_id,
                                                            infohash,
                                                            bitmask,
                                                            msgSentCallback,
                                                            usrCallback))

            except Exception, e:
                print_exc()
                msg = "GET_SUBS not sent: %s" % str(e)
                raise SubtitleMsgHandlerException(e)
 def _get_subs_connect_callback(self, exception, dns, permid, selversion,
                           channel_id, infohash, bitmask, msgSentCallback, usrCallback):
     """
     Called by the Overlay Thread when a connection with permid is established.
     
     Performs the actual action of sending a GET_SUBS request to the peer
     identified by permid. It is called by the OLThread when a connection
     with that peer is established.
 
     """
     
     if exception is not None:
         if DEBUG:
             print >> sys.stderr, SUBS_LOG_PREFIX + \
                   "GET_SUBS not sent. Unable to connect to " + \
                   utilities.show_permid_short(permid)
     else:
         
                 
         if (selversion > 0 and selversion < OLPROTO_VER_FOURTEENTH):
             msg = "GET_SUBS not send, the other peers had an old protocol version: %d" %\
                 selversion
             if DEBUG:
                 print >> sys.stderr, msg
             raise SubtitleMsgHandlerException(msg)
         
         if DEBUG:
             print >> sys.stderr, SUBS_LOG_PREFIX + "sending GET_SUBS to " + \
                   utilities.show_permid_short(permid)
         try :
             message = self._createGETSUBSMessage(channel_id, infohash,
                                                 bitmask)
             
             
             if DEBUG:
                 # Format:
                 # SS|SG, destination, channel, infohash, bitmask, size
                 print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % ("SG",show_permid_short(permid), 
                                                          show_permid_short(channel_id),
                                                          bin2str(infohash),bitmask,len(message))
             
             self._overlay_bridge.send(permid, message,
                                       lambda exc, permid: \
                                         self._sent_callback(exc,permid,
                                                         channel_id,
                                                         infohash,
                                                         bitmask,
                                                         msgSentCallback,
                                                         usrCallback))
     
         except Exception,e:
             print_exc()
             msg = "GET_SUBS not sent: %s" % str(e)
             raise SubtitleMsgHandlerException(e)
Пример #38
0
    def _handleSUBS(self, permid, message, selversion):
        if selversion < OLPROTO_VER_FOURTEENTH:
            if DEBUG:
                print >> sys.stderr, "The peer that sent the SUBS request has an old" \
                     "protcol version: this is strange. Dropping the msg"
            return False

        decoded = self._decodeSUBSMessage(message)

        if decoded is None:
            if DEBUG:
                print >> sys.stderr, "Error decoding a SUBS message from %s" %\
                      utilities.show_permid_short(permid)
            return False

        channel_id, infohash, bitmask, contents = decoded
        #if no subtitle was requested drop the whole message

        if DEBUG:
            print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % (
                "RS", show_permid_short(permid), show_permid_short(channel_id),
                bin2str(infohash), bitmask, len(message))

        requestedSubs = self._checkRequestedSubtitles(channel_id, infohash,
                                                      bitmask)
        if requestedSubs == 0:
            if DEBUG:
                print >> sys.stderr, SUBS_LOG_PREFIX + "Received a SUBS message that was not"\
                      " requested. Dropping"
            return False

        requestedSubsCodes = self._languagesUtility.maskToLangCodes(
            requestedSubs)
        #drop from the contents subtitles that where not requested

        for lang in contents.keys():
            if lang not in requestedSubsCodes:
                del contents[lang]

        #remove the received subtitles from the requested
        callbacks = \
            self._removeFromRequestedSubtitles(channel_id, infohash, bitmask)

        #the receiver does not need the bitmask
        tuple = channel_id, infohash, contents

        # no synch on _listenersList since both this method
        # and the registerListener method are called by
        # the OLThread
        for listener in self._listenersList:
            listener.receivedSubsResponse(permid, tuple, callbacks, selversion)

        return True
 def deleteMetadata(self, channel, infohash):
     """
     Removes all the metadata associated to a channel/infohash.
     
     Everything is dropped from both the Metadata and Subtitles db.
     
     @param channel: the permid of the channel's owner
     @param infohash: the infhoash of the entry
     """
     
     assert channel is not None
     assert infohash is not None
     
     channel = bin2str(channel)
     infohash = bin2str(infohash)
     
     query = QUERIES["SELECT METADATA"]
     
     if DEBUG:
         print >> sys.stderr, "Performing query on db: " + query
     
     res = self._db.fetchall(query, (infohash, channel))
     
     if len(res) == 0 :
         return
     if len(res) > 1 :
         raise IOError("Metadata DB constraint violated")
     
     metadata_fk = res[0][0]
     
     self._deleteAllSubtitles(metadata_fk, False)
     
     query = QUERIES["DELETE METADATA PK"]
     
     self._db.execute_write(query, (metadata_fk,), False)
     
     self._db.commit()
Пример #40
0
    def deleteMetadata(self, channel, infohash):
        """
        Removes all the metadata associated to a channel/infohash.
        
        Everything is dropped from both the Metadata and Subtitles db.
        
        @param channel: the permid of the channel's owner
        @param infohash: the infhoash of the entry
        """

        assert channel is not None
        assert infohash is not None

        channel = bin2str(channel)
        infohash = bin2str(infohash)

        query = QUERIES["SELECT METADATA"]

        if DEBUG:
            print >> sys.stderr, "Performing query on db: " + query

        res = self._db.fetchall(query, (infohash, channel))

        if len(res) == 0:
            return
        if len(res) > 1:
            raise IOError("Metadata DB constraint violated")

        metadata_fk = res[0][0]

        self._deleteAllSubtitles(metadata_fk, False)

        query = QUERIES["DELETE METADATA PK"]

        self._db.execute_write(query, (metadata_fk, ), False)

        self._db.commit()
Пример #41
0
def getSubtitleFileRelativeName(channel_id, infohash, langCode):
    # subtitles filenames are build from the sha1 hash
    # of the triple (channel_id, infohash, langCode)
    
    # channel_id and infohash are binary versions
    
    assert utilities.validPermid(channel_id), "Invalid channel_id %s" % utilities.show_permid_short(channel_id)
    assert utilities.validInfohash(infohash), "Invalid infohash %s" % bin2str(infohash)
    assert LanguagesProvider.getLanguagesInstance().isLangCodeSupported(langCode), "Unsupported language code %s" % langCode
        
    hasher = sha()
    for data in (channel_id, infohash, langCode):
        hasher.update(data)
    subtitleName = hasher.hexdigest() + SUBS_EXTENSION
    return subtitleName
Пример #42
0
    def getAllSubtitles(self, channel, infohash):
        """
        Get all the available subtitles for a channel and infohash.
        
        Returns a list representing subtitles that are available for
        a givenchannel and infohash. 
        
        @param channel: the perm_id of the channel owner (binary)
        @param infohash: the infhash of a channel elements as it
                         is announced in ChannelCast (binary)
        @return: a dictionary of { lang : SubtitleInfo instance}
        """

        query = QUERIES["SELECT SUBS JOIN HASH ALL"]
        infohash = bin2str(infohash)
        channel = bin2str(channel)

        results = self._db.fetchall(query, (infohash, channel))

        subsDict = {}
        for entry in results:
            subsDict[entry[1]] = SubtitleInfo(entry[1], entry[2], entry[3])

        return subsDict
 def getAllSubtitles(self, channel, infohash):
     """
     Get all the available subtitles for a channel and infohash.
     
     Returns a list representing subtitles that are available for
     a givenchannel and infohash. 
     
     @param channel: the perm_id of the channel owner (binary)
     @param infohash: the infhash of a channel elements as it
                      is announced in ChannelCast (binary)
     @return: a dictionary of { lang : SubtitleInfo instance}
     """
     
     query = QUERIES["SELECT SUBS JOIN HASH ALL"]
     infohash = bin2str(infohash)
     channel = bin2str(channel)
       
     results = self._db.fetchall(query, (infohash, channel))
     
     subsDict = {}
     for entry in results:
         subsDict[entry[1]] = SubtitleInfo(entry[1], entry[2], entry[3])
  
     return subsDict
Пример #44
0
    def send_metadata(self, permid, message, selversion):
        try:
            infohash = bdecode(message[1:])
        except:
            print_exc()
            if DEBUG:
                print >> sys.stderr, "metadata: GET_METADATA: error becoding"
            return False
        if not isValidInfohash(infohash):
            if DEBUG:
                print >> sys.stderr, "metadata: GET_METADATA: invalid hash"
            return False

        # TODO:
        res = self.torrent_db.getOne(('torrent_file_name', 'status_id'),
                                     infohash=bin2str(infohash))
        if not res:
            if DEBUG:
                print >> sys.stderr, "metadata: GET_METADATA: not in database", infohash
            return True  # don't close connection because I don't have the torrent
        torrent_file_name, status_id = res
        if status_id == self.torrent_db._getStatusID('dead'):
            if DEBUG:
                print >> sys.stderr, "metadata: GET_METADATA: Torrent was dead"
            return True
        if not torrent_file_name:
            if DEBUG:
                print >> sys.stderr, "metadata: GET_METADATA: no torrent file name"
            return True
        torrent_path = os.path.join(self.torrent_dir, torrent_file_name)
        if not os.path.isfile(torrent_path):
            if DEBUG:
                print >> sys.stderr, "metadata: GET_METADATA: not existing", res, torrent_path
            return True

        task = {
            'permid': permid,
            'infohash': infohash,
            'torrent_path': torrent_path,
            'selversion': selversion
        }
        self.upload_queue.append(task)
        if int(time()) >= self.next_upload_time:
            self.checking_upload_queue()

        return True
 def _insertNewSubtitle(self, metadata_fk, subtitle, commitNow=True) :
     """
     Insert a new subtitle entry in the Subtitles table.
     
     Given a foreign key from the Metadata table, and a SubtitleInfo instance
     describing the subtitle to insert, adds it to the metadata table.
     This method assumes that that entry does not already exist in the
     table.
     NOTICE that sqlite  does not enforce the foreign key constraint,
     so be careful about integrity
     """
     assert metadata_fk is not None
     assert subtitle is not None
     assert isinstance(subtitle, SubtitleInfo)
     
     query = QUERIES["INSERT SUBTITLES"]
     
     checksum = bin2str(subtitle.checksum)
     self._db.execute_write(query, (metadata_fk, subtitle.lang,
                                    subtitle.path, checksum),
                                    commitNow)
Пример #46
0
    def _insertNewSubtitle(self, metadata_fk, subtitle, commitNow=True):
        """
        Insert a new subtitle entry in the Subtitles table.
        
        Given a foreign key from the Metadata table, and a SubtitleInfo instance
        describing the subtitle to insert, adds it to the metadata table.
        This method assumes that that entry does not already exist in the
        table.
        NOTICE that sqlite  does not enforce the foreign key constraint,
        so be careful about integrity
        """
        assert metadata_fk is not None
        assert subtitle is not None
        assert isinstance(subtitle, SubtitleInfo)

        query = QUERIES["INSERT SUBTITLES"]

        checksum = bin2str(subtitle.checksum)
        self._db.execute_write(
            query, (metadata_fk, subtitle.lang, subtitle.path, checksum),
            commitNow)
Пример #47
0
    def send_metadata(self, permid, message, selversion):
        try:
            infohash = bdecode(message[1:])
        except:
            print_exc()
            if DEBUG:
                print >> sys.stderr,"metadata: GET_METADATA: error becoding"
            return False
        if not isValidInfohash(infohash):
            if DEBUG:
                print >> sys.stderr,"metadata: GET_METADATA: invalid hash"
            return False

        # TODO:
        res = self.torrent_db.getOne(('torrent_file_name', 'status_id'), infohash=bin2str(infohash))
        if not res:
            if DEBUG:
                print >> sys.stderr,"metadata: GET_METADATA: not in database", infohash
            return True    # don't close connection because I don't have the torrent
        torrent_file_name, status_id = res
        if status_id == self.torrent_db._getStatusID('dead'):
            if DEBUG:
                print >> sys.stderr,"metadata: GET_METADATA: Torrent was dead"
            return True
        if not torrent_file_name:
            if DEBUG:
                print >> sys.stderr,"metadata: GET_METADATA: no torrent file name"
            return True
        torrent_path = os.path.join(self.torrent_dir, torrent_file_name)
        if not os.path.isfile(torrent_path):
            if DEBUG:
                print >> sys.stderr,"metadata: GET_METADATA: not existing", res, torrent_path
            return True
        
        task = {'permid':permid, 'infohash':infohash, 'torrent_path':torrent_path, 'selversion':selversion}
        self.upload_queue.append(task)
        if int(time()) >= self.next_upload_time:
            self.checking_upload_queue()
        
        return True
 def getAllMetadataForInfohash(self, infohash):
     """
     Returns a list of MetadataDTO instances for a given infohash
     
     Given a torrent infohash returns a list of MetadataDTO instances for
     that infohash. Each one of the MetadataDTO refers to a different
     channel.
     
     @param infohash: the infohash for the torrent (binary)
     @return: a list of MetadataDTO isntances (or empty list if nothing
              is found)
     """
     
     assert infohash is not None
     
     strinfohash = bin2str(infohash)
     
     query = QUERIES["SELECT PUBLISHERS FROM INFOHASH"]
     
     channels = self._db.fetchall(query, (strinfohash,))
     
     return [self.getMetadata(str2bin(entry[0]), infohash) for entry in channels]
Пример #49
0
    def getAllMetadataForInfohash(self, infohash):
        """
        Returns a list of MetadataDTO instances for a given infohash
        
        Given a torrent infohash returns a list of MetadataDTO instances for
        that infohash. Each one of the MetadataDTO refers to a different
        channel.
        
        @param infohash: the infohash for the torrent (binary)
        @return: a list of MetadataDTO isntances (or empty list if nothing
                 is found)
        """

        assert infohash is not None

        strinfohash = bin2str(infohash)

        query = QUERIES["SELECT PUBLISHERS FROM INFOHASH"]

        channels = self._db.fetchall(query, (strinfohash, ))

        return [
            self.getMetadata(str2bin(entry[0]), infohash) for entry in channels
        ]
Пример #50
0
    def addRichMetadataContent(self,
                               channelCastMessage,
                               destPermid=None,
                               fromQuery=False):
        '''
        Takes plain channelcast message (from OLProto v.13) and adds to it
        a 'rich_metadata' field.
        
        @param channelCastMessage: the old channelcast message in the format of
                                   protocol v13
        @param destPermid: the destination of the message. If not None it is used
                            for logging purposes only. If None, nothing bad happens.
        @return: the "enriched" channelcast message
        '''
        if not len(channelCastMessage) > 0:
            if DEBUG:
                print >> sys.stderr, "no entries to enrich with rmd"
            return channelCastMessage

        if DEBUG:
            if fromQuery:
                print >> sys.stderr, "Intercepted a channelcast message as answer to a query"
            else:
                print >> sys.stderr, "Intercepted a channelcast message as normal channelcast"

        doesChannelHaveSubtitles = {}
        # a channelcast message is made up of a dictionary of entries
        # keyed the signature. Every value in the dictionary is itself
        # a dictionary with the item information
        for key, content in channelCastMessage.iteritems():
            channel_id = content['publisher_id']
            infohash = content['infohash']

            if channel_id not in doesChannelHaveSubtitles:
                doesChannelHaveSubtitles[
                    channel_id] = self.rmdDb.getNrMetadata(channel_id) > 0

            if doesChannelHaveSubtitles[channel_id]:
                metadataDTO = self.rmdDb.getMetadata(channel_id, infohash)
                if metadataDTO is not None:
                    try:
                        if DEBUG:
                            print >> sys.stderr, "Enriching a channelcast message with subtitle contents"
                        metadataPack = metadataDTO.serialize()

                        # I can remove from the metadata pack the infohash, and channelId
                        # since they are already in channelcast and they would be redundant
                        metadataPack.pop(0)
                        metadataPack.pop(0)

                        #adding the haveMask at the end of the metadata pack
                        havemask = self.peerHaveManager.retrieveMyHaveMask(
                            channel_id, infohash)
                        binary_havemask = pack("!L", havemask)
                        metadataPack.append(binary_havemask)

                        content['rich_metadata'] = metadataPack

                        if DEBUG:
                            size = self._computeSize(metadataPack)
                            # if available records also the destination of the message
                            dest = "NA" if destPermid is None else show_permid_short(
                                destPermid)

                            id = "SQ" if fromQuery else "S"
                            # format (S (for sent) | SQ (for sent as response to a query), channel, infohash, destination, timestampe, size)
                            print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % \
                                (id, bin2str(metadataDTO.channel), \
                                bin2str(metadataDTO.infohash), \
                                 dest, metadataDTO.timestamp, size)
                    except Exception, e:
                        print >> sys.stderr, "Warning: Error serializing metadata: %s", str(
                            e)
 def _checkingUploadQueue(self):
     """
     Uses a token bucket to control the subtitles upload rate.
     
     Every time this method is called, it will check if there are enough
     tokens in the bucket to send out a SUBS message.
     Currently fragmentation is not implemented: all the reuquested subtitles
     are sent in a single SUBS messages if there are enough tokens:
     too big responses are simply discarded.
     
     The method tries to consume all the available tokens of the token
     bucket until there are no more messages to send. If there are no
     sufficiente tokens to send a message, another call to this method
     is scheduled in a point in time sufficiently distant.
     """
     
     if DEBUG:
         print >> sys.stderr, SUBS_LOG_PREFIX + "Checking the upload queue..."
     
     if not self._tokenBucket.upload_rate > 0:
         return 
     
     if not len(self._uploadQueue) > 0:
         if DEBUG:
             print >> sys.stderr, SUBS_LOG_PREFIX + "Upload queue is empty."
         
     while len(self._uploadQueue) > 0 :
         responseData = self._uploadQueue[0]
         encodedMsg = self._createSingleResponseMessage(responseData)
         
         if encodedMsg is None:
             if DEBUG:
                 print >> sys.stderr, SUBS_LOG_PREFIX + "Nothing to send"
             del self._uploadQueue[0]
             continue #check other messages in the queue
         
         msgSize = len(encodedMsg) / 1024.0 #in kilobytes
         
         if msgSize > self._tokenBucket.capacity: 
             #message is too big, discarding
             print >> sys.stderr, "Warning:" + SUBS_LOG_PREFIX + "SUBS message too big. Discarded!"
             del self._uploadQueue[0]
             continue #check other messages in the queue
         
         #check if there are sufficiente tokens
         if self._tokenBucket.consume(msgSize):
             
             if DEBUG:
                 # Format:
                 # S|G, destination, channel, infohash, bitmask, size
                 keys = responseData['subtitles'].keys()
                 bitmask = self._languagesUtility.langCodesToMask(keys)
                 print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % ("SS",show_permid_short(responseData['permid']),
                                                          show_permid_short(responseData['channel_id']),
                                                          bin2str(responseData['infohash']),bitmask,int(msgSize*1024))
                 
             self._doSendSubtitles(responseData['permid'], encodedMsg, responseData['selversion'])
             del self._uploadQueue[0]
         else: 
             #tokens are insufficient wait the necessary time and check again
             neededCapacity = max(0, msgSize - self._tokenBucket.tokens)
             delay = (neededCapacity / self._tokenBucket.upload_rate)
             self._nextUploadTime = time() + delay
             self.overlay_bridge.add_task(self._checkingUploadQueue, delay)
             return
Пример #52
0
    def send_metadata(self, permid, message, selversion):
        try:
            infohash = bdecode(message[1:])
        except:
            print_exc()
            if DEBUG:
                print >> sys.stderr, "metadata: GET_METADATA: error becoding"
            return False
        if not isValidInfohash(infohash):
            if DEBUG:
                print >> sys.stderr, "metadata: GET_METADATA: invalid hash"
            return False

        # TODO:
        res = self.torrent_db.getOne(('torrent_file_name', 'status_id'),
                                     infohash=bin2str(infohash))
        if not res:
            if DEBUG:
                print >> sys.stderr, "metadata: GET_METADATA: not in database"
            return True  # don't close connection because I don't have the torrent
        torrent_file_name, status_id = res
        if status_id == self.torrent_db._getStatusID('dead'):
            if DEBUG:
                print >> sys.stderr, "metadata: GET_METADATA: Torrent was dead"
            return True
        if not torrent_file_name:
            return True
        torrent_path = os.path.join(self.torrent_dir, torrent_file_name)
        if not os.path.isfile(torrent_path):
            if DEBUG:
                print >> sys.stderr, "metadata: GET_METADATA: not existing", res, torrent_path
            return True
#
#        data = self.torrent_db.getTorrent(infohash)
#        if not data or not data['torrent_name']:
#            return True     # don't close connection
#        live = data.get('status', 'unknown')
#        #print "**************** check live before send metadata", live
#        if live == 'dead':
#            return True    # don't send dead torrents around
#
#        torrent_path = None
#        try:
#            torrent_path = os.path.join(data['torrent_dir'], data['torrent_name'])
#            if not os.path.isfile(torrent_path):
#                torrent_path = None
#        except:
#            print_exc()
#
#        if not torrent_path:
#            if DEBUG:
#                print >> sys.stderr,"metadata: GET_METADATA: not torrent path"
#            return True

        task = {
            'permid': permid,
            'infohash': infohash,
            'torrent_path': torrent_path,
            'selversion': selversion
        }
        self.upload_queue.append(task)
        if int(time()) >= self.next_upload_time:
            self.checking_upload_queue()

        return True
Пример #53
0
    def _checkingUploadQueue(self):
        """
        Uses a token bucket to control the subtitles upload rate.
        
        Every time this method is called, it will check if there are enough
        tokens in the bucket to send out a SUBS message.
        Currently fragmentation is not implemented: all the reuquested subtitles
        are sent in a single SUBS messages if there are enough tokens:
        too big responses are simply discarded.
        
        The method tries to consume all the available tokens of the token
        bucket until there are no more messages to send. If there are no
        sufficiente tokens to send a message, another call to this method
        is scheduled in a point in time sufficiently distant.
        """

        if DEBUG:
            print >> sys.stderr, SUBS_LOG_PREFIX + "Checking the upload queue..."

        if not self._tokenBucket.upload_rate > 0:
            return

        if not len(self._uploadQueue) > 0:
            if DEBUG:
                print >> sys.stderr, SUBS_LOG_PREFIX + "Upload queue is empty."

        while len(self._uploadQueue) > 0:
            responseData = self._uploadQueue[0]
            encodedMsg = self._createSingleResponseMessage(responseData)

            if encodedMsg is None:
                if DEBUG:
                    print >> sys.stderr, SUBS_LOG_PREFIX + "Nothing to send"
                del self._uploadQueue[0]
                continue  #check other messages in the queue

            msgSize = len(encodedMsg) / 1024.0  #in kilobytes

            if msgSize > self._tokenBucket.capacity:
                #message is too big, discarding
                print >> sys.stderr, "Warning:" + SUBS_LOG_PREFIX + "SUBS message too big. Discarded!"
                del self._uploadQueue[0]
                continue  #check other messages in the queue

            #check if there are sufficiente tokens
            if self._tokenBucket.consume(msgSize):

                if DEBUG:
                    # Format:
                    # S|G, destination, channel, infohash, bitmask, size
                    keys = responseData['subtitles'].keys()
                    bitmask = self._languagesUtility.langCodesToMask(keys)
                    print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % (
                        "SS", show_permid_short(responseData['permid']),
                        show_permid_short(responseData['channel_id']),
                        bin2str(responseData['infohash']), bitmask,
                        int(msgSize * 1024))

                self._doSendSubtitles(responseData['permid'], encodedMsg,
                                      responseData['selversion'])
                del self._uploadQueue[0]
            else:
                #tokens are insufficient wait the necessary time and check again
                neededCapacity = max(0, msgSize - self._tokenBucket.tokens)
                delay = (neededCapacity / self._tokenBucket.upload_rate)
                self._nextUploadTime = time() + delay
                self.overlay_bridge.add_task(self._checkingUploadQueue, delay)
                return
 def addRichMetadataContent(self,channelCastMessage, destPermid = None, fromQuery = False):
     '''
     Takes plain channelcast message (from OLProto v.13) and adds to it
     a 'rich_metadata' field.
     
     @param channelCastMessage: the old channelcast message in the format of
                                protocol v13
     @param destPermid: the destination of the message. If not None it is used
                         for logging purposes only. If None, nothing bad happens.
     @return: the "enriched" channelcast message
     '''
     if not len(channelCastMessage) > 0:
         if DEBUG:
             print >> sys.stderr, "no entries to enrich with rmd"
         return channelCastMessage
     
     if DEBUG:
         if fromQuery: 
             print >> sys.stderr, "Intercepted a channelcast message as answer to a query"
         else:
             print >> sys.stderr, "Intercepted a channelcast message as normal channelcast"
     
     doesChannelHaveSubtitles = {}
     # a channelcast message is made up of a dictionary of entries
     # keyed the signature. Every value in the dictionary is itself
     # a dictionary with the item information
     for key, content in channelCastMessage.iteritems():
         channel_id = content['publisher_id']
         infohash = content['infohash']
         
         if channel_id not in doesChannelHaveSubtitles:
             doesChannelHaveSubtitles[channel_id] = self.rmdDb.getNrMetadata(channel_id) > 0
         
         if doesChannelHaveSubtitles[channel_id]:
             metadataDTO = self.rmdDb.getMetadata(channel_id, infohash)
             if metadataDTO is not None:
                 try:
                     if DEBUG:
                         print >> sys.stderr, "Enriching a channelcast message with subtitle contents"
                     metadataPack = metadataDTO.serialize()
                     
                     # I can remove from the metadata pack the infohash, and channelId
                     # since they are already in channelcast and they would be redundant
                     metadataPack.pop(0)
                     metadataPack.pop(0)
                     
                     #adding the haveMask at the end of the metadata pack
                     havemask = self.peerHaveManager.retrieveMyHaveMask(channel_id, infohash)
                     binary_havemask = pack("!L", havemask)
                     metadataPack.append(binary_havemask)
                     
                     content['rich_metadata'] = metadataPack
                     
                     if DEBUG:
                         size = self._computeSize(metadataPack)
                         # if available records also the destination of the message
                         dest = "NA" if destPermid is None else show_permid_short(destPermid)
                     
                         id = "SQ" if fromQuery else "S"
                         # format (S (for sent) | SQ (for sent as response to a query), channel, infohash, destination, timestampe, size)
                         print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % \
                             (id, bin2str(metadataDTO.channel), \
                             bin2str(metadataDTO.infohash), \
                              dest, metadataDTO.timestamp, size)
                 except Exception,e:
                     print >> sys.stderr, "Warning: Error serializing metadata: %s", str(e)
Пример #55
0
    def insertMetadata(self, metadata_dto):
        '''
        Insert the metadata contained in a Metadata DTO in the database.
        
        If an entry relative to the same channel and infohash of the provided 
        dto already exists in the db, the db is updated only if the timestamp
        of the new dto is newer then the entry in the database. 
        If there is no such an entry, a new wan in the Metadata DB is created
        along with the required entries in the SubtitleInfo DB
        
        @type metadata_dto: MetadataDTO 
        @param metada_dto: an instance of MetadataDTO describing metadata
        
        @return True if an existing entry was updated,  false if a new entry
                was interested. Otherwise None.
        
        '''
        assert metadata_dto is not None
        assert isinstance(metadata_dto, MetadataDTO)
        #try to retrieve a correspindng record for channel,infhoash

        #won't do nothing if the metadata_dto is not correctly signed
        if not metadata_dto.verifySignature():
            raise SignatureException("Metadata to insert is not properly" \
                                     "signed")

        select_query = QUERIES["SELECT METADATA"]

        signature = bin2str(metadata_dto.signature)
        infohash = bin2str(metadata_dto.infohash)
        channel = bin2str(metadata_dto.channel)

        res = self._db.fetchall(select_query, (infohash, channel))

        isUpdate = False

        if len(res) != 0:
            #updated if the new message is newer
            if metadata_dto.timestamp > res[0][4]:
                query = QUERIES["UPDATE METADATA"]

                self._db.execute_write(query, (
                    metadata_dto.description,
                    metadata_dto.timestamp,
                    signature,
                    infohash,
                    channel,
                ), False)  #I don't want the transaction to commit now

                fk_key = res[0][0]

                isUpdate = True

            else:
                return

        else:  #if is this a whole new metadata item
            query = QUERIES["INSERT METADATA"]

            self._db.execute_write(query, (
                channel,
                infohash,
                metadata_dto.description,
                metadata_dto.timestamp,
                signature,
            ), True)

            if DEBUG:
                print >> sys.stderr, "Performing query on db: " + query

            newRows = self._db.fetchall(select_query, (infohash, channel))

            if len(newRows) == 0:
                raise IOError("No results, while there should be one")

            fk_key = newRows[0][0]


        self._insertOrUpdateSubtitles(fk_key, metadata_dto.getAllSubtitles(), \
                                      False)

        self._db.commit()  #time to commit everything

        return isUpdate
 def insertMetadata(self, metadata_dto):
     '''
     Insert the metadata contained in a Metadata DTO in the database.
     
     If an entry relative to the same channel and infohash of the provided 
     dto already exists in the db, the db is updated only if the timestamp
     of the new dto is newer then the entry in the database. 
     If there is no such an entry, a new wan in the Metadata DB is created
     along with the required entries in the SubtitleInfo DB
     
     @type metadata_dto: MetadataDTO 
     @param metada_dto: an instance of MetadataDTO describing metadata
     
     @return True if an existing entry was updated,  false if a new entry
             was interested. Otherwise None.
     
     '''
     assert metadata_dto is not None
     assert isinstance(metadata_dto, MetadataDTO)
     #try to retrieve a correspindng record for channel,infhoash
     
     #won't do nothing if the metadata_dto is not correctly signed
     if not metadata_dto.verifySignature():
         raise SignatureException("Metadata to insert is not properly" \
                                  "signed")
     
     select_query = QUERIES["SELECT METADATA"]
       
     signature = bin2str(metadata_dto.signature)
     infohash = bin2str(metadata_dto.infohash)
     channel = bin2str(metadata_dto.channel)
     
     res = self._db.fetchall(select_query,
                             (infohash, channel))
 
     isUpdate = False
 
     if len(res) != 0 :
         #updated if the new message is newer
         if metadata_dto.timestamp > res[0][4] :
             query = QUERIES["UPDATE METADATA"]
             
             
             self._db.execute_write(query,
                                 (metadata_dto.description,
                                 metadata_dto.timestamp,
                                 signature,
                                 infohash,
                                 channel,),
                                False) #I don't want the transaction to commit now
             
             fk_key = res[0][0]
             
             isUpdate = True
     
         else:
             return
             
     else: #if is this a whole new metadata item
         query = QUERIES["INSERT METADATA"]
         
         self._db.execute_write(query,
                                (channel,
                                 infohash,
                                 metadata_dto.description,
                                 metadata_dto.timestamp,
                                 signature,
                                 ),
                                True) 
         
         if DEBUG:
             print >> sys.stderr, "Performing query on db: " + query
         
         newRows = self._db.fetchall(select_query,
                             (infohash, channel))
         
         
         if len(newRows) == 0 : 
             raise IOError("No results, while there should be one")
         
         fk_key = newRows[0][0]
         
         
     self._insertOrUpdateSubtitles(fk_key, metadata_dto.getAllSubtitles(), \
                                   False)
         
     self._db.commit() #time to commit everything
     
     return isUpdate