def handleRMetadata(self, sender_permid, channelCastMessage, fromQuery = False): metadataDTOs = self._splitChannelcastAndRichMetadataContents(channelCastMessage) if DEBUG: print >> sys.stderr, 'Handling rich metadata from %s...' % show_permid_short(sender_permid) for md_and_have in metadataDTOs: md = md_and_have[0] havemask = md_and_have[1] vote = self.votecastDB.getVote(bin2str(md.channel), bin2str(self.my_permid)) if DEBUG: id = 'RQ' if fromQuery else 'R' print >> sys.stderr, '%s, %s, %s, %s, %d' % (id, md.channel, md.infohash, show_permid_short(sender_permid), md.timestamp) if vote == -1: continue isUpdate = self.rmdDb.insertMetadata(md) self.peerHaveManager.newHaveReceived(md.channel, md.infohash, sender_permid, havemask) if isUpdate is not None: md = self.rmdDb.getMetadata(md.channel, md.infohash) self._notifyRichMetadata(md, isUpdate) if vote == 2: if DEBUG: print >> sys.stderr, 'Subscribed to channel %s, trying to retrieveall subtitle contents' % (show_permid_short(md.channel),) self._getAllSubtitles(md)
def insertMetadata(self, metadata_dto): if not metadata_dto.verifySignature(): raise SignatureException( 'Metadata to insert is not properlysigned') select_query = QUERIES['SELECT METADATA'] signature = bin2str(metadata_dto.signature) infohash = bin2str(metadata_dto.infohash) channel = bin2str(metadata_dto.channel) res = self._db.fetchall(select_query, (infohash, channel)) isUpdate = False if len(res) != 0: if metadata_dto.timestamp > res[0][4]: query = QUERIES['UPDATE METADATA'] self._db.execute_write( query, (metadata_dto.description, metadata_dto.timestamp, signature, infohash, channel), False) fk_key = res[0][0] isUpdate = True else: return else: query = QUERIES['INSERT METADATA'] self._db.execute_write( query, (channel, infohash, metadata_dto.description, metadata_dto.timestamp, signature), True) if DEBUG: print >> sys.stderr, 'Performing query on db: ' + query newRows = self._db.fetchall(select_query, (infohash, channel)) if len(newRows) == 0: raise IOError('No results, while there should be one') fk_key = newRows[0][0] self._insertOrUpdateSubtitles(fk_key, metadata_dto.getAllSubtitles(), False) self._db.commit() return isUpdate
def handleRMetadata(self, sender_permid, channelCastMessage, fromQuery=False): metadataDTOs = self._splitChannelcastAndRichMetadataContents( channelCastMessage) if DEBUG: print >> sys.stderr, 'Handling rich metadata from %s...' % show_permid_short( sender_permid) for md_and_have in metadataDTOs: md = md_and_have[0] havemask = md_and_have[1] vote = self.votecastDB.getVote(bin2str(md.channel), bin2str(self.my_permid)) if DEBUG: id = 'RQ' if fromQuery else 'R' print >> sys.stderr, '%s, %s, %s, %s, %d' % ( id, md.channel, md.infohash, show_permid_short(sender_permid), md.timestamp) if vote == -1: continue isUpdate = self.rmdDb.insertMetadata(md) self.peerHaveManager.newHaveReceived(md.channel, md.infohash, sender_permid, havemask) if isUpdate is not None: md = self.rmdDb.getMetadata(md.channel, md.infohash) self._notifyRichMetadata(md, isUpdate) if vote == 2: if DEBUG: print >> sys.stderr, 'Subscribed to channel %s, trying to retrieveall subtitle contents' % ( show_permid_short(md.channel), ) self._getAllSubtitles(md)
def publishSubtitle(self, infohash, lang, pathToSrtSubtitle): channelid = bin2str(self.my_permid) base64infohash = bin2str(infohash) consinstent = self.channelcast_db.isItemInChannel(channelid, base64infohash) if not consinstent: msg = 'Infohash %s not found in my channel. Rejecting subtitle' % base64infohash if DEBUG: print >> sys.stderr, msg raise RichMetadataException(msg) try: filepath = self.subtitlesHandler.copyToSubtitlesFolder(pathToSrtSubtitle, self.my_permid, infohash, lang) except Exception as e: if DEBUG: print >> sys.stderr, 'Failed to read and copy subtitle to appropriate folder: %s' % str(e) metadataDTO = self.richMetadata_db.getMetadata(self.my_permid, infohash) if metadataDTO is None: metadataDTO = MetadataDTO(self.my_permid, infohash) else: metadataDTO.resetTimestamp() newSubtitle = SubtitleInfo(lang, filepath) if newSubtitle.subtitleExists(): newSubtitle.computeChecksum() else: msg = 'Inconsistency found. The subtitle was not published' if DEBUG: print >> sys.stderr, msg raise RichMetadataException(msg) metadataDTO.addSubtitle(newSubtitle) metadataDTO.sign(self.my_keypair) self.richMetadata_db.insertMetadata(metadataDTO)
def getAllSubtitles(self, channel, infohash): query = QUERIES['SELECT SUBS JOIN HASH ALL'] infohash = bin2str(infohash) channel = bin2str(channel) results = self._db.fetchall(query, (infohash, channel)) subsDict = {} for entry in results: subsDict[entry[1]] = SubtitleInfo(entry[1], entry[2], entry[3]) return subsDict
def getAllSubtitles(self, channel, infohash): query = QUERIES['SELECT SUBS JOIN HASH ALL'] infohash = bin2str(infohash) channel = bin2str(channel) results = self._db.fetchall(query, (infohash, channel)) subsDict = {} for entry in results: subsDict[entry[1]] = SubtitleInfo(entry[1], entry[2], entry[3]) return subsDict
def getSubtitle(self, channel, infohash, lang): query = QUERIES['SELECT SUBS JOIN HASH ONE'] infohash = bin2str(infohash) channel = bin2str(channel) res = self._db.fetchall(query, (infohash, channel, lang)) if len(res) == 0: return None if len(res) == 1: checksum = str2bin(res[0][3]) return SubtitleInfo(res[0][1], res[0][2], checksum) raise MetadataDBException('Metadata DB Constraint violeted!')
def getSubtitle(self, channel, infohash, lang): query = QUERIES['SELECT SUBS JOIN HASH ONE'] infohash = bin2str(infohash) channel = bin2str(channel) res = self._db.fetchall(query, (infohash, channel, lang)) if len(res) == 0: return None if len(res) == 1: checksum = str2bin(res[0][3]) return SubtitleInfo(res[0][1], res[0][2], checksum) raise MetadataDBException('Metadata DB Constraint violeted!')
def getHaveMask(self, channel, infohash, peer_id): query = QUERIES['GET ONE HAVE MASK'] channel = bin2str(channel) infohash = bin2str(infohash) peer_id = bin2str(peer_id) res = self._db.fetchall(query, (channel, infohash, peer_id)) if len(res) <= 0: return None if len(res) > 1: raise AssertionError('channel,infohash,peer_id should be unique') else: return res[0][0]
def getHaveMask(self, channel, infohash, peer_id): query = QUERIES['GET ONE HAVE MASK'] channel = bin2str(channel) infohash = bin2str(infohash) peer_id = bin2str(peer_id) res = self._db.fetchall(query, (channel, infohash, peer_id)) if len(res) <= 0: return None if len(res) > 1: raise AssertionError('channel,infohash,peer_id should be unique') else: return res[0][0]
def updateHaveMask(self, channel, infohash, peer_id, newMask, timestamp = None): channel = bin2str(channel) infohash = bin2str(infohash) peer_id = bin2str(peer_id) updateQuery = QUERIES['UPDATE HAVE MASK'] if timestamp is None: timestamp = int(time.time()) self._db.execute_write(updateQuery, (newMask, timestamp, peer_id, channel, infohash))
def getHaveEntries(self, channel, infohash): query = QUERIES['GET ALL HAVE MASK'] channel = bin2str(channel) infohash = bin2str(infohash) res = self._db.fetchall(query, (channel, infohash)) returnlist = list() for entry in res: peer_id = str2bin(entry[0]) haveMask = entry[1] timestamp = entry[2] returnlist.append((peer_id, haveMask, timestamp)) return returnlist
def getHaveEntries(self, channel, infohash): query = QUERIES['GET ALL HAVE MASK'] channel = bin2str(channel) infohash = bin2str(infohash) res = self._db.fetchall(query, (channel, infohash)) returnlist = list() for entry in res: peer_id = str2bin(entry[0]) haveMask = entry[1] timestamp = entry[2] returnlist.append((peer_id, haveMask, timestamp)) return returnlist
def getLocalSubtitles(self, channel, infohash): query = QUERIES['SELECT SUBTITLES WITH PATH BY CHN INFO'] channel = bin2str(channel) infohash = bin2str(infohash) res = self._db.fetchall(query, (channel, infohash)) result = {} for entry in res: location = entry[0] language = entry[1] checksum = str2bin(entry[2]) subInfo = SubtitleInfo(language, location, checksum) result[language] = subInfo return result
def getLocalSubtitles(self, channel, infohash): query = QUERIES['SELECT SUBTITLES WITH PATH BY CHN INFO'] channel = bin2str(channel) infohash = bin2str(infohash) res = self._db.fetchall(query, (channel, infohash)) result = {} for entry in res: location = entry[0] language = entry[1] checksum = str2bin(entry[2]) subInfo = SubtitleInfo(language, location, checksum) result[language] = subInfo return result
def updateHaveMask(self, channel, infohash, peer_id, newMask, timestamp=None): channel = bin2str(channel) infohash = bin2str(infohash) peer_id = bin2str(peer_id) updateQuery = QUERIES['UPDATE HAVE MASK'] if timestamp is None: timestamp = int(time.time()) self._db.execute_write( updateQuery, (newMask, timestamp, peer_id, channel, infohash))
def addRichMetadataContent(self, channelCastMessage, destPermid=None, fromQuery=False): if not len(channelCastMessage) > 0: if DEBUG: print >> sys.stderr, 'no entries to enrich with rmd' return channelCastMessage if DEBUG: if fromQuery: print >> sys.stderr, 'Intercepted a channelcast message as answer to a query' else: print >> sys.stderr, 'Intercepted a channelcast message as normal channelcast' doesChannelHaveSubtitles = {} for key, content in channelCastMessage.iteritems(): channel_id = content['publisher_id'] infohash = content['infohash'] if channel_id not in doesChannelHaveSubtitles: doesChannelHaveSubtitles[ channel_id] = self.rmdDb.getNrMetadata(channel_id) > 0 if doesChannelHaveSubtitles[channel_id]: metadataDTO = self.rmdDb.getMetadata(channel_id, infohash) if metadataDTO is not None: try: if DEBUG: print >> sys.stderr, 'Enriching a channelcast message with subtitle contents' metadataPack = metadataDTO.serialize() metadataPack.pop(0) metadataPack.pop(0) havemask = self.peerHaveManager.retrieveMyHaveMask( channel_id, infohash) binary_havemask = pack('!L', havemask) metadataPack.append(binary_havemask) content['rich_metadata'] = metadataPack if DEBUG: size = self._computeSize(metadataPack) dest = 'NA' if destPermid is None else show_permid_short( destPermid) id = 'SQ' if fromQuery else 'S' print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ( id, bin2str(metadataDTO.channel), bin2str(metadataDTO.infohash), dest, metadataDTO.timestamp, size) except Exception as e: print >> sys.stderr, 'Warning: Error serializing metadata: %s', str( e) return channelCastMessage
def deleteMetadata(self, channel, infohash): channel = bin2str(channel) infohash = bin2str(infohash) query = QUERIES['SELECT METADATA'] if DEBUG: print >> sys.stderr, 'Performing query on db: ' + query res = self._db.fetchall(query, (infohash, channel)) if len(res) == 0: return if len(res) > 1: raise IOError('Metadata DB constraint violated') metadata_fk = res[0][0] self._deleteAllSubtitles(metadata_fk, False) query = QUERIES['DELETE METADATA PK'] self._db.execute_write(query, (metadata_fk, ), False) self._db.commit()
def _handleSUBS(self, permid, message, selversion): if selversion < OLPROTO_VER_FOURTEENTH: if DEBUG: print >> sys.stderr, 'The peer that sent the SUBS request has an oldprotcol version: this is strange. Dropping the msg' return False decoded = self._decodeSUBSMessage(message) if decoded is None: if DEBUG: print >> sys.stderr, 'Error decoding a SUBS message from %s' % utilities.show_permid_short(permid) return False channel_id, infohash, bitmask, contents = decoded if DEBUG: print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ('RS', show_permid_short(permid), show_permid_short(channel_id), bin2str(infohash), bitmask, len(message)) requestedSubs = self._checkRequestedSubtitles(channel_id, infohash, bitmask) if requestedSubs == 0: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Received a SUBS message that was not requested. Dropping' return False requestedSubsCodes = self._languagesUtility.maskToLangCodes(requestedSubs) for lang in contents.keys(): if lang not in requestedSubsCodes: del contents[lang] callbacks = self._removeFromRequestedSubtitles(channel_id, infohash, bitmask) tuple = (channel_id, infohash, contents) for listener in self._listenersList: listener.receivedSubsResponse(permid, tuple, callbacks, selversion) return True
def _get_subs_connect_callback(self, exception, dns, permid, selversion, channel_id, infohash, bitmask, msgSentCallback, usrCallback): if exception is not None: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'GET_SUBS not sent. Unable to connect to ' + utilities.show_permid_short(permid) else: if selversion > 0 and selversion < OLPROTO_VER_FOURTEENTH: msg = 'GET_SUBS not send, the other peers had an old protocol version: %d' % selversion if DEBUG: print >> sys.stderr, msg raise SubtitleMsgHandlerException(msg) if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'sending GET_SUBS to ' + utilities.show_permid_short(permid) try: message = self._createGETSUBSMessage(channel_id, infohash, bitmask) if DEBUG: print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ('SG', show_permid_short(permid), show_permid_short(channel_id), bin2str(infohash), bitmask, len(message)) self._overlay_bridge.send(permid, message, lambda exc, permid: self._sent_callback(exc, permid, channel_id, infohash, bitmask, msgSentCallback, usrCallback)) except Exception as e: print_exc() msg = 'GET_SUBS not sent: %s' % str(e) raise SubtitleMsgHandlerException(e)
def getAllMetadataForInfohash(self, infohash): strinfohash = bin2str(infohash) query = QUERIES['SELECT PUBLISHERS FROM INFOHASH'] channels = self._db.fetchall(query, (strinfohash, )) return [ self.getMetadata(str2bin(entry[0]), infohash) for entry in channels ]
def _handleSUBS(self, permid, message, selversion): if selversion < OLPROTO_VER_FOURTEENTH: if DEBUG: print >> sys.stderr, 'The peer that sent the SUBS request has an oldprotcol version: this is strange. Dropping the msg' return False decoded = self._decodeSUBSMessage(message) if decoded is None: if DEBUG: print >> sys.stderr, 'Error decoding a SUBS message from %s' % utilities.show_permid_short( permid) return False channel_id, infohash, bitmask, contents = decoded if DEBUG: print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ( 'RS', show_permid_short(permid), show_permid_short(channel_id), bin2str(infohash), bitmask, len(message)) requestedSubs = self._checkRequestedSubtitles(channel_id, infohash, bitmask) if requestedSubs == 0: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Received a SUBS message that was not requested. Dropping' return False requestedSubsCodes = self._languagesUtility.maskToLangCodes( requestedSubs) for lang in contents.keys(): if lang not in requestedSubsCodes: del contents[lang] callbacks = self._removeFromRequestedSubtitles(channel_id, infohash, bitmask) tuple = (channel_id, infohash, contents) for listener in self._listenersList: listener.receivedSubsResponse(permid, tuple, callbacks, selversion) return True
def _insertNewSubtitle(self, metadata_fk, subtitle, commitNow = True): query = QUERIES['INSERT SUBTITLES'] checksum = bin2str(subtitle.checksum) self._db.execute_write(query, (metadata_fk, subtitle.lang, subtitle.path, checksum), commitNow)
def _get_subs_connect_callback(self, exception, dns, permid, selversion, channel_id, infohash, bitmask, msgSentCallback, usrCallback): if exception is not None: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'GET_SUBS not sent. Unable to connect to ' + utilities.show_permid_short( permid) else: if selversion > 0 and selversion < OLPROTO_VER_FOURTEENTH: msg = 'GET_SUBS not send, the other peers had an old protocol version: %d' % selversion if DEBUG: print >> sys.stderr, msg raise SubtitleMsgHandlerException(msg) if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'sending GET_SUBS to ' + utilities.show_permid_short( permid) try: message = self._createGETSUBSMessage(channel_id, infohash, bitmask) if DEBUG: print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ( 'SG', show_permid_short(permid), show_permid_short(channel_id), bin2str(infohash), bitmask, len(message)) self._overlay_bridge.send( permid, message, lambda exc, permid: self._sent_callback( exc, permid, channel_id, infohash, bitmask, msgSentCallback, usrCallback)) except Exception as e: print_exc() msg = 'GET_SUBS not sent: %s' % str(e) raise SubtitleMsgHandlerException(e)
def deleteMetadata(self, channel, infohash): channel = bin2str(channel) infohash = bin2str(infohash) query = QUERIES['SELECT METADATA'] if DEBUG: print >> sys.stderr, 'Performing query on db: ' + query res = self._db.fetchall(query, (infohash, channel)) if len(res) == 0: return if len(res) > 1: raise IOError('Metadata DB constraint violated') metadata_fk = res[0][0] self._deleteAllSubtitles(metadata_fk, False) query = QUERIES['DELETE METADATA PK'] self._db.execute_write(query, (metadata_fk,), False) self._db.commit()
def _updateSubtitle(self, metadata_fk, subtitle, commitNow=True): toUpdate = self._getSubtitleByKey(metadata_fk, subtitle.lang) if toUpdate is None: return query = QUERIES['UPDATE SUBTITLES'] checksum = bin2str(subtitle.checksum) self._db.execute_write( query, (subtitle.path, checksum, metadata_fk, subtitle.lang), commitNow)
def updateSubtitlePath(self, channel, infohash, lang, newPath, commitNow = True): query = QUERIES['SELECT SUBS JOIN HASH ONE'] channel = bin2str(channel) infohash = bin2str(infohash) res = self._db.fetchall(query, (infohash, channel, lang)) if len(res) > 1: raise MetadataDBException('Metadata DB constraint violated') else: if len(res) == 0: if DEBUG: print >> sys.stderr, 'Nothing to update for channel %s, infohash %s, lang %s. Doing nothing.' % (channel[-10:], infohash, lang) return False query = QUERIES['UPDATE SUBTITLES'] self._db.execute_write(query, (newPath, res[0][3], res[0][0], lang), commitNow) return True
def addRichMetadataContent(self, channelCastMessage, destPermid = None, fromQuery = False): if not len(channelCastMessage) > 0: if DEBUG: print >> sys.stderr, 'no entries to enrich with rmd' return channelCastMessage if DEBUG: if fromQuery: print >> sys.stderr, 'Intercepted a channelcast message as answer to a query' else: print >> sys.stderr, 'Intercepted a channelcast message as normal channelcast' doesChannelHaveSubtitles = {} for key, content in channelCastMessage.iteritems(): channel_id = content['publisher_id'] infohash = content['infohash'] if channel_id not in doesChannelHaveSubtitles: doesChannelHaveSubtitles[channel_id] = self.rmdDb.getNrMetadata(channel_id) > 0 if doesChannelHaveSubtitles[channel_id]: metadataDTO = self.rmdDb.getMetadata(channel_id, infohash) if metadataDTO is not None: try: if DEBUG: print >> sys.stderr, 'Enriching a channelcast message with subtitle contents' metadataPack = metadataDTO.serialize() metadataPack.pop(0) metadataPack.pop(0) havemask = self.peerHaveManager.retrieveMyHaveMask(channel_id, infohash) binary_havemask = pack('!L', havemask) metadataPack.append(binary_havemask) content['rich_metadata'] = metadataPack if DEBUG: size = self._computeSize(metadataPack) dest = 'NA' if destPermid is None else show_permid_short(destPermid) id = 'SQ' if fromQuery else 'S' print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % (id, bin2str(metadataDTO.channel), bin2str(metadataDTO.infohash), dest, metadataDTO.timestamp, size) except Exception as e: print >> sys.stderr, 'Warning: Error serializing metadata: %s', str(e) return channelCastMessage
def _updateSubtitle(self, metadata_fk, subtitle, commitNow = True): toUpdate = self._getSubtitleByKey(metadata_fk, subtitle.lang) if toUpdate is None: return query = QUERIES['UPDATE SUBTITLES'] checksum = bin2str(subtitle.checksum) self._db.execute_write(query, (subtitle.path, checksum, metadata_fk, subtitle.lang), commitNow)
def insertHaveMask(self, channel, infohash, peer_id, havemask, timestamp = None): query = QUERIES['SELECT METADATA'] if timestamp is None: timestamp = int(time.time()) channel = bin2str(channel) infohash = bin2str(infohash) peer_id = bin2str(peer_id) res = self._db.fetchall(query, (infohash, channel)) if len(res) != 1: raise MetadataDBException('No entry in the MetadataDB for %s, %s' % (channel[-10:], infohash)) metadata_fk = res[0][0] insertQuery = QUERIES['INSERT HAVE MASK'] try: self._db.execute_write(insertQuery, (metadata_fk, peer_id, havemask, timestamp)) except sqlite3.IntegrityError as e: raise MetadataDBException(str(e))
def getMetadata(self, channel, infohash): query = QUERIES['SELECT METADATA'] infohash = bin2str(infohash) channel = bin2str(channel) res = self._db.fetchall(query, (infohash, channel)) if len(res) == 0: return if len(res) > 1: raise MetadataDBException('Metadata DB Constraint violated') metaTuple = res[0] subsDictionary = self._getAllSubtitlesByKey(metaTuple[0]) publisher = str2bin(metaTuple[1]) infohash = str2bin(metaTuple[2]) timestamp = int(metaTuple[4]) description = unicode(metaTuple[3]) signature = str2bin(metaTuple[5]) toReturn = MetadataDTO(publisher, infohash, timestamp, description, None, signature) for sub in subsDictionary.itervalues(): toReturn.addSubtitle(sub) return toReturn
def getMetadata(self, channel, infohash): query = QUERIES['SELECT METADATA'] infohash = bin2str(infohash) channel = bin2str(channel) res = self._db.fetchall(query, (infohash, channel)) if len(res) == 0: return if len(res) > 1: raise MetadataDBException('Metadata DB Constraint violated') metaTuple = res[0] subsDictionary = self._getAllSubtitlesByKey(metaTuple[0]) publisher = str2bin(metaTuple[1]) infohash = str2bin(metaTuple[2]) timestamp = int(metaTuple[4]) description = unicode(metaTuple[3]) signature = str2bin(metaTuple[5]) toReturn = MetadataDTO(publisher, infohash, timestamp, description, None, signature) for sub in subsDictionary.itervalues(): toReturn.addSubtitle(sub) return toReturn
def insertHaveMask(self, channel, infohash, peer_id, havemask, timestamp=None): query = QUERIES['SELECT METADATA'] if timestamp is None: timestamp = int(time.time()) channel = bin2str(channel) infohash = bin2str(infohash) peer_id = bin2str(peer_id) res = self._db.fetchall(query, (infohash, channel)) if len(res) != 1: raise MetadataDBException('No entry in the MetadataDB for %s, %s' % (channel[-10:], infohash)) metadata_fk = res[0][0] insertQuery = QUERIES['INSERT HAVE MASK'] try: self._db.execute_write(insertQuery, (metadata_fk, peer_id, havemask, timestamp)) except sqlite3.IntegrityError as e: raise MetadataDBException(str(e))
def updateSubtitlePath(self, channel, infohash, lang, newPath, commitNow=True): query = QUERIES['SELECT SUBS JOIN HASH ONE'] channel = bin2str(channel) infohash = bin2str(infohash) res = self._db.fetchall(query, (infohash, channel, lang)) if len(res) > 1: raise MetadataDBException('Metadata DB constraint violated') else: if len(res) == 0: if DEBUG: print >> sys.stderr, 'Nothing to update for channel %s, infohash %s, lang %s. Doing nothing.' % ( channel[-10:], infohash, lang) return False query = QUERIES['UPDATE SUBTITLES'] self._db.execute_write(query, (newPath, res[0][3], res[0][0], lang), commitNow) return True
def insertMetadata(self, metadata_dto): if not metadata_dto.verifySignature(): raise SignatureException('Metadata to insert is not properlysigned') select_query = QUERIES['SELECT METADATA'] signature = bin2str(metadata_dto.signature) infohash = bin2str(metadata_dto.infohash) channel = bin2str(metadata_dto.channel) res = self._db.fetchall(select_query, (infohash, channel)) isUpdate = False if len(res) != 0: if metadata_dto.timestamp > res[0][4]: query = QUERIES['UPDATE METADATA'] self._db.execute_write(query, (metadata_dto.description, metadata_dto.timestamp, signature, infohash, channel), False) fk_key = res[0][0] isUpdate = True else: return else: query = QUERIES['INSERT METADATA'] self._db.execute_write(query, (channel, infohash, metadata_dto.description, metadata_dto.timestamp, signature), True) if DEBUG: print >> sys.stderr, 'Performing query on db: ' + query newRows = self._db.fetchall(select_query, (infohash, channel)) if len(newRows) == 0: raise IOError('No results, while there should be one') fk_key = newRows[0][0] self._insertOrUpdateSubtitles(fk_key, metadata_dto.getAllSubtitles(), False) self._db.commit() return isUpdate
def send_metadata(self, permid, message, selversion): try: infohash = bdecode(message[1:]) except: print_exc() if DEBUG: print >> sys.stderr, 'metadata: GET_METADATA: error becoding' return False if not isValidInfohash(infohash): if DEBUG: print >> sys.stderr, 'metadata: GET_METADATA: invalid hash' return False res = self.torrent_db.getOne(('torrent_file_name', 'status_id'), infohash=bin2str(infohash)) if not res: if DEBUG: print >> sys.stderr, 'metadata: GET_METADATA: not in database', infohash return True torrent_file_name, status_id = res if status_id == self.torrent_db._getStatusID('dead'): if DEBUG: print >> sys.stderr, 'metadata: GET_METADATA: Torrent was dead' return True if not torrent_file_name: if DEBUG: print >> sys.stderr, 'metadata: GET_METADATA: no torrent file name' return True torrent_path = os.path.join(self.torrent_dir, torrent_file_name) if not os.path.isfile(torrent_path): if DEBUG: print >> sys.stderr, 'metadata: GET_METADATA: not existing', res, torrent_path return True task = { 'permid': permid, 'infohash': infohash, 'torrent_path': torrent_path, 'selversion': selversion } self.upload_queue.append(task) if int(time()) >= self.next_upload_time: self.checking_upload_queue() return True
def _handleGETSUBS(self, permid, message, selversion): if selversion < OLPROTO_VER_FOURTEENTH: if DEBUG: print >> sys.stderr, 'The peer that sent the GET_SUBS request has an oldprotcol version: this is strange. Dropping the msg' return False decoded = self._decodeGETSUBSMessage(message) if decoded is None: if DEBUG: print >> sys.stderr, 'Error decoding a GET_SUBS message from %s' % utilities.show_permid_short( permid) return False if DEBUG: channel_id, infohash, languages = decoded bitmask = self._languagesUtility.langCodesToMask(languages) print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ( 'RG', show_permid_short(permid), show_permid_short(channel_id), bin2str(infohash), bitmask, len(message)) for listener in self._listenersList: listener.receivedSubsRequest(permid, decoded, selversion) return True
def send_metadata(self, permid, message, selversion): try: infohash = bdecode(message[1:]) except: print_exc() if DEBUG: print >> sys.stderr, 'metadata: GET_METADATA: error becoding' return False if not isValidInfohash(infohash): if DEBUG: print >> sys.stderr, 'metadata: GET_METADATA: invalid hash' return False res = self.torrent_db.getOne(('torrent_file_name', 'status_id'), infohash=bin2str(infohash)) if not res: if DEBUG: print >> sys.stderr, 'metadata: GET_METADATA: not in database', infohash return True torrent_file_name, status_id = res if status_id == self.torrent_db._getStatusID('dead'): if DEBUG: print >> sys.stderr, 'metadata: GET_METADATA: Torrent was dead' return True if not torrent_file_name: if DEBUG: print >> sys.stderr, 'metadata: GET_METADATA: no torrent file name' return True torrent_path = os.path.join(self.torrent_dir, torrent_file_name) if not os.path.isfile(torrent_path): if DEBUG: print >> sys.stderr, 'metadata: GET_METADATA: not existing', res, torrent_path return True task = {'permid': permid, 'infohash': infohash, 'torrent_path': torrent_path, 'selversion': selversion} self.upload_queue.append(task) if int(time()) >= self.next_upload_time: self.checking_upload_queue() return True
def _checkingUploadQueue(self): if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Checking the upload queue...' if not self._tokenBucket.upload_rate > 0: return if not len(self._uploadQueue) > 0: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Upload queue is empty.' while len(self._uploadQueue) > 0: responseData = self._uploadQueue[0] encodedMsg = self._createSingleResponseMessage(responseData) if encodedMsg is None: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Nothing to send' del self._uploadQueue[0] continue msgSize = len(encodedMsg) / 1024.0 if msgSize > self._tokenBucket.capacity: print >> sys.stderr, 'Warning:' + SUBS_LOG_PREFIX + 'SUBS message too big. Discarded!' del self._uploadQueue[0] continue if self._tokenBucket.consume(msgSize): if DEBUG: keys = responseData['subtitles'].keys() bitmask = self._languagesUtility.langCodesToMask(keys) print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ( 'SS', show_permid_short(responseData['permid']), show_permid_short(responseData['channel_id']), bin2str(responseData['infohash']), bitmask, int(msgSize * 1024)) self._doSendSubtitles(responseData['permid'], encodedMsg, responseData['selversion']) del self._uploadQueue[0] else: neededCapacity = max(0, msgSize - self._tokenBucket.tokens) delay = neededCapacity / self._tokenBucket.upload_rate self._nextUploadTime = time() + delay self.overlay_bridge.add_task(self._checkingUploadQueue, delay) return
def _checkingUploadQueue(self): if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Checking the upload queue...' if not self._tokenBucket.upload_rate > 0: return if not len(self._uploadQueue) > 0: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Upload queue is empty.' while len(self._uploadQueue) > 0: responseData = self._uploadQueue[0] encodedMsg = self._createSingleResponseMessage(responseData) if encodedMsg is None: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Nothing to send' del self._uploadQueue[0] continue msgSize = len(encodedMsg) / 1024.0 if msgSize > self._tokenBucket.capacity: print >> sys.stderr, 'Warning:' + SUBS_LOG_PREFIX + 'SUBS message too big. Discarded!' del self._uploadQueue[0] continue if self._tokenBucket.consume(msgSize): if DEBUG: keys = responseData['subtitles'].keys() bitmask = self._languagesUtility.langCodesToMask(keys) print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ('SS', show_permid_short(responseData['permid']), show_permid_short(responseData['channel_id']), bin2str(responseData['infohash']), bitmask, int(msgSize * 1024)) self._doSendSubtitles(responseData['permid'], encodedMsg, responseData['selversion']) del self._uploadQueue[0] else: neededCapacity = max(0, msgSize - self._tokenBucket.tokens) delay = neededCapacity / self._tokenBucket.upload_rate self._nextUploadTime = time() + delay self.overlay_bridge.add_task(self._checkingUploadQueue, delay) return
def _handleGETSUBS(self, permid, message, selversion): if selversion < OLPROTO_VER_FOURTEENTH: if DEBUG: print >> sys.stderr, 'The peer that sent the GET_SUBS request has an oldprotcol version: this is strange. Dropping the msg' return False decoded = self._decodeGETSUBSMessage(message) if decoded is None: if DEBUG: print >> sys.stderr, 'Error decoding a GET_SUBS message from %s' % utilities.show_permid_short(permid) return False if DEBUG: channel_id, infohash, languages = decoded bitmask = self._languagesUtility.langCodesToMask(languages) print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ('RG', show_permid_short(permid), show_permid_short(channel_id), bin2str(infohash), bitmask, len(message)) for listener in self._listenersList: listener.receivedSubsRequest(permid, decoded, selversion) return True
def _insertNewSubtitle(self, metadata_fk, subtitle, commitNow=True): query = QUERIES['INSERT SUBTITLES'] checksum = bin2str(subtitle.checksum) self._db.execute_write( query, (metadata_fk, subtitle.lang, subtitle.path, checksum), commitNow)
def getNrMetadata(self, channel): query = QUERIES['SELECT NRMETADATA'] channel = bin2str(channel) return self._db.fetchone(query, (channel,))
def getAllMetadataForInfohash(self, infohash): strinfohash = bin2str(infohash) query = QUERIES['SELECT PUBLISHERS FROM INFOHASH'] channels = self._db.fetchall(query, (strinfohash,)) return [ self.getMetadata(str2bin(entry[0]), infohash) for entry in channels ]
def _deleteSubtitleByChannel(self, channel, infohash, lang): query = QUERIES['DELETE ONE SUBTITLE JOIN'] infohash = bin2str(infohash) channel = bin2str(channel) self._db.execute_write(query, (channel, infohash, lang))
def deleteHaveEntry(self, channel, infohash, peer_id): channel = bin2str(channel) infohash = bin2str(infohash) peer_id = bin2str(peer_id) deleteQuery = QUERIES['DELETE HAVE'] self._db.execute_write(deleteQuery, (peer_id, channel, infohash))
def getNrMetadata(self, channel): query = QUERIES['SELECT NRMETADATA'] channel = bin2str(channel) return self._db.fetchone(query, (channel, ))
def hasMetadata(self, channel, infohash): query = QUERIES['SELECT METADATA'] infohash = bin2str(infohash) channel = bin2str(channel) res = self._db.fetchall(query, (infohash, channel)) return len(res) != 0
def hasMetadata(self, channel, infohash): query = QUERIES['SELECT METADATA'] infohash = bin2str(infohash) channel = bin2str(channel) res = self._db.fetchall(query, (infohash, channel)) return len(res) != 0
def _deleteSubtitleByChannel(self, channel, infohash, lang): query = QUERIES['DELETE ONE SUBTITLE JOIN'] infohash = bin2str(infohash) channel = bin2str(channel) self._db.execute_write(query, (channel, infohash, lang))
def receivedSubsRequest(self, permid, request, selversion): channel_id, infohash, languages = request allSubtitles = self.subtitlesDb.getAllSubtitles(channel_id, infohash) contentsList = {} for lang in sorted(languages): if lang in allSubtitles.keys(): if allSubtitles[lang].subtitleExists(): content = self._readSubContent(allSubtitles[lang].path) if content is not None: contentsList[lang] = content else: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'File not available for channel %s, infohash %s, lang %s' % (show_permid_short(channel_id), bin2str(infohash), lang) self.subtitlesDb.updateSubtitlePath(channel_id, infohash, lang, None) elif DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'Subtitle not available for channel %s, infohash %s, lang %s' % (show_permid_short(channel_id), bin2str(infohash), lang) if len(contentsList) == 0: if DEBUG: print >> sys.stderr, SUBS_LOG_PREFIX + 'None of the requested subtitles were available. No answer will be sent to %s' % show_permid_short(permid) return True return self._subsMsgHndlr.sendSubtitleResponse(permid, (channel_id, infohash, contentsList), selversion)
def deleteHaveEntry(self, channel, infohash, peer_id): channel = bin2str(channel) infohash = bin2str(infohash) peer_id = bin2str(peer_id) deleteQuery = QUERIES['DELETE HAVE'] self._db.execute_write(deleteQuery, (peer_id, channel, infohash))