def handleRMetadata(self, sender_permid, channelCastMessage, fromQuery = False):
metadataDTOs = self._splitChannelcastAndRichMetadataContents(channelCastMessage)
if DEBUG:
print >> sys.stderr, 'Handling rich metadata from %s...' % show_permid_short(sender_permid)
for md_and_have in metadataDTOs:
md = md_and_have[0]
havemask = md_and_have[1]
vote = self.votecastDB.getVote(bin2str(md.channel), bin2str(self.my_permid))
if DEBUG:
id = 'RQ' if fromQuery else 'R'
print >> sys.stderr, '%s, %s, %s, %s, %d' % (id,
md.channel,
md.infohash,
show_permid_short(sender_permid),
md.timestamp)
if vote == -1:
continue
isUpdate = self.rmdDb.insertMetadata(md)
self.peerHaveManager.newHaveReceived(md.channel, md.infohash, sender_permid, havemask)
if isUpdate is not None:
md = self.rmdDb.getMetadata(md.channel, md.infohash)
self._notifyRichMetadata(md, isUpdate)
if vote == 2:
if DEBUG:
print >> sys.stderr, 'Subscribed to channel %s, trying to retrieveall subtitle contents' % (show_permid_short(md.channel),)
self._getAllSubtitles(md)
def insertMetadata(self, metadata_dto):
if not metadata_dto.verifySignature():
raise SignatureException(
'Metadata to insert is not properlysigned')
select_query = QUERIES['SELECT METADATA']
signature = bin2str(metadata_dto.signature)
infohash = bin2str(metadata_dto.infohash)
channel = bin2str(metadata_dto.channel)
res = self._db.fetchall(select_query, (infohash, channel))
isUpdate = False
if len(res) != 0:
if metadata_dto.timestamp > res[0][4]:
query = QUERIES['UPDATE METADATA']
self._db.execute_write(
query, (metadata_dto.description, metadata_dto.timestamp,
signature, infohash, channel), False)
fk_key = res[0][0]
isUpdate = True
else:
return
else:
query = QUERIES['INSERT METADATA']
self._db.execute_write(
query, (channel, infohash, metadata_dto.description,
metadata_dto.timestamp, signature), True)
if DEBUG:
print >> sys.stderr, 'Performing query on db: ' + query
newRows = self._db.fetchall(select_query, (infohash, channel))
if len(newRows) == 0:
raise IOError('No results, while there should be one')
fk_key = newRows[0][0]
self._insertOrUpdateSubtitles(fk_key, metadata_dto.getAllSubtitles(),
False)
self._db.commit()
return isUpdate
def handleRMetadata(self,
sender_permid,
channelCastMessage,
fromQuery=False):
metadataDTOs = self._splitChannelcastAndRichMetadataContents(
channelCastMessage)
if DEBUG:
print >> sys.stderr, 'Handling rich metadata from %s...' % show_permid_short(
sender_permid)
for md_and_have in metadataDTOs:
md = md_and_have[0]
havemask = md_and_have[1]
vote = self.votecastDB.getVote(bin2str(md.channel),
bin2str(self.my_permid))
if DEBUG:
id = 'RQ' if fromQuery else 'R'
print >> sys.stderr, '%s, %s, %s, %s, %d' % (
id, md.channel, md.infohash,
show_permid_short(sender_permid), md.timestamp)
if vote == -1:
continue
isUpdate = self.rmdDb.insertMetadata(md)
self.peerHaveManager.newHaveReceived(md.channel, md.infohash,
sender_permid, havemask)
if isUpdate is not None:
md = self.rmdDb.getMetadata(md.channel, md.infohash)
self._notifyRichMetadata(md, isUpdate)
if vote == 2:
if DEBUG:
print >> sys.stderr, 'Subscribed to channel %s, trying to retrieveall subtitle contents' % (
show_permid_short(md.channel), )
self._getAllSubtitles(md)
def publishSubtitle(self, infohash, lang, pathToSrtSubtitle):
channelid = bin2str(self.my_permid)
base64infohash = bin2str(infohash)
consinstent = self.channelcast_db.isItemInChannel(channelid, base64infohash)
if not consinstent:
msg = 'Infohash %s not found in my channel. Rejecting subtitle' % base64infohash
if DEBUG:
print >> sys.stderr, msg
raise RichMetadataException(msg)
try:
filepath = self.subtitlesHandler.copyToSubtitlesFolder(pathToSrtSubtitle, self.my_permid, infohash, lang)
except Exception as e:
if DEBUG:
print >> sys.stderr, 'Failed to read and copy subtitle to appropriate folder: %s' % str(e)
metadataDTO = self.richMetadata_db.getMetadata(self.my_permid, infohash)
if metadataDTO is None:
metadataDTO = MetadataDTO(self.my_permid, infohash)
else:
metadataDTO.resetTimestamp()
newSubtitle = SubtitleInfo(lang, filepath)
if newSubtitle.subtitleExists():
newSubtitle.computeChecksum()
else:
msg = 'Inconsistency found. The subtitle was not published'
if DEBUG:
print >> sys.stderr, msg
raise RichMetadataException(msg)
metadataDTO.addSubtitle(newSubtitle)
metadataDTO.sign(self.my_keypair)
self.richMetadata_db.insertMetadata(metadataDTO)
def getAllSubtitles(self, channel, infohash):
query = QUERIES['SELECT SUBS JOIN HASH ALL']
infohash = bin2str(infohash)
channel = bin2str(channel)
results = self._db.fetchall(query, (infohash, channel))
subsDict = {}
for entry in results:
subsDict[entry[1]] = SubtitleInfo(entry[1], entry[2], entry[3])
return subsDict
def getAllSubtitles(self, channel, infohash):
query = QUERIES['SELECT SUBS JOIN HASH ALL']
infohash = bin2str(infohash)
channel = bin2str(channel)
results = self._db.fetchall(query, (infohash, channel))
subsDict = {}
for entry in results:
subsDict[entry[1]] = SubtitleInfo(entry[1], entry[2], entry[3])
return subsDict
def getSubtitle(self, channel, infohash, lang):
query = QUERIES['SELECT SUBS JOIN HASH ONE']
infohash = bin2str(infohash)
channel = bin2str(channel)
res = self._db.fetchall(query, (infohash, channel, lang))
if len(res) == 0:
return None
if len(res) == 1:
checksum = str2bin(res[0][3])
return SubtitleInfo(res[0][1], res[0][2], checksum)
raise MetadataDBException('Metadata DB Constraint violeted!')
def getSubtitle(self, channel, infohash, lang):
query = QUERIES['SELECT SUBS JOIN HASH ONE']
infohash = bin2str(infohash)
channel = bin2str(channel)
res = self._db.fetchall(query, (infohash, channel, lang))
if len(res) == 0:
return None
if len(res) == 1:
checksum = str2bin(res[0][3])
return SubtitleInfo(res[0][1], res[0][2], checksum)
raise MetadataDBException('Metadata DB Constraint violeted!')
def getHaveMask(self, channel, infohash, peer_id):
query = QUERIES['GET ONE HAVE MASK']
channel = bin2str(channel)
infohash = bin2str(infohash)
peer_id = bin2str(peer_id)
res = self._db.fetchall(query, (channel, infohash, peer_id))
if len(res) <= 0:
return None
if len(res) > 1:
raise AssertionError('channel,infohash,peer_id should be unique')
else:
return res[0][0]
def getHaveMask(self, channel, infohash, peer_id):
query = QUERIES['GET ONE HAVE MASK']
channel = bin2str(channel)
infohash = bin2str(infohash)
peer_id = bin2str(peer_id)
res = self._db.fetchall(query, (channel, infohash, peer_id))
if len(res) <= 0:
return None
if len(res) > 1:
raise AssertionError('channel,infohash,peer_id should be unique')
else:
return res[0][0]
def updateHaveMask(self, channel, infohash, peer_id, newMask, timestamp = None):
channel = bin2str(channel)
infohash = bin2str(infohash)
peer_id = bin2str(peer_id)
updateQuery = QUERIES['UPDATE HAVE MASK']
if timestamp is None:
timestamp = int(time.time())
self._db.execute_write(updateQuery, (newMask,
timestamp,
peer_id,
channel,
infohash))
def getHaveEntries(self, channel, infohash):
query = QUERIES['GET ALL HAVE MASK']
channel = bin2str(channel)
infohash = bin2str(infohash)
res = self._db.fetchall(query, (channel, infohash))
returnlist = list()
for entry in res:
peer_id = str2bin(entry[0])
haveMask = entry[1]
timestamp = entry[2]
returnlist.append((peer_id, haveMask, timestamp))
return returnlist
def getHaveEntries(self, channel, infohash):
query = QUERIES['GET ALL HAVE MASK']
channel = bin2str(channel)
infohash = bin2str(infohash)
res = self._db.fetchall(query, (channel, infohash))
returnlist = list()
for entry in res:
peer_id = str2bin(entry[0])
haveMask = entry[1]
timestamp = entry[2]
returnlist.append((peer_id, haveMask, timestamp))
return returnlist
def getLocalSubtitles(self, channel, infohash):
query = QUERIES['SELECT SUBTITLES WITH PATH BY CHN INFO']
channel = bin2str(channel)
infohash = bin2str(infohash)
res = self._db.fetchall(query, (channel, infohash))
result = {}
for entry in res:
location = entry[0]
language = entry[1]
checksum = str2bin(entry[2])
subInfo = SubtitleInfo(language, location, checksum)
result[language] = subInfo
return result
def getLocalSubtitles(self, channel, infohash):
query = QUERIES['SELECT SUBTITLES WITH PATH BY CHN INFO']
channel = bin2str(channel)
infohash = bin2str(infohash)
res = self._db.fetchall(query, (channel, infohash))
result = {}
for entry in res:
location = entry[0]
language = entry[1]
checksum = str2bin(entry[2])
subInfo = SubtitleInfo(language, location, checksum)
result[language] = subInfo
return result
def updateHaveMask(self,
channel,
infohash,
peer_id,
newMask,
timestamp=None):
channel = bin2str(channel)
infohash = bin2str(infohash)
peer_id = bin2str(peer_id)
updateQuery = QUERIES['UPDATE HAVE MASK']
if timestamp is None:
timestamp = int(time.time())
self._db.execute_write(
updateQuery, (newMask, timestamp, peer_id, channel, infohash))
def addRichMetadataContent(self,
channelCastMessage,
destPermid=None,
fromQuery=False):
if not len(channelCastMessage) > 0:
if DEBUG:
print >> sys.stderr, 'no entries to enrich with rmd'
return channelCastMessage
if DEBUG:
if fromQuery:
print >> sys.stderr, 'Intercepted a channelcast message as answer to a query'
else:
print >> sys.stderr, 'Intercepted a channelcast message as normal channelcast'
doesChannelHaveSubtitles = {}
for key, content in channelCastMessage.iteritems():
channel_id = content['publisher_id']
infohash = content['infohash']
if channel_id not in doesChannelHaveSubtitles:
doesChannelHaveSubtitles[
channel_id] = self.rmdDb.getNrMetadata(channel_id) > 0
if doesChannelHaveSubtitles[channel_id]:
metadataDTO = self.rmdDb.getMetadata(channel_id, infohash)
if metadataDTO is not None:
try:
if DEBUG:
print >> sys.stderr, 'Enriching a channelcast message with subtitle contents'
metadataPack = metadataDTO.serialize()
metadataPack.pop(0)
metadataPack.pop(0)
havemask = self.peerHaveManager.retrieveMyHaveMask(
channel_id, infohash)
binary_havemask = pack('!L', havemask)
metadataPack.append(binary_havemask)
content['rich_metadata'] = metadataPack
if DEBUG:
size = self._computeSize(metadataPack)
dest = 'NA' if destPermid is None else show_permid_short(
destPermid)
id = 'SQ' if fromQuery else 'S'
print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % (
id, bin2str(metadataDTO.channel),
bin2str(metadataDTO.infohash), dest,
metadataDTO.timestamp, size)
except Exception as e:
print >> sys.stderr, 'Warning: Error serializing metadata: %s', str(
e)
return channelCastMessage
def deleteMetadata(self, channel, infohash):
channel = bin2str(channel)
infohash = bin2str(infohash)
query = QUERIES['SELECT METADATA']
if DEBUG:
print >> sys.stderr, 'Performing query on db: ' + query
res = self._db.fetchall(query, (infohash, channel))
if len(res) == 0:
return
if len(res) > 1:
raise IOError('Metadata DB constraint violated')
metadata_fk = res[0][0]
self._deleteAllSubtitles(metadata_fk, False)
query = QUERIES['DELETE METADATA PK']
self._db.execute_write(query, (metadata_fk, ), False)
self._db.commit()
def _handleSUBS(self, permid, message, selversion):
if selversion < OLPROTO_VER_FOURTEENTH:
if DEBUG:
print >> sys.stderr, 'The peer that sent the SUBS request has an oldprotcol version: this is strange. Dropping the msg'
return False
decoded = self._decodeSUBSMessage(message)
if decoded is None:
if DEBUG:
print >> sys.stderr, 'Error decoding a SUBS message from %s' % utilities.show_permid_short(permid)
return False
channel_id, infohash, bitmask, contents = decoded
if DEBUG:
print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ('RS',
show_permid_short(permid),
show_permid_short(channel_id),
bin2str(infohash),
bitmask,
len(message))
requestedSubs = self._checkRequestedSubtitles(channel_id, infohash, bitmask)
if requestedSubs == 0:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'Received a SUBS message that was not requested. Dropping'
return False
requestedSubsCodes = self._languagesUtility.maskToLangCodes(requestedSubs)
for lang in contents.keys():
if lang not in requestedSubsCodes:
del contents[lang]
callbacks = self._removeFromRequestedSubtitles(channel_id, infohash, bitmask)
tuple = (channel_id, infohash, contents)
for listener in self._listenersList:
listener.receivedSubsResponse(permid, tuple, callbacks, selversion)
return True
def _get_subs_connect_callback(self, exception, dns, permid, selversion, channel_id, infohash, bitmask, msgSentCallback, usrCallback):
if exception is not None:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'GET_SUBS not sent. Unable to connect to ' + utilities.show_permid_short(permid)
else:
if selversion > 0 and selversion < OLPROTO_VER_FOURTEENTH:
msg = 'GET_SUBS not send, the other peers had an old protocol version: %d' % selversion
if DEBUG:
print >> sys.stderr, msg
raise SubtitleMsgHandlerException(msg)
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'sending GET_SUBS to ' + utilities.show_permid_short(permid)
try:
message = self._createGETSUBSMessage(channel_id, infohash, bitmask)
if DEBUG:
print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ('SG',
show_permid_short(permid),
show_permid_short(channel_id),
bin2str(infohash),
bitmask,
len(message))
self._overlay_bridge.send(permid, message, lambda exc, permid: self._sent_callback(exc, permid, channel_id, infohash, bitmask, msgSentCallback, usrCallback))
except Exception as e:
print_exc()
msg = 'GET_SUBS not sent: %s' % str(e)
raise SubtitleMsgHandlerException(e)
def getAllMetadataForInfohash(self, infohash):
strinfohash = bin2str(infohash)
query = QUERIES['SELECT PUBLISHERS FROM INFOHASH']
channels = self._db.fetchall(query, (strinfohash, ))
return [
self.getMetadata(str2bin(entry[0]), infohash) for entry in channels
]
def _handleSUBS(self, permid, message, selversion):
if selversion < OLPROTO_VER_FOURTEENTH:
if DEBUG:
print >> sys.stderr, 'The peer that sent the SUBS request has an oldprotcol version: this is strange. Dropping the msg'
return False
decoded = self._decodeSUBSMessage(message)
if decoded is None:
if DEBUG:
print >> sys.stderr, 'Error decoding a SUBS message from %s' % utilities.show_permid_short(
permid)
return False
channel_id, infohash, bitmask, contents = decoded
if DEBUG:
print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % (
'RS', show_permid_short(permid), show_permid_short(channel_id),
bin2str(infohash), bitmask, len(message))
requestedSubs = self._checkRequestedSubtitles(channel_id, infohash,
bitmask)
if requestedSubs == 0:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'Received a SUBS message that was not requested. Dropping'
return False
requestedSubsCodes = self._languagesUtility.maskToLangCodes(
requestedSubs)
for lang in contents.keys():
if lang not in requestedSubsCodes:
del contents[lang]
callbacks = self._removeFromRequestedSubtitles(channel_id, infohash,
bitmask)
tuple = (channel_id, infohash, contents)
for listener in self._listenersList:
listener.receivedSubsResponse(permid, tuple, callbacks, selversion)
return True
def _insertNewSubtitle(self, metadata_fk, subtitle, commitNow = True):
query = QUERIES['INSERT SUBTITLES']
checksum = bin2str(subtitle.checksum)
self._db.execute_write(query, (metadata_fk,
subtitle.lang,
subtitle.path,
checksum), commitNow)
def _get_subs_connect_callback(self, exception, dns, permid, selversion,
channel_id, infohash, bitmask,
msgSentCallback, usrCallback):
if exception is not None:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'GET_SUBS not sent. Unable to connect to ' + utilities.show_permid_short(
permid)
else:
if selversion > 0 and selversion < OLPROTO_VER_FOURTEENTH:
msg = 'GET_SUBS not send, the other peers had an old protocol version: %d' % selversion
if DEBUG:
print >> sys.stderr, msg
raise SubtitleMsgHandlerException(msg)
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'sending GET_SUBS to ' + utilities.show_permid_short(
permid)
try:
message = self._createGETSUBSMessage(channel_id, infohash,
bitmask)
if DEBUG:
print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % (
'SG', show_permid_short(permid),
show_permid_short(channel_id), bin2str(infohash),
bitmask, len(message))
self._overlay_bridge.send(
permid, message, lambda exc, permid: self._sent_callback(
exc, permid, channel_id, infohash, bitmask,
msgSentCallback, usrCallback))
except Exception as e:
print_exc()
msg = 'GET_SUBS not sent: %s' % str(e)
raise SubtitleMsgHandlerException(e)
def deleteMetadata(self, channel, infohash):
channel = bin2str(channel)
infohash = bin2str(infohash)
query = QUERIES['SELECT METADATA']
if DEBUG:
print >> sys.stderr, 'Performing query on db: ' + query
res = self._db.fetchall(query, (infohash, channel))
if len(res) == 0:
return
if len(res) > 1:
raise IOError('Metadata DB constraint violated')
metadata_fk = res[0][0]
self._deleteAllSubtitles(metadata_fk, False)
query = QUERIES['DELETE METADATA PK']
self._db.execute_write(query, (metadata_fk,), False)
self._db.commit()
def _updateSubtitle(self, metadata_fk, subtitle, commitNow=True):
toUpdate = self._getSubtitleByKey(metadata_fk, subtitle.lang)
if toUpdate is None:
return
query = QUERIES['UPDATE SUBTITLES']
checksum = bin2str(subtitle.checksum)
self._db.execute_write(
query, (subtitle.path, checksum, metadata_fk, subtitle.lang),
commitNow)
def updateSubtitlePath(self, channel, infohash, lang, newPath, commitNow = True):
query = QUERIES['SELECT SUBS JOIN HASH ONE']
channel = bin2str(channel)
infohash = bin2str(infohash)
res = self._db.fetchall(query, (infohash, channel, lang))
if len(res) > 1:
raise MetadataDBException('Metadata DB constraint violated')
else:
if len(res) == 0:
if DEBUG:
print >> sys.stderr, 'Nothing to update for channel %s, infohash %s, lang %s. Doing nothing.' % (channel[-10:], infohash, lang)
return False
query = QUERIES['UPDATE SUBTITLES']
self._db.execute_write(query, (newPath,
res[0][3],
res[0][0],
lang), commitNow)
return True
def addRichMetadataContent(self, channelCastMessage, destPermid = None, fromQuery = False):
if not len(channelCastMessage) > 0:
if DEBUG:
print >> sys.stderr, 'no entries to enrich with rmd'
return channelCastMessage
if DEBUG:
if fromQuery:
print >> sys.stderr, 'Intercepted a channelcast message as answer to a query'
else:
print >> sys.stderr, 'Intercepted a channelcast message as normal channelcast'
doesChannelHaveSubtitles = {}
for key, content in channelCastMessage.iteritems():
channel_id = content['publisher_id']
infohash = content['infohash']
if channel_id not in doesChannelHaveSubtitles:
doesChannelHaveSubtitles[channel_id] = self.rmdDb.getNrMetadata(channel_id) > 0
if doesChannelHaveSubtitles[channel_id]:
metadataDTO = self.rmdDb.getMetadata(channel_id, infohash)
if metadataDTO is not None:
try:
if DEBUG:
print >> sys.stderr, 'Enriching a channelcast message with subtitle contents'
metadataPack = metadataDTO.serialize()
metadataPack.pop(0)
metadataPack.pop(0)
havemask = self.peerHaveManager.retrieveMyHaveMask(channel_id, infohash)
binary_havemask = pack('!L', havemask)
metadataPack.append(binary_havemask)
content['rich_metadata'] = metadataPack
if DEBUG:
size = self._computeSize(metadataPack)
dest = 'NA' if destPermid is None else show_permid_short(destPermid)
id = 'SQ' if fromQuery else 'S'
print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % (id,
bin2str(metadataDTO.channel),
bin2str(metadataDTO.infohash),
dest,
metadataDTO.timestamp,
size)
except Exception as e:
print >> sys.stderr, 'Warning: Error serializing metadata: %s', str(e)
return channelCastMessage
def _updateSubtitle(self, metadata_fk, subtitle, commitNow = True):
toUpdate = self._getSubtitleByKey(metadata_fk, subtitle.lang)
if toUpdate is None:
return
query = QUERIES['UPDATE SUBTITLES']
checksum = bin2str(subtitle.checksum)
self._db.execute_write(query, (subtitle.path,
checksum,
metadata_fk,
subtitle.lang), commitNow)
def insertHaveMask(self, channel, infohash, peer_id, havemask, timestamp = None):
query = QUERIES['SELECT METADATA']
if timestamp is None:
timestamp = int(time.time())
channel = bin2str(channel)
infohash = bin2str(infohash)
peer_id = bin2str(peer_id)
res = self._db.fetchall(query, (infohash, channel))
if len(res) != 1:
raise MetadataDBException('No entry in the MetadataDB for %s, %s' % (channel[-10:], infohash))
metadata_fk = res[0][0]
insertQuery = QUERIES['INSERT HAVE MASK']
try:
self._db.execute_write(insertQuery, (metadata_fk,
peer_id,
havemask,
timestamp))
except sqlite3.IntegrityError as e:
raise MetadataDBException(str(e))
def getMetadata(self, channel, infohash):
query = QUERIES['SELECT METADATA']
infohash = bin2str(infohash)
channel = bin2str(channel)
res = self._db.fetchall(query, (infohash, channel))
if len(res) == 0:
return
if len(res) > 1:
raise MetadataDBException('Metadata DB Constraint violated')
metaTuple = res[0]
subsDictionary = self._getAllSubtitlesByKey(metaTuple[0])
publisher = str2bin(metaTuple[1])
infohash = str2bin(metaTuple[2])
timestamp = int(metaTuple[4])
description = unicode(metaTuple[3])
signature = str2bin(metaTuple[5])
toReturn = MetadataDTO(publisher, infohash, timestamp, description, None, signature)
for sub in subsDictionary.itervalues():
toReturn.addSubtitle(sub)
return toReturn
def getMetadata(self, channel, infohash):
query = QUERIES['SELECT METADATA']
infohash = bin2str(infohash)
channel = bin2str(channel)
res = self._db.fetchall(query, (infohash, channel))
if len(res) == 0:
return
if len(res) > 1:
raise MetadataDBException('Metadata DB Constraint violated')
metaTuple = res[0]
subsDictionary = self._getAllSubtitlesByKey(metaTuple[0])
publisher = str2bin(metaTuple[1])
infohash = str2bin(metaTuple[2])
timestamp = int(metaTuple[4])
description = unicode(metaTuple[3])
signature = str2bin(metaTuple[5])
toReturn = MetadataDTO(publisher, infohash, timestamp, description,
None, signature)
for sub in subsDictionary.itervalues():
toReturn.addSubtitle(sub)
return toReturn
def insertHaveMask(self,
channel,
infohash,
peer_id,
havemask,
timestamp=None):
query = QUERIES['SELECT METADATA']
if timestamp is None:
timestamp = int(time.time())
channel = bin2str(channel)
infohash = bin2str(infohash)
peer_id = bin2str(peer_id)
res = self._db.fetchall(query, (infohash, channel))
if len(res) != 1:
raise MetadataDBException('No entry in the MetadataDB for %s, %s' %
(channel[-10:], infohash))
metadata_fk = res[0][0]
insertQuery = QUERIES['INSERT HAVE MASK']
try:
self._db.execute_write(insertQuery,
(metadata_fk, peer_id, havemask, timestamp))
except sqlite3.IntegrityError as e:
raise MetadataDBException(str(e))
def updateSubtitlePath(self,
channel,
infohash,
lang,
newPath,
commitNow=True):
query = QUERIES['SELECT SUBS JOIN HASH ONE']
channel = bin2str(channel)
infohash = bin2str(infohash)
res = self._db.fetchall(query, (infohash, channel, lang))
if len(res) > 1:
raise MetadataDBException('Metadata DB constraint violated')
else:
if len(res) == 0:
if DEBUG:
print >> sys.stderr, 'Nothing to update for channel %s, infohash %s, lang %s. Doing nothing.' % (
channel[-10:], infohash, lang)
return False
query = QUERIES['UPDATE SUBTITLES']
self._db.execute_write(query,
(newPath, res[0][3], res[0][0], lang),
commitNow)
return True
def insertMetadata(self, metadata_dto):
if not metadata_dto.verifySignature():
raise SignatureException('Metadata to insert is not properlysigned')
select_query = QUERIES['SELECT METADATA']
signature = bin2str(metadata_dto.signature)
infohash = bin2str(metadata_dto.infohash)
channel = bin2str(metadata_dto.channel)
res = self._db.fetchall(select_query, (infohash, channel))
isUpdate = False
if len(res) != 0:
if metadata_dto.timestamp > res[0][4]:
query = QUERIES['UPDATE METADATA']
self._db.execute_write(query, (metadata_dto.description,
metadata_dto.timestamp,
signature,
infohash,
channel), False)
fk_key = res[0][0]
isUpdate = True
else:
return
else:
query = QUERIES['INSERT METADATA']
self._db.execute_write(query, (channel,
infohash,
metadata_dto.description,
metadata_dto.timestamp,
signature), True)
if DEBUG:
print >> sys.stderr, 'Performing query on db: ' + query
newRows = self._db.fetchall(select_query, (infohash, channel))
if len(newRows) == 0:
raise IOError('No results, while there should be one')
fk_key = newRows[0][0]
self._insertOrUpdateSubtitles(fk_key, metadata_dto.getAllSubtitles(), False)
self._db.commit()
return isUpdate
def send_metadata(self, permid, message, selversion):
try:
infohash = bdecode(message[1:])
except:
print_exc()
if DEBUG:
print >> sys.stderr, 'metadata: GET_METADATA: error becoding'
return False
if not isValidInfohash(infohash):
if DEBUG:
print >> sys.stderr, 'metadata: GET_METADATA: invalid hash'
return False
res = self.torrent_db.getOne(('torrent_file_name', 'status_id'),
infohash=bin2str(infohash))
if not res:
if DEBUG:
print >> sys.stderr, 'metadata: GET_METADATA: not in database', infohash
return True
torrent_file_name, status_id = res
if status_id == self.torrent_db._getStatusID('dead'):
if DEBUG:
print >> sys.stderr, 'metadata: GET_METADATA: Torrent was dead'
return True
if not torrent_file_name:
if DEBUG:
print >> sys.stderr, 'metadata: GET_METADATA: no torrent file name'
return True
torrent_path = os.path.join(self.torrent_dir, torrent_file_name)
if not os.path.isfile(torrent_path):
if DEBUG:
print >> sys.stderr, 'metadata: GET_METADATA: not existing', res, torrent_path
return True
task = {
'permid': permid,
'infohash': infohash,
'torrent_path': torrent_path,
'selversion': selversion
}
self.upload_queue.append(task)
if int(time()) >= self.next_upload_time:
self.checking_upload_queue()
return True
def _handleGETSUBS(self, permid, message, selversion):
if selversion < OLPROTO_VER_FOURTEENTH:
if DEBUG:
print >> sys.stderr, 'The peer that sent the GET_SUBS request has an oldprotcol version: this is strange. Dropping the msg'
return False
decoded = self._decodeGETSUBSMessage(message)
if decoded is None:
if DEBUG:
print >> sys.stderr, 'Error decoding a GET_SUBS message from %s' % utilities.show_permid_short(
permid)
return False
if DEBUG:
channel_id, infohash, languages = decoded
bitmask = self._languagesUtility.langCodesToMask(languages)
print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % (
'RG', show_permid_short(permid), show_permid_short(channel_id),
bin2str(infohash), bitmask, len(message))
for listener in self._listenersList:
listener.receivedSubsRequest(permid, decoded, selversion)
return True
def send_metadata(self, permid, message, selversion):
try:
infohash = bdecode(message[1:])
except:
print_exc()
if DEBUG:
print >> sys.stderr, 'metadata: GET_METADATA: error becoding'
return False
if not isValidInfohash(infohash):
if DEBUG:
print >> sys.stderr, 'metadata: GET_METADATA: invalid hash'
return False
res = self.torrent_db.getOne(('torrent_file_name', 'status_id'), infohash=bin2str(infohash))
if not res:
if DEBUG:
print >> sys.stderr, 'metadata: GET_METADATA: not in database', infohash
return True
torrent_file_name, status_id = res
if status_id == self.torrent_db._getStatusID('dead'):
if DEBUG:
print >> sys.stderr, 'metadata: GET_METADATA: Torrent was dead'
return True
if not torrent_file_name:
if DEBUG:
print >> sys.stderr, 'metadata: GET_METADATA: no torrent file name'
return True
torrent_path = os.path.join(self.torrent_dir, torrent_file_name)
if not os.path.isfile(torrent_path):
if DEBUG:
print >> sys.stderr, 'metadata: GET_METADATA: not existing', res, torrent_path
return True
task = {'permid': permid,
'infohash': infohash,
'torrent_path': torrent_path,
'selversion': selversion}
self.upload_queue.append(task)
if int(time()) >= self.next_upload_time:
self.checking_upload_queue()
return True
def _checkingUploadQueue(self):
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'Checking the upload queue...'
if not self._tokenBucket.upload_rate > 0:
return
if not len(self._uploadQueue) > 0:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'Upload queue is empty.'
while len(self._uploadQueue) > 0:
responseData = self._uploadQueue[0]
encodedMsg = self._createSingleResponseMessage(responseData)
if encodedMsg is None:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'Nothing to send'
del self._uploadQueue[0]
continue
msgSize = len(encodedMsg) / 1024.0
if msgSize > self._tokenBucket.capacity:
print >> sys.stderr, 'Warning:' + SUBS_LOG_PREFIX + 'SUBS message too big. Discarded!'
del self._uploadQueue[0]
continue
if self._tokenBucket.consume(msgSize):
if DEBUG:
keys = responseData['subtitles'].keys()
bitmask = self._languagesUtility.langCodesToMask(keys)
print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % (
'SS', show_permid_short(responseData['permid']),
show_permid_short(responseData['channel_id']),
bin2str(responseData['infohash']), bitmask,
int(msgSize * 1024))
self._doSendSubtitles(responseData['permid'], encodedMsg,
responseData['selversion'])
del self._uploadQueue[0]
else:
neededCapacity = max(0, msgSize - self._tokenBucket.tokens)
delay = neededCapacity / self._tokenBucket.upload_rate
self._nextUploadTime = time() + delay
self.overlay_bridge.add_task(self._checkingUploadQueue, delay)
return
def _checkingUploadQueue(self):
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'Checking the upload queue...'
if not self._tokenBucket.upload_rate > 0:
return
if not len(self._uploadQueue) > 0:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'Upload queue is empty.'
while len(self._uploadQueue) > 0:
responseData = self._uploadQueue[0]
encodedMsg = self._createSingleResponseMessage(responseData)
if encodedMsg is None:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'Nothing to send'
del self._uploadQueue[0]
continue
msgSize = len(encodedMsg) / 1024.0
if msgSize > self._tokenBucket.capacity:
print >> sys.stderr, 'Warning:' + SUBS_LOG_PREFIX + 'SUBS message too big. Discarded!'
del self._uploadQueue[0]
continue
if self._tokenBucket.consume(msgSize):
if DEBUG:
keys = responseData['subtitles'].keys()
bitmask = self._languagesUtility.langCodesToMask(keys)
print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ('SS',
show_permid_short(responseData['permid']),
show_permid_short(responseData['channel_id']),
bin2str(responseData['infohash']),
bitmask,
int(msgSize * 1024))
self._doSendSubtitles(responseData['permid'], encodedMsg, responseData['selversion'])
del self._uploadQueue[0]
else:
neededCapacity = max(0, msgSize - self._tokenBucket.tokens)
delay = neededCapacity / self._tokenBucket.upload_rate
self._nextUploadTime = time() + delay
self.overlay_bridge.add_task(self._checkingUploadQueue, delay)
return
def _handleGETSUBS(self, permid, message, selversion):
if selversion < OLPROTO_VER_FOURTEENTH:
if DEBUG:
print >> sys.stderr, 'The peer that sent the GET_SUBS request has an oldprotcol version: this is strange. Dropping the msg'
return False
decoded = self._decodeGETSUBSMessage(message)
if decoded is None:
if DEBUG:
print >> sys.stderr, 'Error decoding a GET_SUBS message from %s' % utilities.show_permid_short(permid)
return False
if DEBUG:
channel_id, infohash, languages = decoded
bitmask = self._languagesUtility.langCodesToMask(languages)
print >> sys.stderr, '%s, %s, %s, %s, %d, %d' % ('RG',
show_permid_short(permid),
show_permid_short(channel_id),
bin2str(infohash),
bitmask,
len(message))
for listener in self._listenersList:
listener.receivedSubsRequest(permid, decoded, selversion)
return True
def _insertNewSubtitle(self, metadata_fk, subtitle, commitNow=True):
query = QUERIES['INSERT SUBTITLES']
checksum = bin2str(subtitle.checksum)
self._db.execute_write(
query, (metadata_fk, subtitle.lang, subtitle.path, checksum),
commitNow)
def getNrMetadata(self, channel):
query = QUERIES['SELECT NRMETADATA']
channel = bin2str(channel)
return self._db.fetchone(query, (channel,))
def getAllMetadataForInfohash(self, infohash):
strinfohash = bin2str(infohash)
query = QUERIES['SELECT PUBLISHERS FROM INFOHASH']
channels = self._db.fetchall(query, (strinfohash,))
return [ self.getMetadata(str2bin(entry[0]), infohash) for entry in channels ]
def _deleteSubtitleByChannel(self, channel, infohash, lang):
query = QUERIES['DELETE ONE SUBTITLE JOIN']
infohash = bin2str(infohash)
channel = bin2str(channel)
self._db.execute_write(query, (channel, infohash, lang))
def deleteHaveEntry(self, channel, infohash, peer_id):
channel = bin2str(channel)
infohash = bin2str(infohash)
peer_id = bin2str(peer_id)
deleteQuery = QUERIES['DELETE HAVE']
self._db.execute_write(deleteQuery, (peer_id, channel, infohash))
def getNrMetadata(self, channel):
query = QUERIES['SELECT NRMETADATA']
channel = bin2str(channel)
return self._db.fetchone(query, (channel, ))
def hasMetadata(self, channel, infohash):
query = QUERIES['SELECT METADATA']
infohash = bin2str(infohash)
channel = bin2str(channel)
res = self._db.fetchall(query, (infohash, channel))
return len(res) != 0
def hasMetadata(self, channel, infohash):
query = QUERIES['SELECT METADATA']
infohash = bin2str(infohash)
channel = bin2str(channel)
res = self._db.fetchall(query, (infohash, channel))
return len(res) != 0
def _deleteSubtitleByChannel(self, channel, infohash, lang):
query = QUERIES['DELETE ONE SUBTITLE JOIN']
infohash = bin2str(infohash)
channel = bin2str(channel)
self._db.execute_write(query, (channel, infohash, lang))
def receivedSubsRequest(self, permid, request, selversion):
channel_id, infohash, languages = request
allSubtitles = self.subtitlesDb.getAllSubtitles(channel_id, infohash)
contentsList = {}
for lang in sorted(languages):
if lang in allSubtitles.keys():
if allSubtitles[lang].subtitleExists():
content = self._readSubContent(allSubtitles[lang].path)
if content is not None:
contentsList[lang] = content
else:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'File not available for channel %s, infohash %s, lang %s' % (show_permid_short(channel_id), bin2str(infohash), lang)
self.subtitlesDb.updateSubtitlePath(channel_id, infohash, lang, None)
elif DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'Subtitle not available for channel %s, infohash %s, lang %s' % (show_permid_short(channel_id), bin2str(infohash), lang)
if len(contentsList) == 0:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + 'None of the requested subtitles were available. No answer will be sent to %s' % show_permid_short(permid)
return True
return self._subsMsgHndlr.sendSubtitleResponse(permid, (channel_id, infohash, contentsList), selversion)
def deleteHaveEntry(self, channel, infohash, peer_id):
channel = bin2str(channel)
infohash = bin2str(infohash)
peer_id = bin2str(peer_id)
deleteQuery = QUERIES['DELETE HAVE']
self._db.execute_write(deleteQuery, (peer_id, channel, infohash))
