def handleRMetadata(self, sender_permid, channelCastMessage, fromQuery = False):
'''
Handles the reception of rich metadata.
Called when an "erniched" channelCastMessage (v14) is received.
@param sender_permid: the PermId of the peer who sent the message
@param channelCastMessage: the received message
@return: None
'''
metadataDTOs, sizeList = \
self._splitChannelcastAndRichMetadataContents(channelCastMessage)
if DEBUG:
print >> sys.stderr, "Handling rich metadata from %s..." % show_permid_short(sender_permid)
i=0
for md_and_have in metadataDTOs:
md = md_and_have[0]
havemask = md_and_have[1]
vote = self.votecastDB.getVote(bin2str(md.channel),
bin2str(self.my_permid))
# the next if may seem useless, but since sizeList is defined only when
# logging is enabled for debug, I get an error without this conditional statement
# because the argument for the debug() call getsEvaluated before the logging
# system understands that debug is disabled
#if announceStatsLog.isEnabledFor(logging.INFO):
if DEBUG:
id = "RQ" if fromQuery else "R"
print >> sys.stderr, "%c, %s, %s, %s, %d, %d" % \
(id, md.channel, md.infohash, \
show_permid_short(sender_permid), md.timestamp,
sizeList[i])
#format "R|S (R: received - S: sent), channel, infohash, sender|destination,metadataCreationTimestamp"
# 30-06-2010: "RQ" as received from query
i += 1
# check if the record belongs to a channel
# who we have "reported spam" (negative vote)
if vote == -1:
# if so, ignore the incoming record
continue
isUpdate =self.rmdDb.insertMetadata(md)
self.peerHaveManager.newHaveReceived(md.channel,md.infohash,sender_permid,havemask)
if isUpdate is not None:
#retrieve the metadataDTO from the database in the case it is an update
md = self.rmdDb.getMetadata(md.channel,md.infohash)
self._notifyRichMetadata(md, isUpdate)
# if I am a subscriber send immediately a GET_SUBS to the
# sender
if vote == 2:
if DEBUG:
print >> sys.stderr, "Subscribed to channel %s, trying to retrieve" \
"all subtitle contents" % (show_permid_short(md.channel),)
self._getAllSubtitles(md)
def updateHaveMask(self,channel,infohash,peer_id, newMask, timestamp=None):
'''
Store a received have mask in the db
(See insertHaveMask for description)
@type channel: str
@param channel: channel_id (binary)
@type infohash: str
@param infohash: the infohash of a torrent (binary)
@type peer_id: str
@param peer_id: peer from whom the infomask was received.(ie its binary permid)
@type havemask: int
"param havemask: a non-negative integer. It must be smaller
then 2**32.
'''
channel = bin2str(channel)
infohash = bin2str(infohash)
peer_id = bin2str(peer_id)
updateQuery = QUERIES["UPDATE HAVE MASK"]
if timestamp is None:
timestamp = int(time.time())
self._db.execute_write(updateQuery,
(newMask,timestamp,peer_id, channel, infohash))
def deleteHaveEntry(self, channel, infohash, peer_id):
'''
Delete a row from the SubtitlesHave db.
If the row is not in the db nothing happens.
@type channel: str
@param channel: channel_id (binary)
@type infohash: str
@param infohash: the infohash of a torrent (binary)
@type peer_id: str
@param peer_id: peer from whom the infomask was received.(ie its binary permid)
@postcondition: if a row identified by channel, infohash, peer_id
was in the database, it will no longer be there
at the end of this method call
'''
channel = bin2str(channel)
infohash = bin2str(infohash)
peer_id = bin2str(peer_id)
deleteQuery = QUERIES["DELETE HAVE"]
self._db.execute_write(deleteQuery,
(peer_id,channel,infohash))
def getSubtitle(self, channel, infohash, lang):
"""
Get a subtitle for a language for a given item in a given channel.
Returns the details reguarding a subtitles in a given language for a
given item in a given channel, if it exists. Otherwise it returns
None.
@param channel: a perm_id identifying the owner of the channel.
@param infohash: the infohash of an item, as announced in channelcast
messages.
@param lang: a 3 characters ISO 639-2 language code, identifying
the desired subtitle langugage
@return: a SubtitleInfo instance
"""
query = QUERIES["SELECT SUBS JOIN HASH ONE"]
infohash = bin2str(infohash)
channel = bin2str(channel)
res = self._db.fetchall(query, (infohash, channel, lang))
if len(res) == 0 :
return None
elif len(res) == 1 :
checksum = str2bin(res[0][3])
return SubtitleInfo(res[0][1], res[0][2], checksum)
else :
# This should be not possible to database constraints
raise MetadataDBException("Metadata DB Constraint violeted!")
def getHaveMask(self, channel, infohash, peer_id):
'''
Returns the have mask for a single peer if available.
@type channel: str
@param channel: channel_id (binary)
@type infohash: str
@param infohash: the infohash of a torrent (binary)
@type peer_id: str
@param peer_id: peer from whom the infomask was received.(ie its binary permid)
@rtype: int
@return: the have mask relative to channel, infohash, and peer.
If not available returns None
@postcondition: the return value is either None or a non-negative
integer smaller then 2**32
'''
query = QUERIES["GET ONE HAVE MASK"]
channel = bin2str(channel)
infohash = bin2str(infohash)
peer_id = bin2str(peer_id)
res = self._db.fetchall(query,(channel,infohash,peer_id))
if len(res) <= 0:
return None
elif len(res) > 1:
raise AssertionError("channel,infohash,peer_id should be unique")
else:
return res[0][0]
def getLocalSubtitles(self, channel, infohash):
'''
Returns a dictionary containing all the subtitles pointing
to a local pathm for the given channel, infohash
@param channel: binary channel_id(permid)
@param infohash: binary infohash
@rtype: dict
@return: a dictionary like this:
{
...
langCode : SubtitleInfo,
...
}
The dictionary will be empty if no local subtitle
is available.
'''
query = QUERIES["SELECT SUBTITLES WITH PATH BY CHN INFO"]
channel = bin2str(channel)
infohash = bin2str(infohash)
res = self._db.fetchall(query,(channel,infohash))
result = {}
for entry in res:
location = entry[0]
language = entry[1]
checksum = str2bin(entry[2])
subInfo = SubtitleInfo(language, location, checksum)
result[language] = subInfo
return result
def receivedSubsRequest(self, permid, request, selversion):
"""
Reads a received GET_SUBS message and possibly sends a response.
@param permid: the permid of the sender of the GET_SUBS message
@param request: a tuple made of channel_id, infohash, language code
@param selversion: the protocol version of the requesting peer
@return: False if the message had something wrong. (a return value
of False makes the caller close the connection).
Otherwise True
"""
assert self.registered, SUBS_LOG_PREFIX + "Handler not yet registered"
channel_id, infohash, languages = request #happily unpacking
#diction {lang : Subtitle}
allSubtitles = self.subtitlesDb.getAllSubtitles(channel_id,
infohash)
contentsList = {} #{langCode : path}
#for each requested language check if the corresponding subtitle
#is available
for lang in sorted(languages):
if lang in allSubtitles.keys():
if allSubtitles[lang].subtitleExists():
content = self._readSubContent(allSubtitles[lang].path)
if content is not None:
contentsList[lang] = content
else:
if DEBUG:
print >> sys.stderr, time.asctime(),'-', SUBS_LOG_PREFIX + "File not available for " + \
"channel %s, infohash %s, lang %s" % \
(show_permid_short(channel_id), bin2str(infohash),
lang)
self.subtitlesDb.updateSubtitlePath(channel_id,infohash,lang,None)
else:
if DEBUG:
print >> sys.stderr, time.asctime(),'-', SUBS_LOG_PREFIX + "Subtitle not available for " + \
"channel %s, infohash %s, lang %s" % \
(show_permid_short(channel_id), bin2str(infohash),
lang)
if len(contentsList) == 0: #pathlist is empty
if DEBUG:
print >> sys.stderr, time.asctime(),'-', SUBS_LOG_PREFIX + "None of the requested subtitles " + \
" was available. No answer will be sent to %s" % \
show_permid_short(permid)
return True
return self._subsMsgHndlr.sendSubtitleResponse(permid,
(channel_id,infohash,contentsList),
selversion)
def publishSubtitle(self, infohash, lang, pathToSrtSubtitle):
'''
Allows an user to publish an srt subtitle file in his channel.
Called by a channel owner this method inserts a new subtitle for
a torrent published in his channel.
The method assumes that the torrent identified by the infohash
parameter is already in the channel, and that the parameter
pathToSrtSubtitle points to an existing srt file on the local
filesystem.
If a subtitle for the same language was already associated to the
specified infohash and channel, it will be overwritten.
After calling this method the newly inserted subtitle will be
disseminated via Channelcast.
@param infohash: the infohash of the torrent to associate the subtitle
with, binary
@param lang: a 3 characters code for the language of the subtitle as
specified in ISO 639-2. Currently just 32 language codes
will be supported.
@param pathToSrtSubtitle: a path in the local filesystem to a subtitle
in srt format.
@raise RichMetadataException: if something "general" goes wrong while
adding new metadata
@raise IOError: if disk related problems occur
'''
assert utilities.isValidInfohash(infohash), "Invalid Infohash"
assert lang is not None and self.langUtility.isLangCodeSupported(lang)
assert self._registered, "Instance is not registered"
channelid = bin2str(self.my_permid)
base64infohash = bin2str(infohash)
# consisnstency check: I want to assure that this method is called
# for an item that is actually in my channel
consinstent = self.channelcast_db.isItemInChannel(channelid,base64infohash)
if not consinstent:
msg = "Infohash %s not found in my channel. Rejecting subtitle" \
% base64infohash
if DEBUG:
print >> sys.stderr, time.asctime(),'-', msg
raise RichMetadataException(msg)
try:
filepath = \
self.subtitlesHandler.copyToSubtitlesFolder(pathToSrtSubtitle,
self.my_permid,infohash,
lang)
except Exception,e:
if DEBUG:
print >> sys.stderr, time.asctime(),'-', "Failed to read and copy subtitle to appropriate folder: %s" % str(e)
def insertHaveMask(self, channel, infohash, peer_id, havemask, timestamp=None):
'''
Store a received have mask in the db
Each inserted rows represent a delcaration of subtitle
availability from peer_id, for some subtitles for
a torrent identified by infohash in a channel identified
by channel.
@type channel: str
@param channel: channel_id (binary)
@type infohash: str
@param infohash: the infohash of a torrent (binary)
@type peer_id: str
@param peer_id: peer from whom the infomask was received.(ie its binary permid)
@type havemask: int
@param havemask: a non-negative integer. It must be smaller
then 2**32.
@precondition: an entry for (channel, infohash) must already
exist in the database
'''
query = QUERIES["SELECT METADATA"]
if timestamp is None:
timestamp = int(time.time())
channel = bin2str(channel)
infohash = bin2str(infohash)
peer_id = bin2str(peer_id)
res = self._db.fetchall(query, (infohash, channel))
if len(res) != 1:
raise MetadataDBException("No entry in the MetadataDB for %s, %s" %\
(channel[-10:],infohash))
metadata_fk = res[0][0]
insertQuery = QUERIES["INSERT HAVE MASK"]
try:
self._db.execute_write(insertQuery, (metadata_fk, peer_id, havemask, timestamp))
except sqlite3.IntegrityError,e:
raise MetadataDBException(str(e))
def _handleGETSUBS(self,permid, message, selversion):
if selversion < OLPROTO_VER_FOURTEENTH:
if DEBUG:
print >> sys.stderr, "The peer that sent the GET_SUBS request has an old" \
"protcol version: this is strange. Dropping the msg"
return False
decoded = self._decodeGETSUBSMessage(message)
if decoded is None:
if DEBUG:
print >> sys.stderr, "Error decoding a GET_SUBS message from %s" %\
utilities.show_permid_short(permid)
return False
if DEBUG:
channel_id, infohash, languages = decoded
bitmask = self._languagesUtility.langCodesToMask(languages)
print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % ("RG", show_permid_short(permid),
show_permid_short(channel_id),
bin2str(infohash), bitmask, len(message))
# no synch on _listenersList since both this method
# and the registerListener method are called by
# the OLThread
for listener in self._listenersList:
listener.receivedSubsRequest(permid, decoded, selversion)
return True
def _updateSubtitle(self, metadata_fk, subtitle, commitNow=True):
"""
Update an entry in the Subtitles database.
If the entry identified by metadata_fk, subtitle.lang does not exist
in the subtitle database this method does nothing.
@param metadata_fk: foreign key of the metadata table
@param subtitle: instance of Subitle containing the data to insert
@param commitNow: if False, this method does not commit the changes to
the database
"""
assert metadata_fk is not None
assert subtitle is not None
assert isinstance(subtitle, SubtitleInfo)
toUpdate = self._getSubtitleByKey(metadata_fk, subtitle.lang)
if toUpdate is None:
return
query = QUERIES["UPDATE SUBTITLES"]
checksum = bin2str(subtitle.checksum)
self._db.execute_write(query, (subtitle.path,
checksum, metadata_fk, subtitle.lang),
commitNow)
def hasMetadata(self, channel, infohash):
"""
Checks whether there exists some metadata for an item in a channel.
@param channel: a perm_id identifying the owner of the channel.
@param infohash: the infohash of an item, as announced in channelcast
messages.
@return boolean
"""
query = QUERIES["SELECT METADATA"]
infohash = bin2str(infohash)
channel = bin2str(channel)
res = self._db.fetchall(query, (infohash, channel))
return len(res) != 0
def getHaveEntries(self, channel, infohash):
'''
Return a list of have entries for subtitles for a torrent
in a channel.
This method returns a list of tuple, like:
[
...
(peer_id, haveMask, timestamp),
...
]
(peer_id) is the perm_id of a Tribler
Peer, haveMask is an integer value representing a
bitmask of subtitles owned by that peer.
Timestamp is the timestamp at the time the havemask
was received.
The results are ordered by descending timestamp.
If there are no
entris for the givenn channel,infohash pair, the returned
list will be empty
@type channel: str
@param channel: channel_id (binary)
@type infohash: str
@param infohash: the infohash of a torrent (binary)
@rtype: list
@return: see description
'''
query = QUERIES["GET ALL HAVE MASK"]
channel = bin2str(channel)
infohash = bin2str(infohash)
res = self._db.fetchall(query,(channel,infohash))
returnlist = list()
for entry in res:
peer_id = str2bin(entry[0])
haveMask = entry[1]
timestamp = entry[2]
returnlist.append((peer_id, haveMask, timestamp))
return returnlist
def _deleteSubtitleByChannel(self, channel, infohash, lang):
'''
Remove a subtitle for a channel infohash
@param channel: the channel where the subtitle is (binary)
@param infohash: the infohash of the torrent referred by the subtitle
(binary)
@param lang: ISO-639-2 language code of the subtitle to remove
'''
query = QUERIES["DELETE ONE SUBTITLE JOIN"]
infohash = bin2str(infohash)
channel = bin2str(channel)
self._db.execute_write(query,(channel, infohash, lang))
def _handleSUBS(self, permid, message, selversion):
if selversion < OLPROTO_VER_FOURTEENTH:
if DEBUG:
print >> sys.stderr, "The peer that sent the SUBS request has an old" \
"protcol version: this is strange. Dropping the msg"
return False
decoded = self._decodeSUBSMessage(message)
if decoded is None:
if DEBUG:
print >> sys.stderr, "Error decoding a SUBS message from %s" %\
utilities.show_permid_short(permid)
return False
channel_id, infohash, bitmask,contents = decoded
#if no subtitle was requested drop the whole message
if DEBUG:
print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % ("RS", show_permid_short(permid),
show_permid_short(channel_id),
bin2str(infohash), bitmask, len(message))
requestedSubs = self._checkRequestedSubtitles(channel_id,infohash,bitmask)
if requestedSubs == 0:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + "Received a SUBS message that was not"\
" requested. Dropping"
return False
requestedSubsCodes = self._languagesUtility.maskToLangCodes(requestedSubs)
#drop from the contents subtitles that where not requested
for lang in contents.keys():
if lang not in requestedSubsCodes:
del contents[lang]
#remove the received subtitles from the requested
callbacks = \
self._removeFromRequestedSubtitles(channel_id, infohash, bitmask)
#the receiver does not need the bitmask
tuple = channel_id, infohash, contents
# no synch on _listenersList since both this method
# and the registerListener method are called by
# the OLThread
for listener in self._listenersList:
listener.receivedSubsResponse(permid, tuple, callbacks, selversion)
return True
def getMetadata(self, channel, infohash):
"""
Returns a MetadataDTO instance for channel/infohash if available in DB
Given a channel/infhash couple returns a MetadataDTO instance, built
with the values retrieved from the Metadata and Subtitles DB. If
no result returns None
@param channel: the permid of the channel's owner (binary)
@param infohash: the infohash of the item the metadata refers to
(binary)
@return: a MetadataDTO instance comprehensive of subtitles if any
metadata is found in the DB. None otherwise.
"""
query = QUERIES["SELECT METADATA"]
infohash = bin2str(infohash)
channel = bin2str(channel)
res = self._db.fetchall(query, (infohash, channel))
if len(res) == 0:
return None
if len(res) > 1:
raise MetadataDBException("Metadata DB Constraint violated")
metaTuple = res[0]
subsDictionary = self._getAllSubtitlesByKey(metaTuple[0])
publisher = str2bin(metaTuple[1])
infohash = str2bin(metaTuple[2])
timestamp = int(metaTuple[4])
description = unicode(metaTuple[3])
signature = str2bin(metaTuple[5])
toReturn = MetadataDTO(publisher, infohash,
timestamp, description, None,
signature)
for sub in subsDictionary.itervalues():
toReturn.addSubtitle(sub)
return toReturn
def updateSubtitlePath(self, channel, infohash, lang, newPath, commitNow=True):
"""
Updates a subtitle entry in the database if it exists.
Given the channel, the infohash, and a SubtitleInfo instance,
the entry relative to that subtitle is updated accordingly
to the details in the SubtitleInfo instance.
If an instance for the provided channel, infohash, and language
does not already exist in the db, nothing is done.
@param channel: the channel id (permid) of the channel for the
subtitle (binary)
@param infohash: the infohash of the item the subtitle refrs to
(binary)
@param lang: the language of the subtitle to update
@param path: the new path of the subtitle. None to indicate that the
subtitle is not available
@return True if an entry was updated in the db. False if nothing
got written on the db
@precondition: subtitle.lang is not None
"""
query = QUERIES["SELECT SUBS JOIN HASH ONE"]
channel = bin2str(channel)
infohash = bin2str(infohash)
res = self._db.fetchall(query, (infohash, channel, lang))
if len(res) > 1 :
raise MetadataDBException("Metadata DB constraint violated")
elif len(res) == 0 :
if DEBUG:
print >> sys.stderr, time.asctime(),'-', "Nothing to update for channel %s, infohash %s, lang"\
" %s. Doing nothing." % (channel[-10:],\
infohash, lang)
return False
else:
query = QUERIES["UPDATE SUBTITLES"]
self._db.execute_write(query, (newPath,
res[0][3], res[0][0], lang),
commitNow)
return True
def _get_subs_connect_callback(self, exception, dns, permid, selversion,
channel_id, infohash, bitmask, msgSentCallback, usrCallback):
"""
Called by the Overlay Thread when a connection with permid is established.
Performs the actual action of sending a GET_SUBS request to the peer
identified by permid. It is called by the OLThread when a connection
with that peer is established.
"""
if exception is not None:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + \
"GET_SUBS not sent. Unable to connect to " + \
utilities.show_permid_short(permid)
else:
if (selversion > 0 and selversion < OLPROTO_VER_FOURTEENTH):
msg = "GET_SUBS not send, the other peers had an old protocol version: %d" %\
selversion
if DEBUG:
print >> sys.stderr, msg
raise SubtitleMsgHandlerException(msg)
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + "sending GET_SUBS to " + \
utilities.show_permid_short(permid)
try :
message = self._createGETSUBSMessage(channel_id, infohash,
bitmask)
if DEBUG:
# Format:
# SS|SG, destination, channel, infohash, bitmask, size
print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % ("SG",show_permid_short(permid),
show_permid_short(channel_id),
bin2str(infohash),bitmask,len(message))
self._overlay_bridge.send(permid, message,
lambda exc, permid: \
self._sent_callback(exc,permid,
channel_id,
infohash,
bitmask,
msgSentCallback,
usrCallback))
except Exception,e:
print_exc()
msg = "GET_SUBS not sent: %s" % str(e)
raise SubtitleMsgHandlerException(e)
def deleteMetadata(self, channel, infohash):
"""
Removes all the metadata associated to a channel/infohash.
Everything is dropped from both the Metadata and Subtitles db.
@param channel: the permid of the channel's owner
@param infohash: the infhoash of the entry
"""
assert channel is not None
assert infohash is not None
channel = bin2str(channel)
infohash = bin2str(infohash)
query = QUERIES["SELECT METADATA"]
if DEBUG:
print >> sys.stderr, time.asctime(),'-', "Performing query on db: " + query
res = self._db.fetchall(query, (infohash, channel))
if len(res) == 0 :
return
if len(res) > 1 :
raise IOError("Metadata DB constraint violated")
metadata_fk = res[0][0]
self._deleteAllSubtitles(metadata_fk, False)
query = QUERIES["DELETE METADATA PK"]
self._db.execute_write(query, (metadata_fk,), False)
self._db.commit()
def getAllSubtitles(self, channel, infohash):
"""
Get all the available subtitles for a channel and infohash.
Returns a list representing subtitles that are available for
a givenchannel and infohash.
@param channel: the perm_id of the channel owner (binary)
@param infohash: the infhash of a channel elements as it
is announced in ChannelCast (binary)
@return: a dictionary of { lang : SubtitleInfo instance}
"""
query = QUERIES["SELECT SUBS JOIN HASH ALL"]
infohash = bin2str(infohash)
channel = bin2str(channel)
results = self._db.fetchall(query, (infohash, channel))
subsDict = {}
for entry in results:
subsDict[entry[1]] = SubtitleInfo(entry[1], entry[2], entry[3])
return subsDict
def getSubtitleFileRelativeName(channel_id, infohash, langCode):
# subtitles filenames are build from the sha1 hash
# of the triple (channel_id, infohash, langCode)
# channel_id and infohash are binary versions
assert utilities.validPermid(channel_id), "Invalid channel_id %s" % utilities.show_permid_short(channel_id)
assert utilities.validInfohash(infohash), "Invalid infohash %s" % bin2str(infohash)
assert LanguagesProvider.getLanguagesInstance().isLangCodeSupported(langCode), (
"Unsupported language code %s" % langCode
)
hasher = sha()
for data in (channel_id, infohash, langCode):
hasher.update(data)
subtitleName = hasher.hexdigest() + SUBS_EXTENSION
return subtitleName
def send_metadata(self, permid, message, selversion):
try:
infohash = bdecode(message[1:])
except:
print_exc()
if DEBUG:
print >> sys.stderr, "metadata: GET_METADATA: error becoding"
return False
if not isValidInfohash(infohash):
if DEBUG:
print >> sys.stderr, "metadata: GET_METADATA: invalid hash"
return False
# TODO:
res = self.torrent_db.getOne(('torrent_file_name', 'status_id'),
infohash=bin2str(infohash))
if not res:
if DEBUG:
print >> sys.stderr, "metadata: GET_METADATA: not in database", infohash
return True # don't close connection because I don't have the torrent
torrent_file_name, status_id = res
if status_id == self.torrent_db._getStatusID('dead'):
if DEBUG:
print >> sys.stderr, "metadata: GET_METADATA: Torrent was dead"
return True
if not torrent_file_name:
return True
torrent_path = os.path.join(self.torrent_dir, torrent_file_name)
if not os.path.isfile(torrent_path):
if DEBUG:
print >> sys.stderr, "metadata: GET_METADATA: not existing", res, torrent_path
return True
task = {
'permid': permid,
'infohash': infohash,
'torrent_path': torrent_path,
'selversion': selversion
}
self.upload_queue.append(task)
if int(time()) >= self.next_upload_time:
self.checking_upload_queue()
return True
def _insertNewSubtitle(self, metadata_fk, subtitle, commitNow=True) :
"""
Insert a new subtitle entry in the Subtitles table.
Given a foreign key from the Metadata table, and a SubtitleInfo instance
describing the subtitle to insert, adds it to the metadata table.
This method assumes that that entry does not already exist in the
table.
NOTICE that sqlite does not enforce the foreign key constraint,
so be careful about integrity
"""
assert metadata_fk is not None
assert subtitle is not None
assert isinstance(subtitle, SubtitleInfo)
query = QUERIES["INSERT SUBTITLES"]
checksum = bin2str(subtitle.checksum)
self._db.execute_write(query, (metadata_fk, subtitle.lang,
subtitle.path, checksum),
commitNow)
def send_metadata(self, permid, message, selversion):
try:
infohash = bdecode(message[1:])
except:
print_exc()
if DEBUG:
print >> sys.stderr,time.asctime(),'-', "metadata: GET_METADATA: error becoding"
return False
if not isValidInfohash(infohash):
if DEBUG:
print >> sys.stderr,time.asctime(),'-', "metadata: GET_METADATA: invalid hash"
return False
# TODO:
res = self.torrent_db.getOne(('torrent_file_name', 'status_id'), infohash=bin2str(infohash))
if not res:
if DEBUG:
print >> sys.stderr,time.asctime(),'-', "metadata: GET_METADATA: not in database", infohash
return True # don't close connection because I don't have the torrent
torrent_file_name, status_id = res
if status_id == self.torrent_db._getStatusID('dead'):
if DEBUG:
print >> sys.stderr,time.asctime(),'-', "metadata: GET_METADATA: Torrent was dead"
return True
if not torrent_file_name:
if DEBUG:
print >> sys.stderr,time.asctime(),'-', "metadata: GET_METADATA: no torrent file name"
return True
torrent_path = os.path.join(self.torrent_dir, torrent_file_name)
if not os.path.isfile(torrent_path):
if DEBUG:
print >> sys.stderr,time.asctime(),'-', "metadata: GET_METADATA: not existing", res, torrent_path
return True
task = {'permid':permid, 'infohash':infohash, 'torrent_path':torrent_path, 'selversion':selversion}
self.upload_queue.append(task)
if int(time()) >= self.next_upload_time:
self.checking_upload_queue()
return True
def getAllMetadataForInfohash(self, infohash):
"""
Returns a list of MetadataDTO instances for a given infohash
Given a torrent infohash returns a list of MetadataDTO instances for
that infohash. Each one of the MetadataDTO refers to a different
channel.
@param infohash: the infohash for the torrent (binary)
@return: a list of MetadataDTO isntances (or empty list if nothing
is found)
"""
assert infohash is not None
strinfohash = bin2str(infohash)
query = QUERIES["SELECT PUBLISHERS FROM INFOHASH"]
channels = self._db.fetchall(query, (strinfohash,))
return [self.getMetadata(str2bin(entry[0]), infohash) for entry in channels]
def addRichMetadataContent(self,channelCastMessage, destPermid = None, fromQuery = False):
'''
Takes plain channelcast message (from OLProto v.13) and adds to it
a 'rich_metadata' field.
@param channelCastMessage: the old channelcast message in the format of
protocol v13
@param destPermid: the destination of the message. If not None it is used
for logging purposes only. If None, nothing bad happens.
@return: the "enriched" channelcast message
'''
if not len(channelCastMessage) > 0:
if DEBUG:
print >> sys.stderr, "no entries to enrich with rmd"
return channelCastMessage
if DEBUG:
if fromQuery:
print >> sys.stderr, "Intercepted a channelcast message as answer to a query"
else:
print >> sys.stderr, "Intercepted a channelcast message as normal channelcast"
#otherwise I'm modifying the old one (even if there's nothing bad
#it's not good for the caller to see its parameters changed :)
newMessage = dict()
# a channelcast message is made up of a dictionary of entries
# keyed the signature. Every value in the dictionary is itself
# a dictionary with the item informatino
for key in iter(channelCastMessage):
entryContent = copy(channelCastMessage[key])
newMessage[key] = entryContent
channel_id = entryContent['publisher_id']
infohash = entryContent['infohash']
#not clean but the fastest way :(
# TODO: make something more elegant
metadataDTO = self.rmdDb.getMetadata(channel_id, infohash)
if metadataDTO is not None:
try:
if DEBUG:
print >> sys.stderr, "Enriching a channelcast message with subtitle contents"
metadataPack = metadataDTO.serialize()
# I can remove from the metadata pack the infohash, and channelId
# since they are already in channelcast and they would be redundant
metadataPack.pop(0)
metadataPack.pop(0)
#adding the haveMask at the end of the metadata pack
havemask = self.peerHaveManager.retrieveMyHaveMask(channel_id, infohash)
binary_havemask = uintToBinaryString(havemask)
metadataPack.append(binary_havemask)
entryContent['rich_metadata'] = metadataPack
if DEBUG:
size = self._computeSize(metadataPack)
# if available records also the destination of the message
dest = "NA" if destPermid is None else show_permid_short(destPermid)
id = "SQ" if fromQuery else "S"
# format (S (for sent) | SQ (for sent as response to a query), channel, infohash, destination, timestampe, size)
print >> sys.stderr, "%c, %s, %s, %s, %d, %d" % \
(id, bin2str(metadataDTO.channel), \
bin2str(metadataDTO.infohash), \
dest, metadataDTO.timestamp, size)
except Exception,e:
print >> sys.stderr, "Warning: Error serializing metadata: %s", str(e)
return channelCastMessage
else:
# better to put the field to None, or to avoid adding the
# metadata field at all?
##entryContent['rich_metadata'] = None
pass
def _checkingUploadQueue(self):
"""
Uses a token bucket to control the subtitles upload rate.
Every time this method is called, it will check if there are enough
tokens in the bucket to send out a SUBS message.
Currently fragmentation is not implemented: all the reuquested subtitles
are sent in a single SUBS messages if there are enough tokens:
too big responses are simply discarded.
The method tries to consume all the available tokens of the token
bucket until there are no more messages to send. If there are no
sufficiente tokens to send a message, another call to this method
is scheduled in a point in time sufficiently distant.
"""
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + "Checking the upload queue..."
if not self._tokenBucket.upload_rate > 0:
return
if not len(self._uploadQueue) > 0:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + "Upload queue is empty."
while len(self._uploadQueue) > 0 :
responseData = self._uploadQueue[0]
encodedMsg = self._createSingleResponseMessage(responseData)
if encodedMsg is None:
if DEBUG:
print >> sys.stderr, SUBS_LOG_PREFIX + "Nothing to send"
del self._uploadQueue[0]
continue #check other messages in the queue
msgSize = len(encodedMsg) / 1024.0 #in kilobytes
if msgSize > self._tokenBucket.capacity:
#message is too big, discarding
print >> sys.stderr, "Warning:" + SUBS_LOG_PREFIX + "SUBS message too big. Discarded!"
del self._uploadQueue[0]
continue #check other messages in the queue
#check if there are sufficiente tokens
if self._tokenBucket.consume(msgSize):
if DEBUG:
# Format:
# S|G, destination, channel, infohash, bitmask, size
keys = responseData['subtitles'].keys()
bitmask = self._languagesUtility.langCodesToMask(keys)
print >> sys.stderr, "%s, %s, %s, %s, %d, %d" % ("SS",show_permid_short(responseData['permid']),
show_permid_short(responseData['channel_id']),
bin2str(responseData['infohash']),bitmask,int(msgSize*1024))
self._doSendSubtitles(responseData['permid'], encodedMsg, responseData['selversion'])
del self._uploadQueue[0]
else:
#tokens are insufficient wait the necessary time and check again
neededCapacity = max(0, msgSize - self._tokenBucket.tokens)
delay = (neededCapacity / self._tokenBucket.upload_rate)
self._nextUploadTime = time() + delay
self.overlay_bridge.add_task(self._checkingUploadQueue, delay)
return
def insertMetadata(self, metadata_dto):
'''
Insert the metadata contained in a Metadata DTO in the database.
If an entry relative to the same channel and infohash of the provided
dto already exists in the db, the db is updated only if the timestamp
of the new dto is newer then the entry in the database.
If there is no such an entry, a new wan in the Metadata DB is created
along with the required entries in the SubtitleInfo DB
@type metadata_dto: MetadataDTO
@param metada_dto: an instance of MetadataDTO describing metadata
@return True if an existing entry was updated, false if a new entry
was interested. Otherwise None.
'''
assert metadata_dto is not None
assert isinstance(metadata_dto, MetadataDTO)
#try to retrieve a correspindng record for channel,infhoash
#won't do nothing if the metadata_dto is not correctly signed
if not metadata_dto.verifySignature():
raise SignatureException("Metadata to insert is not properly" \
"signed")
select_query = QUERIES["SELECT METADATA"]
signature = bin2str(metadata_dto.signature)
infohash = bin2str(metadata_dto.infohash)
channel = bin2str(metadata_dto.channel)
res = self._db.fetchall(select_query,
(infohash, channel))
isUpdate = False
if len(res) != 0 :
#updated if the new message is newer
if metadata_dto.timestamp > res[0][4] :
query = QUERIES["UPDATE METADATA"]
self._db.execute_write(query,
(metadata_dto.description,
metadata_dto.timestamp,
signature,
infohash,
channel,),
False) #I don't want the transaction to commit now
fk_key = res[0][0]
isUpdate = True
else:
return
else: #if is this a whole new metadata item
query = QUERIES["INSERT METADATA"]
self._db.execute_write(query,
(channel,
infohash,
metadata_dto.description,
metadata_dto.timestamp,
signature,
),
True)
if DEBUG:
print >> sys.stderr, time.asctime(),'-', "Performing query on db: " + query
newRows = self._db.fetchall(select_query,
(infohash, channel))
if len(newRows) == 0 :
raise IOError("No results, while there should be one")
fk_key = newRows[0][0]
self._insertOrUpdateSubtitles(fk_key, metadata_dto.getAllSubtitles(), \
False)
self._db.commit() #time to commit everything
return isUpdate