def getSomeMetadata(self, channel_id, infohash):
s1 = SubtitleInfo("eng", None)
s2 = SubtitleInfo("rus", None)
self.content1 = u"Subtitle Content 1"
self.content2 = u"Subtitle Content 2"
hasher = sha()
hasher.update(self.content1)
s1.checksum = hasher.digest()
hasher = sha()
hasher.update(self.content2)
s2.checksum = hasher.digest()
metadata = MetadataDTO(channel_id, infohash, time.time(),
"", {"eng":s1, "rus":s2})
if self.nextKeypair is None:
metadata.signature = "fake"
else:
metadata.sign(self.nextKeypair)
return metadata
def testDesrialize(self):
badInfohash = str2bin("GEh/o8rtTLB1wZJzFcSZSS4u9qo=")
dto = MetadataDTO(test_perm_id, badInfohash)
dto.description = u"Sample Description"
dto.sign(test_keypair)
serialized = dto.serialize()
newDto = MDUtil.deserialize(serialized)
self.assertEquals(dto,newDto)
def testSerialize(self):
badInfohash = str2bin("GEh/o8rtTLB1wZJzFcSZSS4u9qo=")
dto = MetadataDTO(test_perm_id, badInfohash)
dto.description = u"Sample Description"
dto.sign(test_keypair)
serialized = dto.serialize()
self.assertEquals(7, len(serialized))
signature = serialized[6]
self.assertEquals(dto.signature,signature)
def getMetadata(self, channel, infohash):
"""
Returns a MetadataDTO instance for channel/infohash if available in DB
Given a channel/infhash couple returns a MetadataDTO instance, built
with the values retrieved from the Metadata and Subtitles DB. If
no result returns None
@param channel: the permid of the channel's owner (binary)
@param infohash: the infohash of the item the metadata refers to
(binary)
@return: a MetadataDTO instance comprehensive of subtitles if any
metadata is found in the DB. None otherwise.
"""
query = QUERIES["SELECT METADATA"]
infohash = bin2str(infohash)
channel = bin2str(channel)
res = self._db.fetchall(query, (infohash, channel))
if len(res) == 0:
return None
if len(res) > 1:
raise MetadataDBException("Metadata DB Constraint violated")
metaTuple = res[0]
subsDictionary = self._getAllSubtitlesByKey(metaTuple[0])
publisher = str2bin(metaTuple[1])
infohash = str2bin(metaTuple[2])
timestamp = int(metaTuple[4])
description = unicode(metaTuple[3])
signature = str2bin(metaTuple[5])
toReturn = MetadataDTO(publisher, infohash,
timestamp, description, None,
signature)
for sub in subsDictionary.itervalues():
toReturn.addSubtitle(sub)
return toReturn
def test_packData(self):
badInfohash = str2bin("GEh/o8rtTLB1wZJzFcSZSS4u9qo=")
dto = MetadataDTO(test_perm_id, badInfohash)
dto.description = u"Sample Description\u041f"
bla = dto._packData()
decoded = bdecode(bla)
self.assertTrue(len(decoded) == 6)
decodedChannelId = decoded[0]
decodedInfohash = decoded[1]
decodedDescription = decoded[2].decode("utf-8")
decodedTimestamp = decoded[3]
bin_decodedBitmask = decoded[4]
decodedBitmask = binaryStringToUint(bin_decodedBitmask)
self.assertEquals(dto.channel, decodedChannelId)
self.assertEquals(dto.infohash, decodedInfohash)
self.assertEquals(dto.description,decodedDescription)
self.assertAlmostEquals(dto.timestamp,decodedTimestamp)
self.assertEquals(0,decodedBitmask)
self.assertEquals(0,len(decoded[5]))
def test_packDataWithSubs(self):
badInfohash = str2bin("GEh/o8rtTLB1wZJzFcSZSS4u9qo=")
dto = MetadataDTO(test_perm_id, badInfohash)
subtitles = [SubtitleInfo(lang,path) for lang,path in self._srtSubs.iteritems()]
for sub in subtitles :
sub.computeChecksum()
dto.addSubtitle(sub)
packed = dto._packData()
decoded = bdecode(packed)
self.assertTrue(len(decoded) == 6)
decodedChannelId = decoded[0]
decodedInfohash = decoded[1]
decodedDescription = decoded[2]
decodedTimestamp = decoded[3]
decodedBitmask = decoded[4]
checksums = decoded[5]
expectedMask = \
LanguagesProvider.getLanguagesInstance().langCodesToMask(self._srtSubs.keys())
binaryExpexted = uintToBinaryString(expectedMask)
self.assertEquals(dto.channel, decodedChannelId)
self.assertEquals(dto.infohash, decodedInfohash)
self.assertEquals(dto.description,decodedDescription)
self.assertAlmostEquals(dto.timestamp,decodedTimestamp)
self.assertEquals(binaryExpexted,decodedBitmask)
self.assertEquals(3,len(checksums))
subs = dto.getAllSubtitles()
i=0
for key in sorted(subs.iterkeys()):
self.assertEquals(subs[key].checksum, checksums[i])
i += 1
def testSignature(self):
badInfohash = str2bin("GEh/o8rtTLB1wZJzFcSZSS4u9qo=")
dto = MetadataDTO(test_perm_id, badInfohash)
dto.sign(test_keypair)
self.assertTrue(dto.verifySignature())
dto.timestamp = 2
ok = dto.verifySignature()
self.assertFalse(ok)
def testSignatureOnChecksums(self):
badInfohash = str2bin("GEh/o8rtTLB1wZJzFcSZSS4u9qo=")
dto = MetadataDTO(test_perm_id, badInfohash)
subtitles = [SubtitleInfo(lang,path) for lang,path in self._srtSubs.iteritems()]
for sub in subtitles :
sub.computeChecksum()
dto.addSubtitle(sub)
dto.sign(test_keypair)
self.assertTrue(dto.verifySignature())
dto.getSubtitle("rus").checksum = "ABCDEFGHILMOPQRS"
self.assertFalse(dto.verifySignature())
def setupDB(self,nickname):
TestChannels.setupDB(self,nickname)
try:
self.richMetadata_db = self.session.open_dbhandler(NTFY_RICH_METADATA)
#add some metadata for torrents (they are defined in TestChannels.setupDB()
self.mdto = MetadataDTO(self.hispermid, self.infohash1)
subtitle1 = SubtitleInfo("nld", os.path.join(RES_DIR,"fake.srt"))
subtitle1.computeChecksum()
subtitle2 = SubtitleInfo("eng", os.path.join(RES_DIR, "fake0.srt"))
subtitle2.computeChecksum()
self.mdto.addSubtitle(subtitle1)
self.mdto.addSubtitle(subtitle2)
self.mdto.sign(self.his_keypair)
self.richMetadata_db.insertMetadata(self.mdto)
except:
print_exc()
def testDeserializeWithSubs(self):
badInfohash = str2bin("GEh/o8rtTLB1wZJzFcSZSS4u9qo=")
dto = MetadataDTO(test_perm_id, badInfohash)
subtitles = [SubtitleInfo(lang,path) for lang,path in self._srtSubs.iteritems()]
for sub in subtitles :
sub.computeChecksum()
dto.addSubtitle(sub)
dto.sign(test_keypair)
serial = dto.serialize()
newDto = MDUtil.deserialize(serial)
self.assertEquals(dto,newDto)
def testSerializeWithSubs(self):
badInfohash = str2bin("GEh/o8rtTLB1wZJzFcSZSS4u9qo=")
dto = MetadataDTO(test_perm_id, badInfohash)
subtitles = [SubtitleInfo(lang,path) for lang,path in self._srtSubs.iteritems()]
for sub in subtitles :
sub.computeChecksum()
dto.addSubtitle(sub)
dto.sign(test_keypair)
serial = dto.serialize()
decoded = serial
self.assertEquals(7, len(decoded))
signature = decoded[6]
self.assertEquals(dto.signature,signature)
def setUpDB(self):
try:
self.richMetadata_db = self.session.open_dbhandler(NTFY_RICH_METADATA)
#add some metadata
self.mdto = MetadataDTO(self.anotherpermid, self.testInfohash)
subtitle1 = SubtitleInfo("nld", self.sub1)
subtitle1.computeChecksum()
subtitle2 = SubtitleInfo("eng", os.path.join(RES_DIR, "fake0.srt"))
subtitle2.computeChecksum()
self.mdto.addSubtitle(subtitle1)
self.mdto.addSubtitle(subtitle2)
self.mdto.sign(self.another_keypair)
self.richMetadata_db.insertMetadata(self.mdto)
#hisoermid has the nld subtitle but doesn't have the english one
self.richMetadata_db.updateSubtitlePath(self.mdto.channel,self.mdto.infohash,"eng",None)
except:
print_exc()
class TestChannelsPlusSubtitles(TestChannels):
"""
Testing the rich metadata extension of channelcast.
The test suite defined in this module executes all the old
channelcast tests, plus a test to validate that the rich metadata
(currently subtitles) extension works properly
"""
def setupDB(self,nickname):
TestChannels.setupDB(self,nickname)
try:
self.richMetadata_db = self.session.open_dbhandler(NTFY_RICH_METADATA)
#add some metadata for torrents (they are defined in TestChannels.setupDB()
self.mdto = MetadataDTO(self.hispermid, self.infohash1)
subtitle1 = SubtitleInfo("nld", os.path.join(RES_DIR,"fake.srt"))
subtitle1.computeChecksum()
subtitle2 = SubtitleInfo("eng", os.path.join(RES_DIR, "fake0.srt"))
subtitle2.computeChecksum()
self.mdto.addSubtitle(subtitle1)
self.mdto.addSubtitle(subtitle2)
self.mdto.sign(self.his_keypair)
self.richMetadata_db.insertMetadata(self.mdto)
except:
print_exc()
def tearDown(self):
TestChannels.tearDown(self)
self.session.close_dbhandler(self.richMetadata_db)
def _test_all(self,nickname):
"""
I want to start a Tribler client once and then connect to
it many times. So there must be only one test method
to prevent setUp() from creating a new client every time.
The code is constructed so unittest will show the name of the
(sub)test where the error occured in the traceback it prints.
"""
TestChannels._test_all(self,nickname)
self.subtest_channelcastPlusMetadata()
def subtest_channelcastPlusMetadata(self):
'''
Extends channelcast test to channelcast messages enriched with
metadata (subtitles) informations
'''
print >>sys.stderr,"test: channelcast_subtitles ---------------------------"
s = OLConnection(self.my_keypair,'localhost',self.hisport)
chcast = ChannelCastCore(None, s, self.session, None, log = '', dnsindb = None)
#test send standard channelcast
chdata = {}
print >> sys.stderr, "Test Good ChannelCast Plus Subtitles", `chdata`
msg = CHANNELCAST+bencode(chdata)
s.send(msg)
resp = s.recv()
if len(resp) > 0:
print >>sys.stderr,"test: channelcast_subtitles: got",getMessageName(resp[0])
self.assert_(resp[0]==CHANNELCAST)
print >>sys.stderr, "test: channelcast_subtitles: got msg", `bdecode(resp[1:])`
chdata_rcvd = bdecode(resp[1:])
self.assertTrue(validChannelCastMsg(chdata_rcvd))
for entry in chdata_rcvd.itervalues():
if entry['infohash'] == self.infohash1: #the torrent for which two subtitles exist
self.assertTrue('rich_metadata' in entry.keys())
richMetadata = entry['rich_metadata']
print >> sys.stderr, "test: channelcast_subtitles: richMetadata entry is ", richMetadata
self.assertEquals(6, len(richMetadata))
self.assertEquals(self.mdto.description, richMetadata[0])
self.assertEquals(4, len(richMetadata[2])) #the subtitles mask 4 bytes
self.assertTrue(isinstance(richMetadata[3],list)) #the subtitles checsums
for checksum in richMetadata[3]:
self.assertEquals(20,len(checksum)) #160 bit sha1 checksum
self.assertEquals(self.mdto.signature, richMetadata[4])
self.assertEquals(4,len(richMetadata[5])) #the subtitles have mask 32 bit
#also must (in this case) be equal to the subtitles mask
self.assertEquals(richMetadata[2], richMetadata[5])
print >> sys.stderr, "test: channelcast_subtitles; richMetadata entry is valid and correct"
else:
self.assertFalse('rich_metadata' in entry.keys())
s.close()
#Now, send a bad ChannelCast message.
# The other side should close the connection
# Create bad message by manipulating a good one
#bad bitmask
chdata = deepcopy(chdata_rcvd)
for k,v in chdata.items():
if 'rich_metadata' in v:
v['rich_metadata'][2] = 44 #an integer instead of a 4bytes bitmask
self.subtest_bad_channelcast(chdata)
#Bad message format
chdata = deepcopy(chdata_rcvd)
for k,v in chdata.items():
if 'rich_metadata' in v:
v['rich_metadata'].insert(0, u"asdfafa22")
self.subtest_bad_channelcast(chdata)
#Bad
print>>sys.stderr, "End of channelcast_subtitles test ---------------------------"
class TestSubtitleMessages(TestAsServer):
def setUpPreSession(self):
""" override TestAsServer """
TestAsServer.setUpPreSession(self)
self.config.set_buddycast(True)
BuddyCastCore.TESTASSERVER = True
ChannelCastCore.TESTASSERVER = True
VoteCastCore.TESTASSERVER = True
self.config.set_start_recommender(True)
self.config.set_bartercast(True)
self.config.set_remote_query(True)
self.config.set_crawler(False)
self.config.set_torrent_collecting_dir(os.path.join(self.config_path, "tmp_torrent_collecting"))
self.collecting_dir = os.path.join(self.config_path, "temp_subtitles_collecting")
os.makedirs(self.collecting_dir)
self.config.set_subtitles_collecting(True)
self.config.set_subtitles_collecting_dir(self.collecting_dir)
# # Write superpeers.txt and DB schema
self.install_path = tempfile.mkdtemp()
spdir = os.path.join(self.install_path, LIBRARYNAME, 'Core')
os.makedirs(spdir)
statsdir = os.path.join(self.install_path, LIBRARYNAME, 'Core', 'Statistics')
os.makedirs(statsdir)
superpeerfilename = os.path.join(spdir, 'superpeer.txt')
print >> sys.stderr,time.asctime(),'-', "test: writing empty superpeers to",superpeerfilename
f = open(superpeerfilename, "w")
f.write('# Leeg')
f.close()
self.config.set_install_dir(self.install_path)
srcfiles = []
srcfiles.append(os.path.join(LIBRARYNAME,"schema_sdb_v5.sql"))
for srcfile in srcfiles:
sfn = os.path.join('..','..',srcfile)
dfn = os.path.join(self.install_path,srcfile)
print >>sys.stderr,time.asctime(),'-', "test: copying",sfn,dfn
shutil.copyfile(sfn,dfn)
#copy subtitles files in the appropriate subtitles folder
self.src1 = os.path.join(RES_DIR,'fake.srt')
self.src2 = os.path.join(RES_DIR,'fake0.srt')
def setUpPostSession(self):
""" override TestAsServer """
TestAsServer.setUpPostSession(self)
self.mypermid = str(self.my_keypair.pub().get_der())
self.hispermid = str(self.his_keypair.pub().get_der())
self.another_keypair = generate_keypair()
self.anotherpermid = str(self.another_keypair.pub().get_der())
self.testInfohash = hashlib.sha1("yoman!").digest()
#copy subtitles in the collecting dir
nldName = SubUtils.getSubtitleFileRelativeName(self.anotherpermid, self.testInfohash, "nld")
engName = SubUtils.getSubtitleFileRelativeName(self.anotherpermid, self.testInfohash, "eng")
self.sub1 = os.path.join(self.collecting_dir, nldName)
self.sub2 = os.path.join(self.collecting_dir, engName)
shutil.copyfile(self.src1, self.sub1)
# Let's say that the receiving peer has only the nld subtitle
# avialable locally
# shutil.copyfile(self.src2, self.sub2)
def setUpDB(self):
try:
self.richMetadata_db = self.session.open_dbhandler(NTFY_RICH_METADATA)
#add some metadata
self.mdto = MetadataDTO(self.anotherpermid, self.testInfohash)
subtitle1 = SubtitleInfo("nld", self.sub1)
subtitle1.computeChecksum()
subtitle2 = SubtitleInfo("eng", os.path.join(RES_DIR, "fake0.srt"))
subtitle2.computeChecksum()
self.mdto.addSubtitle(subtitle1)
self.mdto.addSubtitle(subtitle2)
self.mdto.sign(self.another_keypair)
self.richMetadata_db.insertMetadata(self.mdto)
#hisoermid has the nld subtitle but doesn't have the english one
self.richMetadata_db.updateSubtitlePath(self.mdto.channel,self.mdto.infohash,"eng",None)
except:
print_exc()
def tearDown(self):
TestAsServer.tearDown(self)
self.session.close_dbhandler(self.richMetadata_db)
def subtest_receptionOfSUBS(self):
'''
Asking for the single available subtitle. The response should be
a valid SUBS message containing its contents
'''
print >> sys.stderr, time.asctime(),'-', "test: test_subtitles_msgs_1_1 -----------------------"
ol_conn = OLConnection(self.my_keypair,'localhost',self.hisport)
bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(['nld'])
binmask = utilities.uintToBinaryString(bitmask, length=4)
request = GET_SUBS + \
bencode((
self.anotherpermid,
self.testInfohash,
binmask
))
subshandler = SubtitlesHandler()
subshandler.register(ol_conn, self.richMetadata_db, self.session)
ol_conn.send(request)
subs_data = ol_conn.recv()
print >> sys.stderr, time.asctime(),'-', "test: subtitles_messages : received SUBS response: len",len(subs_data)
self.assertEquals(SUBS, subs_data[0])
data = bdecode(subs_data[1:])
print >> sys.stderr, time.asctime(),'-', "test: subtitles_messages : received SUBS response: ", data
#check on the format of the response
self.assertTrue(isinstance(data,list))
self.assertEquals(4, len(data)) # for fields
self.assertEquals(self.mdto.channel,data[0])
self.assertEquals(self.mdto.infohash, data[1])
self.assertEquals(binmask, data[2])
self.assertTrue(isinstance(data[3],list))
self.assertEquals(1, len(data[3]))
with codecs.open(self.sub1, "rb", "utf-8") as sub:
expectedContents = sub.read()
self.assertEquals(expectedContents, data[3][0])
ol_conn.close()
print >> sys.stderr, time.asctime(),'-', "test: subtitles_messages: received content is valid."
print >> sys.stderr, time.asctime(),'-', "End of test_subtitles_msgs_1_1 test --------------------"
def subtest_receptionOfSUBSTwoRequestsOneAvailable(self):
"""
Asking for two subtitles while the recipent of the request has only one.
The response should contain only the one available subtitle content,
plus a bitmask that reflects the contents of the response.
"""
print >> sys.stderr, time.asctime(),'-', "test: test_subtitles_msgs_2_1 -----------------------"
ol_conn = OLConnection(self.my_keypair,'localhost',self.hisport)
bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(['nld','eng'])
binmask = utilities.uintToBinaryString(bitmask, length=4)
request = GET_SUBS + \
bencode((
self.anotherpermid,
self.testInfohash,
binmask
))
subshandler = SubtitlesHandler()
subshandler.register(ol_conn, self.richMetadata_db, self.session)
ol_conn.send(request)
subs_data = ol_conn.recv()
self.assertEquals(SUBS, subs_data[0])
data = bdecode(subs_data[1:])
print >> sys.stderr, time.asctime(),'-', "test: subtitles_messages : received SUBS repsonse: ", data
#check on the format of the response
self.assertTrue(isinstance(data,list))
self.assertEquals(4, len(data)) # for fields
self.assertEquals(self.mdto.channel,data[0])
self.assertEquals(self.mdto.infohash, data[1])
#the receiver had only one of the two requested subtitles
# so I expect a different bitmask
bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(['nld'])
expectedBinarymask = utilities.uintToBinaryString(bitmask, length=4)
self.assertEquals(expectedBinarymask, data[2])
self.assertTrue(isinstance(data[3],list))
self.assertEquals(1, len(data[3]))
with codecs.open(self.sub1, "rb", "utf-8") as sub:
expectedContents = sub.read()
self.assertEquals(expectedContents, data[3][0])
ol_conn.close()
print >> sys.stderr, time.asctime(),'-', "test: subtitles_messages: received content is valid."
print >> sys.stderr, time.asctime(),'-', "End of test_subtitles_msgs_2_1 test --------------------"
def subtest_invalidRequest1(self):
"""
Trying to send an empty message.
The connection should be closed by the receiver
"""
print >> sys.stderr, time.asctime(),'-', "test: test_subtitles_msgs_invalid_request_1 ------------------"
ol_conn = OLConnection(self.my_keypair,'localhost',self.hisport)
request = GET_SUBS + \
bencode({})
ol_conn.send(request)
self.assertEquals(0, len(ol_conn.recv()))
print >> sys.stderr, time.asctime(),'-', "test: test_subtitles_msgs_invalid_request_1: connection closed as expected"
ol_conn.close()
print >> sys.stderr, time.asctime(),'-', "End of test_subtitles_msgs_invalid_request_1 ------------------"
def subtest_invalidRequest2(self):
"""
Trying to send an invalid message (an integer instead of a 4 bytes binary string)
The connection should be closed by the receiver
"""
print >> sys.stderr, time.asctime(),'-', "test: test_subtitles_msgs_invalid_request_2 ------------------"
ol_conn = OLConnection(self.my_keypair,'localhost',self.hisport)
request = GET_SUBS + \
bencode((
self.anotherpermid,
self.testInfohash,
42
))
ol_conn.send(request)
self.assertEquals(0, len(ol_conn.recv()))
print >> sys.stderr, time.asctime(),'-', "test: test_subtitles_msgs_invalid_request_2: connection closed as expected"
ol_conn.close()
print >> sys.stderr, time.asctime(),'-', "End of test_subtitles_msgs_invalid_request_2 ------------------"
def subtest_invalidRequest3(self):
"""
Trying to send an invalid message (valid for everythin except that there is one field more)
The connection should be closed by the receiver
"""
print >> sys.stderr, time.asctime(),'-', "test: test_subtitles_msgs_invalid_request_3 ------------------"
ol_conn = OLConnection(self.my_keypair,'localhost',self.hisport)
bitmask = LanguagesProvider.getLanguagesInstance().langCodesToMask(['nld','eng'])
binmask = utilities.uintToBinaryString(bitmask, length=4)
request = GET_SUBS + \
bencode((
self.anotherpermid,
self.testInfohash,
binmask,
42
))
ol_conn.send(request)
self.assertEquals(0, len(ol_conn.recv()))
print >> sys.stderr, time.asctime(),'-', "test: test_subtitles_msgs_invalid_request_3: connection closed as expected"
ol_conn.close()
print >> sys.stderr, time.asctime(),'-', "End of test_subtitles_msgs_invalid_request_3 ------------------"
def singtest_subs_messages(self):
self.setUpDB()
self.subtest_receptionOfSUBS()
self.subtest_receptionOfSUBSTwoRequestsOneAvailable()
self.subtest_invalidRequest1()
self.subtest_invalidRequest2()
self.subtest_invalidRequest3()
