def test_get_channel_name(self):
"""
Test getting torrent name for a channel to be displayed in the downloads list
"""
infohash = b"\x00" * 20
title = "testchan"
chan = self.mds.ChannelMetadata(title=title,
infohash=database_blob(infohash))
dirname = chan.dirname
self.assertEqual(
title,
self.mds.ChannelMetadata.get_channel_name(dirname, infohash))
self.assertEqual(
title,
self.mds.ChannelMetadata.get_channel_name_cached(
dirname, infohash))
chan.infohash = b"\x11" * 20
self.assertEqual(
"OLD:" + title,
self.mds.ChannelMetadata.get_channel_name(dirname, infohash))
chan.delete()
self.assertEqual(
dirname,
self.mds.ChannelMetadata.get_channel_name(dirname, infohash))
# Check that the cached version of the name is returned even if the channel has been deleted
self.mds.ChannelMetadata.get_channel_name = Mock()
self.assertEqual(
title,
self.mds.ChannelMetadata.get_channel_name_cached(
dirname, infohash))
self.mds.ChannelMetadata.get_channel_name.assert_not_called()
def test_squash_mdblobs_multiple_chunks(metadata_store):
r = random.Random(123)
md_list = [
metadata_store.TorrentMetadata(
title=''.join(
r.choice(string.ascii_uppercase + string.digits)
for _ in range(20)),
infohash=database_blob(random_infohash()),
torrent_date=datetime.utcfromtimestamp(100),
) for _ in range(0, 10)
]
# Test splitting into multiple chunks
chunk, index = entries_to_chunk(md_list, chunk_size=900)
chunk2, _ = entries_to_chunk(md_list, chunk_size=900, start_index=index)
dict_list = [d.to_dict()["signature"] for d in md_list]
for d in md_list:
d.delete()
assert dict_list[:index] == [
d.md_obj.to_dict()["signature"]
for d in metadata_store.process_compressed_mdblob(
chunk, skip_personal_metadata_payload=False)
]
assert dict_list[index:] == [
d.md_obj.to_dict()["signature"]
for d in metadata_store.process_compressed_mdblob(
chunk2, skip_personal_metadata_payload=False)
]
def test_process_channel_dir_file(tmpdir, metadata_store):
"""
Test whether we are able to process files in a directory containing node metadata
"""
test_node_metadata = metadata_store.TorrentMetadata(title='test', infohash=database_blob(random_infohash()))
metadata_path = tmpdir / 'metadata.data'
test_node_metadata.to_file(metadata_path)
# We delete this TorrentMeta info now, it should be added again to the database when loading it
test_node_metadata.delete()
loaded_metadata = metadata_store.process_mdblob_file(metadata_path, skip_personal_metadata_payload=False)
assert loaded_metadata[0][0].title == 'test'
# Test whether we delete existing metadata when loading a DeletedMetadata blob
metadata = metadata_store.TorrentMetadata(infohash=b'1' * 20)
metadata.to_delete_file(metadata_path)
loaded_metadata = metadata_store.process_mdblob_file(metadata_path, skip_personal_metadata_payload=False)
# Make sure the original metadata is deleted
assert loaded_metadata[0] == (None, 6)
assert metadata_store.TorrentMetadata.get(infohash=b'1' * 20) is None
# Test an unknown metadata type, this should raise an exception
invalid_metadata = tmpdir / 'invalidtype.mdblob'
make_wrong_payload(invalid_metadata)
with pytest.raises(UnknownBlobTypeException):
metadata_store.process_mdblob_file(invalid_metadata, skip_personal_metadata_payload=False)
def add_vote_to_votes(self, voter_public_key, module_identifier):
"""
Add vote to votes database
:param voter_public_key: Public key of the voter
:type voter_public_key: bytes
:param module_identifier: module identifier
:type module_identifier: ModuleIdentifier
:return: None
"""
self._logger.debug("persistence: Add vote (%s, %s) to votes", hexlify(voter_public_key), module_identifier)
sql = "INSERT INTO module_votes (voter_public_key, public_key, info_hash) VALUES (?, ?, ?);"
self.execute(sql, (database_blob(voter_public_key), database_blob(module_identifier.creator),
database_blob(module_identifier.content_hash),))
self.commit()
async def test_reject_malformed_channel(self):
with db_session:
channel = self.mock_session.mds.ChannelMetadata(
title="bla1", public_key=database_blob(b'123'), infohash=random_infohash()
)
self.mock_session.config = MockObject()
self.mock_session.config.get_state_dir = lambda: None
self.mock_session.dlmgr = MockObject()
def mock_get_metainfo_bad(*args, **kwargs):
return succeed({b'info': {b'name': b'bla'}})
def mock_get_metainfo_good(*args, **kwargs):
return succeed({b'info': {b'name': channel.dirname.encode('utf-8')}})
self.initiated_download = False
def mock_download_from_tdef(*_, **__):
self.initiated_download = True
mock_dl = MockObject()
mock_dl.future_finished = succeed(None)
return mock_dl
self.mock_session.dlmgr.start_download = mock_download_from_tdef
# Check that we skip channels with incorrect dirnames
self.mock_session.dlmgr.get_metainfo = mock_get_metainfo_bad
await self.chanman.download_channel(channel)
self.assertFalse(self.initiated_download)
with patch.object(TorrentDef, "__init__", lambda *_, **__: None):
# Check that we download channels with correct dirname
self.mock_session.dlmgr.get_metainfo = mock_get_metainfo_good
await self.chanman.download_channel(channel)
self.assertTrue(self.initiated_download)
def make_wrong_payload(filename):
key = default_eccrypto.generate_key("curve25519")
metadata_payload = SignedPayload(
666, 0, database_blob(key.pub().key_to_bin()[10:]), signature=b'\x00' * 64, skip_key_check=True
)
with open(filename, 'wb') as output_file:
output_file.write(metadata_payload.serialized())
def test_skip_processing_of_received_personal_channel_torrents(metadata_store):
"""
Test that personal torrent is ignored by default when processing the torrent metadata payload
"""
channel = metadata_store.ChannelMetadata.create_channel('testchan')
torrent_md = metadata_store.TorrentMetadata(
origin_id=channel.id_, title='test', status=NEW, infohash=database_blob(random_infohash())
)
channel.commit_channel_torrent()
torrent_md.delete()
channel_dir = Path(metadata_store.ChannelMetadata._channels_dir) / channel.dirname
assert os.listdir(str_path(channel_dir))
# By default, personal channel torrent metadata processing is skipped so there should be no torrents
# added to the channel
channel.local_version = 0
metadata_store.process_channel_dir(channel_dir, channel.public_key, channel.id_)
assert not channel.contents
# Enable processing of personal channel torrent metadata
channel.local_version = 0
metadata_store.process_channel_dir(
channel_dir, channel.public_key, channel.id_, skip_personal_metadata_payload=False
)
assert len(channel.contents) == 1
def test_get_channel_name(metadata_store):
"""
Test getting torrent name for a channel to be displayed in the downloads list
"""
infohash = b"\x00" * 20
title = "testchan"
chan = metadata_store.ChannelMetadata(title=title,
infohash=database_blob(infohash))
dirname = chan.dirname
assert title == metadata_store.ChannelMetadata.get_channel_name(
dirname, infohash)
assert title == metadata_store.ChannelMetadata.get_channel_name_cached(
dirname, infohash)
chan.infohash = b"\x11" * 20
assert "OLD:" + title == metadata_store.ChannelMetadata.get_channel_name(
dirname, infohash)
chan.delete()
assert dirname == metadata_store.ChannelMetadata.get_channel_name(
dirname, infohash)
# Check that the cached version of the name is returned even if the channel has been deleted
metadata_store.ChannelMetadata.get_channel_name = Mock()
assert title == metadata_store.ChannelMetadata.get_channel_name_cached(
dirname, infohash)
metadata_store.ChannelMetadata.get_channel_name.assert_not_called()
def test_multiple_squashed_commit_and_read(metadata_store):
"""
Test committing entries into several squashed blobs and reading them back
"""
metadata_store.ChannelMetadata._CHUNK_SIZE_LIMIT = 500
num_entries = 10
channel = metadata_store.ChannelMetadata.create_channel('testchan')
md_list = [
metadata_store.TorrentMetadata(
origin_id=channel.id_, title='test' + str(x), status=NEW, infohash=database_blob(random_infohash())
)
for x in range(0, num_entries)
]
channel.commit_channel_torrent()
channel.local_version = 0
for md in md_list:
md.delete()
channel_dir = Path(metadata_store.ChannelMetadata._channels_dir) / channel.dirname
assert len(os.listdir(channel_dir)) > 1 # make sure it was broken into more than one .mdblob file
metadata_store.process_channel_dir(
channel_dir, channel.public_key, channel.id_, skip_personal_metadata_payload=False
)
assert num_entries == len(channel.contents)
async def is_channel_dirty(self, request):
channel_pk, _ = self.get_channel_from_request(request)
with db_session:
dirty = self.session.mds.MetadataNode.exists(
lambda g: g.public_key == database_blob(
channel_pk) and g.status in DIRTY_STATUSES)
return RESTResponse({"dirty": dirty})
def delete_transaction(self, transaction_id):
"""
Delete a specific transaction from the database
"""
self.execute(u"DELETE FROM transactions WHERE transaction_id = ?",
(database_blob(bytes(transaction_id)), ))
self.delete_payments(transaction_id)
def insert_or_update_transaction(self, transaction):
"""
Inserts or updates a specific transaction in the database, according to the timestamp.
Updates only if the timestamp is more recent than the one in the database.
"""
self.execute(
u"INSERT OR IGNORE INTO transactions (trader_id, transaction_id, order_number,"
u"partner_trader_id, partner_order_number, asset1_amount, asset1_type, asset1_transferred, asset2_amount,"
u"asset2_type, asset2_transferred, transaction_timestamp, sent_wallet_info, received_wallet_info,"
u"incoming_address, outgoing_address, partner_incoming_address, partner_outgoing_address) "
u"VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
transaction.to_database())
self.execute(
u"UPDATE transactions SET asset1_amount = ?, asset1_transferred = ?, asset2_amount = ?, "
u"asset2_transferred = ?, transaction_timestamp = ? WHERE transaction_id = ?"
u"AND transaction_timestamp < ?",
(transaction.assets.first.amount,
transaction.transferred_assets.first.amount,
transaction.assets.second.amount,
transaction.transferred_assets.second.amount,
int(transaction.timestamp),
database_blob(bytes(
transaction.transaction_id)), int(transaction.timestamp)))
self.commit()
def get_votes_for_peer(self, peer):
"""
Get votes for peer
:param peer: public key of peer
:type peer: bytes
:return: vote information
"""
self._logger.debug("persistence: Getting votes for peer (%s)", hexlify(peer))
sql = "SELECT * FROM module_votes WHERE voter_public_key = ?;"
res = list(self.execute(sql, (database_blob(peer),)))
votes = []
for vote in res:
voter_public_key = bytes(vote[0])
creator = bytes(vote[1])
content_hash = str(vote[2])
identifier = ModuleIdentifier(creator, content_hash)
votes.append({
'voter': voter_public_key,
'identifier': identifier,
})
return votes
async def on_search_request(self, peer, request):
# Caution: beware of potential SQL injection!
# Since this string 'txt_filter' is passed as it is to fetch the results, there could be a chance for
# SQL injection. But since we use Pony which is supposed to be doing proper variable bindings, it should
# be relatively safe.
txt_filter = request.txt_filter.decode('utf8')
# Check if the txt_filter is a simple query
if not is_simple_match_query(txt_filter):
self.logger.error("Dropping a complex remote search query:%s",
txt_filter)
return
metadata_type = v1_md_field_to_metadata_type.get(
request.metadata_type.decode('utf8'),
frozenset((REGULAR_TORRENT, CHANNEL_TORRENT)))
# If we get a hex-encoded public key in the txt_filter field, we drop the filter,
# and instead query by public_key. However, we only do this if there is no channel_pk or
# origin_id attributes set, because it is only for support of GigaChannel v1.0 channel preview requests.
channel_pk = None
normal_filter = txt_filter.replace('"', '').replace("*", "")
if (metadata_type == frozenset((REGULAR_TORRENT, COLLECTION_NODE))
and is_hex_string(normal_filter)
and len(normal_filter) % 2 == 0
and is_channel_public_key(normal_filter)):
channel_pk = database_blob(unhexlify(normal_filter))
txt_filter = None
request_dict = {
"first": 1,
"last": max_entries,
"sort_by": request.sort_by.decode('utf8'),
"sort_desc":
not request.sort_asc if request.sort_asc is not None else None,
"txt_filter": txt_filter,
"hide_xxx": request.hide_xxx,
"metadata_type": metadata_type,
"exclude_legacy": True,
"channel_pk": channel_pk,
}
def _get_search_results():
with db_session:
db_results = self.metadata_store.MetadataNode.get_entries(
**request_dict)
result = entries_to_chunk(
db_results[:max_entries],
maximum_payload_size)[0] if db_results else None
self.metadata_store.disconnect_thread()
return result
result_blob = await get_event_loop().run_in_executor(
None, _get_search_results)
if result_blob:
self.endpoint.send(
peer.address,
self.ezr_pack(self.SEARCH_RESPONSE,
SearchResponsePayload(request.id, result_blob)))
def fake_get_metainfo(infohash, **_):
return {
'info': {
'name':
session.mds.ChannelMetadata.get(
infohash=database_blob(infohash)).dirname
}
}
def get_payments(self, transaction_id):
"""
Return all payment tied to a specific transaction.
"""
db_result = self.execute(
u"SELECT * FROM payments WHERE transaction_id = ? ORDER BY timestamp ASC",
(database_blob(bytes(transaction_id)), ))
return [Payment.from_database(db_item) for db_item in db_result]
def check_update_opportunity():
# Check for possible update sending opportunity.
node = self.TorrentMetadata.get(
lambda g: g.public_key == database_blob(
payload.public_key) and g.id_ == payload.id_ and g.
timestamp > payload.timestamp)
return [(node, GOT_NEWER_VERSION)] if node else [(None,
NO_ACTION)]
def delete_transaction(self, transaction_id):
"""
Delete a specific transaction from the database
"""
self.execute(u"DELETE FROM transactions WHERE trader_id = ? AND transaction_number = ?",
(database_blob(bytes(transaction_id.trader_id)),
text_type(transaction_id.transaction_number)))
self.delete_payments(transaction_id)
def delete_reserved_ticks(self, order_id):
"""
Delete all reserved ticks from a specific order
"""
self.execute(
u"DELETE FROM orders_reserved_ticks WHERE trader_id = ? AND order_number = ?",
(database_blob(bytes(
order_id.trader_id)), str(order_id.order_number)))
def needle_in_haystack(enable_chant, enable_api, session): # pylint: disable=unused-argument
num_hay = 100
with db_session:
_ = session.mds.ChannelMetadata(title='test',
tags='test',
subscribed=True,
infohash=random_infohash())
for x in range(0, num_hay):
session.mds.TorrentMetadata(title='hay ' + str(x),
infohash=random_infohash())
session.mds.TorrentMetadata(title='needle',
infohash=database_blob(
bytearray(random_infohash())))
session.mds.TorrentMetadata(title='needle2',
infohash=database_blob(
bytearray(random_infohash())))
return session
async def copy_channel(self, request):
with db_session:
channel_pk, channel_id = self.get_channel_from_request(request)
personal_root = channel_id == 0 and channel_pk == self.session.mds.my_key.pub(
).key_to_bin()[10:]
# TODO: better error handling
target_collection = self.session.mds.CollectionNode.get(
public_key=database_blob(channel_pk), id_=channel_id)
try:
request_parsed = await request.json()
except (ContentTypeError, ValueError):
return RESTResponse({"error": "Bad JSON"},
status=HTTP_BAD_REQUEST)
if not target_collection and not personal_root:
return RESTResponse({"error": "Target channel not found"},
status=HTTP_NOT_FOUND)
results_list = []
for entry in request_parsed:
public_key, id_ = database_blob(unhexlify(
entry["public_key"])), entry["id"]
source = self.session.mds.ChannelNode.get(
public_key=public_key, id_=id_)
if not source:
return RESTResponse({"error": "Source entry not found"},
status=HTTP_BAD_REQUEST)
# We must upgrade Collections to Channels when moving them to root channel, and, vice-versa,
# downgrade Channels to Collections when moving them into existing channels
if isinstance(source, self.session.mds.CollectionNode):
src_dict = source.to_dict()
if channel_id == 0:
rslt = self.session.mds.ChannelMetadata.create_channel(
title=source.title)
else:
dst_dict = {'origin_id': channel_id, "status": NEW}
for k in self.session.mds.CollectionNode.nonpersonal_attributes:
dst_dict[k] = src_dict[k]
dst_dict.pop("metadata_type")
rslt = self.session.mds.CollectionNode(**dst_dict)
for child in source.actual_contents:
child.make_copy(rslt.id_)
else:
rslt = source.make_copy(channel_id)
results_list.append(rslt.to_simple_dict())
return RESTResponse(results_list)
def get_updated_channels(cls):
return select(
g
for g in cls
if g.subscribed == 1
and g.status != LEGACY_ENTRY
and (g.local_version < g.timestamp)
and g.public_key != database_blob(cls._my_key.pub().key_to_bin()[10:])
) # don't simplify `g.subscribed == 1` to bool form, it is used by partial index!
def get_valid_trackers_of_torrent(self, torrent_id):
""" Get a set of valid trackers for torrent. Also remove any invalid torrent."""
db_tracker_list = self.tribler_session.mds.TorrentState.get(
infohash=database_blob(torrent_id)).trackers
return {
tracker.url
for tracker in db_tracker_list if is_valid_url(tracker.url)
and not self.is_blacklisted_tracker(tracker.url)
}
def to_database(self):
"""
Returns a database representation of a Transaction object.
:rtype: tuple
"""
return (database_blob(bytes(self.order_id.trader_id)),
database_blob(bytes(self.transaction_id)),
int(self.order_id.order_number),
database_blob(bytes(self.partner_order_id.trader_id)),
int(self.partner_order_id.order_number),
self.assets.first.amount, str(self.assets.first.asset_id),
self.transferred_assets.first.amount,
self.assets.second.amount, str(self.assets.second.asset_id),
self.transferred_assets.second.amount, int(self.timestamp),
self.sent_wallet_info, self.received_wallet_info,
str(self.incoming_address), str(self.outgoing_address),
str(self.partner_incoming_address),
str(self.partner_outgoing_address))
def torrent_exists(self, infohash):
"""
Return True if torrent with given infohash exists in the user channel
:param infohash: The infohash of the torrent
:return: True if torrent exists else False
"""
return db.TorrentMetadata.exists(
lambda g: g.public_key == self.public_key and g.infohash ==
database_blob(infohash) and g.status != LEGACY_ENTRY)
def torrent_exists_in_personal_channel(self, infohash):
"""
Return True if torrent with given infohash exists in any of user's channels
:param infohash: The infohash of the torrent
:return: True if torrent exists else False
"""
return self.TorrentMetadata.exists(
lambda g: g.public_key == self.my_public_key_bin and g.infohash ==
database_blob(infohash) and g.status != LEGACY_ENTRY)
def get_updated_channels(cls):
return select(
g
for g in cls
if g.subscribed
and g.status != LEGACY_ENTRY
and (g.local_version < g.timestamp)
and g.public_key != database_blob(cls._my_key.pub().key_to_bin()[10:])
)
def test_skip_processing_mdblob_with_forbidden_terms(metadata_store):
"""
Test that an mdblob with forbidden terms cannot ever get into the local database
"""
key = default_eccrypto.generate_key("curve25519")
chan_entry = metadata_store.ChannelMetadata(title="12yo", infohash=database_blob(random_infohash()), sign_with=key)
chan_payload = chan_entry._payload_class(**chan_entry.to_dict())
chan_entry.delete()
assert metadata_store.process_payload(chan_payload) == [(None, NO_ACTION)]
def delete_order(self, order_id):
"""
Delete a specific order from the database
"""
self.execute(
u"DELETE FROM orders WHERE trader_id = ? AND order_number = ?",
(database_blob(bytes(
order_id.trader_id)), text_type(order_id.order_number)))
self.delete_reserved_ticks(order_id)
def test_process_payload_reject_older_entry(metadata_store):
"""
Test rejecting and returning GOT_NEWER_VERSION upon receiving an older version
of an already known metadata entry
"""
torrent_old = metadata_store.TorrentMetadata(
title='blabla', timestamp=11, id_=3, infohash=database_blob(random_infohash())
)
payload_old = torrent_old._payload_class(**torrent_old.to_dict())
torrent_old.delete()
torrent_updated = metadata_store.TorrentMetadata(
title='blabla', timestamp=12, id_=3, infohash=database_blob(random_infohash())
)
# Test rejecting older version of the same entry
assert [(torrent_updated, GOT_NEWER_VERSION)] == metadata_store.process_payload(
payload_old, skip_personal_metadata_payload=False
)
