def test_cleanup_pony_experimental_db(self):
# Assert True is returned for a garbled db and nothing is done with it
garbled_db = self.session_base_dir / 'garbled.db'
with open(garbled_db, 'w') as f:
f.write("123")
self.assertRaises(sqlite3.DatabaseError, cleanup_pony_experimental_db, garbled_db)
self.assertTrue(garbled_db.exists())
# Create a Pony database of older experimental version
pony_db = self.session_base_dir / 'pony.db'
pony_db_bak = self.session_base_dir / 'pony2.db'
my_key = default_eccrypto.generate_key(u"curve25519")
mds = MetadataStore(pony_db, self.session_base_dir, my_key)
mds.shutdown()
shutil.copyfile(pony_db, pony_db_bak)
with contextlib.closing(sqlite3.connect(pony_db)) as connection, connection:
cursor = connection.cursor()
cursor.execute("DROP TABLE MiscData")
# Assert older experimental version is deleted
cleanup_pony_experimental_db(pony_db)
self.assertFalse(pony_db.exists())
# Assert recent database version is left untouched
cleanup_pony_experimental_db(pony_db_bak)
self.assertTrue(pony_db_bak.exists())
class TestTrackerState(TriblerCoreTest):
"""
Contains various tests for the TrackerState class.
"""
async def setUp(self):
await super(TestTrackerState, self).setUp()
self.my_key = default_eccrypto.generate_key(u"curve25519")
self.mds = MetadataStore(":memory:", self.session_base_dir,
self.my_key)
async def tearDown(self):
self.mds.shutdown()
await super(TestTrackerState, self).tearDown()
@db_session
def test_create_tracker_state(self):
ts = self.mds.TrackerState(
url='http://tracker.tribler.org:80/announce')
self.assertEqual(list(self.mds.TrackerState.select())[0], ts)
@db_session
def test_canonicalize_tracker_state(self):
ts = self.mds.TrackerState(
url='http://tracker.tribler.org:80/announce/')
self.assertEqual(
self.mds.TrackerState.get(
url='http://tracker.tribler.org/announce'), ts)
@db_session
def test_canonicalize_raise_on_malformed_url(self):
self.assertRaises(MalformedTrackerURLException,
self.mds.TrackerState,
url='udp://tracker.tribler.org/announce/')
def metadata_store(tmpdir):
metadata_store_path = Path(tmpdir) / 'test.db'
mds = MetadataStore(metadata_store_path,
tmpdir,
TEST_PERSONAL_KEY,
disable_sync=True)
yield mds
mds.shutdown()
async def setUp(self):
await super(TestUpgradeDB72ToPony, self).setUp()
self.my_key = default_eccrypto.generate_key(u"curve25519")
mds_db = self.session_base_dir / 'test.db'
mds_channels_dir = self.session_base_dir
self.mds = MetadataStore(mds_db, mds_channels_dir, self.my_key)
self.m = DispersyToPonyMigration(OLD_DB_SAMPLE)
self.m.initialize(self.mds)
async def setUp(self):
await super(TestChannelMetadata, self).setUp()
self.torrent_template = {
"title": "",
"infohash": b"",
"torrent_date": datetime(1970, 1, 1),
"tags": "video"
}
self.my_key = default_eccrypto.generate_key(u"curve25519")
self.mds = MetadataStore(":memory:", self.session_base_dir,
self.my_key)
async def test_upgrade_72_to_pony(upgrader, session):
old_db_sample = TESTS_DATA_DIR / 'upgrade_databases' / 'tribler_v29.sdb'
old_database_path = session.config.get_state_dir() / 'sqlite' / 'tribler.sdb'
new_database_path = session.config.get_state_dir() / 'sqlite' / 'metadata.db'
shutil.copyfile(old_db_sample, old_database_path)
await upgrader.run()
channels_dir = session.config.get_chant_channels_dir()
mds = MetadataStore(new_database_path, channels_dir, session.trustchain_keypair, db_version=6)
with db_session:
assert mds.TorrentMetadata.select().count() == 24
mds.shutdown()
def test_already_upgraded(tmpdir, metadata_store):
pony_db_path = Path(tmpdir) / 'test.db'
my_key = default_eccrypto.generate_key("curve25519")
metadata_store.shutdown()
assert not already_upgraded(pony_db_path)
mds = MetadataStore(pony_db_path, tmpdir, my_key)
with db_session:
mds.MiscData(name=CONVERSION_FROM_72, value=CONVERSION_FINISHED)
mds.shutdown()
assert already_upgraded(pony_db_path)
async def test_upgrade_pony_10to11(upgrader, session):
old_db_sample = TESTS_DATA_DIR / 'upgrade_databases' / 'pony_v10.db'
database_path = session.config.get_state_dir() / 'sqlite' / 'metadata.db'
shutil.copyfile(old_db_sample, database_path)
upgrader.upgrade_pony_db_10to11()
channels_dir = session.config.get_chant_channels_dir()
mds = MetadataStore(database_path, channels_dir, session.trustchain_keypair, check_tables=False, db_version=11)
with db_session:
# pylint: disable=protected-access
assert upgrader.column_exists_in_table(mds._db, 'TorrentState', 'self_checked')
assert int(mds.MiscData.get(name="db_version").value) == 11
mds.shutdown()
def test_new_db_version_ok(self):
pony_db = self.session_base_dir / 'pony.db'
my_key = default_eccrypto.generate_key(u"curve25519")
mds = MetadataStore(pony_db, self.session_base_dir, my_key)
mds.shutdown()
# Correct new dabatase
self.assertTrue(new_db_version_ok(pony_db))
# Wrong new database version
with contextlib.closing(sqlite3.connect(pony_db)) as connection, connection:
cursor = connection.cursor()
cursor.execute("UPDATE MiscData SET value = 12313512 WHERE name == 'db_version'")
self.assertFalse(new_db_version_ok(pony_db))
async def test_upgrade_pony_8to10(upgrader, session):
old_db_sample = TESTS_DATA_DIR / 'upgrade_databases' / 'pony_v6.db'
database_path = session.config.get_state_dir() / 'sqlite' / 'metadata.db'
shutil.copyfile(old_db_sample, database_path)
upgrader.upgrade_pony_db_6to7()
upgrader.upgrade_pony_db_7to8()
await upgrader.upgrade_pony_db_8to10()
channels_dir = session.config.get_chant_channels_dir()
mds = MetadataStore(database_path, channels_dir, session.trustchain_keypair, check_tables=False, db_version=10)
with db_session:
assert int(mds.MiscData.get(name="db_version").value) == 10
assert mds.ChannelNode.select().count() == 23
mds.shutdown()
async def test_upgrade_72_to_pony(self):
OLD_DB_SAMPLE = TESTS_DATA_DIR / 'upgrade_databases' / 'tribler_v29.sdb'
old_database_path = self.session.config.get_state_dir(
) / 'sqlite' / 'tribler.sdb'
new_database_path = self.session.config.get_state_dir(
) / 'sqlite' / 'metadata.db'
shutil.copyfile(OLD_DB_SAMPLE, old_database_path)
await self.upgrader.run()
channels_dir = self.session.config.get_chant_channels_dir()
mds = MetadataStore(new_database_path, channels_dir,
self.session.trustchain_keypair)
with db_session:
self.assertEqual(mds.TorrentMetadata.select().count(), 24)
mds.shutdown()
def test_upgrade_pony11to12(upgrader, session):
old_db_sample = TESTS_DATA_DIR / 'upgrade_databases' / 'pony_v11.db'
database_path = session.config.get_state_dir() / 'sqlite' / 'metadata.db'
shutil.copyfile(old_db_sample, database_path)
upgrader.upgrade_pony_db_11to12()
channels_dir = session.config.get_chant_channels_dir()
mds = MetadataStore(database_path, channels_dir, session.trustchain_keypair, check_tables=False, db_version=11)
with db_session:
# pylint: disable=protected-access
assert upgrader.column_exists_in_table(mds._db, 'ChannelNode', 'json_text')
assert upgrader.column_exists_in_table(mds._db, 'ChannelNode', 'binary_data')
assert upgrader.column_exists_in_table(mds._db, 'ChannelNode', 'data_type')
assert int(mds.MiscData.get(name="db_version").value) == 12
mds.shutdown()
def upgrade_pony_db_7to8(self):
"""
Upgrade GigaChannel DB from version 7 (7.4.x) to version 8 (7.5.x).
Migration should be relatively fast, so we do it in the foreground.
"""
# We have to create the Metadata Store object because Session-managed Store has not been started yet
database_path = self.session.config.get_state_dir(
) / 'sqlite' / 'metadata.db'
channels_dir = self.session.config.get_chant_channels_dir()
if not database_path.exists():
return
mds = MetadataStore(database_path,
channels_dir,
self.session.trustchain_keypair,
disable_sync=True)
self.do_upgrade_pony_db_7to8(mds)
mds.shutdown()
def test_upgrade_pony_db_7to8(upgrader, session):
"""
Test that proper additional index is created.
Also, check that the DB version is upgraded.
"""
old_db_sample = TESTS_DATA_DIR / 'upgrade_databases' / 'pony_v7.db'
old_database_path = session.config.get_state_dir() / 'sqlite' / 'metadata.db'
shutil.copyfile(old_db_sample, old_database_path)
upgrader.upgrade_pony_db_7to8()
channels_dir = session.config.get_chant_channels_dir()
mds = MetadataStore(old_database_path, channels_dir, session.trustchain_keypair, check_tables=False, db_version=8)
with db_session:
assert int(mds.MiscData.get(name="db_version").value) == 8
assert mds.Vsids[0].exp_period == 24.0 * 60 * 60 * 3
assert list(mds._db.execute('PRAGMA index_info("idx_channelnode__metadata_type")'))
mds.shutdown()
def test_already_upgraded(self):
pony_db = self.session_base_dir / 'pony.db'
my_key = default_eccrypto.generate_key(u"curve25519")
mds = MetadataStore(pony_db, self.session_base_dir, my_key)
mds.shutdown()
self.assertFalse(already_upgraded(pony_db))
mds = MetadataStore(pony_db, self.session_base_dir, my_key)
with db_session:
mds.MiscData(name=CONVERSION_FROM_72, value=CONVERSION_FINISHED)
mds.shutdown()
self.assertTrue(already_upgraded(pony_db))
async def setUp(self):
await super(TestGigaChannelManager, self).setUp()
self.torrent_template = {"title": "", "infohash": b"", "torrent_date": datetime(1970, 1, 1), "tags": "video"}
my_key = default_eccrypto.generate_key(u"curve25519")
self.mock_session = Mock()
self.mock_session.mds = MetadataStore(":memory:", self.session_base_dir, my_key)
self.chanman = GigaChannelManager(self.mock_session)
self.torrents_added = 0
def upgrade_pony_db_6to7(self):
"""
Upgrade GigaChannel DB from version 6 (7.3.0) to version 7 (7.3.1).
Migration should be relatively fast, so we do it in the foreground, without notifying the user
and breaking it in smaller chunks as we do with 72_to_pony.
"""
# We have to create the Metadata Store object because Session-managed Store has not been started yet
database_path = self.session.config.get_state_dir(
) / 'sqlite' / 'metadata.db'
channels_dir = self.session.config.get_chant_channels_dir()
if not database_path.exists():
return
mds = MetadataStore(database_path,
channels_dir,
self.session.trustchain_keypair,
disable_sync=True)
self.do_upgrade_pony_db_6to7(mds)
mds.shutdown()
def test_upgrade_pony_db_6to7(upgrader, session):
"""
Test that channels and torrents with forbidden words are cleaned up during upgrade from Pony db ver 6 to 7.
Also, check that the DB version is upgraded.
:return:
"""
old_db_sample = TESTS_DATA_DIR / 'upgrade_databases' / 'pony_v6.db'
old_database_path = session.config.get_state_dir() / 'sqlite' / 'metadata.db'
shutil.copyfile(old_db_sample, old_database_path)
upgrader.upgrade_pony_db_6to7()
channels_dir = session.config.get_chant_channels_dir()
mds = MetadataStore(old_database_path, channels_dir, session.trustchain_keypair, check_tables=False, db_version=7)
with db_session:
assert mds.TorrentMetadata.select().count() == 23
assert mds.ChannelMetadata.select().count() == 2
assert int(mds.MiscData.get(name="db_version").value) == 7
mds.shutdown()
def upgrade_pony_db_12to13(self):
"""
Upgrade GigaChannel DB from version 12 (7.9.x) to version 13 (7.11.x).
Version 12 adds index for TorrentState.last_check attribute.
"""
# We have to create the Metadata Store object because Session-managed Store has not been started yet
database_path = self.session.config.get_state_dir(
) / 'sqlite' / 'metadata.db'
channels_dir = self.session.config.get_chant_channels_dir()
if database_path.exists():
mds = MetadataStore(database_path,
channels_dir,
self.session.trustchain_keypair,
disable_sync=True,
check_tables=False,
db_version=12)
self.do_upgrade_pony_db_12to13(mds)
mds.shutdown()
async def test_upgrade_pony_db_complete(upgrader, session):
"""
Test complete update sequence for Pony DB (e.g. 6->7->8)
"""
old_db_sample = TESTS_DATA_DIR / 'upgrade_databases' / 'pony_v6.db'
old_database_path = session.config.get_state_dir() / 'sqlite' / 'metadata.db'
shutil.copyfile(old_db_sample, old_database_path)
await upgrader.run()
channels_dir = session.config.get_chant_channels_dir()
mds = MetadataStore(old_database_path, channels_dir, session.trustchain_keypair)
db = mds._db # pylint: disable=protected-access
existing_indexes = [
'idx_channelnode__metadata_type__partial',
'idx_channelnode__metadata_subscribed__partial',
'idx_torrentstate__last_check__partial',
]
removed_indexes = [
'idx_channelnode__public_key',
'idx_channelnode__status',
'idx_channelnode__size',
'idx_channelnode__share',
'idx_channelnode__subscribed',
'idx_channelnode__votes',
'idx_channelnode__tags',
'idx_channelnode__title',
'idx_channelnode__num_entries',
'idx_channelnode__metadata_type',
]
with db_session:
assert mds.TorrentMetadata.select().count() == 23
assert mds.ChannelMetadata.select().count() == 2
assert int(mds.MiscData.get(name="db_version").value) == CURRENT_DB_VERSION
for index_name in existing_indexes:
assert list(db.execute(f'PRAGMA index_info("{index_name}")'))
for index_name in removed_indexes:
assert not list(db.execute(f'PRAGMA index_info("{index_name}")'))
assert upgrader.trigger_exists(db, 'torrentstate_ai')
assert upgrader.trigger_exists(db, 'torrentstate_au')
mds.shutdown()
def create_node(self, *args, **kwargs):
metadata_store = MetadataStore(
Path(self.temporary_directory()) / ("%d.db" % self.count),
Path(self.temporary_directory()),
default_eccrypto.generate_key(u"curve25519"),
)
kwargs['metadata_store'] = metadata_store
node = super(TestGigaChannelUnits, self).create_node(*args, **kwargs)
self.count += 1
return node
def upgrade_pony_db_10to11(self):
"""
Upgrade GigaChannel DB from version 10 (7.6.x) to version 11 (7.7.x).
Version 11 adds a `self_checked` field to TorrentState table if it
already does not exist.
"""
# We have to create the Metadata Store object because Session-managed Store has not been started yet
database_path = self.session.config.get_state_dir(
) / 'sqlite' / 'metadata.db'
channels_dir = self.session.config.get_chant_channels_dir()
if not database_path.exists():
return
mds = MetadataStore(database_path,
channels_dir,
self.session.trustchain_keypair,
disable_sync=True,
check_tables=False)
self.do_upgrade_pony_db_10to11(mds)
mds.shutdown()
def upgrade_pony_db_11to12(self):
"""
Upgrade GigaChannel DB from version 11 (7.8.x) to version 12 (7.9.x).
Version 12 adds a `json_text`, `binary_data` and `data_type` fields
to TorrentState table if it already does not exist.
"""
# We have to create the Metadata Store object because Session-managed Store has not been started yet
database_path = self.session.config.get_state_dir(
) / 'sqlite' / 'metadata.db'
channels_dir = self.session.config.get_chant_channels_dir()
if not database_path.exists():
return
mds = MetadataStore(database_path,
channels_dir,
self.session.trustchain_keypair,
disable_sync=True,
check_tables=False,
db_version=11)
self.do_upgrade_pony_db_11to12(mds)
mds.shutdown()
def create_node(self, *args, **kwargs):
metadata_store = MetadataStore(
Path(self.temporary_directory()) / f"{self.count}.db",
Path(self.temporary_directory()),
default_eccrypto.generate_key("curve25519"),
disable_sync=True,
)
kwargs['metadata_store'] = metadata_store
node = super().create_node(*args, **kwargs)
self.count += 1
return node
def test_upgrade_pony_db_6to7(self):
"""
Test that channels and torrents with forbidden words are cleaned up during upgrade from Pony db ver 6 to 7.
Also, check that the DB version is upgraded.
:return:
"""
OLD_DB_SAMPLE = TESTS_DATA_DIR / 'upgrade_databases' / 'pony_v6.db'
old_database_path = self.session.config.get_state_dir(
) / 'sqlite' / 'metadata.db'
shutil.copyfile(OLD_DB_SAMPLE, old_database_path)
self.upgrader.upgrade_pony_db_6to7()
channels_dir = self.session.config.get_chant_channels_dir()
mds = MetadataStore(old_database_path, channels_dir,
self.session.trustchain_keypair)
with db_session:
self.assertEqual(mds.TorrentMetadata.select().count(), 23)
self.assertEqual(mds.ChannelMetadata.select().count(), 2)
self.assertEqual(int(mds.MiscData.get(name="db_version").value), 7)
mds.shutdown()
def create_node(self, *args, **kwargs):
metadata_store = MetadataStore(
Path(self.temporary_directory()) / f"{self.count}.db",
Path(self.temporary_directory()),
default_eccrypto.generate_key("curve25519"),
disable_sync=True,
)
kwargs['metadata_store'] = metadata_store
with mock.patch('tribler_core.modules.metadata_store.community.gigachannel_community.DiscoveryBooster'):
node = super().create_node(*args, **kwargs)
self.count += 1
return node
async def upgrade_pony_db_8to10(self):
"""
Upgrade GigaChannel DB from version 8 (7.5.x) to version 10 (7.6.x).
This will recreate the database anew, which can take quite some time.
The code is based on the copy-pasted upgrade_72_to_pony routine which is asynchronous and
reports progress to the user.
"""
database_path = self.session.config.get_state_dir(
) / 'sqlite' / 'metadata.db'
if not database_path.exists() or get_db_version(database_path) >= 10:
# Either no old db exists, or the old db version is up to date - nothing to do
return
# Otherwise, start upgrading
self.notify_starting()
tmp_database_path = database_path.parent / 'metadata_upgraded.db'
# Clean the previous temp database
if tmp_database_path.exists():
tmp_database_path.unlink()
# Create the new database
mds = MetadataStore(tmp_database_path,
None,
self.session.trustchain_keypair,
disable_sync=True,
db_version=10)
with db_session(ddl=True):
mds.drop_indexes()
mds.drop_fts_triggers()
mds.shutdown()
self._pony2pony = PonyToPonyMigration(database_path,
tmp_database_path,
self.update_status,
logger=self._logger)
duration_base = await self._pony2pony.do_migration()
await self._pony2pony.recreate_indexes(mds, duration_base)
# Remove the old DB
database_path.unlink()
if not self._pony2pony.shutting_down:
# Move the upgraded db in its place
tmp_database_path.rename(database_path)
else:
# The upgrade process was either skipped or interrupted. Delete the temp upgrade DB.
if tmp_database_path.exists():
tmp_database_path.unlink()
self.notify_done()
def create_node(self, *args, **kwargs):
mds = MetadataStore(
Path(self.temporary_directory()) / ("%d.db" % self.count),
Path(self.temporary_directory()),
default_eccrypto.generate_key("curve25519"))
torrent_checker = MockObject()
torrent_checker.torrents_checked = set()
return MockIPv8("curve25519",
PopularityCommunity,
metadata_store=mds,
torrent_checker=torrent_checker)
async def test_upgrade_pony_db_complete(upgrader, session):
"""
Test complete update sequence for Pony DB (e.g. 6->7->8)
"""
old_db_sample = TESTS_DATA_DIR / 'upgrade_databases' / 'pony_v6.db'
old_database_path = session.config.get_state_dir(
) / 'sqlite' / 'metadata.db'
shutil.copyfile(old_db_sample, old_database_path)
await upgrader.run()
channels_dir = session.config.get_chant_channels_dir()
mds = MetadataStore(old_database_path, channels_dir,
session.trustchain_keypair)
with db_session:
assert mds.TorrentMetadata.select().count() == 23
assert mds.ChannelMetadata.select().count() == 2
assert int(
mds.MiscData.get(name="db_version").value) == CURRENT_DB_VERSION
assert list(
mds._db.execute(
'PRAGMA index_info("idx_channelnode__metadata_type")'))
mds.shutdown()
async def test_upgrade_pony_db_complete(self):
"""
Test complete update sequence for Pony DB (e.g. 6->7->8)
"""
OLD_DB_SAMPLE = TESTS_DATA_DIR / 'upgrade_databases' / 'pony_v6.db'
old_database_path = self.session.config.get_state_dir(
) / 'sqlite' / 'metadata.db'
shutil.copyfile(OLD_DB_SAMPLE, old_database_path)
await self.upgrader.run()
channels_dir = self.session.config.get_chant_channels_dir()
mds = MetadataStore(old_database_path, channels_dir,
self.session.trustchain_keypair)
with db_session:
self.assertEqual(mds.TorrentMetadata.select().count(), 23)
self.assertEqual(mds.ChannelMetadata.select().count(), 2)
self.assertEqual(int(mds.MiscData.get(name="db_version").value), 8)
self.assertTrue(
list(
mds._db.execute(
'PRAGMA index_info("idx_channelnode__metadata_type")'))
)
mds.shutdown()
