def test_migration_3(rotkehlchen_api_server):
"""
Test that the third data migration for rotki works. This migration removes icons of assets
that are not valid images and update the list of ignored assets using cryptoscamdb.
"""
rotki = rotkehlchen_api_server.rest_api.rotkehlchen
icon_manager = rotki.icon_manager
btc_iconpath = _create_invalid_icon(A_BTC.identifier, icon_manager.icons_dir)
eth_iconpath = _create_invalid_icon(A_ETH.identifier, icon_manager.icons_dir)
migration_patch = patch(
'rotkehlchen.data_migrations.manager.MIGRATION_LIST',
new=MIGRATION_LIST[2:],
)
with migration_patch:
DataMigrationManager(rotki).maybe_migrate_data()
assert btc_iconpath.is_file() is False
assert eth_iconpath.is_file() is False
def test_failed_migration(rotkehlchen_api_server):
"""Test that a failed migration does not update DB setting and logs error"""
rotki = rotkehlchen_api_server.rest_api.rotkehlchen
db = rotki.data.db
def botched_migration(rotki) -> None:
raise ValueError('ngmi')
botched_list = [MigrationRecord(version=1, function=botched_migration)]
migrate_mock = patch(
'rotkehlchen.data_migrations.manager.MIGRATION_LIST',
new=botched_list,
)
with migrate_mock:
DataMigrationManager(rotki).maybe_migrate_data()
settings = db.get_settings()
assert settings.last_data_migration == 0, 'no upgrade should have happened'
errors = rotki.msg_aggregator.consume_errors()
warnings = rotki.msg_aggregator.consume_warnings()
assert len(warnings) == 0
assert len(errors) == 1
assert errors[0] == 'Failed to run soft data migration to version 1 due to ngmi'
def unlock_user(
self,
user: str,
password: str,
create_new: bool,
sync_approval: Literal['yes', 'no', 'unknown'],
premium_credentials: Optional[PremiumCredentials],
initial_settings: Optional[ModifiableDBSettings] = None,
sync_database: bool = True,
) -> None:
"""Unlocks an existing user or creates a new one if `create_new` is True
May raise:
- PremiumAuthenticationError if the password can't unlock the database.
- AuthenticationError if premium_credentials are given and are invalid
or can't authenticate with the server
- DBUpgradeError if the rotki DB version is newer than the software or
there is a DB upgrade and there is an error.
- SystemPermissionError if the directory or DB file can not be accessed
"""
log.info(
'Unlocking user',
user=user,
create_new=create_new,
sync_approval=sync_approval,
sync_database=sync_database,
initial_settings=initial_settings,
)
# unlock or create the DB
self.password = password
self.user_directory = self.data.unlock(user, password, create_new, initial_settings)
self.data_importer = DataImporter(db=self.data.db)
self.last_data_upload_ts = self.data.db.get_last_data_upload_ts()
self.premium_sync_manager = PremiumSyncManager(data=self.data, password=password)
# set the DB in the external services instances that need it
self.cryptocompare.set_database(self.data.db)
# Anything that was set above here has to be cleaned in case of failure in the next step
# by reset_after_failed_account_creation_or_login()
try:
self.premium = self.premium_sync_manager.try_premium_at_start(
given_premium_credentials=premium_credentials,
username=user,
create_new=create_new,
sync_approval=sync_approval,
sync_database=sync_database,
)
except PremiumAuthenticationError:
# Reraise it only if this is during the creation of a new account where
# the premium credentials were given by the user
if create_new:
raise
self.msg_aggregator.add_warning(
'Could not authenticate the rotki premium API keys found in the DB.'
' Has your subscription expired?',
)
# else let's just continue. User signed in succesfully, but he just
# has unauthenticable/invalid premium credentials remaining in his DB
settings = self.get_settings()
self.greenlet_manager.spawn_and_track(
after_seconds=None,
task_name='submit_usage_analytics',
exception_is_error=False,
method=maybe_submit_usage_analytics,
data_dir=self.data_dir,
should_submit=settings.submit_usage_analytics,
)
self.etherscan = Etherscan(database=self.data.db, msg_aggregator=self.msg_aggregator)
self.beaconchain = BeaconChain(database=self.data.db, msg_aggregator=self.msg_aggregator)
eth_rpc_endpoint = settings.eth_rpc_endpoint
# Initialize the price historian singleton
PriceHistorian(
data_directory=self.data_dir,
cryptocompare=self.cryptocompare,
coingecko=self.coingecko,
)
PriceHistorian().set_oracles_order(settings.historical_price_oracles)
self.accountant = Accountant(
db=self.data.db,
user_directory=self.user_directory,
msg_aggregator=self.msg_aggregator,
create_csv=True,
premium=self.premium,
)
exchange_credentials = self.data.db.get_exchange_credentials()
self.exchange_manager.initialize_exchanges(
exchange_credentials=exchange_credentials,
database=self.data.db,
)
# Initialize blockchain querying modules
ethereum_manager = EthereumManager(
ethrpc_endpoint=eth_rpc_endpoint,
etherscan=self.etherscan,
msg_aggregator=self.msg_aggregator,
greenlet_manager=self.greenlet_manager,
connect_at_start=ETHEREUM_NODES_TO_CONNECT_AT_START,
)
kusama_manager = SubstrateManager(
chain=SubstrateChain.KUSAMA,
msg_aggregator=self.msg_aggregator,
greenlet_manager=self.greenlet_manager,
connect_at_start=KUSAMA_NODES_TO_CONNECT_AT_START,
connect_on_startup=self._connect_ksm_manager_on_startup(),
own_rpc_endpoint=settings.ksm_rpc_endpoint,
)
polkadot_manager = SubstrateManager(
chain=SubstrateChain.POLKADOT,
msg_aggregator=self.msg_aggregator,
greenlet_manager=self.greenlet_manager,
connect_at_start=POLKADOT_NODES_TO_CONNECT_AT_START,
connect_on_startup=self._connect_dot_manager_on_startup(),
own_rpc_endpoint=settings.dot_rpc_endpoint,
)
self.covalent_avalanche = Covalent(
database=self.data.db,
msg_aggregator=self.msg_aggregator,
chain_id=chains_id['avalanche'],
)
avalanche_manager = AvalancheManager(
avaxrpc_endpoint="https://api.avax.network/ext/bc/C/rpc",
covalent=self.covalent_avalanche,
msg_aggregator=self.msg_aggregator,
)
Inquirer().inject_ethereum(ethereum_manager)
Inquirer().set_oracles_order(settings.current_price_oracles)
self.chain_manager = ChainManager(
blockchain_accounts=self.data.db.get_blockchain_accounts(),
ethereum_manager=ethereum_manager,
kusama_manager=kusama_manager,
polkadot_manager=polkadot_manager,
avalanche_manager=avalanche_manager,
msg_aggregator=self.msg_aggregator,
database=self.data.db,
greenlet_manager=self.greenlet_manager,
premium=self.premium,
eth_modules=settings.active_modules,
data_directory=self.data_dir,
beaconchain=self.beaconchain,
btc_derivation_gap_limit=settings.btc_derivation_gap_limit,
)
self.events_historian = EventsHistorian(
user_directory=self.user_directory,
db=self.data.db,
msg_aggregator=self.msg_aggregator,
exchange_manager=self.exchange_manager,
chain_manager=self.chain_manager,
)
self.task_manager = TaskManager(
max_tasks_num=DEFAULT_MAX_TASKS_NUM,
greenlet_manager=self.greenlet_manager,
api_task_greenlets=self.api_task_greenlets,
database=self.data.db,
cryptocompare=self.cryptocompare,
premium_sync_manager=self.premium_sync_manager,
chain_manager=self.chain_manager,
exchange_manager=self.exchange_manager,
)
DataMigrationManager(self).maybe_migrate_data()
self.greenlet_manager.spawn_and_track(
after_seconds=5,
task_name='periodically_query_icons_until_all_cached',
exception_is_error=False,
method=self.icon_manager.periodically_query_icons_until_all_cached,
batch_size=ICONS_BATCH_SIZE,
sleep_time_secs=ICONS_QUERY_SLEEP,
)
self.user_is_logged_in = True
log.debug('User unlocking complete')
def test_migration_1(rotkehlchen_api_server):
"""
Test that the first data migration for rotki works. This migration removes information about
some exchanges when there is more than one instance or if it is kraken.
In the test we setup instances of the exchanges to trigger the updates and one exchange
(POLONIEX) that shouldn't be affected.
"""
rotki = rotkehlchen_api_server.rest_api.rotkehlchen
db = rotki.data.db
for exchange_location in [Location.BINANCE, Location.KRAKEN, Location.POLONIEX]:
check_saved_events_for_exchange(
exchange_location=exchange_location,
db=rotki.data.db,
should_exist=True,
queryrange_formatstr='{exchange}_{type}',
)
cursor = db.conn.cursor()
result = cursor.execute(
'SELECT COUNT(*) from used_query_ranges WHERE name="bittrex_trades"',
)
assert result.fetchone()[0] == 1
migration_patch = patch(
'rotkehlchen.data_migrations.manager.MIGRATION_LIST',
new=MIGRATION_LIST[:1],
)
with migration_patch:
DataMigrationManager(rotki).maybe_migrate_data()
errors = rotki.msg_aggregator.consume_errors()
warnings = rotki.msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
check_saved_events_for_exchange(Location.BINANCE, rotki.data.db, should_exist=False)
check_saved_events_for_exchange(Location.POLONIEX, rotki.data.db, should_exist=True)
check_saved_events_for_exchange(Location.KRAKEN, rotki.data.db, should_exist=False)
cursor = db.conn.cursor()
result = cursor.execute(
'SELECT COUNT(*) from used_query_ranges WHERE name="bittrex_trades"',
)
assert result.fetchone()[0] == 0
assert rotki.data.db.get_settings().last_data_migration == 1
# Migration shouldn't execute and information should stay in database
for exchange_location in [Location.BINANCE, Location.KRAKEN]:
db.add_trades([Trade(
timestamp=Timestamp(1),
location=exchange_location,
base_asset=A_BTC,
quote_asset=A_ETH,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_ETH,
link='foo',
notes='boo',
)])
db.update_used_query_range(name=f'{str(exchange_location)}_trades_{str(exchange_location)}', start_ts=0, end_ts=9999) # noqa: E501
db.update_used_query_range(name=f'{str(exchange_location)}_margins_{str(exchange_location)}', start_ts=0, end_ts=9999) # noqa: E501
db.update_used_query_range(name=f'{str(exchange_location)}_asset_movements_{str(exchange_location)}', start_ts=0, end_ts=9999) # noqa: E501
with migration_patch:
DataMigrationManager(rotki).maybe_migrate_data()
errors = rotki.msg_aggregator.consume_errors()
warnings = rotki.msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
check_saved_events_for_exchange(Location.BINANCE, rotki.data.db, should_exist=True)
check_saved_events_for_exchange(Location.POLONIEX, rotki.data.db, should_exist=True)
check_saved_events_for_exchange(Location.KRAKEN, rotki.data.db, should_exist=True)
assert rotki.data.db.get_settings().last_data_migration == 1