def run_upgrades(self) -> None:
"""Run all required database upgrades
May raise:
- DBUpgradeError if the user uses a newer version than the one we
upgrade to or if there is a problem during upgrade.
"""
our_version = self.db.get_version()
if our_version > ROTKEHLCHEN_DB_VERSION:
raise DBUpgradeError(
'Your database version is newer than the version expected by the '
'executable. Did you perhaps try to revert to an older rotki version? '
'Please only use the latest version of the software.',
)
for upgrade in UPGRADES_LIST:
self._perform_single_upgrade(upgrade)
# Finally make sure to always have latest version in the DB
cursor = self.db.conn.cursor()
cursor.execute(
'INSERT OR REPLACE INTO settings(name, value) VALUES(?, ?)',
('version', str(ROTKEHLCHEN_DB_VERSION)),
)
self.db.conn.commit()
def _migrate_fiat_balances(db: 'DBHandler') -> None:
"""Migrates fiat balances from the old current_balances table to manually tracked balances"""
cursor = db.conn.cursor()
query = cursor.execute('SELECT asset, amount FROM current_balances;')
for entry in query.fetchall(
): # fetchall() here since the same cursors can't be used later
asset = entry[0]
amount = entry[1]
try:
cursor.execute(
'INSERT INTO manually_tracked_balances(asset, label, amount, location) '
'VALUES(?, ?, ?, ?);',
(asset, f'My {asset} bank', amount, 'I'),
)
except sqlcipher.IntegrityError: # pylint: disable=no-member
# Assume it failed since the label already exists. Then use a much more temporary label
try:
cursor.execute(
'INSERT INTO manually_tracked_balances(asset, label, amount, location) '
'VALUES(?, ?, ?, ?);',
(asset, f'Migrated from fiat balances. My {asset} bank',
amount, 'I'),
)
except sqlcipher.IntegrityError as e: # pylint: disable=no-member
raise DBUpgradeError(
f'Failed to migrate {asset} fiat balance to '
f'manually tracked balances. Error: {str(e)}', ) from e
db.conn.commit()
# Once any fiat balances got migrated, we can delete the table
cursor.execute('DROP TABLE IF EXISTS current_balances;')
db.conn.commit()
def run_upgrades(self) -> None:
our_version = self.db.get_version()
if our_version > ROTKEHLCHEN_DB_VERSION:
raise DBUpgradeError(
'Your database version is newer than the version expected by the '
'executable. Did you perhaps try to revert to an older rotkehlchen version?'
'Please only use the latest version of the software.', )
self._perform_single_upgrade(1, 2, self._checksum_eth_accounts)
self._perform_single_upgrade(
from_version=2,
to_version=3,
upgrade_action=rename_assets_in_db,
cursor=self.db.conn.cursor(),
rename_pairs=[('BCHSV', 'BSV')],
)
self._perform_single_upgrade(3, 4,
self._eth_rpc_port_to_eth_rpc_endpoint)
self._perform_single_upgrade(
from_version=4,
to_version=5,
upgrade_action=rename_assets_in_db,
cursor=self.db.conn.cursor(),
rename_pairs=[('BCC', 'BCH')],
)
def v6_deserialize_location_from_db(symbol: str) -> Location:
"""We copy the deserialize_location_from_db() function at v6
This is done in case the function ever changes in the future. Also another
difference is that instead of DeserializationError this throws a DBUpgradeError
"""
if symbol == 'A':
return Location.EXTERNAL
elif symbol == 'B':
return Location.KRAKEN
elif symbol == 'C':
return Location.POLONIEX
elif symbol == 'D':
return Location.BITTREX
elif symbol == 'E':
return Location.BINANCE
elif symbol == 'F':
return Location.BITMEX
elif symbol == 'G':
return Location.COINBASE
elif symbol == 'H':
return Location.TOTAL
elif symbol == 'I':
return Location.BANKS
elif symbol == 'J':
return Location.BLOCKCHAIN
else:
raise DBUpgradeError(
f'Failed to deserialize location. Unknown symbol {symbol} for location found in DB',
)
def _location_to_enum_location(location: str) -> str:
"""Serialize location strings to DB location enums
The reason we have a specialized function here and not just using
deserialize_location(location).serialize_for_db() is that this code
should work in the future if either of the two functions change or dissapear.
"""
if location == 'external':
return 'A'
if location == 'kraken':
return 'B'
if location == 'poloniex':
return 'C'
if location == 'bittrex':
return 'D'
if location == 'binance':
return 'E'
if location == 'bitmex':
return 'F'
if location == 'coinbase':
return 'G'
if location == 'total':
return 'H'
if location == 'banks':
return 'I'
if location == 'blockchain':
return 'J'
# else
raise DBUpgradeError(
f'Invalid location {location} encountered during DB v5->v6 upgrade')
def v7_deserialize_asset_movement_category(
symbol: str) -> AssetMovementCategory:
"""We copy the deserialize_asset_movement_category_from_db() function at v6
This is done in case the function ever changes in the future. Also another
difference is that instead of DeserializationError this throws a DBUpgradeError
"""
if not isinstance(symbol, str):
raise DBUpgradeError(
f'Failed to deserialize asset movement category symbol from '
f'{type(symbol)} DB enum entry', )
if symbol == 'A':
return AssetMovementCategory.DEPOSIT
elif symbol == 'B':
return AssetMovementCategory.WITHDRAWAL
# else
raise DBUpgradeError(
f'Failed to deserialize asset movement category symbol from DB enum entry.'
f'Unknown symbol {symbol}', )
def _perform_single_upgrade(self, upgrade: UpgradeRecord) -> None:
"""
This is the wrapper function that performs each DB upgrade
The logic is:
1. Check version, if not at from_version get out.
2. If at from_version make a DB backup before performing the upgrade
3. Perform the upgrade action
4. If something went wrong during upgrade restore backup and quit
5. If all went well set version and delete the backup
"""
current_version = self.db.get_version()
if current_version != upgrade.from_version:
return
to_version = upgrade.from_version + 1
# First make a backup of the DB
with TemporaryDirectory() as tmpdirname:
tmp_db_filename = f'{ts_now()}_rotkehlchen_db_v{upgrade.from_version}.backup'
tmp_db_path = os.path.join(tmpdirname, tmp_db_filename)
shutil.copyfile(
os.path.join(self.db.user_data_dir, 'rotkehlchen.db'),
tmp_db_path,
)
try:
kwargs = upgrade.kwargs if upgrade.kwargs is not None else {}
upgrade.function(db=self.db, **kwargs)
except BaseException as e:
# Problem .. restore DB backup and bail out
error_message = (
f'Failed at database upgrade from version {upgrade.from_version} to '
f'{to_version}: {str(e)}'
)
log.error(error_message)
shutil.copyfile(
tmp_db_path,
os.path.join(self.db.user_data_dir, 'rotkehlchen.db'),
)
raise DBUpgradeError(error_message) from e
# for some upgrades even for success keep the backup of the previous db
if upgrade.from_version in (24, 25):
shutil.copyfile(
tmp_db_path,
os.path.join(self.db.user_data_dir, tmp_db_filename),
)
# Upgrade success all is good
self.db.set_version(to_version)
def _upgrade_multisettings_table(db: 'DBHandler') -> None:
"""Upgrade the owned ETH tokens for DAI->SAI renaming"""
cursor = db.conn.cursor()
has_dai = cursor.execute(
'SELECT count(*) FROM multisettings WHERE name="eth_token" AND value="DAI"',
).fetchone()[0] > 0
has_sai = cursor.execute(
'SELECT count(*) FROM multisettings WHERE name="eth_token" AND value="SAI"',
).fetchone()[0] > 0
if has_sai:
raise DBUpgradeError('SAI eth_token detected in DB before the DAI->SAI renaming upgrade')
if has_dai:
cursor.execute('INSERT INTO multisettings(name, value) VALUES("eth_token", "SAI");')
db.conn.commit()
def _perform_single_upgrade(
self,
from_version: int,
to_version: int,
upgrade_action: Callable,
**kwargs: Any,
) -> None:
"""
This is the wrapper function that performs each DB upgrade
The logic is:
1. Check version, if not at from_version get out.
2. If at from_version make a DB backup before performing the upgrade
3. Perform the upgrade action
4. If something went wrong during upgrade restore backup and quit
5. If all went well set version and delete the backup
"""
current_version = self.db.get_version()
if current_version != from_version:
return
# First make a backup of the DB
with TemporaryDirectory() as tmpdirname:
tmp_db_filename = os.path.join(tmpdirname,
f'rotkehlchen_db.backup')
shutil.copyfile(
os.path.join(self.db.user_data_dir, 'rotkehlchen.db'),
tmp_db_filename,
)
try:
upgrade_action(**kwargs)
except BaseException as e:
# Problem .. restore DB backup and bail out
error_message = (
f'Failed at database upgrade from version {from_version} to '
f'{to_version}: {str(e)}', )
log.error(error_message)
shutil.copyfile(
tmp_db_filename,
os.path.join(self.db.user_data_dir, 'rotkehlchen.db'),
)
raise DBUpgradeError(error_message)
# Upgrade success all is good
self.db.set_version(to_version)
def run_upgrades(self) -> bool:
"""Run all required database upgrades
Returns true for fresh database and false otherwise.
May raise:
- DBUpgradeError if the user uses a newer version than the one we
upgrade to or if there is a problem during upgrade.
"""
try:
our_version = self.db.get_version()
except sqlcipher.OperationalError: # pylint: disable=no-member
return True # fresh database. Nothing to upgrade.
if our_version > ROTKEHLCHEN_DB_VERSION:
raise DBUpgradeError(
'Your database version is newer than the version expected by the '
'executable. Did you perhaps try to revert to an older rotki version? '
'Please only use the latest version of the software.', )
cursor = self.db.conn.cursor()
version_query = cursor.execute(
'SELECT value FROM settings WHERE name=?;',
('version', ),
)
if version_query.fetchone() is None:
# temporary due to https://github.com/rotki/rotki/issues/3744.
# Figure out if an upgrade needs to actually run.
cursor = self.db.conn.cursor()
result = cursor.execute(
'SELECT COUNT(*) FROM sqlite_master WHERE type="table" AND name="eth2_validators"'
) # noqa: E501
if result.fetchone()[0] == 0: # it's wrong and at least v30
self.db.set_version(30)
for upgrade in UPGRADES_LIST:
self._perform_single_upgrade(upgrade)
# Finally make sure to always have latest version in the DB
cursor = self.db.conn.cursor()
cursor.execute(
'INSERT OR REPLACE INTO settings(name, value) VALUES(?, ?)',
('version', str(ROTKEHLCHEN_DB_VERSION)),
)
self.db.conn.commit()
return False
def v6_deserialize_trade_type_from_db(symbol: str) -> TradeType:
"""We copy the deserialize_trade_type_from_db() function at v6
This is done in case the function ever changes in the future. Also another
difference is that instead of DeserializationError this throws a DBUpgradeError
"""
if symbol == 'A':
return TradeType.BUY
elif symbol == 'B':
return TradeType.SELL
elif symbol == 'C':
return TradeType.SETTLEMENT_BUY
elif symbol == 'D':
return TradeType.SETTLEMENT_SELL
else:
raise DBUpgradeError(
f'Failed to deserialize trade type. Unknown DB symbol {symbol} for trade type in DB',
)
def _upgrade_trades_table(db: 'DBHandler') -> None:
cursor = db.conn.cursor()
# This is the data trades table at v5
query = cursor.execute(
"""SELECT time, location, pair, type, amount, rate, fee, fee_currency,
link, notes FROM trades;""", )
trade_tuples = []
for result in query:
# This is the logic of trade addition in v6 of the DB
time = result[0]
pair = result[2]
old_trade_type = result[3]
# hand deserialize trade type from DB enum since this code is going to stay
# here even if deserialize_trade_type_from_db() changes
if old_trade_type == 'buy':
trade_type = 'A'
elif old_trade_type == 'sell':
trade_type = 'B'
else:
raise DBUpgradeError(
f'Unexpected trade_type "{trade_type}" found while upgrading '
f'from DB version 5 to 6', )
trade_id = sha3(('external' + str(time) + str(old_trade_type) +
pair).encode()).hex()
trade_tuples.append((
trade_id,
time,
'A', # Symbolizes external in the location enum
pair,
trade_type,
result[4],
result[5],
result[6],
result[7],
result[8],
result[9],
))
# We got all the external trades data. Now delete the old table and create
# the new one
cursor.execute('DROP TABLE trades;')
db.conn.commit()
# This is the scheme of the trades table at v6 from db/utils.py
cursor.execute("""
CREATE TABLE IF NOT EXISTS trades (
id TEXT PRIMARY KEY,
time INTEGER,
location VARCHAR[24],
pair VARCHAR[24],
type CHAR(1) NOT NULL DEFAULT ('B') REFERENCES trade_type(type),
amount TEXT,
rate TEXT,
fee TEXT,
fee_currency VARCHAR[10],
link TEXT,
notes TEXT
);""")
db.conn.commit()
# and finally move the data to the new table
cursor.executemany(
'INSERT INTO trades('
' id, '
' time,'
' location,'
' pair,'
' type,'
' amount,'
' rate,'
' fee,'
' fee_currency,'
' link,'
' notes)'
'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
trade_tuples,
)
db.conn.commit()
