def run_upgrades(self) -> None: """Run all required database upgrades May raise: - DBUpgradeError if the user uses a newer version than the one we upgrade to or if there is a problem during upgrade. """ our_version = self.db.get_version() if our_version > ROTKEHLCHEN_DB_VERSION: raise DBUpgradeError( 'Your database version is newer than the version expected by the ' 'executable. Did you perhaps try to revert to an older rotki version? ' 'Please only use the latest version of the software.', ) for upgrade in UPGRADES_LIST: self._perform_single_upgrade(upgrade) # Finally make sure to always have latest version in the DB cursor = self.db.conn.cursor() cursor.execute( 'INSERT OR REPLACE INTO settings(name, value) VALUES(?, ?)', ('version', str(ROTKEHLCHEN_DB_VERSION)), ) self.db.conn.commit()
def _migrate_fiat_balances(db: 'DBHandler') -> None: """Migrates fiat balances from the old current_balances table to manually tracked balances""" cursor = db.conn.cursor() query = cursor.execute('SELECT asset, amount FROM current_balances;') for entry in query.fetchall( ): # fetchall() here since the same cursors can't be used later asset = entry[0] amount = entry[1] try: cursor.execute( 'INSERT INTO manually_tracked_balances(asset, label, amount, location) ' 'VALUES(?, ?, ?, ?);', (asset, f'My {asset} bank', amount, 'I'), ) except sqlcipher.IntegrityError: # pylint: disable=no-member # Assume it failed since the label already exists. Then use a much more temporary label try: cursor.execute( 'INSERT INTO manually_tracked_balances(asset, label, amount, location) ' 'VALUES(?, ?, ?, ?);', (asset, f'Migrated from fiat balances. My {asset} bank', amount, 'I'), ) except sqlcipher.IntegrityError as e: # pylint: disable=no-member raise DBUpgradeError( f'Failed to migrate {asset} fiat balance to ' f'manually tracked balances. Error: {str(e)}', ) from e db.conn.commit() # Once any fiat balances got migrated, we can delete the table cursor.execute('DROP TABLE IF EXISTS current_balances;') db.conn.commit()
def run_upgrades(self) -> None: our_version = self.db.get_version() if our_version > ROTKEHLCHEN_DB_VERSION: raise DBUpgradeError( 'Your database version is newer than the version expected by the ' 'executable. Did you perhaps try to revert to an older rotkehlchen version?' 'Please only use the latest version of the software.', ) self._perform_single_upgrade(1, 2, self._checksum_eth_accounts) self._perform_single_upgrade( from_version=2, to_version=3, upgrade_action=rename_assets_in_db, cursor=self.db.conn.cursor(), rename_pairs=[('BCHSV', 'BSV')], ) self._perform_single_upgrade(3, 4, self._eth_rpc_port_to_eth_rpc_endpoint) self._perform_single_upgrade( from_version=4, to_version=5, upgrade_action=rename_assets_in_db, cursor=self.db.conn.cursor(), rename_pairs=[('BCC', 'BCH')], )
def v6_deserialize_location_from_db(symbol: str) -> Location: """We copy the deserialize_location_from_db() function at v6 This is done in case the function ever changes in the future. Also another difference is that instead of DeserializationError this throws a DBUpgradeError """ if symbol == 'A': return Location.EXTERNAL elif symbol == 'B': return Location.KRAKEN elif symbol == 'C': return Location.POLONIEX elif symbol == 'D': return Location.BITTREX elif symbol == 'E': return Location.BINANCE elif symbol == 'F': return Location.BITMEX elif symbol == 'G': return Location.COINBASE elif symbol == 'H': return Location.TOTAL elif symbol == 'I': return Location.BANKS elif symbol == 'J': return Location.BLOCKCHAIN else: raise DBUpgradeError( f'Failed to deserialize location. Unknown symbol {symbol} for location found in DB', )
def _location_to_enum_location(location: str) -> str: """Serialize location strings to DB location enums The reason we have a specialized function here and not just using deserialize_location(location).serialize_for_db() is that this code should work in the future if either of the two functions change or dissapear. """ if location == 'external': return 'A' if location == 'kraken': return 'B' if location == 'poloniex': return 'C' if location == 'bittrex': return 'D' if location == 'binance': return 'E' if location == 'bitmex': return 'F' if location == 'coinbase': return 'G' if location == 'total': return 'H' if location == 'banks': return 'I' if location == 'blockchain': return 'J' # else raise DBUpgradeError( f'Invalid location {location} encountered during DB v5->v6 upgrade')
def v7_deserialize_asset_movement_category( symbol: str) -> AssetMovementCategory: """We copy the deserialize_asset_movement_category_from_db() function at v6 This is done in case the function ever changes in the future. Also another difference is that instead of DeserializationError this throws a DBUpgradeError """ if not isinstance(symbol, str): raise DBUpgradeError( f'Failed to deserialize asset movement category symbol from ' f'{type(symbol)} DB enum entry', ) if symbol == 'A': return AssetMovementCategory.DEPOSIT elif symbol == 'B': return AssetMovementCategory.WITHDRAWAL # else raise DBUpgradeError( f'Failed to deserialize asset movement category symbol from DB enum entry.' f'Unknown symbol {symbol}', )
def _perform_single_upgrade(self, upgrade: UpgradeRecord) -> None: """ This is the wrapper function that performs each DB upgrade The logic is: 1. Check version, if not at from_version get out. 2. If at from_version make a DB backup before performing the upgrade 3. Perform the upgrade action 4. If something went wrong during upgrade restore backup and quit 5. If all went well set version and delete the backup """ current_version = self.db.get_version() if current_version != upgrade.from_version: return to_version = upgrade.from_version + 1 # First make a backup of the DB with TemporaryDirectory() as tmpdirname: tmp_db_filename = f'{ts_now()}_rotkehlchen_db_v{upgrade.from_version}.backup' tmp_db_path = os.path.join(tmpdirname, tmp_db_filename) shutil.copyfile( os.path.join(self.db.user_data_dir, 'rotkehlchen.db'), tmp_db_path, ) try: kwargs = upgrade.kwargs if upgrade.kwargs is not None else {} upgrade.function(db=self.db, **kwargs) except BaseException as e: # Problem .. restore DB backup and bail out error_message = ( f'Failed at database upgrade from version {upgrade.from_version} to ' f'{to_version}: {str(e)}' ) log.error(error_message) shutil.copyfile( tmp_db_path, os.path.join(self.db.user_data_dir, 'rotkehlchen.db'), ) raise DBUpgradeError(error_message) from e # for some upgrades even for success keep the backup of the previous db if upgrade.from_version in (24, 25): shutil.copyfile( tmp_db_path, os.path.join(self.db.user_data_dir, tmp_db_filename), ) # Upgrade success all is good self.db.set_version(to_version)
def _upgrade_multisettings_table(db: 'DBHandler') -> None: """Upgrade the owned ETH tokens for DAI->SAI renaming""" cursor = db.conn.cursor() has_dai = cursor.execute( 'SELECT count(*) FROM multisettings WHERE name="eth_token" AND value="DAI"', ).fetchone()[0] > 0 has_sai = cursor.execute( 'SELECT count(*) FROM multisettings WHERE name="eth_token" AND value="SAI"', ).fetchone()[0] > 0 if has_sai: raise DBUpgradeError('SAI eth_token detected in DB before the DAI->SAI renaming upgrade') if has_dai: cursor.execute('INSERT INTO multisettings(name, value) VALUES("eth_token", "SAI");') db.conn.commit()
def _perform_single_upgrade( self, from_version: int, to_version: int, upgrade_action: Callable, **kwargs: Any, ) -> None: """ This is the wrapper function that performs each DB upgrade The logic is: 1. Check version, if not at from_version get out. 2. If at from_version make a DB backup before performing the upgrade 3. Perform the upgrade action 4. If something went wrong during upgrade restore backup and quit 5. If all went well set version and delete the backup """ current_version = self.db.get_version() if current_version != from_version: return # First make a backup of the DB with TemporaryDirectory() as tmpdirname: tmp_db_filename = os.path.join(tmpdirname, f'rotkehlchen_db.backup') shutil.copyfile( os.path.join(self.db.user_data_dir, 'rotkehlchen.db'), tmp_db_filename, ) try: upgrade_action(**kwargs) except BaseException as e: # Problem .. restore DB backup and bail out error_message = ( f'Failed at database upgrade from version {from_version} to ' f'{to_version}: {str(e)}', ) log.error(error_message) shutil.copyfile( tmp_db_filename, os.path.join(self.db.user_data_dir, 'rotkehlchen.db'), ) raise DBUpgradeError(error_message) # Upgrade success all is good self.db.set_version(to_version)
def run_upgrades(self) -> bool: """Run all required database upgrades Returns true for fresh database and false otherwise. May raise: - DBUpgradeError if the user uses a newer version than the one we upgrade to or if there is a problem during upgrade. """ try: our_version = self.db.get_version() except sqlcipher.OperationalError: # pylint: disable=no-member return True # fresh database. Nothing to upgrade. if our_version > ROTKEHLCHEN_DB_VERSION: raise DBUpgradeError( 'Your database version is newer than the version expected by the ' 'executable. Did you perhaps try to revert to an older rotki version? ' 'Please only use the latest version of the software.', ) cursor = self.db.conn.cursor() version_query = cursor.execute( 'SELECT value FROM settings WHERE name=?;', ('version', ), ) if version_query.fetchone() is None: # temporary due to https://github.com/rotki/rotki/issues/3744. # Figure out if an upgrade needs to actually run. cursor = self.db.conn.cursor() result = cursor.execute( 'SELECT COUNT(*) FROM sqlite_master WHERE type="table" AND name="eth2_validators"' ) # noqa: E501 if result.fetchone()[0] == 0: # it's wrong and at least v30 self.db.set_version(30) for upgrade in UPGRADES_LIST: self._perform_single_upgrade(upgrade) # Finally make sure to always have latest version in the DB cursor = self.db.conn.cursor() cursor.execute( 'INSERT OR REPLACE INTO settings(name, value) VALUES(?, ?)', ('version', str(ROTKEHLCHEN_DB_VERSION)), ) self.db.conn.commit() return False
def v6_deserialize_trade_type_from_db(symbol: str) -> TradeType: """We copy the deserialize_trade_type_from_db() function at v6 This is done in case the function ever changes in the future. Also another difference is that instead of DeserializationError this throws a DBUpgradeError """ if symbol == 'A': return TradeType.BUY elif symbol == 'B': return TradeType.SELL elif symbol == 'C': return TradeType.SETTLEMENT_BUY elif symbol == 'D': return TradeType.SETTLEMENT_SELL else: raise DBUpgradeError( f'Failed to deserialize trade type. Unknown DB symbol {symbol} for trade type in DB', )
def _upgrade_trades_table(db: 'DBHandler') -> None: cursor = db.conn.cursor() # This is the data trades table at v5 query = cursor.execute( """SELECT time, location, pair, type, amount, rate, fee, fee_currency, link, notes FROM trades;""", ) trade_tuples = [] for result in query: # This is the logic of trade addition in v6 of the DB time = result[0] pair = result[2] old_trade_type = result[3] # hand deserialize trade type from DB enum since this code is going to stay # here even if deserialize_trade_type_from_db() changes if old_trade_type == 'buy': trade_type = 'A' elif old_trade_type == 'sell': trade_type = 'B' else: raise DBUpgradeError( f'Unexpected trade_type "{trade_type}" found while upgrading ' f'from DB version 5 to 6', ) trade_id = sha3(('external' + str(time) + str(old_trade_type) + pair).encode()).hex() trade_tuples.append(( trade_id, time, 'A', # Symbolizes external in the location enum pair, trade_type, result[4], result[5], result[6], result[7], result[8], result[9], )) # We got all the external trades data. Now delete the old table and create # the new one cursor.execute('DROP TABLE trades;') db.conn.commit() # This is the scheme of the trades table at v6 from db/utils.py cursor.execute(""" CREATE TABLE IF NOT EXISTS trades ( id TEXT PRIMARY KEY, time INTEGER, location VARCHAR[24], pair VARCHAR[24], type CHAR(1) NOT NULL DEFAULT ('B') REFERENCES trade_type(type), amount TEXT, rate TEXT, fee TEXT, fee_currency VARCHAR[10], link TEXT, notes TEXT );""") db.conn.commit() # and finally move the data to the new table cursor.executemany( 'INSERT INTO trades(' ' id, ' ' time,' ' location,' ' pair,' ' type,' ' amount,' ' rate,' ' fee,' ' fee_currency,' ' link,' ' notes)' 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', trade_tuples, ) db.conn.commit()