def _process_entry(entry: Any) -> Union[str, List[Any], Dict[str, Any], Any]: if isinstance(entry, FVal): return str(entry) elif isinstance(entry, list): new_list = [] for new_entry in entry: new_list.append(_process_entry(new_entry)) return new_list elif isinstance(entry, dict): new_dict = {} for k, v in entry.items(): if isinstance(k, Asset): k = k.identifier new_dict[k] = _process_entry(v) return new_dict elif isinstance(entry, Location): return str(entry) elif isinstance(entry, LocationData): return { 'time': entry.time, 'location': str(deserialize_location_from_db(entry.location)), 'usd_value': entry.usd_value, } elif isinstance(entry, SingleAssetBalance): return { 'time': entry.time, 'amount': entry.amount, 'usd_value': entry.usd_value } elif isinstance(entry, AssetBalance): return { 'time': entry.time, 'asset': entry.asset.identifier, 'amount': entry.amount, 'usd_value': entry.usd_value, } elif isinstance(entry, Trade): return entry.serialize() elif isinstance(entry, DBSettings): return process_result(entry._asdict()) elif isinstance(entry, EthTokenInfo): return process_result(entry._asdict()) elif isinstance(entry, VersionCheckResult): return process_result(entry._asdict()) elif isinstance(entry, DSRCurrentBalances): return process_result(entry._asdict()) elif isinstance(entry, DBSettings): return process_result(entry._asdict()) elif isinstance(entry, ManuallyTrackedBalanceWithValue): return process_result(entry._asdict()) elif isinstance(entry, tuple): raise ValueError('Query results should not contain plain tuples') elif isinstance(entry, Asset): return entry.identifier elif isinstance(entry, (TradeType, Location, KrakenAccountType)): return str(entry) else: return entry
def get_ledger_actions( self, from_ts: Optional[Timestamp], to_ts: Optional[Timestamp], location: Optional[Location], ) -> List[LedgerAction]: cursor = self.db.conn.cursor() query = ( 'SELECT identifier,' ' timestamp,' ' type,' ' location,' ' amount,' ' asset,' ' link,' ' notes FROM ledger_actions ' ) if location is not None: query += f'WHERE location="{location.serialize_for_db()}" ' query, bindings = form_query_to_filter_timestamps(query, 'timestamp', from_ts, to_ts) results = cursor.execute(query, bindings) actions = [] for result in results: try: action = LedgerAction( identifier=result[0], timestamp=deserialize_timestamp(result[1]), action_type=deserialize_ledger_action_type_from_db(result[2]), location=deserialize_location_from_db(result[3]), amount=deserialize_asset_amount(result[4]), asset=Asset(result[5]), link=result[6], notes=result[7], ) except DeserializationError as e: self.msg_aggregator.add_error( f'Error deserializing Ledger Action from the DB. Skipping it.' f'Error was: {str(e)}', ) continue except UnknownAsset as e: self.msg_aggregator.add_error( f'Error deserializing Ledger Action from the DB. Skipping it. ' f'Unknown asset {e.asset_name} found', ) continue actions.append(action) return actions
def _process_entry(entry: Any) -> Union[str, List[Any], Dict[str, Any], Any]: if isinstance(entry, FVal): return str(entry) elif isinstance(entry, list): new_list = list() for new_entry in entry: new_list.append(_process_entry(new_entry)) return new_list elif isinstance(entry, dict): new_dict = dict() for k, v in entry.items(): if isinstance(k, Asset): k = k.identifier new_dict[k] = _process_entry(v) return new_dict elif isinstance(entry, LocationData): return { 'time': entry.time, 'location': str(deserialize_location_from_db(entry.location)), 'usd_value': entry.usd_value, } elif isinstance(entry, SingleAssetBalance): return { 'time': entry.time, 'amount': entry.amount, 'usd_value': entry.usd_value } elif isinstance(entry, AssetBalance): return { 'time': entry.time, 'asset': entry.asset.identifier, 'amount': entry.amount, 'usd_value': entry.usd_value, } elif isinstance(entry, Trade): return entry.serialize() elif isinstance(entry, EthTokenInfo): return entry._asdict() elif isinstance(entry, tuple): raise ValueError('Query results should not contain tuples') elif isinstance(entry, Asset): return entry.identifier elif isinstance(entry, (TradeType, Location)): return str(entry) else: return entry
def deserialize_from_db( cls, data: LedgerActionDBTupleWithIdentifier) -> 'LedgerAction': """May raise: - DeserializationError - UnknownAsset """ return cls( identifier=data[0], timestamp=deserialize_timestamp(data[1]), action_type=LedgerActionType.deserialize_from_db(data[2]), location=deserialize_location_from_db(data[3]), amount=deserialize_asset_amount(data[4]), asset=Asset(data[5]), rate=deserialize_optional(data[6], deserialize_price), rate_asset=deserialize_optional(data[7], Asset), link=data[8], notes=data[9], )
def update_margin_positions(self, cursor: 'Cursor') -> None: """Upgrades the margin positions table to use the new asset ids if they are ethereum tokens And also makes sure the new primary key id matches the rules used in the app """ query = cursor.execute( 'SELECT id, location, open_time, close_time, profit_loss,' 'pl_currency,fee,fee_currency,link,notes from margin_positions;', ) m_tuples = query.fetchall() cursor.execute('DELETE from margin_positions;') new_tuples = [] for entry in m_tuples: new_pl_currency = self.get_new_asset_identifier_if_existing( entry[5]) new_fee_currency = self.get_new_asset_identifier_if_existing( entry[7]) # formulate the new DB identifier primary key. Copy the identifier() functionality open_time_str = 'None' if entry[2] == 0 else str(entry[2]) new_id_string = (str(deserialize_location_from_db(entry[1])) + open_time_str + str(entry[3]) + new_pl_currency + new_fee_currency + entry[8]) new_id = hash_id(new_id_string) new_tuples.append(( new_id, entry[1], entry[2], entry[3], entry[4], new_pl_currency, entry[6], new_fee_currency, entry[8], entry[9], )) cursor.executemany( 'INSERT INTO margin_positions(' 'id, location, open_time, close_time, profit_loss,' 'pl_currency,fee,fee_currency,link,notes) ' 'VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?);', new_tuples, )
def update_asset_movements(self, cursor: 'Cursor') -> None: """Upgrades the asset movements table to use the new asset ids if they are ethereum tokens And also makes sure the new primary key id matches the rules used in the app """ query = cursor.execute( 'SELECT id, location, category, address, transaction_id, time,' 'asset,amount,fee_asset,fee,link from asset_movements;', ) m_tuples = query.fetchall() cursor.execute('DELETE from asset_movements;') new_tuples = [] for entry in m_tuples: new_asset = self.get_new_asset_identifier_if_existing(entry[6]) new_fee_asset = self.get_new_asset_identifier_if_existing(entry[8]) # formulate the new DB identifier primary key. Copy the identifier() functionality new_id_string = ( str(deserialize_location_from_db(entry[1])) + str(deserialize_asset_movement_category_from_db(entry[2])) + str(entry[5]) + new_asset + new_fee_asset + entry[10]) new_id = hash_id(new_id_string) new_tuples.append(( new_id, entry[1], entry[2], entry[3], entry[4], entry[5], new_asset, entry[7], new_fee_asset, entry[9], entry[10], )) cursor.executemany( 'INSERT INTO asset_movements(' 'id, location, category, address, transaction_id, time,' 'asset, amount, fee_asset, fee, link) ' 'VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);', new_tuples, )
def _process_entry(entry: Any) -> Union[str, List[Any], Dict[str, Any], Any]: if isinstance(entry, FVal): return str(entry) if isinstance(entry, list): new_list = [] for new_entry in entry: new_list.append(_process_entry(new_entry)) return new_list if isinstance(entry, (dict, AttributeDict)): new_dict = {} for k, v in entry.items(): if isinstance(k, Asset): k = k.identifier new_dict[k] = _process_entry(v) return new_dict if isinstance(entry, HexBytes): return entry.hex() if isinstance(entry, LocationData): return { 'time': entry.time, 'location': str(deserialize_location_from_db(entry.location)), 'usd_value': entry.usd_value, } if isinstance(entry, SingleDBAssetBalance): return { 'time': entry.time, 'category': str(entry.category), 'amount': entry.amount, 'usd_value': entry.usd_value, } if isinstance(entry, DBAssetBalance): return { 'time': entry.time, 'category': str(entry.category), 'asset': entry.asset.identifier, 'amount': entry.amount, 'usd_value': entry.usd_value, } if isinstance(entry, ( DefiProtocol, MakerdaoVault, XpubData, )): return entry.serialize() if isinstance(entry, ( Trade, EthereumTransaction, MakerdaoVault, DSRAccountReport, Balance, AaveLendingBalance, AaveBorrowingBalance, CompoundBalance, YearnVaultEvent, YearnVaultBalance, AaveEvent, UniswapPool, UniswapPoolAsset, UnknownEthereumToken, AMMTrade, UniswapPoolEventsBalance, ADXStakingHistory, BalancerBPTEventPoolToken, BalancerEvent, BalancerPoolEventsBalance, BalancerPoolBalance, BalancerPoolTokenBalance, )): return process_result(entry.serialize()) if isinstance(entry, ( DBSettings, CompoundEvent, VersionCheckResult, DBSettings, DSRCurrentBalances, ManuallyTrackedBalanceWithValue, VaultEvent, MakerdaoVaultDetails, AaveBalances, AaveHistory, DefiBalance, DefiProtocolBalances, YearnVaultHistory, BlockchainAccountData, Eth2Deposit, )): return process_result(entry._asdict()) if isinstance(entry, tuple): raise ValueError('Query results should not contain plain tuples') if isinstance(entry, Asset): return entry.identifier if isinstance(entry, ( TradeType, Location, KrakenAccountType, Location, VaultEventType, AssetMovementCategory, CurrentPriceOracle, HistoricalPriceOracle, LedgerActionType, )): return str(entry) # else return entry
def _process_entry(entry: Any) -> Union[str, List[Any], Dict[str, Any], Any]: if isinstance(entry, FVal): return str(entry) elif isinstance(entry, list): new_list = [] for new_entry in entry: new_list.append(_process_entry(new_entry)) return new_list elif isinstance(entry, (dict, AttributeDict)): new_dict = {} for k, v in entry.items(): if isinstance(k, Asset): k = k.identifier new_dict[k] = _process_entry(v) return new_dict elif isinstance(entry, HexBytes): return entry.hex() elif isinstance(entry, LocationData): return { 'time': entry.time, 'location': str(deserialize_location_from_db(entry.location)), 'usd_value': entry.usd_value, } elif isinstance(entry, SingleAssetBalance): return { 'time': entry.time, 'amount': entry.amount, 'usd_value': entry.usd_value } elif isinstance(entry, AssetBalance): return { 'time': entry.time, 'asset': entry.asset.identifier, 'amount': entry.amount, 'usd_value': entry.usd_value, } elif isinstance(entry, (DefiProtocol, MakerDAOVault, XpubData)): return entry.serialize() elif isinstance(entry, ( Trade, EthereumTransaction, MakerDAOVault, DSRAccountReport, Balance, AaveLendingBalance, AaveBorrowingBalance, CompoundBalance, YearnVaultEvent, YearnVaultBalance, AaveEvent, )): return process_result(entry.serialize()) elif isinstance(entry, ( DBSettings, EthTokenInfo, CompoundEvent, VersionCheckResult, DBSettings, DSRCurrentBalances, ManuallyTrackedBalanceWithValue, VaultEvent, MakerDAOVaultDetails, AaveBalances, AaveHistory, DefiBalance, DefiProtocolBalances, YearnVaultHistory, BlockchainAccountData, )): return process_result(entry._asdict()) elif isinstance(entry, tuple): raise ValueError('Query results should not contain plain tuples') elif isinstance(entry, Asset): return entry.identifier elif isinstance(entry, ( TradeType, Location, KrakenAccountType, Location, VaultEventType, AssetMovementCategory, )): return str(entry) else: return entry
def update_trades(self, cursor: 'Cursor') -> None: """Upgrades the trades table to use base/quote asset instead of a pair Also upgrades all asset ids if they are ethereum tokens And also makes sure the new primary key id matches the rules used in the app """ # Get all old data and transform it to the new schema query = cursor.execute( 'SELECT id, ' ' time, ' ' location, ' ' pair, ' ' type, ' ' amount, ' ' rate, ' ' fee, ' ' fee_currency, ' ' link, ' ' notes from trades; ', ) new_trade_tuples = [] for entry in query: try: base, quote = pair_get_asset_ids(entry[3]) except ValueError as e: self.msg_aggregator.add_warning( f'During v24 -> v25 DB upgrade {str(e)}. This should not have happened.' f' Removing the trade with id {entry[0]} at timestamp {entry[1]} ' f'and location {str(deserialize_location_from_db(entry[2]))} that ' f'contained the offending pair from the DB.', ) continue new_id = self.get_new_asset_identifier(base) new_base = new_id if new_id else base new_id = self.get_new_asset_identifier(quote) new_quote = new_id if new_id else quote new_id = self.get_new_asset_identifier(entry[8]) new_fee_currency = new_id if new_id else entry[8] timestamp = entry[1] amount = entry[5] rate = entry[6] old_link = entry[9] link = None if old_link == '' else old_link notes = None if entry[10] == '' else entry[10] # Copy the identifier() functionality. This identifier does not sound like a good idea new_trade_id_string = ( str(deserialize_location_from_db(entry[2])) + str(timestamp) + str(deserialize_trade_type_from_db(entry[4])) + new_base + new_quote + amount + rate + old_link) new_trade_id = hash_id(new_trade_id_string) new_trade_tuples.append(( new_trade_id, entry[1], # time entry[2], # location new_base, new_quote, entry[4], # type amount, rate, entry[7], # fee new_fee_currency, link, notes, )) # Upgrade the table cursor.execute('DROP TABLE IF EXISTS trades;') cursor.execute(""" CREATE TABLE IF NOT EXISTS trades ( id TEXT PRIMARY KEY NOT NULL, time INTEGER NOT NULL, location CHAR(1) NOT NULL DEFAULT('A') REFERENCES location(location), base_asset TEXT NOT NULL, quote_asset TEXT NOT NULL, type CHAR(1) NOT NULL DEFAULT ('A') REFERENCES trade_type(type), amount TEXT NOT NULL, rate TEXT NOT NULL, fee TEXT, fee_currency TEXT, link TEXT, notes TEXT ); """) # Insert the new data executestr = """ INSERT INTO trades( id, time, location, base_asset, quote_asset, type, amount, rate, fee, fee_currency, link, notes) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """ cursor.executemany(executestr, new_trade_tuples)
def deserialize_from_db( cls, trade_tuple: AMMSwapDBTuple, ) -> 'AMMSwap': """Turns a tuple read from DB into an appropriate Swap. May raise a DeserializationError if something is wrong with the DB data Trade_tuple index - Schema columns ---------------------------------- 0 - tx_hash 1 - log_index 2 - address 3 - from_address 4 - to_address 5 - timestamp 6 - location 7 - is_token0_unknown 8 - token0_address 9 - token0_symbol 10 - token0_name 11 - token0_decimals 12 - is_token1_unknown 13 - token1_address 14 - token1_symbol 15 - token1_name 16 - token1_decimals 17 - amount0_in 18 - amount1_in 19 - amount0_out 20 - amount1_out """ address = deserialize_ethereum_address(trade_tuple[2]) from_address = deserialize_ethereum_address(trade_tuple[3]) to_address = deserialize_ethereum_address(trade_tuple[4]) is_token0_unknown = trade_tuple[7] is_token1_unknown = trade_tuple[12] token0: Union[EthereumToken, UnknownEthereumToken] token1: Union[EthereumToken, UnknownEthereumToken] if is_token0_unknown: token0 = deserialize_unknown_ethereum_token_from_db( ethereum_address=trade_tuple[8], symbol=trade_tuple[9], name=trade_tuple[10], decimals=trade_tuple[11], ) else: token0 = deserialize_ethereum_token_from_db( identifier=trade_tuple[9]) if is_token1_unknown: token1 = deserialize_unknown_ethereum_token_from_db( ethereum_address=trade_tuple[13], symbol=trade_tuple[14], name=trade_tuple[15], decimals=trade_tuple[16], ) else: token1 = deserialize_ethereum_token_from_db( identifier=trade_tuple[14]) return cls( tx_hash=trade_tuple[0], log_index=trade_tuple[1], address=address, from_address=from_address, to_address=to_address, timestamp=deserialize_timestamp(trade_tuple[5]), location=deserialize_location_from_db(trade_tuple[6]), token0=token0, token1=token1, amount0_in=deserialize_asset_amount(trade_tuple[17]), amount1_in=deserialize_asset_amount(trade_tuple[18]), amount0_out=deserialize_asset_amount(trade_tuple[19]), amount1_out=deserialize_asset_amount(trade_tuple[20]), )