def _process_entry(entry: Any) -> Union[str, List[Any], Dict[str, Any], Any]:
if isinstance(entry, FVal):
return str(entry)
elif isinstance(entry, list):
new_list = []
for new_entry in entry:
new_list.append(_process_entry(new_entry))
return new_list
elif isinstance(entry, dict):
new_dict = {}
for k, v in entry.items():
if isinstance(k, Asset):
k = k.identifier
new_dict[k] = _process_entry(v)
return new_dict
elif isinstance(entry, Location):
return str(entry)
elif isinstance(entry, LocationData):
return {
'time': entry.time,
'location': str(deserialize_location_from_db(entry.location)),
'usd_value': entry.usd_value,
}
elif isinstance(entry, SingleAssetBalance):
return {
'time': entry.time,
'amount': entry.amount,
'usd_value': entry.usd_value
}
elif isinstance(entry, AssetBalance):
return {
'time': entry.time,
'asset': entry.asset.identifier,
'amount': entry.amount,
'usd_value': entry.usd_value,
}
elif isinstance(entry, Trade):
return entry.serialize()
elif isinstance(entry, DBSettings):
return process_result(entry._asdict())
elif isinstance(entry, EthTokenInfo):
return process_result(entry._asdict())
elif isinstance(entry, VersionCheckResult):
return process_result(entry._asdict())
elif isinstance(entry, DSRCurrentBalances):
return process_result(entry._asdict())
elif isinstance(entry, DBSettings):
return process_result(entry._asdict())
elif isinstance(entry, ManuallyTrackedBalanceWithValue):
return process_result(entry._asdict())
elif isinstance(entry, tuple):
raise ValueError('Query results should not contain plain tuples')
elif isinstance(entry, Asset):
return entry.identifier
elif isinstance(entry, (TradeType, Location, KrakenAccountType)):
return str(entry)
else:
return entry
def get_ledger_actions(
self,
from_ts: Optional[Timestamp],
to_ts: Optional[Timestamp],
location: Optional[Location],
) -> List[LedgerAction]:
cursor = self.db.conn.cursor()
query = (
'SELECT identifier,'
' timestamp,'
' type,'
' location,'
' amount,'
' asset,'
' link,'
' notes FROM ledger_actions '
)
if location is not None:
query += f'WHERE location="{location.serialize_for_db()}" '
query, bindings = form_query_to_filter_timestamps(query, 'timestamp', from_ts, to_ts)
results = cursor.execute(query, bindings)
actions = []
for result in results:
try:
action = LedgerAction(
identifier=result[0],
timestamp=deserialize_timestamp(result[1]),
action_type=deserialize_ledger_action_type_from_db(result[2]),
location=deserialize_location_from_db(result[3]),
amount=deserialize_asset_amount(result[4]),
asset=Asset(result[5]),
link=result[6],
notes=result[7],
)
except DeserializationError as e:
self.msg_aggregator.add_error(
f'Error deserializing Ledger Action from the DB. Skipping it.'
f'Error was: {str(e)}',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_error(
f'Error deserializing Ledger Action from the DB. Skipping it. '
f'Unknown asset {e.asset_name} found',
)
continue
actions.append(action)
return actions
def _process_entry(entry: Any) -> Union[str, List[Any], Dict[str, Any], Any]:
if isinstance(entry, FVal):
return str(entry)
elif isinstance(entry, list):
new_list = list()
for new_entry in entry:
new_list.append(_process_entry(new_entry))
return new_list
elif isinstance(entry, dict):
new_dict = dict()
for k, v in entry.items():
if isinstance(k, Asset):
k = k.identifier
new_dict[k] = _process_entry(v)
return new_dict
elif isinstance(entry, LocationData):
return {
'time': entry.time,
'location': str(deserialize_location_from_db(entry.location)),
'usd_value': entry.usd_value,
}
elif isinstance(entry, SingleAssetBalance):
return {
'time': entry.time,
'amount': entry.amount,
'usd_value': entry.usd_value
}
elif isinstance(entry, AssetBalance):
return {
'time': entry.time,
'asset': entry.asset.identifier,
'amount': entry.amount,
'usd_value': entry.usd_value,
}
elif isinstance(entry, Trade):
return entry.serialize()
elif isinstance(entry, EthTokenInfo):
return entry._asdict()
elif isinstance(entry, tuple):
raise ValueError('Query results should not contain tuples')
elif isinstance(entry, Asset):
return entry.identifier
elif isinstance(entry, (TradeType, Location)):
return str(entry)
else:
return entry
def deserialize_from_db(
cls, data: LedgerActionDBTupleWithIdentifier) -> 'LedgerAction':
"""May raise:
- DeserializationError
- UnknownAsset
"""
return cls(
identifier=data[0],
timestamp=deserialize_timestamp(data[1]),
action_type=LedgerActionType.deserialize_from_db(data[2]),
location=deserialize_location_from_db(data[3]),
amount=deserialize_asset_amount(data[4]),
asset=Asset(data[5]),
rate=deserialize_optional(data[6], deserialize_price),
rate_asset=deserialize_optional(data[7], Asset),
link=data[8],
notes=data[9],
)
def update_margin_positions(self, cursor: 'Cursor') -> None:
"""Upgrades the margin positions table to use the new asset ids if they are ethereum tokens
And also makes sure the new primary key id matches the rules used in the app
"""
query = cursor.execute(
'SELECT id, location, open_time, close_time, profit_loss,'
'pl_currency,fee,fee_currency,link,notes from margin_positions;', )
m_tuples = query.fetchall()
cursor.execute('DELETE from margin_positions;')
new_tuples = []
for entry in m_tuples:
new_pl_currency = self.get_new_asset_identifier_if_existing(
entry[5])
new_fee_currency = self.get_new_asset_identifier_if_existing(
entry[7])
# formulate the new DB identifier primary key. Copy the identifier() functionality
open_time_str = 'None' if entry[2] == 0 else str(entry[2])
new_id_string = (str(deserialize_location_from_db(entry[1])) +
open_time_str + str(entry[3]) + new_pl_currency +
new_fee_currency + entry[8])
new_id = hash_id(new_id_string)
new_tuples.append((
new_id,
entry[1],
entry[2],
entry[3],
entry[4],
new_pl_currency,
entry[6],
new_fee_currency,
entry[8],
entry[9],
))
cursor.executemany(
'INSERT INTO margin_positions('
'id, location, open_time, close_time, profit_loss,'
'pl_currency,fee,fee_currency,link,notes) '
'VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?);',
new_tuples,
)
def update_asset_movements(self, cursor: 'Cursor') -> None:
"""Upgrades the asset movements table to use the new asset ids if they are ethereum tokens
And also makes sure the new primary key id matches the rules used in the app
"""
query = cursor.execute(
'SELECT id, location, category, address, transaction_id, time,'
'asset,amount,fee_asset,fee,link from asset_movements;', )
m_tuples = query.fetchall()
cursor.execute('DELETE from asset_movements;')
new_tuples = []
for entry in m_tuples:
new_asset = self.get_new_asset_identifier_if_existing(entry[6])
new_fee_asset = self.get_new_asset_identifier_if_existing(entry[8])
# formulate the new DB identifier primary key. Copy the identifier() functionality
new_id_string = (
str(deserialize_location_from_db(entry[1])) +
str(deserialize_asset_movement_category_from_db(entry[2])) +
str(entry[5]) + new_asset + new_fee_asset + entry[10])
new_id = hash_id(new_id_string)
new_tuples.append((
new_id,
entry[1],
entry[2],
entry[3],
entry[4],
entry[5],
new_asset,
entry[7],
new_fee_asset,
entry[9],
entry[10],
))
cursor.executemany(
'INSERT INTO asset_movements('
'id, location, category, address, transaction_id, time,'
'asset, amount, fee_asset, fee, link) '
'VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);',
new_tuples,
)
def _process_entry(entry: Any) -> Union[str, List[Any], Dict[str, Any], Any]:
if isinstance(entry, FVal):
return str(entry)
if isinstance(entry, list):
new_list = []
for new_entry in entry:
new_list.append(_process_entry(new_entry))
return new_list
if isinstance(entry, (dict, AttributeDict)):
new_dict = {}
for k, v in entry.items():
if isinstance(k, Asset):
k = k.identifier
new_dict[k] = _process_entry(v)
return new_dict
if isinstance(entry, HexBytes):
return entry.hex()
if isinstance(entry, LocationData):
return {
'time': entry.time,
'location': str(deserialize_location_from_db(entry.location)),
'usd_value': entry.usd_value,
}
if isinstance(entry, SingleDBAssetBalance):
return {
'time': entry.time,
'category': str(entry.category),
'amount': entry.amount,
'usd_value': entry.usd_value,
}
if isinstance(entry, DBAssetBalance):
return {
'time': entry.time,
'category': str(entry.category),
'asset': entry.asset.identifier,
'amount': entry.amount,
'usd_value': entry.usd_value,
}
if isinstance(entry, (
DefiProtocol,
MakerdaoVault,
XpubData,
)):
return entry.serialize()
if isinstance(entry, (
Trade,
EthereumTransaction,
MakerdaoVault,
DSRAccountReport,
Balance,
AaveLendingBalance,
AaveBorrowingBalance,
CompoundBalance,
YearnVaultEvent,
YearnVaultBalance,
AaveEvent,
UniswapPool,
UniswapPoolAsset,
UnknownEthereumToken,
AMMTrade,
UniswapPoolEventsBalance,
ADXStakingHistory,
BalancerBPTEventPoolToken,
BalancerEvent,
BalancerPoolEventsBalance,
BalancerPoolBalance,
BalancerPoolTokenBalance,
)):
return process_result(entry.serialize())
if isinstance(entry, (
DBSettings,
CompoundEvent,
VersionCheckResult,
DBSettings,
DSRCurrentBalances,
ManuallyTrackedBalanceWithValue,
VaultEvent,
MakerdaoVaultDetails,
AaveBalances,
AaveHistory,
DefiBalance,
DefiProtocolBalances,
YearnVaultHistory,
BlockchainAccountData,
Eth2Deposit,
)):
return process_result(entry._asdict())
if isinstance(entry, tuple):
raise ValueError('Query results should not contain plain tuples')
if isinstance(entry, Asset):
return entry.identifier
if isinstance(entry, (
TradeType,
Location,
KrakenAccountType,
Location,
VaultEventType,
AssetMovementCategory,
CurrentPriceOracle,
HistoricalPriceOracle,
LedgerActionType,
)):
return str(entry)
# else
return entry
def _process_entry(entry: Any) -> Union[str, List[Any], Dict[str, Any], Any]:
if isinstance(entry, FVal):
return str(entry)
elif isinstance(entry, list):
new_list = []
for new_entry in entry:
new_list.append(_process_entry(new_entry))
return new_list
elif isinstance(entry, (dict, AttributeDict)):
new_dict = {}
for k, v in entry.items():
if isinstance(k, Asset):
k = k.identifier
new_dict[k] = _process_entry(v)
return new_dict
elif isinstance(entry, HexBytes):
return entry.hex()
elif isinstance(entry, LocationData):
return {
'time': entry.time,
'location': str(deserialize_location_from_db(entry.location)),
'usd_value': entry.usd_value,
}
elif isinstance(entry, SingleAssetBalance):
return {
'time': entry.time,
'amount': entry.amount,
'usd_value': entry.usd_value
}
elif isinstance(entry, AssetBalance):
return {
'time': entry.time,
'asset': entry.asset.identifier,
'amount': entry.amount,
'usd_value': entry.usd_value,
}
elif isinstance(entry, (DefiProtocol, MakerDAOVault, XpubData)):
return entry.serialize()
elif isinstance(entry, (
Trade,
EthereumTransaction,
MakerDAOVault,
DSRAccountReport,
Balance,
AaveLendingBalance,
AaveBorrowingBalance,
CompoundBalance,
YearnVaultEvent,
YearnVaultBalance,
AaveEvent,
)):
return process_result(entry.serialize())
elif isinstance(entry, (
DBSettings,
EthTokenInfo,
CompoundEvent,
VersionCheckResult,
DBSettings,
DSRCurrentBalances,
ManuallyTrackedBalanceWithValue,
VaultEvent,
MakerDAOVaultDetails,
AaveBalances,
AaveHistory,
DefiBalance,
DefiProtocolBalances,
YearnVaultHistory,
BlockchainAccountData,
)):
return process_result(entry._asdict())
elif isinstance(entry, tuple):
raise ValueError('Query results should not contain plain tuples')
elif isinstance(entry, Asset):
return entry.identifier
elif isinstance(entry, (
TradeType,
Location,
KrakenAccountType,
Location,
VaultEventType,
AssetMovementCategory,
)):
return str(entry)
else:
return entry
def update_trades(self, cursor: 'Cursor') -> None:
"""Upgrades the trades table to use base/quote asset instead of a pair
Also upgrades all asset ids if they are ethereum tokens
And also makes sure the new primary key id matches the rules used in the app
"""
# Get all old data and transform it to the new schema
query = cursor.execute(
'SELECT id, '
' time, '
' location, '
' pair, '
' type, '
' amount, '
' rate, '
' fee, '
' fee_currency, '
' link, '
' notes from trades; ', )
new_trade_tuples = []
for entry in query:
try:
base, quote = pair_get_asset_ids(entry[3])
except ValueError as e:
self.msg_aggregator.add_warning(
f'During v24 -> v25 DB upgrade {str(e)}. This should not have happened.'
f' Removing the trade with id {entry[0]} at timestamp {entry[1]} '
f'and location {str(deserialize_location_from_db(entry[2]))} that '
f'contained the offending pair from the DB.', )
continue
new_id = self.get_new_asset_identifier(base)
new_base = new_id if new_id else base
new_id = self.get_new_asset_identifier(quote)
new_quote = new_id if new_id else quote
new_id = self.get_new_asset_identifier(entry[8])
new_fee_currency = new_id if new_id else entry[8]
timestamp = entry[1]
amount = entry[5]
rate = entry[6]
old_link = entry[9]
link = None if old_link == '' else old_link
notes = None if entry[10] == '' else entry[10]
# Copy the identifier() functionality. This identifier does not sound like a good idea
new_trade_id_string = (
str(deserialize_location_from_db(entry[2])) + str(timestamp) +
str(deserialize_trade_type_from_db(entry[4])) + new_base +
new_quote + amount + rate + old_link)
new_trade_id = hash_id(new_trade_id_string)
new_trade_tuples.append((
new_trade_id,
entry[1], # time
entry[2], # location
new_base,
new_quote,
entry[4], # type
amount,
rate,
entry[7], # fee
new_fee_currency,
link,
notes,
))
# Upgrade the table
cursor.execute('DROP TABLE IF EXISTS trades;')
cursor.execute("""
CREATE TABLE IF NOT EXISTS trades (
id TEXT PRIMARY KEY NOT NULL,
time INTEGER NOT NULL,
location CHAR(1) NOT NULL DEFAULT('A') REFERENCES location(location),
base_asset TEXT NOT NULL,
quote_asset TEXT NOT NULL,
type CHAR(1) NOT NULL DEFAULT ('A') REFERENCES trade_type(type),
amount TEXT NOT NULL,
rate TEXT NOT NULL,
fee TEXT,
fee_currency TEXT,
link TEXT,
notes TEXT
);
""")
# Insert the new data
executestr = """
INSERT INTO trades(
id,
time,
location,
base_asset,
quote_asset,
type,
amount,
rate,
fee,
fee_currency,
link,
notes)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
"""
cursor.executemany(executestr, new_trade_tuples)
def deserialize_from_db(
cls,
trade_tuple: AMMSwapDBTuple,
) -> 'AMMSwap':
"""Turns a tuple read from DB into an appropriate Swap.
May raise a DeserializationError if something is wrong with the DB data
Trade_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - from_address
4 - to_address
5 - timestamp
6 - location
7 - is_token0_unknown
8 - token0_address
9 - token0_symbol
10 - token0_name
11 - token0_decimals
12 - is_token1_unknown
13 - token1_address
14 - token1_symbol
15 - token1_name
16 - token1_decimals
17 - amount0_in
18 - amount1_in
19 - amount0_out
20 - amount1_out
"""
address = deserialize_ethereum_address(trade_tuple[2])
from_address = deserialize_ethereum_address(trade_tuple[3])
to_address = deserialize_ethereum_address(trade_tuple[4])
is_token0_unknown = trade_tuple[7]
is_token1_unknown = trade_tuple[12]
token0: Union[EthereumToken, UnknownEthereumToken]
token1: Union[EthereumToken, UnknownEthereumToken]
if is_token0_unknown:
token0 = deserialize_unknown_ethereum_token_from_db(
ethereum_address=trade_tuple[8],
symbol=trade_tuple[9],
name=trade_tuple[10],
decimals=trade_tuple[11],
)
else:
token0 = deserialize_ethereum_token_from_db(
identifier=trade_tuple[9])
if is_token1_unknown:
token1 = deserialize_unknown_ethereum_token_from_db(
ethereum_address=trade_tuple[13],
symbol=trade_tuple[14],
name=trade_tuple[15],
decimals=trade_tuple[16],
)
else:
token1 = deserialize_ethereum_token_from_db(
identifier=trade_tuple[14])
return cls(
tx_hash=trade_tuple[0],
log_index=trade_tuple[1],
address=address,
from_address=from_address,
to_address=to_address,
timestamp=deserialize_timestamp(trade_tuple[5]),
location=deserialize_location_from_db(trade_tuple[6]),
token0=token0,
token1=token1,
amount0_in=deserialize_asset_amount(trade_tuple[17]),
amount1_in=deserialize_asset_amount(trade_tuple[18]),
amount0_out=deserialize_asset_amount(trade_tuple[19]),
amount1_out=deserialize_asset_amount(trade_tuple[20]),
)
