def query_timed_balances_data(self, given_asset: str, start_ts: int, end_ts: int) -> Dict:
try:
from_ts = deserialize_timestamp(start_ts)
to_ts = deserialize_timestamp(end_ts)
asset = Asset(given_asset)
except (UnknownAsset, DeserializationError) as e:
return {'result': False, 'message': str(e)}
res = self.rotkehlchen.data.db.query_timed_balances(
from_ts=from_ts,
to_ts=to_ts,
asset=asset,
)
result = {'result': res, 'message': ''}
return process_result(result)
def deserialize_from_db(
cls,
data: LedgerActionDBTupleWithIdentifier,
given_gitcoin_map: Optional[Dict[int, GitcoinEventDataDB]] = None,
) -> 'LedgerAction':
"""May raise:
- DeserializationError
- UnknownAsset
"""
extra_data = None
gitcoin_map = {} if not given_gitcoin_map else given_gitcoin_map
gitcoin_data = gitcoin_map.get(data[0], None)
if gitcoin_data is not None:
extra_data = GitcoinEventData.deserialize_from_db(
data=gitcoin_data)
return cls(
identifier=data[0],
timestamp=deserialize_timestamp(data[1]),
action_type=LedgerActionType.deserialize_from_db(data[2]),
location=Location.deserialize_from_db(data[3]),
amount=deserialize_asset_amount(data[4]),
asset=Asset(data[5]),
rate=deserialize_optional(data[6], deserialize_price),
rate_asset=deserialize_optional(data[7], Asset),
link=data[8],
notes=data[9],
extra_data=extra_data,
)
def asset_movements_from_dictlist(
given_data: List[Dict[str, Any]],
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
""" Gets a list of dict asset movements, most probably read from the json files and
a time period. Returns it as a list of the AssetMovement tuples that are inside the time period
May raise:
- DeserializationError: If the given_data dict contains data in an unexpected format
- KeyError: If the given_data dict contains data in an unexpected format
"""
returned_movements = list()
for movement in given_data:
timestamp = deserialize_timestamp(movement['timestamp'])
if timestamp < start_ts:
continue
if timestamp > end_ts:
break
category = deserialize_asset_movement_category(movement['category'])
amount = deserialize_asset_amount(movement['amount'])
fee = deserialize_fee(movement['fee'])
returned_movements.append(AssetMovement(
exchange=deserialize_exchange_name(movement['exchange']),
category=category,
timestamp=timestamp,
asset=Asset(movement['asset']),
amount=amount,
fee=fee,
))
return returned_movements
def deserialize_transaction_from_etherscan(
data: Dict[str, Any],
internal: bool,
) -> EthereumTransaction:
"""Reads dict data of a transaction from etherscan and deserializes it
Can raise DeserializationError if something is wrong
"""
try:
# internal tx list contains no gasprice
gas_price = -1 if internal else read_integer(data, 'gasPrice')
tx_hash = read_hash(data, 'hash')
input_data = read_hash(data, 'input')
timestamp = deserialize_timestamp(data['timeStamp'])
block_number = read_integer(data, 'blockNumber')
nonce = -1 if internal else read_integer(data, 'nonce')
return EthereumTransaction(
timestamp=timestamp,
block_number=block_number,
tx_hash=tx_hash,
from_address=to_checksum_address(data['from']),
to_address=to_checksum_address(data['to']) if data['to'] != '' else None,
value=read_integer(data, 'value'),
gas=read_integer(data, 'gas'),
gas_price=gas_price,
gas_used=read_integer(data, 'gasUsed'),
input_data=input_data,
nonce=nonce,
)
except KeyError as e:
raise DeserializationError(f'Etherscan ethereum transaction missing expected key {str(e)}')
def _deserialize(self, value, attr, data, **kwargs): # pylint: disable=unused-argument
try:
timestamp = deserialize_timestamp(value)
except DeserializationError as e:
raise ValidationError(str(e))
return timestamp
def deserialize_from_db(
cls,
event_tuple: BalancerEventDBTuple,
) -> 'BalancerEvent':
"""May raise DeserializationError
Event_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - timestamp
4 - type
5 - pool_address_token
6 - lp_amount
7 - usd_value
8 - amount0
9 - amount1
10 - amount2
11 - amount3
12 - amount4
13 - amount5
14 - amount6
15 - amount7
"""
event_tuple_type = event_tuple[4]
try:
event_type = getattr(BalancerBPTEventType,
event_tuple_type.upper())
except AttributeError as e:
raise DeserializationError(
f'Unexpected event type: {event_tuple_type}.') from e
pool_address_token = EthereumToken.from_identifier(
event_tuple[5],
form_with_incomplete_data=
True, # since some may not have decimals input correctly
)
if pool_address_token is None:
raise DeserializationError(
f'Balancer event pool token: {event_tuple[5]} not found in the DB.',
)
amounts: List[AssetAmount] = [
deserialize_asset_amount(item) for item in event_tuple[8:16]
if item is not None
]
return cls(
tx_hash=event_tuple[0],
log_index=event_tuple[1],
address=string_to_ethereum_address(event_tuple[2]),
timestamp=deserialize_timestamp(event_tuple[3]),
event_type=event_type,
pool_address_token=pool_address_token,
lp_balance=Balance(
amount=deserialize_asset_amount(event_tuple[6]),
usd_value=deserialize_price(event_tuple[7]),
),
amounts=amounts,
)
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
result = self._get_paginated_query(
endpoint='transfers',
start_ts=start_ts,
end_ts=end_ts,
)
movements = []
for entry in result:
try:
timestamp = deserialize_timestamp(entry['timestampms'])
timestamp = Timestamp(int(timestamp / 1000))
asset = Asset(entry['currency'])
movement = AssetMovement(
location=Location.GEMINI,
category=deserialize_asset_movement_category(
entry['type']),
address=deserialize_asset_movement_address(
entry, 'destination', asset),
transaction_id=get_key_if_has_val(entry, 'txHash'),
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
entry['amount']),
fee_asset=asset,
# Gemini does not include withdrawal fees neither in the API nor in their UI
fee=Fee(ZERO),
link=str(entry['eid']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Error processing a gemini deposit/withdrawal. Check logs '
'for details. Ignoring it.', )
log.error(
'Error processing a gemini deposit_withdrawal',
asset_movement=entry,
error=msg,
)
continue
movements.append(movement)
return movements
def trades_from_dictlist(
given_trades: List[Dict[str, Any]],
start_ts: Timestamp,
end_ts: Timestamp,
location: str,
msg_aggregator: MessagesAggregator,
) -> List[Trade]:
""" Gets a list of dict trades, most probably read from the json files and
a time period. Returns it as a list of the Trade tuples that are inside the time period
Can raise:
- KeyError: If a trade dict does not have a key as we expect it
- DeserializationError: If a trade dict entry is of an unexpected format
"""
returned_trades = list()
for given_trade in given_trades:
timestamp = deserialize_timestamp(given_trade['timestamp'])
if timestamp < start_ts:
continue
if timestamp > end_ts:
break
try:
returned_trades.append(deserialize_trade(given_trade))
except UnknownAsset as e:
msg_aggregator.add_warning(
f'When processing {location} trades found a trade containing unknown '
f'asset {e.asset_name}. Ignoring it.')
continue
return returned_trades
def deserialize_invest_event(
raw_event: Dict[str, Any],
event_type: Literal[BalancerInvestEventType.ADD_LIQUIDITY,
BalancerInvestEventType.REMOVE_LIQUIDITY, ],
) -> BalancerInvestEvent:
"""May raise DeserializationError"""
try:
tx_hash, log_index = deserialize_transaction_id(raw_event['id'])
timestamp = deserialize_timestamp(raw_event['timestamp'])
raw_user_address = raw_event['userAddress']['id']
raw_pool_address = raw_event['poolAddress']['id']
if event_type == BalancerInvestEventType.ADD_LIQUIDITY:
raw_token_address = raw_event['tokenIn']['address']
amount = deserialize_asset_amount(raw_event['tokenAmountIn'])
elif event_type == BalancerInvestEventType.REMOVE_LIQUIDITY:
raw_token_address = raw_event['tokenOut']['address']
amount = deserialize_asset_amount(raw_event['tokenAmountOut'])
else:
raise AssertionError(f'Unexpected event type: {event_type}.')
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
try:
user_address = to_checksum_address(raw_user_address)
except ValueError as e:
raise DeserializationError(
f'Invalid ethereum address: {raw_user_address} in userAddress.',
) from e
try:
pool_address = to_checksum_address(raw_pool_address)
except ValueError as e:
raise DeserializationError(
f'Invalid ethereum address: {raw_pool_address} in poolAddress.',
) from e
try:
token_address = to_checksum_address(raw_token_address)
except ValueError as e:
raise DeserializationError(
f'Invalid ethereum address: {raw_token_address} in tokenIn.',
) from e
invest_event = BalancerInvestEvent(
tx_hash=tx_hash,
log_index=log_index,
address=user_address,
timestamp=timestamp,
event_type=event_type,
pool_address=pool_address,
token_address=token_address,
amount=amount,
)
return invest_event
def get_ethereum_transactions(
self,
filter_: ETHTransactionsFilterQuery,
) -> Tuple[List[EthereumTransaction], int]:
"""Returns a tuple with 2 entries.
First entry is a list of ethereum transactions optionally filtered by
time and/or from address and pagination.
Second is the number of entries found for the current filter ignoring pagination.
This function can raise:
- pysqlcipher3.dbapi2.OperationalError if the SQL query fails due to invalid
filtering arguments.
"""
cursor = self.db.conn.cursor()
query, bindings = filter_.prepare()
query = 'SELECT * FROM ethereum_transactions ' + query
results = cursor.execute(query, bindings)
ethereum_transactions = []
for result in results:
try:
tx = EthereumTransaction(
tx_hash=result[0],
timestamp=deserialize_timestamp(result[1]),
block_number=result[2],
from_address=result[3],
to_address=result[4],
value=int(result[5]),
gas=int(result[6]),
gas_price=int(result[7]),
gas_used=int(result[8]),
input_data=result[9],
nonce=result[10],
)
except DeserializationError as e:
self.db.msg_aggregator.add_error(
f'Error deserializing ethereum transaction from the DB. '
f'Skipping it. Error was: {str(e)}',
)
continue
ethereum_transactions.append(tx)
if filter_.pagination is not None:
no_pagination_filter = deepcopy(filter_)
no_pagination_filter.pagination = None
query, bindings = no_pagination_filter.prepare()
query = 'SELECT COUNT(*) FROM ethereum_transactions ' + query
results = cursor.execute(query, bindings).fetchone()
total_filter_count = results[0]
else:
total_filter_count = len(ethereum_transactions)
return ethereum_transactions, total_filter_count
def get_token_transaction_hashes(
self,
account: ChecksumEthAddress,
from_ts: Optional[Timestamp] = None,
to_ts: Optional[Timestamp] = None,
) -> Iterator[List[str]]:
options = {'address': str(account), 'sort': 'asc'}
if from_ts is not None:
from_block = self.get_blocknumber_by_time(from_ts)
options['startBlock'] = str(from_block)
if to_ts is not None:
to_block = self.get_blocknumber_by_time(to_ts)
options['endBlock'] = str(to_block)
hashes: Set[Tuple[str, Timestamp]] = set()
while True:
result = self._query(module='account',
action='tokentx',
options=options)
last_ts = deserialize_timestamp(
result[0]['timeStamp']) if len(result) != 0 else None # noqa: E501 pylint: disable=unsubscriptable-object
for entry in result:
gevent.sleep(0)
timestamp = deserialize_timestamp(entry['timeStamp'])
if timestamp > last_ts and len(
hashes) >= TRANSACTIONS_BATCH_NUM: # type: ignore
yield _hashes_tuple_to_list(hashes)
hashes = set()
last_ts = timestamp
hashes.add((entry['hash'], timestamp))
if len(result) != ETHERSCAN_TX_QUERY_LIMIT:
break
# else we hit the limit. Query once more with startBlock being the last
# block we got. There may be duplicate entries if there are more than one
# transactions for that last block but they should be filtered
# out when we input all of these in the DB
last_block = result[-1]['blockNumber'] # pylint: disable=unsubscriptable-object
options['startBlock'] = last_block
yield _hashes_tuple_to_list(hashes)
def deserialize_from_db(cls,
entry: MarginPositionDBTuple) -> 'MarginPosition':
"""May raise:
- DeserializationError
- UnknownAsset
"""
if entry[2] == 0:
open_time = None
else:
open_time = deserialize_timestamp(entry[2])
return MarginPosition(
location=Location.deserialize_from_db(entry[1]),
open_time=open_time,
close_time=deserialize_timestamp(entry[3]),
profit_loss=deserialize_asset_amount(entry[4]),
pl_currency=Asset(entry[5]),
fee=deserialize_fee(entry[6]),
fee_currency=Asset(entry[7]),
link=entry[8],
notes=entry[9],
)
def _deserialize_bond(
self,
raw_event: Dict[str, Any],
identity_address_map: Dict[ChecksumAddress, ChecksumAddress],
) -> Bond:
"""Deserialize a bond event.
May raise DeserializationError.
"""
try:
adex_event = self._deserialize_adex_staking_event(
raw_event=raw_event,
identity_address_map=identity_address_map,
case='bond',
)
amount_int = int(raw_event['amount'])
amount = FVal(raw_event['amount']) / ADX_AMOUNT_MANTISSA
pool_id = raw_event['poolId']
nonce = int(raw_event['nonce'])
bond_id = self._get_bond_id(
identity_address=adex_event.identity_address,
amount=amount_int,
pool_id=pool_id,
nonce=nonce,
)
slashed_at = deserialize_timestamp(raw_event['slashedAtStart'])
except (DeserializationError, KeyError, ValueError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key in event: {msg}.'
log.error(
'Failed to deserialize an AdEx bond event',
error=msg,
raw_event=raw_event,
identity_address_map=identity_address_map,
)
raise DeserializationError(
'Failed to deserialize an AdEx bond event. Check logs for more details',
) from e
return Bond(
tx_hash=adex_event.tx_hash,
address=adex_event.address,
identity_address=adex_event.identity_address,
timestamp=adex_event.timestamp,
bond_id=bond_id,
value=Balance(amount=amount),
pool_id=pool_id,
nonce=nonce,
slashed_at=slashed_at,
)
def _deserialize(
self,
value: str,
attr: Optional[str], # pylint: disable=unused-argument
data: Optional[Mapping[str, Any]], # pylint: disable=unused-argument
**_kwargs: Any,
) -> Timestamp:
try:
timestamp = deserialize_timestamp(value)
except DeserializationError as e:
raise ValidationError(str(e))
return timestamp
def deserialize_from_db(
cls,
event_tuple: LiquidityPoolEventDBTuple,
) -> 'LiquidityPoolEvent':
"""Turns a tuple read from DB into an appropriate LiquidityPoolEvent.
May raise a DeserializationError if something is wrong with the DB data
Event_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - timestamp
4 - type
5 - pool_address
6 - token0_identifier
7 - token1_identifier
8 - amount0
9 - amount1
10 - usd_price
11 - lp_amount
"""
db_event_type = event_tuple[4]
if db_event_type not in {str(event_type) for event_type in EventType}:
raise DeserializationError(
f'Failed to deserialize event type. Unknown event: {db_event_type}.',
)
if db_event_type == str(EventType.MINT):
event_type = EventType.MINT
elif db_event_type == str(EventType.BURN):
event_type = EventType.BURN
else:
raise ValueError(f'Unexpected event type case: {db_event_type}.')
token0 = deserialize_ethereum_token_from_db(identifier=event_tuple[6])
token1 = deserialize_ethereum_token_from_db(identifier=event_tuple[7])
return cls(
tx_hash=event_tuple[0],
log_index=event_tuple[1],
address=string_to_ethereum_address(event_tuple[2]),
timestamp=deserialize_timestamp(event_tuple[3]),
event_type=event_type,
pool_address=string_to_ethereum_address(event_tuple[5]),
token0=token0,
token1=token1,
amount0=deserialize_asset_amount(event_tuple[8]),
amount1=deserialize_asset_amount(event_tuple[9]),
usd_price=deserialize_price(event_tuple[10]),
lp_amount=deserialize_asset_amount(event_tuple[11]),
)
def _deserialize_asset_movement(
self,
movement_type: AssetMovementCategory,
movement_data: Dict[str, Any],
) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from polo and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if movement_type == AssetMovementCategory.DEPOSIT:
fee = Fee(ZERO)
uid_key = 'depositNumber'
else:
fee = deserialize_fee(movement_data['fee'])
uid_key = 'withdrawalNumber'
asset = asset_from_poloniex(movement_data['currency'])
return AssetMovement(
location=Location.POLONIEX,
category=movement_type,
timestamp=deserialize_timestamp(movement_data['timestamp']),
asset=asset,
amount=deserialize_asset_amount(movement_data['amount']),
fee_asset=asset,
fee=fee,
link=str(movement_data[uid_key]),
)
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(movement_type)} of unsupported poloniex asset '
f'{e.asset_name}. Ignoring it.',
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(movement_type)} of unknown poloniex asset '
f'{e.asset_name}. Ignoring it.',
)
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
f'Unexpected data encountered during deserialization of a poloniex '
f'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of poloniex '
f'{str(movement_type)}: {movement_data}. Error was: {str(e)}',
)
return None
def deserialize_from_db(
cls,
event_tuple: BalancerEventDBTuple,
) -> 'BalancerEvent':
"""May raise DeserializationError
Event_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - timestamp
4 - type
5 - pool_address
6 - lp_amount
7 - usd_value
8 - amount0
9 - amount1
10 - amount2
11 - amount3
12 - amount4
13 - amount5
14 - amount6
15 - amount7
"""
event_tuple_type = event_tuple[4]
try:
event_type = getattr(BalancerBPTEventType,
event_tuple_type.upper())
except AttributeError as e:
raise DeserializationError(
f'Unexpected event type: {event_tuple_type}.') from e
amounts: List[AssetAmount] = [
deserialize_asset_amount(item) for item in event_tuple[8:16]
if item is not None
]
return cls(
tx_hash=event_tuple[0],
log_index=event_tuple[1],
address=string_to_ethereum_address(event_tuple[2]),
timestamp=deserialize_timestamp(event_tuple[3]),
event_type=event_type,
pool_address=string_to_ethereum_address(event_tuple[5]),
lp_balance=Balance(
amount=deserialize_asset_amount(event_tuple[6]),
usd_value=deserialize_price(event_tuple[7]),
),
amounts=amounts,
)
def get_ledger_actions(
self,
from_ts: Optional[Timestamp],
to_ts: Optional[Timestamp],
location: Optional[Location],
) -> List[LedgerAction]:
cursor = self.db.conn.cursor()
query = (
'SELECT identifier,'
' timestamp,'
' type,'
' location,'
' amount,'
' asset,'
' link,'
' notes FROM ledger_actions '
)
if location is not None:
query += f'WHERE location="{location.serialize_for_db()}" '
query, bindings = form_query_to_filter_timestamps(query, 'timestamp', from_ts, to_ts)
results = cursor.execute(query, bindings)
actions = []
for result in results:
try:
action = LedgerAction(
identifier=result[0],
timestamp=deserialize_timestamp(result[1]),
action_type=deserialize_ledger_action_type_from_db(result[2]),
location=deserialize_location_from_db(result[3]),
amount=deserialize_asset_amount(result[4]),
asset=Asset(result[5]),
link=result[6],
notes=result[7],
)
except DeserializationError as e:
self.msg_aggregator.add_error(
f'Error deserializing Ledger Action from the DB. Skipping it.'
f'Error was: {str(e)}',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_error(
f'Error deserializing Ledger Action from the DB. Skipping it. '
f'Unknown asset {e.asset_name} found',
)
continue
actions.append(action)
return actions
def import_snapshot(
self,
balances_snapshot_file: Path,
location_data_snapshot_file: Path,
) -> Tuple[bool, str]:
"""
Converts the snapshot files to list to dictionaries.
Performs a series of validation checks on the list before importing.
"""
balances_list = self._csv_to_dict(balances_snapshot_file)
location_data_list = self._csv_to_dict(location_data_snapshot_file)
# check if the headers match the type stored in the db
has_invalid_headers = (
tuple(balances_list[0].keys()) !=
('timestamp', 'category', 'asset_identifier', 'amount',
'usd_value') or # noqa: E501
tuple(location_data_list[0].keys()) !=
('timestamp', 'location', 'usd_value') # noqa: E501
)
if has_invalid_headers:
return False, 'csv file has invalid headers'
# check if all timestamps are the same.
balances_timestamps = [
int(entry['timestamp']) for entry in balances_list
]
location_data_timestamps = [
int(entry['timestamp']) for entry in location_data_list
]
has_different_timestamps = (
balances_timestamps.count(balances_timestamps[0]) !=
len(balances_timestamps) or location_data_timestamps.count(
location_data_timestamps[0]) != len(location_data_timestamps)
or # noqa: E501
balances_timestamps[0] != location_data_timestamps[0])
if has_different_timestamps:
return False, 'csv file has different timestamps'
# check if the timestamp can be converted to int
try:
_ = deserialize_timestamp(balances_list[0]['timestamp'])
except DeserializationError:
return False, 'csv file contains invalid timestamp format'
return self._import_snapshot(
balances_list=balances_list,
location_data_list=location_data_list,
)
def get_ethereum_transactions(
self,
filter_: ETHTransactionsFilterQuery,
has_premium: bool,
) -> List[EthereumTransaction]:
"""Returns a list of ethereum transactions optionally filtered by
the given filter query
This function can raise:
- pysqlcipher3.dbapi2.OperationalError if the SQL query fails due to invalid
filtering arguments.
"""
cursor = self.db.conn.cursor()
query, bindings = filter_.prepare()
if has_premium:
query = 'SELECT DISTINCT ethereum_transactions.tx_hash, timestamp, block_number, from_address, to_address, value, gas, gas_price, gas_used, input_data, nonce FROM ethereum_transactions ' + query # noqa: E501
results = cursor.execute(query, bindings)
else:
query = 'SELECT DISTINCT ethereum_transactions.tx_hash, timestamp, block_number, from_address, to_address, value, gas, gas_price, gas_used, input_data, nonce FROM (SELECT * from ethereum_transactions ORDER BY timestamp DESC LIMIT ?) ethereum_transactions ' + query # noqa: E501
results = cursor.execute(query, [FREE_ETH_TX_LIMIT] + bindings)
ethereum_transactions = []
for result in results:
try:
tx = EthereumTransaction(
tx_hash=make_evm_tx_hash(result[0]),
timestamp=deserialize_timestamp(result[1]),
block_number=result[2],
from_address=result[3],
to_address=result[4],
value=int(result[5]),
gas=int(result[6]),
gas_price=int(result[7]),
gas_used=int(result[8]),
input_data=result[9],
nonce=result[10],
)
except DeserializationError as e:
self.db.msg_aggregator.add_error(
f'Error deserializing ethereum transaction from the DB. '
f'Skipping it. Error was: {str(e)}',
)
continue
ethereum_transactions.append(tx)
return ethereum_transactions
def deserialize_invest_event(
raw_event: Dict[str, Any],
event_type: Literal[
BalancerInvestEventType.ADD_LIQUIDITY,
BalancerInvestEventType.REMOVE_LIQUIDITY,
],
) -> BalancerInvestEvent:
"""May raise DeserializationError"""
try:
tx_hash, log_index = deserialize_transaction_id(raw_event['id'])
timestamp = deserialize_timestamp(raw_event['timestamp'])
raw_user_address = raw_event['userAddress']['id']
raw_pool_address = raw_event['poolAddress']['id']
if event_type == BalancerInvestEventType.ADD_LIQUIDITY:
raw_token_address = raw_event['tokenIn']['address']
amount = deserialize_asset_amount(raw_event['tokenAmountIn'])
elif event_type == BalancerInvestEventType.REMOVE_LIQUIDITY:
raw_token_address = raw_event['tokenOut']['address']
amount = deserialize_asset_amount(raw_event['tokenAmountOut'])
else:
raise AssertionError(f'Unexpected event type: {event_type}.')
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
user_address = deserialize_ethereum_address(raw_user_address)
pool_address = deserialize_ethereum_address(raw_pool_address)
try:
pool_address_token = EthereumToken(pool_address)
except UnknownAsset as e:
raise DeserializationError(
f'Balancer pool token with address {pool_address} should have been in the DB',
) from e
token_address = deserialize_ethereum_address(raw_token_address)
invest_event = BalancerInvestEvent(
tx_hash=tx_hash,
log_index=log_index,
address=user_address,
timestamp=timestamp,
event_type=event_type,
pool_address_token=pool_address_token,
token_address=token_address,
amount=amount,
)
return invest_event
def deserialize_from_db(
cls,
trade_tuple: AMMSwapDBTuple,
) -> 'AMMSwap':
"""Turns a tuple read from DB into an appropriate Swap.
May raise a DeserializationError if something is wrong with the DB data
Trade_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - from_address
4 - to_address
5 - timestamp
6 - location
7 - token0_identifier
8 - token1_identifier
9 - amount0_in
10 - amount1_in
11 - amount0_out
12 - amount1_out
"""
address = deserialize_ethereum_address(trade_tuple[2])
from_address = deserialize_ethereum_address(trade_tuple[3])
to_address = deserialize_ethereum_address(trade_tuple[4])
token0 = deserialize_ethereum_token_from_db(identifier=trade_tuple[7])
token1 = deserialize_ethereum_token_from_db(identifier=trade_tuple[8])
return cls(
tx_hash=trade_tuple[0],
log_index=trade_tuple[1],
address=address,
from_address=from_address,
to_address=to_address,
timestamp=deserialize_timestamp(trade_tuple[5]),
location=Location.deserialize_from_db(trade_tuple[6]),
token0=token0,
token1=token1,
amount0_in=deserialize_asset_amount(trade_tuple[9]),
amount1_in=deserialize_asset_amount(trade_tuple[10]),
amount0_out=deserialize_asset_amount(trade_tuple[11]),
amount1_out=deserialize_asset_amount(trade_tuple[12]),
)
def deserialize_from_db(
cls, data: LedgerActionDBTupleWithIdentifier) -> 'LedgerAction':
"""May raise:
- DeserializationError
- UnknownAsset
"""
return cls(
identifier=data[0],
timestamp=deserialize_timestamp(data[1]),
action_type=LedgerActionType.deserialize_from_db(data[2]),
location=Location.deserialize_from_db(data[3]),
amount=deserialize_asset_amount(data[4]),
asset=Asset(data[5]),
rate=deserialize_optional(data[6], deserialize_price),
rate_asset=deserialize_optional(data[7], Asset),
link=data[8],
notes=data[9],
)
def deserialize_from_db(cls, entry: TradeDBTuple) -> 'Trade':
"""May raise:
- DeserializationError
- UnknownAsset
"""
return Trade(
timestamp=deserialize_timestamp(entry[1]),
location=Location.deserialize_from_db(entry[2]),
base_asset=Asset(entry[3]),
quote_asset=Asset(entry[4]),
trade_type=TradeType.deserialize_from_db(entry[5]),
amount=deserialize_asset_amount(entry[6]),
rate=deserialize_price(entry[7]),
fee=deserialize_optional(entry[8], deserialize_fee),
fee_currency=deserialize_optional(entry[9], Asset),
link=entry[10],
notes=entry[11],
)
def _deserialize_unbond_request(
self,
raw_event: Dict[str, Any],
identity_address_map: Dict[ChecksumAddress, ChecksumAddress],
) -> UnbondRequest:
"""Deserialize an unbond request event.
It may raise KeyError.
"""
try:
adex_event = self._deserialize_adex_staking_event(
raw_event=raw_event,
identity_address_map=identity_address_map,
case='unbond_request',
)
bond_id = raw_event['bondId']
unlock_at = deserialize_timestamp(raw_event['willUnlock'])
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key in event: {msg}.'
log.error(
'Failed to deserialize an AdEx unbond request event',
error=msg,
raw_event=raw_event,
identity_address_map=identity_address_map,
)
raise DeserializationError(
'Failed to deserialize an AdEx unbond request event. Check logs for more details',
) from e
return UnbondRequest(
tx_hash=adex_event.tx_hash,
address=adex_event.address,
identity_address=adex_event.identity_address,
timestamp=adex_event.timestamp,
bond_id=bond_id,
value=Balance(),
unlock_at=unlock_at,
)
def deserialize_from_db(
cls,
event_tuple: LiquidityPoolEventDBTuple,
) -> 'LiquidityPoolEvent':
"""Turns a tuple read from DB into an appropriate LiquidityPoolEvent.
May raise a DeserializationError if something is wrong with the DB data
Event_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - timestamp
4 - type
5 - pool_address
6 - token0_identifier
7 - token1_identifier
8 - amount0
9 - amount1
10 - usd_price
11 - lp_amount
"""
event_type = EventType.deserialize_from_db(event_tuple[4])
token0 = deserialize_ethereum_token_from_db(identifier=event_tuple[6])
token1 = deserialize_ethereum_token_from_db(identifier=event_tuple[7])
return cls(
tx_hash=event_tuple[0],
log_index=event_tuple[1],
address=string_to_ethereum_address(event_tuple[2]),
timestamp=deserialize_timestamp(event_tuple[3]),
event_type=event_type,
pool_address=string_to_ethereum_address(event_tuple[5]),
token0=token0,
token1=token1,
amount0=deserialize_asset_amount(event_tuple[8]),
amount1=deserialize_asset_amount(event_tuple[9]),
usd_price=deserialize_price(event_tuple[10]),
lp_amount=deserialize_asset_amount(event_tuple[11]),
)
def query_online_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[Trade]:
"""Queries gemini for trades
"""
log.debug('Query gemini trade history',
start_ts=start_ts,
end_ts=end_ts)
trades = []
gemini_trades = []
for symbol in self.symbols:
gemini_trades = self._get_trades_for_symbol(
symbol=symbol,
start_ts=start_ts,
end_ts=end_ts,
)
for entry in gemini_trades:
try:
timestamp = deserialize_timestamp(entry['timestamp'])
if timestamp > end_ts:
break
trades.append(
Trade(
timestamp=timestamp,
location=Location.GEMINI,
pair=gemini_symbol_to_pair(symbol),
trade_type=deserialize_trade_type(entry['type']),
amount=deserialize_asset_amount(entry['amount']),
rate=deserialize_price(entry['price']),
fee=deserialize_fee(entry['fee_amount']),
fee_currency=Asset(entry['fee_currency']),
link=str(entry['tid']),
notes='',
))
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Gemini pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Gemini asset {e.asset_name}. '
f'Ignoring the trade.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a gemini trade. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a gemini trade.',
raw_trade=entry,
error=msg,
)
continue
return trades
def deserialize_from_db(
cls,
event_tuple: LiquidityPoolEventDBTuple,
) -> 'LiquidityPoolEvent':
"""Turns a tuple read from DB into an appropriate LiquidityPoolEvent.
May raise a DeserializationError if something is wrong with the DB data
Event_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - timestamp
4 - type
5 - pool_address
6 - is_token0_unknown
7 - token0_address
8 - token0_symbol
9 - token0_name
10 - token0_decimals
11 - is_token1_unknown
12 - token1_address
13 - token1_symbol
14 - token1_name
15 - token1_decimals
16 - amount0
17 - amount1
18 - usd_price
19 - lp_amount
"""
db_event_type = event_tuple[4]
if db_event_type not in {str(event_type) for event_type in EventType}:
raise DeserializationError(
f'Failed to deserialize event type. Unknown event: {db_event_type}.',
)
if db_event_type == str(EventType.MINT):
event_type = EventType.MINT
elif db_event_type == str(EventType.BURN):
event_type = EventType.BURN
else:
raise ValueError(f'Unexpected event type case: {db_event_type}.')
is_token0_unknown = event_tuple[6]
is_token1_unknown = event_tuple[11]
token0: Union[EthereumToken, UnknownEthereumToken]
token1: Union[EthereumToken, UnknownEthereumToken]
if is_token0_unknown:
token0 = deserialize_unknown_ethereum_token_from_db(
ethereum_address=event_tuple[7],
symbol=event_tuple[8],
name=event_tuple[9],
decimals=event_tuple[10],
)
else:
token0 = deserialize_ethereum_token_from_db(
identifier=event_tuple[8])
if is_token1_unknown:
token1 = deserialize_unknown_ethereum_token_from_db(
ethereum_address=event_tuple[12],
symbol=event_tuple[13],
name=event_tuple[14],
decimals=event_tuple[15],
)
else:
token1 = deserialize_ethereum_token_from_db(
identifier=event_tuple[13])
return cls(
tx_hash=event_tuple[0],
log_index=event_tuple[1],
address=string_to_ethereum_address(event_tuple[2]),
timestamp=deserialize_timestamp(event_tuple[3]),
event_type=event_type,
pool_address=string_to_ethereum_address(event_tuple[5]),
token0=token0,
token1=token1,
amount0=deserialize_asset_amount(event_tuple[16]),
amount1=deserialize_asset_amount(event_tuple[17]),
usd_price=deserialize_price(event_tuple[18]),
lp_amount=deserialize_asset_amount(event_tuple[19]),
)
def deserialize_swap(raw_swap: Dict[str, Any]) -> AMMSwap:
"""May raise DeserializationError"""
try:
tx_hash, log_index = deserialize_transaction_id(raw_swap['id'])
timestamp = deserialize_timestamp(raw_swap['timestamp'])
raw_tokens = raw_swap['poolAddress']['tokens']
token0_symbol = raw_swap['tokenInSym']
token1_symbol = raw_swap['tokenOutSym']
amount0_in = deserialize_asset_amount(raw_swap['tokenAmountIn'])
amount1_out = deserialize_asset_amount(raw_swap['tokenAmountOut'])
raw_user_address = raw_swap['userAddress']['id'] # address
raw_caller_address = raw_swap['caller'] # from_address
raw_pool_address = raw_swap['poolAddress']['id'] # to_address
raw_token_in_address = raw_swap['tokenIn'] # token0_address
raw_token_out_address = raw_swap['tokenOut'] # token1_address
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
if amount0_in == ZERO:
# Prevent a division by zero error when creating the trade
raise DeserializationError('TokenAmountIn balance is zero.')
# Checksum addresses
try:
user_address = to_checksum_address(raw_user_address)
except ValueError as e:
raise DeserializationError(
f'Invalid ethereum address: {raw_user_address} in userAddress.id.',
) from e
try:
caller_address = to_checksum_address(raw_caller_address)
except ValueError as e:
raise DeserializationError(
f'Invalid ethereum address: {raw_caller_address} in caller.',
) from e
try:
pool_address = to_checksum_address(raw_pool_address)
except ValueError as e:
raise DeserializationError(
f'Invalid ethereum address: {raw_pool_address} in poolAddress.id.',
) from e
try:
token_in_address = to_checksum_address(raw_token_in_address)
except ValueError as e:
raise DeserializationError(
f'Invalid ethereum address: {raw_token_in_address} in tokenIn.',
) from e
try:
token_out_address = to_checksum_address(raw_token_out_address)
except ValueError as e:
raise DeserializationError(
f'Invalid ethereum address: {raw_token_out_address} in tokenOut.',
) from e
# Get token0 and token1
# When the controller removes all the tokens from a pool, `raw_tokens` will
# be an empty list. Therefore it won't be possible to get their names and
# decimals. In case of having to instantiate an UnknownEthereumToken both
# params will be None.
if len(raw_tokens) != 0:
try:
raw_address_tokens = {
raw_token['address']: raw_token
for raw_token in raw_tokens
}
raw_token0 = raw_address_tokens[raw_token_in_address]
raw_token1 = raw_address_tokens[raw_token_out_address]
token0_name = raw_token0['name']
token0_decimals = raw_token0['decimals']
token1_name = raw_token1['name']
token1_decimals = raw_token1['decimals']
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
else:
token0_name = None
token0_decimals = None
token1_name = None
token1_decimals = None
token0 = get_ethereum_token(
symbol=token0_symbol,
ethereum_address=token_in_address,
name=token0_name,
decimals=token0_decimals,
)
token1 = get_ethereum_token(
symbol=token1_symbol,
ethereum_address=token_out_address,
name=token1_name,
decimals=token1_decimals,
)
amm_swap = AMMSwap(
tx_hash=tx_hash,
log_index=log_index,
address=user_address,
from_address=caller_address,
to_address=pool_address,
timestamp=timestamp,
location=Location.BALANCER,
token0=token0,
token1=token1,
amount0_in=amount0_in,
amount1_in=AssetAmount(ZERO),
amount0_out=AssetAmount(ZERO),
amount1_out=amount1_out,
)
return amm_swap
"""Process an asset movement result and deserialize it
May raise:
- DeserializationError
- UnknownAsset
- UnsupportedAsset
"""
if case == KucoinCase.DEPOSITS:
category = AssetMovementCategory.DEPOSIT
elif case == KucoinCase.WITHDRAWALS:
category = AssetMovementCategory.WITHDRAWAL
else:
raise AssertionError(f'Unexpected case: {case}')
try:
timestamp_ms = deserialize_timestamp(raw_result['createdAt'])
timestamp = Timestamp(int(timestamp_ms / 1000))
address = raw_result['address']
# The transaction id can have an @ which we should just get rid of
transaction_id = raw_result['walletTxId'].split('@')[0]
amount = deserialize_asset_amount(raw_result['amount'])
fee = deserialize_fee(raw_result['fee'])
fee_currency_symbol = raw_result['currency']
link_id = raw_result.get('id',
'') # NB: id only exists for withdrawals
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
fee_asset = asset_from_kucoin(fee_currency_symbol)
asset_movement = AssetMovement(