def deserialize_from_db(
cls,
event_tuple: BalancerEventDBTuple,
) -> 'BalancerEvent':
"""May raise DeserializationError
Event_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - timestamp
4 - type
5 - pool_address_token
6 - lp_amount
7 - usd_value
8 - amount0
9 - amount1
10 - amount2
11 - amount3
12 - amount4
13 - amount5
14 - amount6
15 - amount7
"""
event_tuple_type = event_tuple[4]
try:
event_type = getattr(BalancerBPTEventType,
event_tuple_type.upper())
except AttributeError as e:
raise DeserializationError(
f'Unexpected event type: {event_tuple_type}.') from e
pool_address_token = EthereumToken.from_identifier(
event_tuple[5],
form_with_incomplete_data=
True, # since some may not have decimals input correctly
)
if pool_address_token is None:
raise DeserializationError(
f'Balancer event pool token: {event_tuple[5]} not found in the DB.',
)
amounts: List[AssetAmount] = [
deserialize_asset_amount(item) for item in event_tuple[8:16]
if item is not None
]
return cls(
tx_hash=event_tuple[0],
log_index=event_tuple[1],
address=string_to_ethereum_address(event_tuple[2]),
timestamp=deserialize_timestamp(event_tuple[3]),
event_type=event_type,
pool_address_token=pool_address_token,
lp_balance=Balance(
amount=deserialize_asset_amount(event_tuple[6]),
usd_value=deserialize_price(event_tuple[7]),
),
amounts=amounts,
)
def parse_loan_csv(self) -> List:
"""Parses (if existing) the lendingHistory.csv and returns the history in a list
It can throw OSError, IOError if the file does not exist and csv.Error if
the file is not proper CSV"""
# the default filename, and should be (if at all) inside the data directory
path = os.path.join(self.db.user_data_dir, "lendingHistory.csv")
lending_history = []
with open(path, 'r') as csvfile:
history = csv.reader(csvfile, delimiter=',', quotechar='|')
next(history) # skip header row
for row in history:
try:
lending_history.append({
'currency': asset_from_poloniex(row[0]),
'earned': deserialize_asset_amount(row[6]),
'amount': deserialize_asset_amount(row[2]),
'fee': deserialize_asset_amount(row[5]),
'open': row[7],
'close': row[8],
})
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found loan with asset {e.asset_name}. Ignoring it.',
)
continue
except DeserializationError as e:
self.msg_aggregator.add_warning(
f'Failed to deserialize amount from loan due to {str(e)}. Ignoring it.',
)
continue
return lending_history
def query_balances(self) -> ExchangeQueryBalances:
try:
balances = self._private_api_query('balances')
balances.extend(self._private_api_query('balances/earn'))
except (GeminiPermissionError, RemoteError) as e:
msg = f'Gemini API request failed. {str(e)}'
log.error(msg)
return None, msg
returned_balances: DefaultDict[Asset, Balance] = defaultdict(Balance)
for entry in balances:
try:
balance_type = entry['type']
if balance_type == 'exchange':
amount = deserialize_asset_amount(entry['amount'])
else: # should be 'Earn'
amount = deserialize_asset_amount(entry['balance'])
# ignore empty balances
if amount == ZERO:
continue
asset = asset_from_gemini(entry['currency'])
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing gemini {balance_type} balance result due to '
f'inability to query USD price: {str(e)}. Skipping balance entry',
)
continue
returned_balances[asset] += Balance(
amount=amount,
usd_value=amount * usd_price,
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini balance result with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini {balance_type} balance result with unsupported '
f'asset {e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
f'Error processing a gemini {balance_type} balance. Check logs '
f'for details. Ignoring it.', )
log.error(
f'Error processing a gemini {balance_type} balance',
error=msg,
)
continue
return returned_balances, ''
def process_polo_loans(
msg_aggregator: MessagesAggregator,
data: List[Dict],
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[Loan]:
"""Takes in the list of loans from poloniex as returned by the return_lending_history
api call, processes it and returns it into our loan format
"""
new_data = []
for loan in reversed(data):
log.debug('processing poloniex loan', **make_sensitive(loan))
try:
close_time = deserialize_timestamp_from_poloniex_date(
loan['close'])
open_time = deserialize_timestamp_from_poloniex_date(loan['open'])
if open_time < start_ts:
continue
if close_time > end_ts:
continue
our_loan = Loan(
location=Location.POLONIEX,
open_time=open_time,
close_time=close_time,
currency=asset_from_poloniex(loan['currency']),
fee=deserialize_fee(loan['fee']),
earned=deserialize_asset_amount(loan['earned']),
amount_lent=deserialize_asset_amount(loan['amount']),
)
except UnsupportedAsset as e:
msg_aggregator.add_warning(
f'Found poloniex loan with unsupported asset'
f' {e.asset_name}. Ignoring it.', )
continue
except UnknownAsset as e:
msg_aggregator.add_warning(
f'Found poloniex loan with unknown asset'
f' {e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
msg_aggregator.add_error(
'Deserialization error while reading a poloniex loan. Check '
'logs for more details. Ignoring it.', )
log.error(
'Deserialization error while reading a poloniex loan',
loan=loan,
error=msg,
)
continue
new_data.append(our_loan)
new_data.sort(key=lambda loan: loan.open_time)
return new_data
def deserialize_bpt_event(
raw_event: Dict[str, Any],
event_type: Literal[BalancerBPTEventType.MINT, BalancerBPTEventType.BURN],
) -> BalancerBPTEvent:
"""May raise DeserializationError"""
try:
tx_hash, log_index = deserialize_transaction_id(raw_event['id'])
raw_user_address = raw_event['user']['id']
amount = deserialize_asset_amount(raw_event['amount'])
raw_pool = raw_event['pool']
raw_pool_address = raw_pool['id']
raw_pool_tokens = raw_pool['tokens']
total_weight = deserialize_asset_amount(raw_pool['totalWeight'])
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
if total_weight == ZERO:
raise DeserializationError('Pool weight is zero.')
user_address = deserialize_ethereum_address(raw_user_address)
pool_address = deserialize_ethereum_address(raw_pool_address)
pool_tokens = []
for raw_token in raw_pool_tokens:
try:
raw_token_address = raw_token['address']
token_symbol = raw_token['symbol']
token_name = raw_token['name']
token_decimals = raw_token['decimals']
token_weight = deserialize_asset_amount(raw_token['denormWeight'])
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
token_address = deserialize_ethereum_address(raw_token_address)
token = get_ethereum_token(
symbol=token_symbol,
ethereum_address=token_address,
name=token_name,
decimals=token_decimals,
)
pool_token = BalancerBPTEventPoolToken(
token=token,
weight=token_weight * 100 / total_weight,
)
pool_tokens.append(pool_token)
pool_tokens.sort(key=lambda pool_token: pool_token.token.ethereum_address)
bpt_event = BalancerBPTEvent(
tx_hash=tx_hash,
log_index=log_index,
address=user_address,
event_type=event_type,
pool_address=pool_address,
pool_tokens=pool_tokens,
amount=amount,
)
return bpt_event
def deserialize_invest_event(
raw_event: Dict[str, Any],
event_type: Literal[BalancerInvestEventType.ADD_LIQUIDITY,
BalancerInvestEventType.REMOVE_LIQUIDITY, ],
) -> BalancerInvestEvent:
"""May raise DeserializationError"""
try:
tx_hash, log_index = deserialize_transaction_id(raw_event['id'])
timestamp = deserialize_timestamp(raw_event['timestamp'])
raw_user_address = raw_event['userAddress']['id']
raw_pool_address = raw_event['poolAddress']['id']
if event_type == BalancerInvestEventType.ADD_LIQUIDITY:
raw_token_address = raw_event['tokenIn']['address']
amount = deserialize_asset_amount(raw_event['tokenAmountIn'])
elif event_type == BalancerInvestEventType.REMOVE_LIQUIDITY:
raw_token_address = raw_event['tokenOut']['address']
amount = deserialize_asset_amount(raw_event['tokenAmountOut'])
else:
raise AssertionError(f'Unexpected event type: {event_type}.')
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
try:
user_address = to_checksum_address(raw_user_address)
except ValueError as e:
raise DeserializationError(
f'Invalid ethereum address: {raw_user_address} in userAddress.',
) from e
try:
pool_address = to_checksum_address(raw_pool_address)
except ValueError as e:
raise DeserializationError(
f'Invalid ethereum address: {raw_pool_address} in poolAddress.',
) from e
try:
token_address = to_checksum_address(raw_token_address)
except ValueError as e:
raise DeserializationError(
f'Invalid ethereum address: {raw_token_address} in tokenIn.',
) from e
invest_event = BalancerInvestEvent(
tx_hash=tx_hash,
log_index=log_index,
address=user_address,
timestamp=timestamp,
event_type=event_type,
pool_address=pool_address,
token_address=token_address,
amount=amount,
)
return invest_event
def trade_from_conversion(trade_a: Dict[str, Any], trade_b: Dict[str, Any]) -> Optional[Trade]:
"""Turn information from a conversion into a trade
Mary raise:
- UnknownAsset due to Asset instantiation
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entires missing an expected entry
"""
# Check that the status is complete
if trade_a['status'] != 'completed':
return None
# Trade b will represent the asset we are converting to
if trade_b['amount']['amount'].startswith('-'):
trade_a, trade_b = trade_b, trade_a
timestamp = deserialize_timestamp_from_date(trade_a['updated_at'], 'iso8601', 'coinbase')
trade_type = deserialize_trade_type('sell')
tx_amount = AssetAmount(abs(deserialize_asset_amount(trade_a['amount']['amount'])))
tx_asset = asset_from_coinbase(trade_a['amount']['currency'], time=timestamp)
native_amount = deserialize_asset_amount(trade_b['amount']['amount'])
native_asset = asset_from_coinbase(trade_b['amount']['currency'], time=timestamp)
amount = tx_amount
# The rate is how much you get/give in quotecurrency if you buy/sell 1 unit of base currency
rate = Price(native_amount / tx_amount)
# Obtain fee amount in the native currency using data from both trades
amount_after_fee = deserialize_asset_amount(trade_b['native_amount']['amount'])
amount_before_fee = deserialize_asset_amount(trade_a['native_amount']['amount'])
# amount_after_fee + amount_before_fee is a negative amount and the fee needs to be positive
conversion_native_fee_amount = abs(amount_after_fee + amount_before_fee)
if ZERO not in (tx_amount, conversion_native_fee_amount, amount_before_fee):
# We have the fee amount in the native currency. To get it in the
# converted asset we have to get the rate
asset_native_rate = tx_amount / abs(amount_before_fee)
fee_amount = Fee(conversion_native_fee_amount / asset_native_rate)
else:
fee_amount = Fee(ZERO)
fee_asset = asset_from_coinbase(trade_a['amount']['currency'], time=timestamp)
return Trade(
timestamp=timestamp,
location=Location.COINBASE,
# in coinbase you are buying/selling tx_asset for native_asset
base_asset=tx_asset,
quote_asset=native_asset,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee_amount,
fee_currency=fee_asset,
link=str(trade_a['trade']['id']),
)
def trade_from_bittrex(bittrex_trade: Dict[str, Any]) -> Trade:
"""Turn a bittrex trade returned from bittrex trade history to our common trade
history format
As we saw in https://github.com/rotki/rotki/issues/1281 it's quite possible
that some keys don't exist in a trade. The required fields are here:
https://bittrex.github.io/api/v3#definition-Order
May raise:
- UnknownAsset/UnsupportedAsset due to bittrex_pair_to_world()
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entries missing an expected entry
"""
amount = deserialize_asset_amount(bittrex_trade['fillQuantity'])
timestamp = deserialize_timestamp_from_date(
date=bittrex_trade['closedAt'], # we only check closed orders
formatstr='iso8601',
location='bittrex',
)
if 'limit' in bittrex_trade:
rate = deserialize_price(bittrex_trade['limit'])
else:
rate = Price(
deserialize_asset_amount(bittrex_trade['proceeds']) /
deserialize_asset_amount(bittrex_trade['fillQuantity']),
)
order_type = deserialize_trade_type(bittrex_trade['direction'])
fee = deserialize_fee(bittrex_trade['commission'])
base_asset, quote_asset = bittrex_pair_to_world(bittrex_trade['marketSymbol'])
log.debug(
'Processing bittrex Trade',
sensitive_log=True,
amount=amount,
rate=rate,
order_type=order_type,
fee=fee,
bittrex_pair=bittrex_trade['marketSymbol'],
base_asset=base_asset,
quote_asset=quote_asset,
)
return Trade(
timestamp=timestamp,
location=Location.BITTREX,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=quote_asset,
link=str(bittrex_trade['id']),
)
def trade_from_coinbase(raw_trade: Dict[str, Any]) -> Optional[Trade]:
"""Turns a coinbase transaction into a rotkehlchen Trade.
https://developers.coinbase.com/api/v2?python#buys
If the coinbase transaction is not a trade related transaction returns None
Mary raise:
- UnknownAsset due to Asset instantiation
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entires missing an expected entry
"""
if raw_trade['status'] != 'completed':
# We only want to deal with completed trades
return None
# Contrary to the Coinbase documentation we will use created_at, and never
# payout_at. It seems like payout_at is not actually the time where the trade is settled.
# Reports generated by Coinbase use created_at as well
if raw_trade.get('created_at') is not None:
raw_time = raw_trade['created_at']
else:
raw_time = raw_trade['payout_at']
timestamp = deserialize_timestamp_from_date(raw_time, 'iso8601',
'coinbase')
trade_type = TradeType.deserialize(raw_trade['resource'])
tx_amount = deserialize_asset_amount(raw_trade['amount']['amount'])
tx_asset = asset_from_coinbase(raw_trade['amount']['currency'],
time=timestamp)
native_amount = deserialize_asset_amount(raw_trade['subtotal']['amount'])
native_asset = asset_from_coinbase(raw_trade['subtotal']['currency'],
time=timestamp)
amount = tx_amount
# The rate is how much you get/give in quotecurrency if you buy/sell 1 unit of base currency
rate = Price(native_amount / tx_amount)
fee_amount = deserialize_fee(raw_trade['fee']['amount'])
fee_asset = asset_from_coinbase(raw_trade['fee']['currency'],
time=timestamp)
return Trade(
timestamp=timestamp,
location=Location.COINBASE,
# in coinbase you are buying/selling tx_asset for native_asset
base_asset=tx_asset,
quote_asset=native_asset,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee_amount,
fee_currency=fee_asset,
link=str(raw_trade['id']),
)
def deserialize_from_db(
cls,
event_tuple: LiquidityPoolEventDBTuple,
) -> 'LiquidityPoolEvent':
"""Turns a tuple read from DB into an appropriate LiquidityPoolEvent.
May raise a DeserializationError if something is wrong with the DB data
Event_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - timestamp
4 - type
5 - pool_address
6 - token0_identifier
7 - token1_identifier
8 - amount0
9 - amount1
10 - usd_price
11 - lp_amount
"""
db_event_type = event_tuple[4]
if db_event_type not in {str(event_type) for event_type in EventType}:
raise DeserializationError(
f'Failed to deserialize event type. Unknown event: {db_event_type}.',
)
if db_event_type == str(EventType.MINT):
event_type = EventType.MINT
elif db_event_type == str(EventType.BURN):
event_type = EventType.BURN
else:
raise ValueError(f'Unexpected event type case: {db_event_type}.')
token0 = deserialize_ethereum_token_from_db(identifier=event_tuple[6])
token1 = deserialize_ethereum_token_from_db(identifier=event_tuple[7])
return cls(
tx_hash=event_tuple[0],
log_index=event_tuple[1],
address=string_to_ethereum_address(event_tuple[2]),
timestamp=deserialize_timestamp(event_tuple[3]),
event_type=event_type,
pool_address=string_to_ethereum_address(event_tuple[5]),
token0=token0,
token1=token1,
amount0=deserialize_asset_amount(event_tuple[8]),
amount1=deserialize_asset_amount(event_tuple[9]),
usd_price=deserialize_price(event_tuple[10]),
lp_amount=deserialize_asset_amount(event_tuple[11]),
)
def deserialize_from_db(
cls,
event_tuple: BalancerEventDBTuple,
) -> 'BalancerEvent':
"""May raise DeserializationError
Event_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - timestamp
4 - type
5 - pool_address
6 - lp_amount
7 - usd_value
8 - amount0
9 - amount1
10 - amount2
11 - amount3
12 - amount4
13 - amount5
14 - amount6
15 - amount7
"""
event_tuple_type = event_tuple[4]
try:
event_type = getattr(BalancerBPTEventType,
event_tuple_type.upper())
except AttributeError as e:
raise DeserializationError(
f'Unexpected event type: {event_tuple_type}.') from e
amounts: List[AssetAmount] = [
deserialize_asset_amount(item) for item in event_tuple[8:16]
if item is not None
]
return cls(
tx_hash=event_tuple[0],
log_index=event_tuple[1],
address=string_to_ethereum_address(event_tuple[2]),
timestamp=deserialize_timestamp(event_tuple[3]),
event_type=event_type,
pool_address=string_to_ethereum_address(event_tuple[5]),
lp_balance=Balance(
amount=deserialize_asset_amount(event_tuple[6]),
usd_value=deserialize_price(event_tuple[7]),
),
amounts=amounts,
)
def trade_from_coinbase(raw_trade: Dict[str, Any]) -> Optional[Trade]:
"""Turns a coinbase transaction into a rotkehlchen Trade.
https://developers.coinbase.com/api/v2?python#buys
If the coinbase transaction is not a trade related transaction returns None
Throws:
- UnknownAsset due to Asset instantiation
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entires missing an expected entry
"""
if raw_trade['status'] != 'completed':
# We only want to deal with completed trades
return None
if raw_trade['instant']:
raw_time = raw_trade['created_at']
else:
raw_time = raw_trade['payout_at']
timestamp = deserialize_timestamp_from_date(raw_time, 'iso8601',
'coinbase')
trade_type = deserialize_trade_type(raw_trade['resource'])
tx_amount = deserialize_asset_amount(raw_trade['amount']['amount'])
tx_asset = asset_from_coinbase(raw_trade['amount']['currency'],
time=timestamp)
native_amount = deserialize_asset_amount(raw_trade['subtotal']['amount'])
native_asset = asset_from_coinbase(raw_trade['subtotal']['currency'],
time=timestamp)
# in coinbase you are buying/selling tx_asset for native_asset
pair = TradePair(f'{tx_asset.identifier}_{native_asset.identifier}')
amount = tx_amount
# The rate is how much you get/give in quotecurrency if you buy/sell 1 unit of base currency
rate = Price(native_amount / tx_amount)
fee_amount = deserialize_fee(raw_trade['fee']['amount'])
fee_asset = asset_from_coinbase(raw_trade['fee']['currency'],
time=timestamp)
return Trade(
timestamp=timestamp,
location=Location.COINBASE,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee_amount,
fee_currency=fee_asset,
link=str(raw_trade['id']),
)
def query_balances(self, **kwargs: Any) -> ExchangeQueryBalances:
assets_balance: Dict[Asset, Balance] = {}
try:
resp_info = self._api_query('get', 'user/balance')
except RemoteError as e:
msg = (
'ICONOMI API request failed. Could not reach ICONOMI due '
'to {}'.format(e)
)
log.error(msg)
return None, msg
if resp_info['currency'] != 'USD':
raise RemoteError('Iconomi API did not return values in USD')
for balance_info in resp_info['assetList']:
ticker = balance_info['ticker']
try:
asset = asset_from_iconomi(ticker)
# There seems to be a bug in the ICONOMI API regarding balance_info['value'].
# The value is supposed to be in USD, but is actually returned
# in EUR. So let's use the Inquirer for now.
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing ICONOMI balance entry due to inability to '
f'query USD price: {str(e)}. Skipping balance entry',
)
continue
try:
amount = deserialize_asset_amount(balance_info['balance'])
except DeserializationError as e:
self.msg_aggregator.add_warning(
f'Skipping iconomi balance entry {balance_info} due to {str(e)}',
)
continue
assets_balance[asset] = Balance(
amount=amount,
usd_value=amount * usd_price,
)
except (UnknownAsset, UnsupportedAsset) as e:
asset_tag = 'unknown' if isinstance(e, UnknownAsset) else 'unsupported'
self.msg_aggregator.add_warning(
f'Found {asset_tag} ICONOMI asset {ticker}. '
f' Ignoring its balance query.',
)
continue
for balance_info in resp_info['daaList']:
ticker = balance_info['ticker']
self.msg_aggregator.add_warning(
f'Found unsupported ICONOMI strategy {ticker}. '
f' Ignoring its balance query.',
)
return assets_balance, ''
def query_balances(self) -> ExchangeQueryBalances:
returned_balances: Dict[Asset, Balance] = {}
try:
resp = self._api_query_dict('get', 'user/wallet',
{'currency': 'XBt'})
# Bitmex shows only BTC balance
usd_price = Inquirer().find_usd_price(A_BTC)
except RemoteError as e:
msg = f'Bitmex API request failed due to: {str(e)}'
log.error(msg)
return None, msg
# result is in satoshis
try:
amount = satoshis_to_btc(deserialize_asset_amount(resp['amount']))
except DeserializationError as e:
msg = f'Bitmex API request failed. Failed to deserialized amount due to {str(e)}'
log.error(msg)
return None, msg
usd_value = amount * usd_price
returned_balances[A_BTC] = Balance(
amount=amount,
usd_value=usd_value,
)
log.debug(
'Bitmex balance query result',
currency='BTC',
amount=amount,
usd_value=usd_value,
)
return returned_balances, ''
def asset_movements_from_dictlist(
given_data: List[Dict[str, Any]],
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
""" Gets a list of dict asset movements, most probably read from the json files and
a time period. Returns it as a list of the AssetMovement tuples that are inside the time period
May raise:
- DeserializationError: If the given_data dict contains data in an unexpected format
- KeyError: If the given_data dict contains data in an unexpected format
"""
returned_movements = list()
for movement in given_data:
timestamp = deserialize_timestamp(movement['timestamp'])
if timestamp < start_ts:
continue
if timestamp > end_ts:
break
category = deserialize_asset_movement_category(movement['category'])
amount = deserialize_asset_amount(movement['amount'])
fee = deserialize_fee(movement['fee'])
returned_movements.append(AssetMovement(
exchange=deserialize_exchange_name(movement['exchange']),
category=category,
timestamp=timestamp,
asset=Asset(movement['asset']),
amount=amount,
fee=fee,
))
return returned_movements
def trade_from_ftx(raw_trade: Dict[str, Any]) -> Optional[Trade]:
"""Turns an FTX transaction into a rotki Trade.
May raise:
- UnknownAsset due to Asset instantiation
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entries missing an expected key
"""
# In the case of perpetuals and futures this fields can be None
if raw_trade.get('baseCurrency', None) is None:
return None
if raw_trade.get('quoteCurrency', None) is None:
return None
timestamp = deserialize_timestamp_from_date(raw_trade['time'], 'iso8601', 'FTX')
trade_type = deserialize_trade_type(raw_trade['side'])
base_asset = asset_from_ftx(raw_trade['baseCurrency'])
quote_asset = asset_from_ftx(raw_trade['quoteCurrency'])
amount = deserialize_asset_amount(raw_trade['size'])
rate = deserialize_price(raw_trade['price'])
fee_currency = asset_from_ftx(raw_trade['feeCurrency'])
fee = deserialize_fee(raw_trade['fee'])
return Trade(
timestamp=timestamp,
location=Location.FTX,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(raw_trade['id']),
)
def deserialize_trade(data: Dict[str, Any]) -> Trade:
"""
Takes a dict trade representation of our common trade format and serializes
it into the Trade object
May raise:
- UnknownAsset: If the fee_currency string is not a known asset
- DeserializationError: If any of the trade dict entries is not as expected
"""
pair = data['pair']
rate = deserialize_price(data['rate'])
amount = deserialize_asset_amount(data['amount'])
trade_type = deserialize_trade_type(data['trade_type'])
trade_link = ''
if 'link' in data:
trade_link = data['link']
trade_notes = ''
if 'notes' in data:
trade_notes = data['notes']
return Trade(
timestamp=data['timestamp'],
location=data['location'],
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=deserialize_fee(data['fee']),
fee_currency=Asset(data['fee_currency']),
link=trade_link,
notes=trade_notes,
)
def deserialize_from_db(
cls,
data: LedgerActionDBTupleWithIdentifier,
given_gitcoin_map: Optional[Dict[int, GitcoinEventDataDB]] = None,
) -> 'LedgerAction':
"""May raise:
- DeserializationError
- UnknownAsset
"""
extra_data = None
gitcoin_map = {} if not given_gitcoin_map else given_gitcoin_map
gitcoin_data = gitcoin_map.get(data[0], None)
if gitcoin_data is not None:
extra_data = GitcoinEventData.deserialize_from_db(
data=gitcoin_data)
return cls(
identifier=data[0],
timestamp=deserialize_timestamp(data[1]),
action_type=LedgerActionType.deserialize_from_db(data[2]),
location=Location.deserialize_from_db(data[3]),
amount=deserialize_asset_amount(data[4]),
asset=Asset(data[5]),
rate=deserialize_optional(data[6], deserialize_price),
rate_asset=deserialize_optional(data[7], Asset),
link=data[8],
notes=data[9],
extra_data=extra_data,
)
def query_balances(
self) -> Tuple[Optional[Dict[Asset, Dict[str, Any]]], str]:
try:
resp = self._api_query('accounts')
except RemoteError as e:
msg = ('Coinbase API request failed. Could not reach coinbase due '
'to {}'.format(e))
log.error(msg)
return None, msg
returned_balances: Dict[Asset, Dict[str, Any]] = dict()
for account in resp:
try:
if not account['balance']:
continue
amount = deserialize_asset_amount(account['balance']['amount'])
# ignore empty balances. Coinbase returns zero balances for everything
# a user does not own
if amount == ZERO:
continue
asset = Asset(account['balance']['currency'])
usd_price = Inquirer().find_usd_price(asset=asset)
if asset in returned_balances:
amount = returned_balances[asset]['amount'] + amount
else:
returned_balances[asset] = {}
returned_balances[asset]['amount'] = amount
usd_value = returned_balances[asset]['amount'] * usd_price
returned_balances[asset]['usd_value'] = usd_value
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase balance result with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase balance result with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Error processing a coinbase account balance. Check logs '
'for details. Ignoring it.', )
log.error(
'Error processing a coinbase account balance',
account_balance=account,
error=msg,
)
continue
return returned_balances, ''
def _deserialize_asset_movement(
self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from binance and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if 'insertTime' in raw_data:
category = AssetMovementCategory.DEPOSIT
time_key = 'insertTime'
fee = Fee(ZERO)
else:
category = AssetMovementCategory.WITHDRAWAL
time_key = 'applyTime'
fee = Fee(deserialize_asset_amount(raw_data['transactionFee']))
timestamp = deserialize_timestamp_from_binance(raw_data[time_key])
asset = asset_from_binance(raw_data['asset'])
tx_id = get_key_if_has_val(raw_data, 'txId')
internal_id = get_key_if_has_val(raw_data, 'id')
link_str = str(internal_id) if internal_id else str(
tx_id) if tx_id else ''
return AssetMovement(
location=self.location,
category=category,
address=deserialize_asset_movement_address(
raw_data, 'address', asset),
transaction_id=tx_id,
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
raw_data['amount']),
fee_asset=asset,
fee=fee,
link=link_str,
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(self.location)} deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(self.location)} deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
f'Error processing a {str(self.location)} deposit/withdrawal. Check logs '
f'for details. Ignoring it.', )
log.error(
f'Error processing a {str(self.location)} deposit/withdrawal',
asset_movement=raw_data,
error=msg,
)
return None
def _query_cross_collateral_futures_balances(
self,
balances: DefaultDict[Asset, Balance],
) -> DefaultDict[Asset, Balance]:
"""Queries binance collateral future balances and if any found adds them to `balances`
May raise:
- RemoteError
"""
futures_response = self.api_query_dict('sapi', 'futures/loan/wallet')
try:
cross_collaterals = futures_response['crossCollaterals']
for entry in cross_collaterals:
amount = deserialize_asset_amount(entry['locked'])
if amount == ZERO:
continue
try:
asset = asset_from_binance(entry['collateralCoin'])
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found unsupported {self.name} asset {e.asset_name}. '
f'Ignoring its futures balance query.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown {self.name} asset {e.asset_name}. '
f'Ignoring its futures balance query.',
)
continue
except DeserializationError:
self.msg_aggregator.add_error(
f'Found {self.name} asset with non-string type '
f'{type(entry["asset"])}. Ignoring its futures balance query.',
)
continue
try:
usd_price = Inquirer().find_usd_price(asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing {self.name} balance entry due to inability to '
f'query USD price: {str(e)}. Skipping balance entry',
)
continue
balances[asset] += Balance(
amount=amount,
usd_value=amount * usd_price,
)
except KeyError as e:
self.msg_aggregator.add_error(
f'At {self.name} futures balance query did not find expected key '
f'{str(e)}. Skipping futures query...',
)
return balances
def _deserialize_asset_movement(self, raw_result: List[Any]) -> AssetMovement:
"""Process an asset movement (i.e. deposit or withdrawal) from Bitfinex
and deserialize it.
Bitfinex support confirmed the following in regards to
DESTINATION_ADDRESS and TRANSACTION_ID:
- Fiat movement: both attributes won't show any value.
- Cryptocurrency movement: address and tx id on the blockchain.
Timestamp is from MTS_STARTED (when the movement was created), and not
from MTS_UPDATED (when it was completed/cancelled).
Can raise DeserializationError.
Schema reference in:
https://docs.bitfinex.com/reference#rest-auth-movements
"""
if raw_result[9] != 'COMPLETED':
raise DeserializationError(
f'Unexpected bitfinex movement with status: {raw_result[5]}. '
f'Only completed movements are processed. Raw movement: {raw_result}',
)
try:
fee_asset = asset_from_bitfinex(
bitfinex_name=raw_result[1],
currency_map=self.currency_map,
)
except (UnknownAsset, UnsupportedAsset) as e:
asset_tag = 'Unknown' if isinstance(e, UnknownAsset) else 'Unsupported'
raise DeserializationError(
f'{asset_tag} {e.asset_name} found while processing movement asset '
f'due to: {str(e)}',
) from e
amount = deserialize_asset_amount(raw_result[12])
category = (
AssetMovementCategory.DEPOSIT
if amount > ZERO
else AssetMovementCategory.WITHDRAWAL
)
address = None
transaction_id = None
if fee_asset.is_fiat() is False:
address = str(raw_result[16])
transaction_id = str(raw_result[20])
asset_movement = AssetMovement(
timestamp=Timestamp(int(raw_result[5] / 1000)),
location=Location.BITFINEX,
category=category,
address=address,
transaction_id=transaction_id,
asset=fee_asset,
amount=abs(amount),
fee_asset=fee_asset,
fee=Fee(abs(deserialize_fee(raw_result[13]))),
link=str(raw_result[0]),
)
return asset_movement
def deserialize_invest_event(
raw_event: Dict[str, Any],
event_type: Literal[
BalancerInvestEventType.ADD_LIQUIDITY,
BalancerInvestEventType.REMOVE_LIQUIDITY,
],
) -> BalancerInvestEvent:
"""May raise DeserializationError"""
try:
tx_hash, log_index = deserialize_transaction_id(raw_event['id'])
timestamp = deserialize_timestamp(raw_event['timestamp'])
raw_user_address = raw_event['userAddress']['id']
raw_pool_address = raw_event['poolAddress']['id']
if event_type == BalancerInvestEventType.ADD_LIQUIDITY:
raw_token_address = raw_event['tokenIn']['address']
amount = deserialize_asset_amount(raw_event['tokenAmountIn'])
elif event_type == BalancerInvestEventType.REMOVE_LIQUIDITY:
raw_token_address = raw_event['tokenOut']['address']
amount = deserialize_asset_amount(raw_event['tokenAmountOut'])
else:
raise AssertionError(f'Unexpected event type: {event_type}.')
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
user_address = deserialize_ethereum_address(raw_user_address)
pool_address = deserialize_ethereum_address(raw_pool_address)
try:
pool_address_token = EthereumToken(pool_address)
except UnknownAsset as e:
raise DeserializationError(
f'Balancer pool token with address {pool_address} should have been in the DB',
) from e
token_address = deserialize_ethereum_address(raw_token_address)
invest_event = BalancerInvestEvent(
tx_hash=tx_hash,
log_index=log_index,
address=user_address,
timestamp=timestamp,
event_type=event_type,
pool_address_token=pool_address_token,
token_address=token_address,
amount=amount,
)
return invest_event
def deserialize_from_db(
cls,
trade_tuple: AMMSwapDBTuple,
) -> 'AMMSwap':
"""Turns a tuple read from DB into an appropriate Swap.
May raise a DeserializationError if something is wrong with the DB data
Trade_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - from_address
4 - to_address
5 - timestamp
6 - location
7 - token0_identifier
8 - token1_identifier
9 - amount0_in
10 - amount1_in
11 - amount0_out
12 - amount1_out
"""
address = deserialize_ethereum_address(trade_tuple[2])
from_address = deserialize_ethereum_address(trade_tuple[3])
to_address = deserialize_ethereum_address(trade_tuple[4])
token0 = deserialize_ethereum_token_from_db(identifier=trade_tuple[7])
token1 = deserialize_ethereum_token_from_db(identifier=trade_tuple[8])
return cls(
tx_hash=trade_tuple[0],
log_index=trade_tuple[1],
address=address,
from_address=from_address,
to_address=to_address,
timestamp=deserialize_timestamp(trade_tuple[5]),
location=Location.deserialize_from_db(trade_tuple[6]),
token0=token0,
token1=token1,
amount0_in=deserialize_asset_amount(trade_tuple[9]),
amount1_in=deserialize_asset_amount(trade_tuple[10]),
amount0_out=deserialize_asset_amount(trade_tuple[11]),
amount1_out=deserialize_asset_amount(trade_tuple[12]),
)
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
result = self.query_until_finished(
endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict=dict(type='deposit'),
)
result.extend(self.query_until_finished(
endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict=dict(type='withdrawal'),
))
log.debug('Kraken deposit/withdrawals query result', num_results=len(result))
movements = list()
for movement in result:
try:
asset = asset_from_kraken(movement['asset'])
movements.append(AssetMovement(
location=Location.KRAKEN,
category=deserialize_asset_movement_category(movement['type']),
timestamp=deserialize_timestamp_from_kraken(movement['time']),
asset=asset,
amount=deserialize_asset_amount(movement['amount']),
fee_asset=asset,
fee=deserialize_fee(movement['fee']),
link=str(movement['refid']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown kraken asset {e.asset_name}. '
f'Ignoring its deposit/withdrawals query.',
)
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a kraken deposit/withdrawals. '
'Check logs for details. Ignoring it.',
)
log.error(
'Error processing a kraken deposit/withdrawal.',
raw_asset_movement=movement,
error=msg,
)
continue
return movements
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
result = self._get_paginated_query(
endpoint='transfers',
start_ts=start_ts,
end_ts=end_ts,
)
movements = []
for entry in result:
try:
timestamp = deserialize_timestamp(entry['timestampms'])
timestamp = Timestamp(int(timestamp / 1000))
asset = Asset(entry['currency'])
movement = AssetMovement(
location=Location.GEMINI,
category=deserialize_asset_movement_category(entry['type']),
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount(entry['amount']),
fee_asset=asset,
# Gemini does not include withdrawal fees neither in the API nor in their UI
fee=Fee(ZERO),
link=str(entry['eid']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.',
)
continue
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.',
)
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Error processing a gemini deposit/withdrawal. Check logs '
'for details. Ignoring it.',
)
log.error(
'Error processing a gemini deposit_withdrawal',
asset_movement=entry,
error=msg,
)
continue
movements.append(movement)
return movements
def trade_from_bitcoinde(raw_trade: Dict) -> Trade:
"""Convert bitcoin.de raw data to a trade
May raise:
- DeserializationError
- UnknownAsset
- KeyError
"""
try:
timestamp = deserialize_timestamp_from_date(
raw_trade['successfully_finished_at'],
'iso8601',
'bitcoinde',
)
except KeyError:
# For very old trades (2013) bitcoin.de does not return 'successfully_finished_at'
timestamp = deserialize_timestamp_from_date(
raw_trade['trade_marked_as_paid_at'],
'iso8601',
'bitcoinde',
)
trade_type = TradeType.deserialize(raw_trade['type'])
tx_amount = deserialize_asset_amount(raw_trade['amount_currency_to_trade'])
native_amount = deserialize_asset_amount(
raw_trade['volume_currency_to_pay'])
tx_asset, native_asset = bitcoinde_pair_to_world(raw_trade['trading_pair'])
amount = tx_amount
rate = Price(native_amount / tx_amount)
fee_amount = deserialize_fee(raw_trade['fee_currency_to_pay'])
fee_asset = A_EUR
return Trade(
timestamp=timestamp,
location=Location.BITCOINDE,
base_asset=tx_asset,
quote_asset=native_asset,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee_amount,
fee_currency=fee_asset,
link=str(raw_trade['trade_id']),
)
def _deserialize_asset_movement(self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from bittrex and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data['status'] != 'COMPLETED':
# Don't mind failed/in progress asset movements
return None
if 'source' in raw_data:
category = AssetMovementCategory.DEPOSIT
fee = Fee(ZERO)
else:
category = AssetMovementCategory.WITHDRAWAL
fee = deserialize_fee(raw_data.get('txCost', 0))
timestamp = deserialize_timestamp_from_date(
date=raw_data['completedAt'], # we only check completed orders
formatstr='iso8601',
location='bittrex',
)
asset = asset_from_bittrex(raw_data['currencySymbol'])
return AssetMovement(
location=Location.BITTREX,
category=category,
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount(raw_data['quantity']),
fee_asset=asset,
fee=fee,
link=str(raw_data.get('txId', '')),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found bittrex deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.',
)
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found bittrex deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.',
)
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a bittrex '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of bittrex '
f'asset_movement {raw_data}. Error was: {str(e)}',
)
return None
def query_balances(self) -> Tuple[Optional[Dict[Asset, Dict[str, Any]]], str]:
try:
balances = self._private_api_query('balances')
except (GeminiPermissionError, RemoteError) as e:
msg = f'Gemini API request failed. {str(e)}'
log.error(msg)
return None, msg
returned_balances: Dict[Asset, Dict[str, Any]] = {}
for entry in balances:
try:
amount = deserialize_asset_amount(entry['amount'])
# ignore empty balances
if amount == ZERO:
continue
asset = Asset(entry['currency'])
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing gemini balance result due to inability to '
f'query USD price: {str(e)}. Skipping balance entry',
)
continue
returned_balances[asset] = {
'amount': amount,
'usd_value': amount * usd_price,
}
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini balance result with unknown asset '
f'{e.asset_name}. Ignoring it.',
)
continue
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini balance result with unsupported asset '
f'{e.asset_name}. Ignoring it.',
)
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Error processing a gemini balance. Check logs '
'for details. Ignoring it.',
)
log.error(
'Error processing a gemini balance',
error=msg,
)
continue
return returned_balances, ''
def _deserialize_trade(self, raw_result: List[Any]) -> Trade:
"""Process a trade result from Bitfinex and deserialize it.
The base and quote assets are instantiated using the `fee_currency_symbol`
(from raw_result[10]) over the pair (from raw_result[1]).
Known pairs format: 'tETHUST', 'tETH:UST'.
Can raise DeserializationError.
Schema reference in:
https://docs.bitfinex.com/reference#rest-auth-trades
"""
amount = deserialize_asset_amount(raw_result[4])
trade_type = TradeType.BUY if amount >= ZERO else TradeType.SELL
bfx_pair = self._process_bfx_pair(raw_result[1])
if bfx_pair not in self.pair_bfx_symbols_map:
raise DeserializationError(
f'Could not deserialize bitfinex trade pair {raw_result[1]}. '
f'Raw trade: {raw_result}', )
bfx_base_asset_symbol, bfx_quote_asset_symbol = self.pair_bfx_symbols_map[
bfx_pair]
try:
base_asset = asset_from_bitfinex(
bitfinex_name=bfx_base_asset_symbol,
currency_map=self.currency_map,
)
quote_asset = asset_from_bitfinex(
bitfinex_name=bfx_quote_asset_symbol,
currency_map=self.currency_map,
)
fee_asset = asset_from_bitfinex(
bitfinex_name=raw_result[10],
currency_map=self.currency_map,
)
except (UnknownAsset, UnsupportedAsset) as e:
asset_tag = 'Unknown' if isinstance(
e, UnknownAsset) else 'Unsupported'
raise DeserializationError(
f'{asset_tag} {e.asset_name} found while processing trade pair due to: {str(e)}',
) from e
trade = Trade(
timestamp=Timestamp(int(raw_result[2] / 1000)),
location=Location.BITFINEX,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=trade_type,
amount=AssetAmount(abs(amount)),
rate=deserialize_price(raw_result[5]),
fee=Fee(abs(deserialize_fee(raw_result[9]))),
fee_currency=fee_asset,
link=str(raw_result[0]),
notes='',
)
return trade
