def test_coinbase_query_income_loss_expense(function_scope_coinbase):
"""Test that coinbase deposit/withdrawals history query works fine for the happy path"""
coinbase = function_scope_coinbase
with patch.object(coinbase.session, 'get', side_effect=mock_normal_coinbase_query):
ledger_actions = coinbase.query_online_income_loss_expense(
start_ts=0,
end_ts=1611426233,
)
warnings = coinbase.msg_aggregator.consume_warnings()
errors = coinbase.msg_aggregator.consume_errors()
assert len(warnings) == 0
assert len(errors) == 0
assert len(ledger_actions) == 2
expected_ledger_actions = [LedgerAction(
identifier=ledger_actions[0].identifier,
location=Location.COINBASE,
action_type=LedgerActionType.INCOME,
timestamp=1609877514,
asset=asset_from_coinbase('NMR'),
amount=FVal('0.02762431'),
rate=FVal('36.56199919563601769600761069'),
rate_asset=A_USD,
link='id4',
notes=('Received Numeraire '
'From Coinbase Earn '
'Received 0.02762431 NMR ($1.01)'),
), LedgerAction(
identifier=ledger_actions[1].identifier,
location=Location.COINBASE,
action_type=LedgerActionType.INCOME,
timestamp=1611426233,
asset=asset_from_coinbase('ALGO'),
amount=FVal('0.000076'),
rate=ZERO,
rate_asset=A_USD,
link='id5',
notes=('Algorand reward '
'From Coinbase '
'Received 0.000076 ALGO ($0.00)'),
)]
assert expected_ledger_actions == ledger_actions
# and now try to query within a specific range
with patch.object(coinbase.session, 'get', side_effect=mock_normal_coinbase_query):
ledger_actions = coinbase.query_online_income_loss_expense(
start_ts=0,
end_ts=1609877514,
)
warnings = coinbase.msg_aggregator.consume_warnings()
errors = coinbase.msg_aggregator.consume_errors()
assert len(warnings) == 0
assert len(errors) == 0
assert len(ledger_actions) == 1
assert ledger_actions[0].action_type == LedgerActionType.INCOME
assert ledger_actions[0].timestamp == 1609877514
def trade_from_coinbase(raw_trade: Dict[str, Any]) -> Optional[Trade]:
"""Turns a coinbase transaction into a rotkehlchen Trade.
https://developers.coinbase.com/api/v2?python#buys
If the coinbase transaction is not a trade related transaction returns None
Mary raise:
- UnknownAsset due to Asset instantiation
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entires missing an expected entry
"""
if raw_trade['status'] != 'completed':
# We only want to deal with completed trades
return None
# Contrary to the Coinbase documentation we will use created_at, and never
# payout_at. It seems like payout_at is not actually the time where the trade is settled.
# Reports generated by Coinbase use created_at as well
if raw_trade.get('created_at') is not None:
raw_time = raw_trade['created_at']
else:
raw_time = raw_trade['payout_at']
timestamp = deserialize_timestamp_from_date(raw_time, 'iso8601',
'coinbase')
trade_type = TradeType.deserialize(raw_trade['resource'])
tx_amount = deserialize_asset_amount(raw_trade['amount']['amount'])
tx_asset = asset_from_coinbase(raw_trade['amount']['currency'],
time=timestamp)
native_amount = deserialize_asset_amount(raw_trade['subtotal']['amount'])
native_asset = asset_from_coinbase(raw_trade['subtotal']['currency'],
time=timestamp)
amount = tx_amount
# The rate is how much you get/give in quotecurrency if you buy/sell 1 unit of base currency
rate = Price(native_amount / tx_amount)
fee_amount = deserialize_fee(raw_trade['fee']['amount'])
fee_asset = asset_from_coinbase(raw_trade['fee']['currency'],
time=timestamp)
return Trade(
timestamp=timestamp,
location=Location.COINBASE,
# in coinbase you are buying/selling tx_asset for native_asset
base_asset=tx_asset,
quote_asset=native_asset,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee_amount,
fee_currency=fee_asset,
link=str(raw_trade['id']),
)
def trade_from_conversion(trade_a: Dict[str, Any], trade_b: Dict[str, Any]) -> Optional[Trade]:
"""Turn information from a conversion into a trade
Mary raise:
- UnknownAsset due to Asset instantiation
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entires missing an expected entry
"""
# Check that the status is complete
if trade_a['status'] != 'completed':
return None
# Trade b will represent the asset we are converting to
if trade_b['amount']['amount'].startswith('-'):
trade_a, trade_b = trade_b, trade_a
timestamp = deserialize_timestamp_from_date(trade_a['updated_at'], 'iso8601', 'coinbase')
trade_type = deserialize_trade_type('sell')
tx_amount = AssetAmount(abs(deserialize_asset_amount(trade_a['amount']['amount'])))
tx_asset = asset_from_coinbase(trade_a['amount']['currency'], time=timestamp)
native_amount = deserialize_asset_amount(trade_b['amount']['amount'])
native_asset = asset_from_coinbase(trade_b['amount']['currency'], time=timestamp)
amount = tx_amount
# The rate is how much you get/give in quotecurrency if you buy/sell 1 unit of base currency
rate = Price(native_amount / tx_amount)
# Obtain fee amount in the native currency using data from both trades
amount_after_fee = deserialize_asset_amount(trade_b['native_amount']['amount'])
amount_before_fee = deserialize_asset_amount(trade_a['native_amount']['amount'])
# amount_after_fee + amount_before_fee is a negative amount and the fee needs to be positive
conversion_native_fee_amount = abs(amount_after_fee + amount_before_fee)
if ZERO not in (tx_amount, conversion_native_fee_amount, amount_before_fee):
# We have the fee amount in the native currency. To get it in the
# converted asset we have to get the rate
asset_native_rate = tx_amount / abs(amount_before_fee)
fee_amount = Fee(conversion_native_fee_amount / asset_native_rate)
else:
fee_amount = Fee(ZERO)
fee_asset = asset_from_coinbase(trade_a['amount']['currency'], time=timestamp)
return Trade(
timestamp=timestamp,
location=Location.COINBASE,
# in coinbase you are buying/selling tx_asset for native_asset
base_asset=tx_asset,
quote_asset=native_asset,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee_amount,
fee_currency=fee_asset,
link=str(trade_a['trade']['id']),
)
def trade_from_coinbase(raw_trade: Dict[str, Any]) -> Optional[Trade]:
"""Turns a coinbase transaction into a rotkehlchen Trade.
https://developers.coinbase.com/api/v2?python#buys
If the coinbase transaction is not a trade related transaction returns None
Throws:
- UnknownAsset due to Asset instantiation
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entires missing an expected entry
"""
if raw_trade['status'] != 'completed':
# We only want to deal with completed trades
return None
if raw_trade['instant']:
raw_time = raw_trade['created_at']
else:
raw_time = raw_trade['payout_at']
timestamp = deserialize_timestamp_from_date(raw_time, 'iso8601',
'coinbase')
trade_type = deserialize_trade_type(raw_trade['resource'])
tx_amount = deserialize_asset_amount(raw_trade['amount']['amount'])
tx_asset = asset_from_coinbase(raw_trade['amount']['currency'],
time=timestamp)
native_amount = deserialize_asset_amount(raw_trade['subtotal']['amount'])
native_asset = asset_from_coinbase(raw_trade['subtotal']['currency'],
time=timestamp)
# in coinbase you are buying/selling tx_asset for native_asset
pair = TradePair(f'{tx_asset.identifier}_{native_asset.identifier}')
amount = tx_amount
# The rate is how much you get/give in quotecurrency if you buy/sell 1 unit of base currency
rate = Price(native_amount / tx_amount)
fee_amount = deserialize_fee(raw_trade['fee']['amount'])
fee_asset = asset_from_coinbase(raw_trade['fee']['currency'],
time=timestamp)
return Trade(
timestamp=timestamp,
location=Location.COINBASE,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee_amount,
fee_currency=fee_asset,
link=str(raw_trade['id']),
)
def create_or_return_account_to_currency_map(self) -> Dict[str, Asset]:
if self.account_to_currency is not None:
return self.account_to_currency
accounts, _ = self._api_query('accounts')
self.account_to_currency = {}
for account in accounts:
try:
asset = asset_from_coinbase(account['currency'])
self.account_to_currency[account['id']] = asset
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase pro account with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase pro account result with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except KeyError as e:
self.msg_aggregator.add_warning(
f'Found coinbase pro account entry with missing {str(e)} field. '
f'Ignoring it', )
continue
return self.account_to_currency
def query_balances(self) -> ExchangeQueryBalances:
try:
accounts = self._api_query('accounts')
except (CoinbaseProPermissionError, RemoteError) as e:
msg = f'Coinbase Pro API request failed. {str(e)}'
log.error(msg)
return None, msg
assets_balance: DefaultDict[Asset, Balance] = defaultdict(Balance)
for account in accounts:
try:
amount = deserialize_asset_amount(account['balance'])
# ignore empty balances. Coinbase returns zero balances for everything
# a user does not own
if amount == ZERO:
continue
asset = asset_from_coinbase(account['currency'])
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing coinbasepro balance result due to inability to '
f'query USD price: {str(e)}. Skipping balance entry', )
continue
assets_balance[asset] += Balance(
amount=amount,
usd_value=amount * usd_price,
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase pro balance result with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase pro balance result with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Error processing a coinbase pro account balance. Check logs '
'for details. Ignoring it.', )
log.error(
'Error processing a coinbase pro account balance',
account_balance=account,
error=msg,
)
continue
return dict(assets_balance), ''
def _deserialize_asset_movement(
self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from coinbase and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data['status'] != 'completed':
return None
payout_date = raw_data.get('payout_at', None)
if payout_date:
timestamp = deserialize_timestamp_from_date(
payout_date, 'iso8601', 'coinbase')
else:
timestamp = deserialize_timestamp_from_date(
raw_data['created_at'],
'iso8601',
'coinbase',
)
# Only get address/transaction id for "send" type of transactions
address = None
transaction_id = None
# movement_category: Union[Literal['deposit'], Literal['withdrawal']]
if 'type' in raw_data:
# Then this should be a "send" which is the way Coinbase uses to send
# crypto outside of the exchange
# https://developers.coinbase.com/api/v2?python#transaction-resource
msg = 'Non "send" type found in coinbase deposit/withdrawal processing'
assert raw_data['type'] == 'send', msg
movement_category = AssetMovementCategory.WITHDRAWAL
# Can't see the fee being charged from the "send" resource
amount = deserialize_asset_amount_force_positive(
raw_data['amount']['amount'])
asset = asset_from_coinbase(raw_data['amount']['currency'],
time=timestamp)
# Fees dont appear in the docs but from an experiment of sending ETH
# to an address from coinbase there is the network fee in the response
fee = Fee(ZERO)
raw_network = raw_data.get('network', None)
if raw_network:
raw_fee = raw_network.get('transaction_fee', None)
if raw_fee:
# Since this is a withdrawal the fee should be the same as the moved asset
if asset != asset_from_coinbase(raw_fee['currency'],
time=timestamp):
# If not we set ZERO fee and ignore
log.error(
f'In a coinbase withdrawal of {asset.identifier} the fee'
f'is denoted in {raw_fee["currency"]}', )
else:
fee = deserialize_fee(raw_fee['amount'])
if 'network' in raw_data:
transaction_id = get_key_if_has_val(
raw_data['network'], 'hash')
if 'to' in raw_data:
address = deserialize_asset_movement_address(
raw_data['to'], 'address', asset)
else:
movement_category = deserialize_asset_movement_category(
raw_data['resource'])
amount = deserialize_asset_amount_force_positive(
raw_data['amount']['amount'])
fee = deserialize_fee(raw_data['fee']['amount'])
asset = asset_from_coinbase(raw_data['amount']['currency'],
time=timestamp)
return AssetMovement(
location=Location.COINBASE,
category=movement_category,
address=address,
transaction_id=transaction_id,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=str(raw_data['id']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a coinbase '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of coinbase '
f'asset_movement {raw_data}. Error was: {str(e)}', )
return None
def query_balances(
self) -> Tuple[Optional[Dict[Asset, Dict[str, Any]]], str]:
try:
resp = self._api_query('accounts')
except RemoteError as e:
msg = ('Coinbase API request failed. Could not reach coinbase due '
'to {}'.format(e))
log.error(msg)
return None, msg
returned_balances: Dict[Asset, Dict[str, Any]] = {}
for account in resp:
try:
if not account['balance']:
continue
amount = deserialize_asset_amount(account['balance']['amount'])
# ignore empty balances. Coinbase returns zero balances for everything
# a user does not own
if amount == ZERO:
continue
asset = asset_from_coinbase(account['balance']['currency'])
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing coinbase balance entry due to inability to '
f'query USD price: {str(e)}. Skipping balance entry', )
continue
if asset in returned_balances:
amount = returned_balances[asset]['amount'] + amount
else:
returned_balances[asset] = {}
returned_balances[asset]['amount'] = amount
usd_value = returned_balances[asset]['amount'] * usd_price
returned_balances[asset]['usd_value'] = usd_value
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase balance result with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase balance result with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Error processing a coinbase account balance. Check logs '
'for details. Ignoring it.', )
log.error(
'Error processing a coinbase account balance',
account_balance=account,
error=msg,
)
continue
return returned_balances, ''
def _deserialize_ledger_action(
self, raw_data: Dict[str, Any]) -> Optional[LedgerAction]:
"""Processes a single transaction from coinbase and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data.get('status', '') != 'completed':
return None
payout_date = raw_data.get('payout_at', None)
if payout_date:
timestamp = deserialize_timestamp_from_date(
payout_date, 'iso8601', 'coinbase')
else:
timestamp = deserialize_timestamp_from_date(
get_key_if_has_val(raw_data, 'created_at'),
'iso8601',
'coinbase',
)
if 'type' in raw_data:
# The parent method filtered with 'from' attribute, so it is from another user.
# https://developers.coinbase.com/api/v2?python#transaction-resource
action_type = LedgerActionType.INCOME
if raw_data.get('type',
'') not in ('send', 'inflation_reward'):
msg = ('Non "send" or "inflation_reward" type '
'found in coinbase transactions processing')
raise DeserializationError(msg)
amount_data = raw_data.get('amount', {})
amount = deserialize_asset_amount(amount_data['amount'])
asset = asset_from_coinbase(amount_data['currency'],
time=timestamp)
native_amount_data = raw_data.get('native_amount', {})
native_amount = deserialize_asset_amount(
native_amount_data['amount'])
native_asset = asset_from_coinbase(
native_amount_data['currency'])
rate = ZERO
if amount_data and native_amount_data and native_amount and amount != ZERO:
rate = native_amount / amount
if 'details' in raw_data and 'title' in raw_data['details'] \
and 'subtitle' in raw_data['details'] and 'header' in raw_data['details']:
details = raw_data.get('details', {})
notes = (f"{details.get('title', '')} "
f"{details.get('subtitle', '')} "
f"{details.get('header', '')}")
else:
notes = ''
return LedgerAction(identifier=0,
location=Location.COINBASE,
action_type=action_type,
timestamp=timestamp,
asset=asset,
amount=amount,
rate=Price(rate),
rate_asset=native_asset,
link=str(raw_data['id']),
notes=notes)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase transaction with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase transaction with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a coinbase '
'ledger action. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of coinbase '
f'ledger action {raw_data}. Error was: {msg}', )
return None
