def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
result = self._get_paginated_query(
endpoint='transfers',
start_ts=start_ts,
end_ts=end_ts,
)
movements = []
for entry in result:
try:
timestamp = deserialize_timestamp(entry['timestampms'])
timestamp = Timestamp(int(timestamp / 1000))
asset = Asset(entry['currency'])
movement = AssetMovement(
location=Location.GEMINI,
category=deserialize_asset_movement_category(
entry['type']),
address=deserialize_asset_movement_address(
entry, 'destination', asset),
transaction_id=get_key_if_has_val(entry, 'txHash'),
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
entry['amount']),
fee_asset=asset,
# Gemini does not include withdrawal fees neither in the API nor in their UI
fee=Fee(ZERO),
link=str(entry['eid']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Error processing a gemini deposit/withdrawal. Check logs '
'for details. Ignoring it.', )
log.error(
'Error processing a gemini deposit_withdrawal',
asset_movement=entry,
error=msg,
)
continue
movements.append(movement)
return movements
def _deserialize_asset_movement(
self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from bittrex and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data['status'] != 'COMPLETED':
# Don't mind failed/in progress asset movements
return None
if 'source' in raw_data:
category = AssetMovementCategory.DEPOSIT
fee = Fee(ZERO)
else:
category = AssetMovementCategory.WITHDRAWAL
fee = deserialize_fee(raw_data.get('txCost', 0))
timestamp = deserialize_timestamp_from_date(
date=raw_data['completedAt'], # we only check completed orders
formatstr='iso8601',
location='bittrex',
)
asset = asset_from_bittrex(raw_data['currencySymbol'])
return AssetMovement(
location=Location.BITTREX,
category=category,
address=deserialize_asset_movement_address(
raw_data, 'cryptoAddress', asset),
transaction_id=get_key_if_has_val(raw_data, 'txId'),
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
raw_data['quantity']),
fee_asset=asset,
fee=fee,
link=str(raw_data.get('txId', '')),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found bittrex deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found bittrex deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a bittrex '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of bittrex '
f'asset_movement {raw_data}. Error was: {str(e)}', )
return None
def _deserialize_asset_movement(
self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from binance and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if 'insertTime' in raw_data:
category = AssetMovementCategory.DEPOSIT
time_key = 'insertTime'
fee = Fee(ZERO)
else:
category = AssetMovementCategory.WITHDRAWAL
time_key = 'applyTime'
fee = Fee(deserialize_asset_amount(raw_data['transactionFee']))
timestamp = deserialize_timestamp_from_binance(raw_data[time_key])
asset = asset_from_binance(raw_data['asset'])
tx_id = get_key_if_has_val(raw_data, 'txId')
internal_id = get_key_if_has_val(raw_data, 'id')
link_str = str(internal_id) if internal_id else str(
tx_id) if tx_id else ''
return AssetMovement(
location=self.location,
category=category,
address=deserialize_asset_movement_address(
raw_data, 'address', asset),
transaction_id=tx_id,
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
raw_data['amount']),
fee_asset=asset,
fee=fee,
link=link_str,
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(self.location)} deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(self.location)} deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
f'Error processing a {str(self.location)} deposit/withdrawal. Check logs '
f'for details. Ignoring it.', )
log.error(
f'Error processing a {str(self.location)} deposit/withdrawal',
asset_movement=raw_data,
error=msg,
)
return None
def _deserialize_asset_movement(
self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from binance and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if 'insertTime' in raw_data:
category = AssetMovementCategory.DEPOSIT
time_key = 'insertTime'
else:
category = AssetMovementCategory.WITHDRAWAL
time_key = 'applyTime'
timestamp = deserialize_timestamp_from_binance(raw_data[time_key])
asset = asset_from_binance(raw_data['asset'])
location = Location.BINANCE if self.name == str(
Location.BINANCE) else Location.BINANCE_US # noqa: E501
return AssetMovement(
location=location,
category=category,
address=deserialize_asset_movement_address(
raw_data, 'address', asset),
transaction_id=get_key_if_has_val(raw_data, 'txId'),
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
raw_data['amount']),
fee_asset=asset,
# Binance does not include withdrawal fees neither in the API nor in their UI
fee=Fee(ZERO),
link=str(raw_data['txId']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found {self.name} deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found {self.name} deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
f'Error processing a {self.name} deposit/withdrawal. Check logs '
f'for details. Ignoring it.', )
log.error(
f'Error processing a {self.name} deposit_withdrawal',
asset_movement=raw_data,
error=msg,
)
return None
def _read_asset_movements(self, filepath: str) -> List[AssetMovement]:
"""Reads a csv account type report and extracts the AssetMovements"""
with open(filepath, newline='') as csvfile:
reader = csv.DictReader(csvfile)
movements = []
for row in reader:
try:
if row['type'] in ('withdrawal', 'deposit'):
timestamp = deserialize_timestamp_from_date(
row['time'],
'iso8601',
'coinbasepro',
)
asset = Asset(row['amount/balance unit'])
movements.append(
AssetMovement(
location=Location.COINBASEPRO,
category=deserialize_asset_movement_category(
row['type']),
address=None, # can't get it from csv data
transaction_id=
None, # can't get it from csv data
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
row['amount']),
fee_asset=asset,
# I don't see any fee in deposit withdrawals in coinbasepro
fee=Fee(ZERO),
link=str(row['transfer id']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring its deposit/withdrawal.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a Coinbasepro deposit/withdrawal. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a coinbasepro deposit/withdrawal.',
raw_asset_movement=row,
error=msg,
)
continue
return movements
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
"""Queries coinbase pro for asset movements"""
log.debug('Query coinbasepro asset movements',
start_ts=start_ts,
end_ts=end_ts)
movements = []
raw_movements = []
for batch in self._paginated_query(
endpoint='transfers',
query_options={'type': 'withdraw'},
):
raw_movements.extend(batch)
for batch in self._paginated_query(
endpoint='transfers',
query_options={'type': 'deposit'},
):
raw_movements.extend(batch)
account_to_currency = self.create_or_return_account_to_currency_map()
for entry in raw_movements:
try:
# Check if the transaction has not been completed. If so it should be skipped
if entry.get('completed_at', None) is None:
log.warning(
f'Skipping coinbase pro deposit/withdrawal '
f'due to not having been completed: {entry}', )
continue
timestamp = coinbasepro_deserialize_timestamp(
entry, 'completed_at')
if timestamp < start_ts or timestamp > end_ts:
continue
category = deserialize_asset_movement_category(entry['type'])
asset = account_to_currency.get(entry['account_id'], None)
if asset is None:
log.warning(
f'Skipping coinbase pro asset_movement {entry} due to '
f'inability to match account id to an asset', )
continue
address = None
transaction_id = None
fee = Fee(ZERO)
if category == AssetMovementCategory.DEPOSIT:
try:
address = entry['details']['crypto_address']
transaction_id = entry['details'][
'crypto_transaction_hash']
except KeyError:
pass
else: # withdrawal
try:
address = entry['details']['sent_to_address']
transaction_id = entry['details'][
'crypto_transaction_hash']
fee = deserialize_fee(entry['details']['fee'])
except KeyError:
pass
if transaction_id and (
asset == A_ETH or asset.asset_type
== AssetType.ETHEREUM_TOKEN): # noqa: E501
transaction_id = '0x' + transaction_id
movements.append(
AssetMovement(
location=Location.COINBASEPRO,
category=category,
address=address,
transaction_id=transaction_id,
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
entry['amount']),
fee_asset=asset,
fee=fee,
link=str(entry['id']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring its deposit/withdrawal.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a Coinbasepro deposit/withdrawal. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a coinbasepro deposit/withdrawal.',
raw_asset_movement=entry,
error=msg,
)
continue
return movements
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
result = self.query_until_finished(
endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict={'type': 'deposit'},
)
result.extend(
self.query_until_finished(
endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict={'type': 'withdrawal'},
))
log.debug('Kraken deposit/withdrawals query result',
num_results=len(result))
movements = []
for movement in result:
try:
asset = asset_from_kraken(movement['asset'])
movement_type = movement['type']
if movement_type not in ('deposit', 'withdrawal'):
# Other known types: 'transfer'
continue # Can be for moving funds from spot to stake etc.
movements.append(
AssetMovement(
location=Location.KRAKEN,
category=deserialize_asset_movement_category(
movement_type),
timestamp=deserialize_timestamp_from_kraken(
movement['time']),
address=None, # no data from kraken ledger endpoint
transaction_id=
None, # no data from kraken ledger endpoint
asset=asset,
amount=deserialize_asset_amount_force_positive(
movement['amount']),
fee_asset=asset,
fee=deserialize_fee(movement['fee']),
link=str(movement['refid']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown kraken asset {e.asset_name}. '
f'Ignoring its deposit/withdrawals query.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a kraken deposit/withdrawals. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a kraken deposit/withdrawal.',
raw_asset_movement=movement,
error=msg,
)
continue
return movements
def _consume_cointracking_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cointracking entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCointrackingEntry if importing of this entry is not supported.
- IndexError if the CSV file is corrupt
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Type']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date'],
formatstr='%d.%m.%Y %H:%M:%S',
location='cointracking.info',
)
notes = csv_row['Comment']
location = exchange_row_to_location(csv_row['Exchange'])
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if csv_row['Fee'] != '':
fee = deserialize_fee(csv_row['Fee'])
fee_currency = Asset(csv_row['Cur.Fee'])
if row_type in ('Gift/Tip', 'Trade', 'Income'):
base_asset = Asset(csv_row['Cur.Buy'])
quote_asset = None if csv_row['Cur.Sell'] == '' else Asset(
csv_row['Cur.Sell'])
if quote_asset is None and row_type not in ('Gift/Tip', 'Income'):
raise DeserializationError(
'Got a trade entry with an empty quote asset')
if quote_asset is None:
# Really makes no difference as this is just a gift and the amount is zero
quote_asset = A_USD
pair = TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}')
base_amount_bought = deserialize_asset_amount(csv_row['Buy'])
if csv_row['Sell'] != '-':
quote_amount_sold = deserialize_asset_amount(csv_row['Sell'])
else:
quote_amount_sold = AssetAmount(ZERO)
rate = Price(quote_amount_sold / base_amount_bought)
trade = Trade(
timestamp=timestamp,
location=location,
pair=pair,
trade_type=TradeType.
BUY, # It's always a buy during cointracking import
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type == 'Deposit' or row_type == 'Withdrawal':
category = deserialize_asset_movement_category(row_type.lower())
if category == AssetMovementCategory.DEPOSIT:
amount = deserialize_asset_amount(csv_row['Buy'])
asset = Asset(csv_row['Cur.Buy'])
else:
amount = deserialize_asset_amount_force_positive(
csv_row['Sell'])
asset = Asset(csv_row['Cur.Sell'])
asset_movement = AssetMovement(
location=location,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_currency,
link='',
)
self.db.add_asset_movements([asset_movement])
else:
raise UnsupportedCointrackingEntry(
f'Unknown entrype type "{row_type}" encountered during cointracking '
f'data import. Ignoring entry', )
def _consume_cryptocom_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cryptocom entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCryptocomEntry if importing of this entry is not supported.
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Transaction Kind']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='crypto.com',
)
description = csv_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees info until (Nov 2020) on crypto.com
# fees are not displayed in the export data
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if row_type in (
'crypto_purchase',
'crypto_exchange',
'referral_gift',
'referral_bonus',
'crypto_earn_interest_paid',
'referral_card_cashback',
'card_cashback_reverted',
'reimbursement',
):
# variable mapping to raw data
currency = csv_row['Currency']
to_currency = csv_row['To Currency']
native_currency = csv_row['Native Currency']
amount = csv_row['Amount']
to_amount = csv_row['To Amount']
native_amount = csv_row['Native Amount']
trade_type = TradeType.BUY if to_currency != native_currency else TradeType.SELL
if row_type == 'crypto_exchange':
# trades crypto to crypto
base_asset = Asset(to_currency)
quote_asset = Asset(currency)
if quote_asset is None:
raise DeserializationError(
'Got a trade entry with an empty quote asset')
base_amount_bought = deserialize_asset_amount(to_amount)
quote_amount_sold = deserialize_asset_amount(amount)
else:
base_asset = Asset(currency)
quote_asset = Asset(native_currency)
base_amount_bought = deserialize_asset_amount(amount)
quote_amount_sold = deserialize_asset_amount(native_amount)
rate = Price(abs(quote_amount_sold / base_amount_bought))
pair = TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}')
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
pair=pair,
trade_type=trade_type,
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type == 'crypto_withdrawal' or row_type == 'crypto_deposit':
if row_type == 'crypto_withdrawal':
category = AssetMovementCategory.WITHDRAWAL
amount = deserialize_asset_amount_force_positive(
csv_row['Amount'])
else:
category = AssetMovementCategory.DEPOSIT
amount = deserialize_asset_amount(csv_row['Amount'])
asset = Asset(csv_row['Currency'])
asset_movement = AssetMovement(
location=Location.CRYPTOCOM,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif row_type in (
'crypto_earn_program_created',
'lockup_lock',
'lockup_unlock',
'dynamic_coin_swap_bonus_exchange_deposit',
'crypto_wallet_swap_debited',
'crypto_wallet_swap_credited',
'lockup_swap_debited',
'lockup_swap_credited',
'lockup_swap_rebate',
'dynamic_coin_swap_bonus_exchange_deposit',
# we don't handle cryto.com exchange yet
'crypto_to_exchange_transfer',
'exchange_to_crypto_transfer',
# supercharger actions
'supercharger_deposit',
'supercharger_withdrawal',
# already handled using _import_cryptocom_double_entries
'dynamic_coin_swap_debited',
'dynamic_coin_swap_credited',
'dust_conversion_debited',
'dust_conversion_credited',
):
# those types are ignored because it doesn't affect the wallet balance
# or are not handled here
return
else:
raise UnsupportedCryptocomEntry(
f'Unknown entrype type "{row_type}" encountered during '
f'cryptocom data import. Ignoring entry', )
def _deserialize_asset_movement(
self,
movement_type: AssetMovementCategory,
movement_data: Dict[str, Any],
) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from polo and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if movement_type == AssetMovementCategory.DEPOSIT:
fee = Fee(ZERO)
uid_key = 'depositNumber'
transaction_id = get_key_if_has_val(movement_data, 'txid')
else:
fee = deserialize_fee(movement_data['fee'])
uid_key = 'withdrawalNumber'
split = movement_data['status'].split(':')
if len(split) != 2:
transaction_id = None
else:
transaction_id = split[1].lstrip()
if transaction_id == '':
transaction_id = None
asset = asset_from_poloniex(movement_data['currency'])
return AssetMovement(
location=Location.POLONIEX,
category=movement_type,
address=deserialize_asset_movement_address(movement_data, 'address', asset),
transaction_id=transaction_id,
timestamp=deserialize_timestamp(movement_data['timestamp']),
asset=asset,
amount=deserialize_asset_amount_force_positive(movement_data['amount']),
fee_asset=asset,
fee=fee,
link=str(movement_data[uid_key]),
)
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(movement_type)} of unsupported poloniex asset '
f'{e.asset_name}. Ignoring it.',
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(movement_type)} of unknown poloniex asset '
f'{e.asset_name}. Ignoring it.',
)
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a poloniex '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of poloniex '
f'{str(movement_type)}: {movement_data}. Error was: {str(e)}',
)
return None
def _deserialize_asset_movement(
self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from coinbase and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data['status'] != 'completed':
return None
payout_date = raw_data.get('payout_at', None)
if payout_date:
timestamp = deserialize_timestamp_from_date(
payout_date, 'iso8601', 'coinbase')
else:
timestamp = deserialize_timestamp_from_date(
raw_data['created_at'],
'iso8601',
'coinbase',
)
# Only get address/transaction id for "send" type of transactions
address = None
transaction_id = None
# movement_category: Union[Literal['deposit'], Literal['withdrawal']]
if 'type' in raw_data:
# Then this should be a "send" which is the way Coinbase uses to send
# crypto outside of the exchange
# https://developers.coinbase.com/api/v2?python#transaction-resource
msg = 'Non "send" type found in coinbase deposit/withdrawal processing'
assert raw_data['type'] == 'send', msg
movement_category = AssetMovementCategory.WITHDRAWAL
# Can't see the fee being charged from the "send" resource
amount = deserialize_asset_amount_force_positive(
raw_data['amount']['amount'])
asset = asset_from_coinbase(raw_data['amount']['currency'],
time=timestamp)
# Fees dont appear in the docs but from an experiment of sending ETH
# to an address from coinbase there is the network fee in the response
fee = Fee(ZERO)
raw_network = raw_data.get('network', None)
if raw_network:
raw_fee = raw_network.get('transaction_fee', None)
if raw_fee:
# Since this is a withdrawal the fee should be the same as the moved asset
if asset != asset_from_coinbase(raw_fee['currency'],
time=timestamp):
# If not we set ZERO fee and ignore
log.error(
f'In a coinbase withdrawal of {asset.identifier} the fee'
f'is denoted in {raw_fee["currency"]}', )
else:
fee = deserialize_fee(raw_fee['amount'])
if 'network' in raw_data:
transaction_id = get_key_if_has_val(
raw_data['network'], 'hash')
if 'to' in raw_data:
address = deserialize_asset_movement_address(
raw_data['to'], 'address', asset)
else:
movement_category = deserialize_asset_movement_category(
raw_data['resource'])
amount = deserialize_asset_amount_force_positive(
raw_data['amount']['amount'])
fee = deserialize_fee(raw_data['fee']['amount'])
asset = asset_from_coinbase(raw_data['amount']['currency'],
time=timestamp)
return AssetMovement(
location=Location.COINBASE,
category=movement_category,
address=address,
transaction_id=transaction_id,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=str(raw_data['id']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a coinbase '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of coinbase '
f'asset_movement {raw_data}. Error was: {str(e)}', )
return None
def _consume_nexo(self, csv_row: Dict[str, Any]) -> None:
"""
Consume CSV file from NEXO.
This method can raise:
- UnsupportedNexoEntry
- UnknownAsset
- DeserializationError
"""
ignored_entries = ('ExchangeToWithdraw', 'DepositToExchange')
if 'rejected' not in csv_row['Details']:
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date / Time'],
formatstr='%Y-%m-%d %H:%M',
location='NEXO',
)
else:
log.debug(f'Ignoring rejected nexo entry {csv_row}')
return
asset = symbol_to_asset_or_token(csv_row['Currency'])
amount = deserialize_asset_amount_force_positive(csv_row['Amount'])
entry_type = csv_row['Type']
transaction = csv_row['Transaction']
if entry_type in ('Deposit', 'ExchangeDepositedOn',
'LockingTermDeposit'):
asset_movement = AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=Fee(ZERO),
fee_asset=A_USD,
link=transaction,
)
self.db.add_asset_movements([asset_movement])
elif entry_type in ('Withdrawal', 'WithdrawExchanged'):
asset_movement = AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=Fee(ZERO),
fee_asset=A_USD,
link=transaction,
)
self.db.add_asset_movements([asset_movement])
elif entry_type == 'Withdrawal Fee':
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.EXPENSE,
location=Location.NEXO,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from Nexo',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ('Interest', 'Bonus', 'Dividend'):
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.INCOME,
location=Location.NEXO,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=transaction,
notes=f'{entry_type} from Nexo',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ignored_entries:
pass
else:
raise UnsupportedCSVEntry(
f'Unsuported entry {entry_type}. Data: {csv_row}')
def _consume_blockfi_entry(self, csv_row: Dict[str, Any]) -> None:
"""
Process entry for BlockFi transaction history. Trades for this file are ignored
and istead should be extracted from the file containing only trades.
This method can raise:
- UnsupportedBlockFiEntry
- UnknownAsset
- DeserializationError
"""
if len(csv_row['Confirmed At']) != 0:
timestamp = deserialize_timestamp_from_date(
date=csv_row['Confirmed At'],
formatstr='%Y-%m-%d %H:%M:%S',
location='BlockFi',
)
else:
log.debug(f'Ignoring unconfirmed BlockFi entry {csv_row}')
return
asset = symbol_to_asset_or_token(csv_row['Cryptocurrency'])
amount = deserialize_asset_amount_force_positive(csv_row['Amount'])
entry_type = csv_row['Transaction Type']
# BlockFI doesn't provide information about fees
fee = Fee(ZERO)
fee_asset = A_USD # Can be whatever
if entry_type in ('Deposit', 'Wire Deposit', 'ACH Deposit'):
asset_movement = AssetMovement(
location=Location.BLOCKFI,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif entry_type in ('Withdrawal', 'Wire Withdrawal', 'ACH Withdrawal'):
asset_movement = AssetMovement(
location=Location.BLOCKFI,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif entry_type == 'Withdrawal Fee':
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.EXPENSE,
location=Location.BLOCKFI,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from BlockFi',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ('Interest Payment', 'Bonus Payment',
'Referral Bonus'):
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.INCOME,
location=Location.BLOCKFI,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from BlockFi',
)
self.db_ledger.add_ledger_action(action)
elif entry_type == 'Trade':
pass
else:
raise UnsupportedCSVEntry(
f'Unsuported entry {entry_type}. Data: {csv_row}')
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List:
resp = self._api_query_list('get', 'user/walletHistory')
log.debug('Bitmex deposit/withdrawals query', results_num=len(resp))
movements = []
for movement in resp:
try:
transaction_type = movement['transactType']
if transaction_type == 'Deposit':
transaction_type = AssetMovementCategory.DEPOSIT
elif transaction_type == 'Withdrawal':
transaction_type = AssetMovementCategory.WITHDRAWAL
else:
continue
timestamp = iso8601ts_to_timestamp(movement['timestamp'])
if timestamp < start_ts:
continue
if timestamp > end_ts:
continue
asset = bitmex_to_world(movement['currency'])
amount = deserialize_asset_amount_force_positive(
movement['amount'])
fee = deserialize_fee(movement['fee'])
if asset == A_BTC:
# bitmex stores amounts in satoshis
amount = AssetAmount(satoshis_to_btc(amount))
fee = Fee(satoshis_to_btc(fee))
movements.append(
AssetMovement(
location=Location.BITMEX,
category=transaction_type,
address=deserialize_asset_movement_address(
movement, 'address', asset),
transaction_id=get_key_if_has_val(movement, 'tx'),
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=str(movement['transactID']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found bitmex deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a bitmex '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of bitmex '
f'asset_movement {movement}. Error was: {msg}', )
continue
return movements
def _deserialize_asset_movement(
self,
raw_data: Dict[str, Any],
movement_type: AssetMovementCategory,
) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from FTX and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data['status'] not in ('complete', 'confirmed'):
return None
timestamp = deserialize_timestamp_from_date(
raw_data['time'], 'iso8601', 'FTX')
amount = deserialize_asset_amount_force_positive(raw_data['size'])
asset = asset_from_ftx(raw_data['coin'])
fee = Fee(ZERO)
movement_category = movement_type
if raw_data.get('fee', None) is not None:
fee = deserialize_fee(raw_data['fee'])
address = raw_data.get('address', None)
if isinstance(address, dict):
address = raw_data['address'].get('address', None)
transaction_id = raw_data.get('txid', None)
return AssetMovement(
location=Location.FTX,
category=movement_category,
address=address,
transaction_id=transaction_id,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=str(raw_data['id']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found FTX deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found FTX deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of an FTX '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
'Error processing FTX trade',
trade=raw_data,
error=msg,
)
return None
def _get_trades_graph_v3_for_address(
self,
address: ChecksumEthAddress,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AMMTrade]:
"""Get the address' trades data querying the Uniswap subgraph
Each trade (swap) instantiates an <AMMTrade>.
The trade pair (i.e. BASE_QUOTE) is determined by `reserve0_reserve1`.
Translated to Uniswap lingo:
Trade type BUY:
- `amount1` (QUOTE, reserve1) is gt 0.
- `amount0` (BASE, reserve0) is lt 0.
Trade type SELL:
- `amount0` (BASE, reserve0) is gt 0.
- `amount1` (QUOTE, reserve1) is lt 0.
May raise:
- RemoteError
"""
trades: List[AMMTrade] = []
param_types = {
'$limit': 'Int!',
'$offset': 'Int!',
'$address': 'Bytes!',
'$start_ts': 'BigInt!',
'$end_ts': 'BigInt!',
}
param_values = {
'limit': GRAPH_QUERY_LIMIT,
'offset': 0,
'address': address.lower(),
'start_ts': str(start_ts),
'end_ts': str(end_ts),
}
querystr = format_query_indentation(V3_SWAPS_QUERY.format())
while True:
try:
result = self.graph_v3.query(
querystr=querystr,
param_types=param_types,
param_values=param_values,
)
except RemoteError as e:
self.msg_aggregator.add_error(SUBGRAPH_REMOTE_ERROR_MSG.format(error_msg=str(e)))
raise
result_data = result['swaps']
for entry in result_data:
swaps = []
for swap in entry['transaction']['swaps']:
timestamp = swap['timestamp']
swap_token0 = swap['token0']
swap_token1 = swap['token1']
try:
token0_deserialized = deserialize_ethereum_address(swap_token0['id'])
token1_deserialized = deserialize_ethereum_address(swap_token1['id'])
from_address_deserialized = deserialize_ethereum_address(swap['sender'])
to_address_deserialized = deserialize_ethereum_address(swap['recipient'])
except DeserializationError:
msg = (
f'Failed to deserialize addresses in trade from uniswap graph with '
f'token 0: {swap_token0["id"]}, token 1: {swap_token1["id"]}, '
f'swap sender: {swap["sender"]}, swap receiver {swap["to"]}'
)
log.error(msg)
continue
token0 = get_or_create_ethereum_token(
userdb=self.database,
symbol=swap_token0['symbol'],
ethereum_address=token0_deserialized,
name=swap_token0['name'],
decimals=swap_token0['decimals'],
)
token1 = get_or_create_ethereum_token(
userdb=self.database,
symbol=swap_token1['symbol'],
ethereum_address=token1_deserialized,
name=swap_token1['name'],
decimals=int(swap_token1['decimals']),
)
try:
if swap['amount0'].startswith('-'):
amount0_in = AssetAmount(FVal(ZERO))
amount0_out = deserialize_asset_amount_force_positive(swap['amount0'])
amount1_in = deserialize_asset_amount_force_positive(swap['amount1'])
amount1_out = AssetAmount(FVal(ZERO))
else:
amount0_in = deserialize_asset_amount_force_positive(swap['amount0'])
amount0_out = AssetAmount(FVal(ZERO))
amount1_in = AssetAmount(FVal(ZERO))
amount1_out = deserialize_asset_amount_force_positive(swap['amount1'])
except ValueError as e:
log.error(
f'Failed to read amounts in Uniswap V3 swap {str(swap)}. '
f'{str(e)}.',
)
continue
swaps.append(AMMSwap(
tx_hash=swap['id'].split('#')[0],
log_index=int(swap['logIndex']),
address=address,
from_address=from_address_deserialized,
to_address=to_address_deserialized,
timestamp=Timestamp(int(timestamp)),
location=Location.UNISWAP,
token0=token0,
token1=token1,
amount0_in=amount0_in,
amount1_in=amount1_in,
amount0_out=amount0_out,
amount1_out=amount1_out,
))
# with the new logic the list of swaps can be empty, in that case don't try
# to make trades from the swaps
if len(swaps) == 0:
continue
# Now that we got all swaps for a transaction, create the trade object
trades.extend(self._tx_swaps_to_trades(swaps))
# Check whether an extra request is needed
if len(result_data) < GRAPH_QUERY_LIMIT:
break
# Update pagination step
param_values = {
**param_values,
'offset': param_values['offset'] + GRAPH_QUERY_LIMIT, # type: ignore
}
return trades
