def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
result = self._get_paginated_query(
endpoint='transfers',
start_ts=start_ts,
end_ts=end_ts,
)
movements = []
for entry in result:
try:
timestamp = deserialize_timestamp(entry['timestampms'])
timestamp = Timestamp(int(timestamp / 1000))
asset = Asset(entry['currency'])
movement = AssetMovement(
location=Location.GEMINI,
category=deserialize_asset_movement_category(
entry['type']),
address=deserialize_asset_movement_address(
entry, 'destination', asset),
transaction_id=get_key_if_has_val(entry, 'txHash'),
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
entry['amount']),
fee_asset=asset,
# Gemini does not include withdrawal fees neither in the API nor in their UI
fee=Fee(ZERO),
link=str(entry['eid']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Error processing a gemini deposit/withdrawal. Check logs '
'for details. Ignoring it.', )
log.error(
'Error processing a gemini deposit_withdrawal',
asset_movement=entry,
error=msg,
)
continue
movements.append(movement)
return movements
def asset_movements_from_dictlist(
given_data: List[Dict[str, Any]],
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
""" Gets a list of dict asset movements, most probably read from the json files and
a time period. Returns it as a list of the AssetMovement tuples that are inside the time period
May raise:
- DeserializationError: If the given_data dict contains data in an unexpected format
- KeyError: If the given_data dict contains data in an unexpected format
"""
returned_movements = list()
for movement in given_data:
timestamp = deserialize_timestamp(movement['timestamp'])
if timestamp < start_ts:
continue
if timestamp > end_ts:
break
category = deserialize_asset_movement_category(movement['category'])
amount = deserialize_asset_amount(movement['amount'])
fee = deserialize_fee(movement['fee'])
returned_movements.append(AssetMovement(
exchange=deserialize_exchange_name(movement['exchange']),
category=category,
timestamp=timestamp,
asset=Asset(movement['asset']),
amount=amount,
fee=fee,
))
return returned_movements
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
result = self.query_until_finished(
endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict=dict(type='deposit'),
)
result.extend(self.query_until_finished(
endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict=dict(type='withdrawal'),
))
log.debug('Kraken deposit/withdrawals query result', num_results=len(result))
movements = list()
for movement in result:
try:
asset = asset_from_kraken(movement['asset'])
movements.append(AssetMovement(
location=Location.KRAKEN,
category=deserialize_asset_movement_category(movement['type']),
timestamp=deserialize_timestamp_from_kraken(movement['time']),
asset=asset,
amount=deserialize_asset_amount(movement['amount']),
fee_asset=asset,
fee=deserialize_fee(movement['fee']),
link=str(movement['refid']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown kraken asset {e.asset_name}. '
f'Ignoring its deposit/withdrawals query.',
)
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a kraken deposit/withdrawals. '
'Check logs for details. Ignoring it.',
)
log.error(
'Error processing a kraken deposit/withdrawal.',
raw_asset_movement=movement,
error=msg,
)
continue
return movements
def _asset_movement_from_independentreserve(
raw_tx: Dict) -> Optional[AssetMovement]:
"""Convert IndependentReserve raw data to an AssetMovement
https://www.independentreserve.com/products/api#GetTransactions
May raise:
- DeserializationError
- UnknownAsset
- KeyError
"""
log.debug(f'Processing raw IndependentReserve transaction: {raw_tx}')
movement_type = deserialize_asset_movement_category(raw_tx['Type'])
asset = independentreserve_asset(raw_tx['CurrencyCode'])
bitcoin_tx_id = raw_tx.get('BitcoinTransactionId')
eth_tx_id = raw_tx.get('EthereumTransactionId')
if asset == A_BTC and bitcoin_tx_id is not None:
transaction_id = raw_tx['BitcoinTransactionId']
elif eth_tx_id is not None:
transaction_id = eth_tx_id
else:
transaction_id = None
timestamp = deserialize_timestamp_from_date(
date=raw_tx['CreatedTimestampUtc'],
formatstr='iso8601',
location='IndependentReserve',
)
comment = raw_tx.get('Comment')
address = None
if comment is not None and comment.startswith('Withdrawing to'):
address = comment.rsplit()[-1]
raw_amount = raw_tx.get(
'Credit'
) if movement_type == AssetMovementCategory.DEPOSIT else raw_tx.get(
'Debit') # noqa: E501
if raw_amount is None: # skip
return None # Can end up being None for some things like this: 'Comment': 'Initial balance after Bitcoin fork' # noqa: E501
amount = deserialize_asset_amount(raw_amount)
return AssetMovement(
location=Location.INDEPENDENTRESERVE,
category=movement_type,
address=address,
transaction_id=transaction_id,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset, # whatever -- no fee
fee=Fee(ZERO), # we can't get fee from this exchange
link=raw_tx['CreatedTimestampUtc'] + str(amount) + str(movement_type) +
asset.identifier,
)
def _read_asset_movements(self, filepath: str) -> List[AssetMovement]:
"""Reads a csv account type report and extracts the AssetMovements"""
with open(filepath, newline='') as csvfile:
reader = csv.DictReader(csvfile)
movements = []
for row in reader:
try:
if row['type'] in ('withdrawal', 'deposit'):
timestamp = deserialize_timestamp_from_date(
row['time'],
'iso8601',
'coinbasepro',
)
amount = deserialize_asset_amount(row['amount'])
if row['type'] == 'withdrawal':
# For withdrawals the withdraw amount is negative
amount = AssetAmount(amount * FVal('-1'))
asset = Asset(row['amount/balance unit'])
movements.append(
AssetMovement(
location=Location.COINBASEPRO,
category=deserialize_asset_movement_category(
row['type']),
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
# I don't see any fee in deposit withdrawals in coinbasepro
fee=Fee(ZERO),
link=str(row['transfer id']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring its deposit/withdrawal.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a Coinbasepro deposit/withdrawal. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a coinbasepro deposit/withdrawal.',
raw_asset_movement=row,
error=msg,
)
continue
return movements
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
"""Queries coinbase pro for asset movements"""
log.debug('Query coinbasepro asset movements',
start_ts=start_ts,
end_ts=end_ts)
movements = []
raw_movements = []
for batch in self._paginated_query(
endpoint='transfers',
query_options={'type': 'withdraw'},
):
raw_movements.extend(batch)
for batch in self._paginated_query(
endpoint='transfers',
query_options={'type': 'deposit'},
):
raw_movements.extend(batch)
account_to_currency = self.create_or_return_account_to_currency_map()
for entry in raw_movements:
try:
# Check if the transaction has not been completed. If so it should be skipped
if entry.get('completed_at', None) is None:
log.warning(
f'Skipping coinbase pro deposit/withdrawal '
f'due to not having been completed: {entry}', )
continue
timestamp = coinbasepro_deserialize_timestamp(
entry, 'completed_at')
if timestamp < start_ts or timestamp > end_ts:
continue
category = deserialize_asset_movement_category(entry['type'])
asset = account_to_currency.get(entry['account_id'], None)
if asset is None:
log.warning(
f'Skipping coinbase pro asset_movement {entry} due to '
f'inability to match account id to an asset', )
continue
address = None
transaction_id = None
fee = Fee(ZERO)
if category == AssetMovementCategory.DEPOSIT:
try:
address = entry['details']['crypto_address']
transaction_id = entry['details'][
'crypto_transaction_hash']
except KeyError:
pass
else: # withdrawal
try:
address = entry['details']['sent_to_address']
transaction_id = entry['details'][
'crypto_transaction_hash']
fee = deserialize_fee(entry['details']['fee'])
except KeyError:
pass
if transaction_id and (
asset == A_ETH or asset.asset_type
== AssetType.ETHEREUM_TOKEN): # noqa: E501
transaction_id = '0x' + transaction_id
movements.append(
AssetMovement(
location=Location.COINBASEPRO,
category=category,
address=address,
transaction_id=transaction_id,
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
entry['amount']),
fee_asset=asset,
fee=fee,
link=str(entry['id']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring its deposit/withdrawal.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a Coinbasepro deposit/withdrawal. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a coinbasepro deposit/withdrawal.',
raw_asset_movement=entry,
error=msg,
)
continue
return movements
def _consume_cointracking_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cointracking entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCointrackingEntry if importing of this entry is not supported.
- IndexError if the CSV file is corrupt
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Type']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date'],
formatstr='%d.%m.%Y %H:%M:%S',
location='cointracking.info',
)
notes = csv_row['Comment']
location = exchange_row_to_location(csv_row['Exchange'])
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if csv_row['Fee'] != '':
fee = deserialize_fee(csv_row['Fee'])
fee_currency = Asset(csv_row['Cur.Fee'])
if row_type in ('Gift/Tip', 'Trade', 'Income'):
base_asset = Asset(csv_row['Cur.Buy'])
quote_asset = None if csv_row['Cur.Sell'] == '' else Asset(
csv_row['Cur.Sell'])
if quote_asset is None and row_type not in ('Gift/Tip', 'Income'):
raise DeserializationError(
'Got a trade entry with an empty quote asset')
if quote_asset is None:
# Really makes no difference as this is just a gift and the amount is zero
quote_asset = A_USD
pair = TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}')
base_amount_bought = deserialize_asset_amount(csv_row['Buy'])
if csv_row['Sell'] != '-':
quote_amount_sold = deserialize_asset_amount(csv_row['Sell'])
else:
quote_amount_sold = AssetAmount(ZERO)
rate = Price(quote_amount_sold / base_amount_bought)
trade = Trade(
timestamp=timestamp,
location=location,
pair=pair,
trade_type=TradeType.
BUY, # It's always a buy during cointracking import
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type == 'Deposit' or row_type == 'Withdrawal':
category = deserialize_asset_movement_category(row_type.lower())
if category == AssetMovementCategory.DEPOSIT:
amount = deserialize_asset_amount(csv_row['Buy'])
asset = Asset(csv_row['Cur.Buy'])
else:
amount = deserialize_asset_amount_force_positive(
csv_row['Sell'])
asset = Asset(csv_row['Cur.Sell'])
asset_movement = AssetMovement(
location=location,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_currency,
link='',
)
self.db.add_asset_movements([asset_movement])
else:
raise UnsupportedCointrackingEntry(
f'Unknown entrype type "{row_type}" encountered during cointracking '
f'data import. Ignoring entry', )
def _deserialize_asset_movement(
self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from coinbase and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data['status'] != 'completed':
return None
payout_date = raw_data.get('payout_at', None)
if payout_date:
timestamp = deserialize_timestamp_from_date(
payout_date, 'iso8601', 'coinbase')
else:
timestamp = deserialize_timestamp_from_date(
raw_data['created_at'],
'iso8601',
'coinbase',
)
# Only get address/transaction id for "send" type of transactions
address = None
transaction_id = None
# movement_category: Union[Literal['deposit'], Literal['withdrawal']]
if 'type' in raw_data:
# Then this should be a "send" which is the way Coinbase uses to send
# crypto outside of the exchange
# https://developers.coinbase.com/api/v2?python#transaction-resource
msg = 'Non "send" type found in coinbase deposit/withdrawal processing'
assert raw_data['type'] == 'send', msg
movement_category = AssetMovementCategory.WITHDRAWAL
# Can't see the fee being charged from the "send" resource
amount = deserialize_asset_amount_force_positive(
raw_data['amount']['amount'])
asset = asset_from_coinbase(raw_data['amount']['currency'],
time=timestamp)
# Fees dont appear in the docs but from an experiment of sending ETH
# to an address from coinbase there is the network fee in the response
fee = Fee(ZERO)
raw_network = raw_data.get('network', None)
if raw_network:
raw_fee = raw_network.get('transaction_fee', None)
if raw_fee:
# Since this is a withdrawal the fee should be the same as the moved asset
if asset != asset_from_coinbase(raw_fee['currency'],
time=timestamp):
# If not we set ZERO fee and ignore
log.error(
f'In a coinbase withdrawal of {asset.identifier} the fee'
f'is denoted in {raw_fee["currency"]}', )
else:
fee = deserialize_fee(raw_fee['amount'])
if 'network' in raw_data:
transaction_id = get_key_if_has_val(
raw_data['network'], 'hash')
if 'to' in raw_data:
address = deserialize_asset_movement_address(
raw_data['to'], 'address', asset)
else:
movement_category = deserialize_asset_movement_category(
raw_data['resource'])
amount = deserialize_asset_amount_force_positive(
raw_data['amount']['amount'])
fee = deserialize_fee(raw_data['fee']['amount'])
asset = asset_from_coinbase(raw_data['amount']['currency'],
time=timestamp)
return AssetMovement(
location=Location.COINBASE,
category=movement_category,
address=address,
transaction_id=transaction_id,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=str(raw_data['id']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a coinbase '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of coinbase '
f'asset_movement {raw_data}. Error was: {str(e)}', )
return None
def _consume_cointracking_entry(self, csv_row: List[str]) -> None:
"""Consumes a cointracking entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCointrackingEntry if importing of this entry is not supported.
- IndexError if the CSV file is corrupt
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row[1] # Type
timestamp = deserialize_timestamp_from_date(
date=csv_row[9],
formatstr='%d.%m.%Y %H:%M',
location='cointracking.info',
)
notes = csv_row[8]
location = exchange_row_to_location(csv_row[6])
if row_type == 'Gift/Tip' or row_type == 'Trade':
base_asset = Asset(csv_row[3])
quote_asset = None if csv_row[5] == '' else Asset(csv_row[5])
if not quote_asset and row_type != 'Gift/Tip':
raise DeserializationError(
'Got a trade entry with an empty quote asset')
if quote_asset is None:
# Really makes no difference as this is just a gift and the amount is zero
quote_asset = A_USD
pair = TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}')
base_amount_bought = deserialize_asset_amount(csv_row[2])
if csv_row[4] != '-':
quote_amount_sold = deserialize_asset_amount(csv_row[4])
else:
quote_amount_sold = AssetAmount(ZERO)
rate = Price(quote_amount_sold / base_amount_bought)
trade = Trade(
timestamp=timestamp,
location=location,
pair=pair,
trade_type=TradeType.
BUY, # It's always a buy during cointracking import
amount=base_amount_bought,
rate=rate,
fee=Fee(
ZERO), # There are no fees when import from cointracking
fee_currency=base_asset,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type == 'Deposit' or row_type == 'Withdrawal':
category = deserialize_asset_movement_category(row_type.lower())
if category == AssetMovementCategory.DEPOSIT:
amount = deserialize_asset_amount(csv_row[2])
asset = Asset(csv_row[3])
else:
amount = deserialize_asset_amount(csv_row[4])
asset = Asset(csv_row[5])
asset_movement = AssetMovement(
location=location,
category=category,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=Fee(ZERO),
link='',
)
self.db.add_asset_movements([asset_movement])
else:
raise UnsupportedCointrackingEntry(
f'Unknown entrype type "{row_type}" encountered during cointracking '
f'data import. Ignoring entry', )
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
self.query_kraken_ledgers(start_ts=start_ts, end_ts=end_ts)
filter_query = HistoryEventFilterQuery.make(
from_ts=Timestamp(start_ts),
to_ts=Timestamp(end_ts),
event_types=[
HistoryEventType.DEPOSIT,
HistoryEventType.WITHDRAWAL,
],
location=Location.KRAKEN,
location_label=self.name,
)
events = self.history_events_db.get_history_events(
filter_query=filter_query,
has_premium=True,
)
log.debug('Kraken deposit/withdrawals query result',
num_results=len(events))
movements = []
get_attr = operator.attrgetter('event_identifier')
# Create a list of lists where each sublist has the events for the same event identifier
grouped_events = [
list(g) for k, g in itertools.groupby(sorted(events, key=get_attr),
get_attr)
] # noqa: E501
for movement_events in grouped_events:
if len(movement_events) == 2:
if movement_events[0].event_subtype == HistoryEventSubType.FEE:
fee = Fee(movement_events[0].balance.amount)
movement = movement_events[1]
elif movement_events[
1].event_subtype == HistoryEventSubType.FEE:
fee = Fee(movement_events[1].balance.amount)
movement = movement_events[0]
else:
self.msg_aggregator.add_error(
f'Failed to process deposit/withdrawal. {grouped_events}. Ignoring ...',
)
continue
else:
movement = movement_events[0]
fee = Fee(ZERO)
amount = movement.balance.amount
if movement.event_type == HistoryEventType.WITHDRAWAL:
amount = amount * -1
try:
asset = movement.asset
movement_type = movement.event_type
movements.append(
AssetMovement(
location=Location.KRAKEN,
category=deserialize_asset_movement_category(
movement_type),
timestamp=ts_ms_to_sec(movement.timestamp),
address=None, # no data from kraken ledger endpoint
transaction_id=
None, # no data from kraken ledger endpoint
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=movement.event_identifier,
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown kraken asset {e.asset_name}. '
f'Ignoring its deposit/withdrawals query.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a kraken deposit/withdrawal. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a kraken deposit/withdrawal.',
raw_asset_movement=movement_events,
error=msg,
)
continue
return movements
def _deserialize_wallettx(
self,
entry: Dict[str, Any],
from_ts: Timestamp,
to_ts: Timestamp,
) -> Optional[AssetMovement]:
"""Deserializes a bitpanda fiatwallets/transactions or wallets/transactions
entry to a deposit/withdrawal
Returns None and logs error is there is a problem or simpy None if
it's not a type of entry we are interested in
"""
try:
transaction_type = entry['type']
if (transaction_type
not in ('fiat_wallet_transaction', 'wallet_transaction')
or entry['attributes']['status'] != 'finished'):
return None
time = Timestamp(
deserialize_int_from_str(
symbol=entry['attributes']['time']['unix'],
location='bitpanda wallet transaction',
))
if time < from_ts or time > to_ts:
# should we also stop querying from calling method?
# Probably yes but docs don't mention anything about results
# being ordered by time so let's be conservative
return None
try:
movement_category = deserialize_asset_movement_category(
entry['attributes']['type']) # noqa: E501
except DeserializationError:
return None # not a deposit/withdrawal
if transaction_type == 'fiat_wallet_transaction':
asset_id = entry['attributes']['fiat_id']
asset = self.fiat_map.get(asset_id)
else:
asset_id = entry['attributes']['cryptocoin_id']
asset = self.cryptocoin_map.get(asset_id)
if asset is None:
self.msg_aggregator.add_error(
f'While deserializing Bitpanda fiat transaction, could not find '
f'bitpanda asset with id {asset_id} in the mapping', )
return None
amount = deserialize_asset_amount(entry['attributes']['amount'])
fee = deserialize_fee(entry['attributes']['fee'])
tx_id = entry['id']
transaction_id = entry['attributes'].get('tx_id')
address = entry['attributes'].get('recipient')
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key {msg} for wallet transaction entry'
self.msg_aggregator.add_error(
f'Error processing bitpanda wallet transaction entry due to {msg}'
) # noqa: E501
log.error(
'Error processing bitpanda wallet transaction entry',
error=msg,
entry=entry,
)
return None
return AssetMovement(
location=Location.BITPANDA,
category=movement_category,
address=address,
transaction_id=transaction_id,
timestamp=time,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=tx_id,
)
