def _deserialize_asset_movement(
self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from binance and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if 'insertTime' in raw_data:
category = AssetMovementCategory.DEPOSIT
time_key = 'insertTime'
fee = Fee(ZERO)
else:
category = AssetMovementCategory.WITHDRAWAL
time_key = 'applyTime'
fee = Fee(deserialize_asset_amount(raw_data['transactionFee']))
timestamp = deserialize_timestamp_from_binance(raw_data[time_key])
asset = asset_from_binance(raw_data['asset'])
tx_id = get_key_if_has_val(raw_data, 'txId')
internal_id = get_key_if_has_val(raw_data, 'id')
link_str = str(internal_id) if internal_id else str(
tx_id) if tx_id else ''
return AssetMovement(
location=self.location,
category=category,
address=deserialize_asset_movement_address(
raw_data, 'address', asset),
transaction_id=tx_id,
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
raw_data['amount']),
fee_asset=asset,
fee=fee,
link=link_str,
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(self.location)} deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(self.location)} deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
f'Error processing a {str(self.location)} deposit/withdrawal. Check logs '
f'for details. Ignoring it.', )
log.error(
f'Error processing a {str(self.location)} deposit/withdrawal',
asset_movement=raw_data,
error=msg,
)
return None
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
result = self._get_paginated_query(
endpoint='transfers',
start_ts=start_ts,
end_ts=end_ts,
)
movements = []
for entry in result:
try:
timestamp = deserialize_timestamp(entry['timestampms'])
timestamp = Timestamp(int(timestamp / 1000))
asset = Asset(entry['currency'])
movement = AssetMovement(
location=Location.GEMINI,
category=deserialize_asset_movement_category(
entry['type']),
address=deserialize_asset_movement_address(
entry, 'destination', asset),
transaction_id=get_key_if_has_val(entry, 'txHash'),
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
entry['amount']),
fee_asset=asset,
# Gemini does not include withdrawal fees neither in the API nor in their UI
fee=Fee(ZERO),
link=str(entry['eid']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Error processing a gemini deposit/withdrawal. Check logs '
'for details. Ignoring it.', )
log.error(
'Error processing a gemini deposit_withdrawal',
asset_movement=entry,
error=msg,
)
continue
movements.append(movement)
return movements
def _deserialize_asset_movement(
self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from bittrex and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data['status'] != 'COMPLETED':
# Don't mind failed/in progress asset movements
return None
if 'source' in raw_data:
category = AssetMovementCategory.DEPOSIT
fee = Fee(ZERO)
else:
category = AssetMovementCategory.WITHDRAWAL
fee = deserialize_fee(raw_data.get('txCost', 0))
timestamp = deserialize_timestamp_from_date(
date=raw_data['completedAt'], # we only check completed orders
formatstr='iso8601',
location='bittrex',
)
asset = asset_from_bittrex(raw_data['currencySymbol'])
return AssetMovement(
location=Location.BITTREX,
category=category,
address=deserialize_asset_movement_address(
raw_data, 'cryptoAddress', asset),
transaction_id=get_key_if_has_val(raw_data, 'txId'),
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
raw_data['quantity']),
fee_asset=asset,
fee=fee,
link=str(raw_data.get('txId', '')),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found bittrex deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found bittrex deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a bittrex '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of bittrex '
f'asset_movement {raw_data}. Error was: {str(e)}', )
return None
def _deserialize_asset_movement(
self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from binance and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if 'insertTime' in raw_data:
category = AssetMovementCategory.DEPOSIT
time_key = 'insertTime'
else:
category = AssetMovementCategory.WITHDRAWAL
time_key = 'applyTime'
timestamp = deserialize_timestamp_from_binance(raw_data[time_key])
asset = asset_from_binance(raw_data['asset'])
location = Location.BINANCE if self.name == str(
Location.BINANCE) else Location.BINANCE_US # noqa: E501
return AssetMovement(
location=location,
category=category,
address=deserialize_asset_movement_address(
raw_data, 'address', asset),
transaction_id=get_key_if_has_val(raw_data, 'txId'),
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
raw_data['amount']),
fee_asset=asset,
# Binance does not include withdrawal fees neither in the API nor in their UI
fee=Fee(ZERO),
link=str(raw_data['txId']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found {self.name} deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found {self.name} deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
f'Error processing a {self.name} deposit/withdrawal. Check logs '
f'for details. Ignoring it.', )
log.error(
f'Error processing a {self.name} deposit_withdrawal',
asset_movement=raw_data,
error=msg,
)
return None
def _deserialize_asset_movement(
self,
movement_type: AssetMovementCategory,
movement_data: Dict[str, Any],
) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from polo and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if movement_type == AssetMovementCategory.DEPOSIT:
fee = Fee(ZERO)
uid_key = 'depositNumber'
transaction_id = get_key_if_has_val(movement_data, 'txid')
else:
fee = deserialize_fee(movement_data['fee'])
uid_key = 'withdrawalNumber'
split = movement_data['status'].split(':')
if len(split) != 2:
transaction_id = None
else:
transaction_id = split[1].lstrip()
if transaction_id == '':
transaction_id = None
asset = asset_from_poloniex(movement_data['currency'])
return AssetMovement(
location=Location.POLONIEX,
category=movement_type,
address=deserialize_asset_movement_address(movement_data, 'address', asset),
transaction_id=transaction_id,
timestamp=deserialize_timestamp(movement_data['timestamp']),
asset=asset,
amount=deserialize_asset_amount_force_positive(movement_data['amount']),
fee_asset=asset,
fee=fee,
link=str(movement_data[uid_key]),
)
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(movement_type)} of unsupported poloniex asset '
f'{e.asset_name}. Ignoring it.',
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(movement_type)} of unknown poloniex asset '
f'{e.asset_name}. Ignoring it.',
)
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a poloniex '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of poloniex '
f'{str(movement_type)}: {movement_data}. Error was: {str(e)}',
)
return None
def _deserialize_asset_movement(
self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from coinbase and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data['status'] != 'completed':
return None
payout_date = raw_data.get('payout_at', None)
if payout_date:
timestamp = deserialize_timestamp_from_date(
payout_date, 'iso8601', 'coinbase')
else:
timestamp = deserialize_timestamp_from_date(
raw_data['created_at'],
'iso8601',
'coinbase',
)
# Only get address/transaction id for "send" type of transactions
address = None
transaction_id = None
# movement_category: Union[Literal['deposit'], Literal['withdrawal']]
if 'type' in raw_data:
# Then this should be a "send" which is the way Coinbase uses to send
# crypto outside of the exchange
# https://developers.coinbase.com/api/v2?python#transaction-resource
msg = 'Non "send" type found in coinbase deposit/withdrawal processing'
assert raw_data['type'] == 'send', msg
movement_category = AssetMovementCategory.WITHDRAWAL
# Can't see the fee being charged from the "send" resource
amount = deserialize_asset_amount_force_positive(
raw_data['amount']['amount'])
asset = asset_from_coinbase(raw_data['amount']['currency'],
time=timestamp)
# Fees dont appear in the docs but from an experiment of sending ETH
# to an address from coinbase there is the network fee in the response
fee = Fee(ZERO)
raw_network = raw_data.get('network', None)
if raw_network:
raw_fee = raw_network.get('transaction_fee', None)
if raw_fee:
# Since this is a withdrawal the fee should be the same as the moved asset
if asset != asset_from_coinbase(raw_fee['currency'],
time=timestamp):
# If not we set ZERO fee and ignore
log.error(
f'In a coinbase withdrawal of {asset.identifier} the fee'
f'is denoted in {raw_fee["currency"]}', )
else:
fee = deserialize_fee(raw_fee['amount'])
if 'network' in raw_data:
transaction_id = get_key_if_has_val(
raw_data['network'], 'hash')
if 'to' in raw_data:
address = deserialize_asset_movement_address(
raw_data['to'], 'address', asset)
else:
movement_category = deserialize_asset_movement_category(
raw_data['resource'])
amount = deserialize_asset_amount_force_positive(
raw_data['amount']['amount'])
fee = deserialize_fee(raw_data['fee']['amount'])
asset = asset_from_coinbase(raw_data['amount']['currency'],
time=timestamp)
return AssetMovement(
location=Location.COINBASE,
category=movement_category,
address=address,
transaction_id=transaction_id,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=str(raw_data['id']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a coinbase '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of coinbase '
f'asset_movement {raw_data}. Error was: {str(e)}', )
return None
def _deserialize_ledger_action(
self, raw_data: Dict[str, Any]) -> Optional[LedgerAction]:
"""Processes a single transaction from coinbase and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data.get('status', '') != 'completed':
return None
payout_date = raw_data.get('payout_at', None)
if payout_date:
timestamp = deserialize_timestamp_from_date(
payout_date, 'iso8601', 'coinbase')
else:
timestamp = deserialize_timestamp_from_date(
get_key_if_has_val(raw_data, 'created_at'),
'iso8601',
'coinbase',
)
if 'type' in raw_data:
# The parent method filtered with 'from' attribute, so it is from another user.
# https://developers.coinbase.com/api/v2?python#transaction-resource
action_type = LedgerActionType.INCOME
if raw_data.get('type',
'') not in ('send', 'inflation_reward'):
msg = ('Non "send" or "inflation_reward" type '
'found in coinbase transactions processing')
raise DeserializationError(msg)
amount_data = raw_data.get('amount', {})
amount = deserialize_asset_amount(amount_data['amount'])
asset = asset_from_coinbase(amount_data['currency'],
time=timestamp)
native_amount_data = raw_data.get('native_amount', {})
native_amount = deserialize_asset_amount(
native_amount_data['amount'])
native_asset = asset_from_coinbase(
native_amount_data['currency'])
rate = ZERO
if amount_data and native_amount_data and native_amount and amount != ZERO:
rate = native_amount / amount
if 'details' in raw_data and 'title' in raw_data['details'] \
and 'subtitle' in raw_data['details'] and 'header' in raw_data['details']:
details = raw_data.get('details', {})
notes = (f"{details.get('title', '')} "
f"{details.get('subtitle', '')} "
f"{details.get('header', '')}")
else:
notes = ''
return LedgerAction(identifier=0,
location=Location.COINBASE,
action_type=action_type,
timestamp=timestamp,
asset=asset,
amount=amount,
rate=Price(rate),
rate_asset=native_asset,
link=str(raw_data['id']),
notes=notes)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase transaction with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase transaction with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a coinbase '
'ledger action. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of coinbase '
f'ledger action {raw_data}. Error was: {msg}', )
return None
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List:
resp = self._api_query_list('get', 'user/walletHistory')
log.debug('Bitmex deposit/withdrawals query', results_num=len(resp))
movements = []
for movement in resp:
try:
transaction_type = movement['transactType']
if transaction_type == 'Deposit':
transaction_type = AssetMovementCategory.DEPOSIT
elif transaction_type == 'Withdrawal':
transaction_type = AssetMovementCategory.WITHDRAWAL
else:
continue
timestamp = iso8601ts_to_timestamp(movement['timestamp'])
if timestamp < start_ts:
continue
if timestamp > end_ts:
continue
asset = bitmex_to_world(movement['currency'])
amount = deserialize_asset_amount_force_positive(
movement['amount'])
fee = deserialize_fee(movement['fee'])
if asset == A_BTC:
# bitmex stores amounts in satoshis
amount = AssetAmount(satoshis_to_btc(amount))
fee = Fee(satoshis_to_btc(fee))
movements.append(
AssetMovement(
location=Location.BITMEX,
category=transaction_type,
address=deserialize_asset_movement_address(
movement, 'address', asset),
transaction_id=get_key_if_has_val(movement, 'tx'),
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=str(movement['transactID']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found bitmex deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a bitmex '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of bitmex '
f'asset_movement {movement}. Error was: {msg}', )
continue
return movements