def _consume_cointracking_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cointracking entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCointrackingEntry if importing of this entry is not supported.
- IndexError if the CSV file is corrupt
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Type']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date'],
formatstr='%d.%m.%Y %H:%M:%S',
location='cointracking.info',
)
notes = csv_row['Comment']
location = exchange_row_to_location(csv_row['Exchange'])
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if csv_row['Fee'] != '':
fee = deserialize_fee(csv_row['Fee'])
fee_currency = Asset(csv_row['Cur.Fee'])
if row_type in ('Gift/Tip', 'Trade', 'Income'):
base_asset = Asset(csv_row['Cur.Buy'])
quote_asset = None if csv_row['Cur.Sell'] == '' else Asset(
csv_row['Cur.Sell'])
if quote_asset is None and row_type not in ('Gift/Tip', 'Income'):
raise DeserializationError(
'Got a trade entry with an empty quote asset')
if quote_asset is None:
# Really makes no difference as this is just a gift and the amount is zero
quote_asset = A_USD
pair = TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}')
base_amount_bought = deserialize_asset_amount(csv_row['Buy'])
if csv_row['Sell'] != '-':
quote_amount_sold = deserialize_asset_amount(csv_row['Sell'])
else:
quote_amount_sold = AssetAmount(ZERO)
rate = Price(quote_amount_sold / base_amount_bought)
trade = Trade(
timestamp=timestamp,
location=location,
pair=pair,
trade_type=TradeType.
BUY, # It's always a buy during cointracking import
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type == 'Deposit' or row_type == 'Withdrawal':
category = deserialize_asset_movement_category(row_type.lower())
if category == AssetMovementCategory.DEPOSIT:
amount = deserialize_asset_amount(csv_row['Buy'])
asset = Asset(csv_row['Cur.Buy'])
else:
amount = deserialize_asset_amount_force_positive(
csv_row['Sell'])
asset = Asset(csv_row['Cur.Sell'])
asset_movement = AssetMovement(
location=location,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_currency,
link='',
)
self.db.add_asset_movements([asset_movement])
else:
raise UnsupportedCointrackingEntry(
f'Unknown entrype type "{row_type}" encountered during cointracking '
f'data import. Ignoring entry', )
def _deserialize_asset_movement(
self,
movement_type: AssetMovementCategory,
movement_data: Dict[str, Any],
) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from polo and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if movement_type == AssetMovementCategory.DEPOSIT:
fee = Fee(ZERO)
uid_key = 'depositNumber'
transaction_id = get_key_if_has_val(movement_data, 'txid')
else:
fee = deserialize_fee(movement_data['fee'])
uid_key = 'withdrawalNumber'
split = movement_data['status'].split(':')
if len(split) != 2:
transaction_id = None
else:
transaction_id = split[1].lstrip()
if transaction_id == '':
transaction_id = None
asset = asset_from_poloniex(movement_data['currency'])
return AssetMovement(
location=Location.POLONIEX,
category=movement_type,
address=deserialize_asset_movement_address(movement_data, 'address', asset),
transaction_id=transaction_id,
timestamp=deserialize_timestamp(movement_data['timestamp']),
asset=asset,
amount=deserialize_asset_amount_force_positive(movement_data['amount']),
fee_asset=asset,
fee=fee,
link=str(movement_data[uid_key]),
)
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(movement_type)} of unsupported poloniex asset '
f'{e.asset_name}. Ignoring it.',
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(movement_type)} of unknown poloniex asset '
f'{e.asset_name}. Ignoring it.',
)
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a poloniex '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of poloniex '
f'{str(movement_type)}: {movement_data}. Error was: {str(e)}',
)
return None
def trade_from_poloniex(poloniex_trade: Dict[str, Any], pair: TradePair) -> Trade:
"""Turn a poloniex trade returned from poloniex trade history to our common trade
history format
Throws:
- UnsupportedAsset due to asset_from_poloniex()
- DeserializationError due to the data being in unexpected format
- UnprocessableTradePair due to the pair data being in an unexpected format
"""
try:
trade_type = deserialize_trade_type(poloniex_trade['type'])
amount = deserialize_asset_amount(poloniex_trade['amount'])
rate = deserialize_price(poloniex_trade['rate'])
perc_fee = deserialize_fee(poloniex_trade['fee'])
base_currency = asset_from_poloniex(get_pair_position_str(pair, 'first'))
quote_currency = asset_from_poloniex(get_pair_position_str(pair, 'second'))
timestamp = deserialize_timestamp_from_poloniex_date(poloniex_trade['date'])
except KeyError as e:
raise DeserializationError(
f'Poloniex trade deserialization error. Missing key entry for {str(e)} in trade dict',
)
cost = rate * amount
if trade_type == TradeType.BUY:
fee = Fee(amount * perc_fee)
fee_currency = quote_currency
elif trade_type == TradeType.SELL:
fee = Fee(cost * perc_fee)
fee_currency = base_currency
else:
raise DeserializationError(f'Got unexpected trade type "{trade_type}" for poloniex trade')
if poloniex_trade['category'] == 'settlement':
if trade_type == TradeType.BUY:
trade_type = TradeType.SETTLEMENT_BUY
else:
trade_type = TradeType.SETTLEMENT_SELL
log.debug(
'Processing poloniex Trade',
sensitive_log=True,
timestamp=timestamp,
order_type=trade_type,
pair=pair,
base_currency=base_currency,
quote_currency=quote_currency,
amount=amount,
fee=fee,
rate=rate,
)
# Use the converted assets in our pair
pair = trade_pair_from_assets(base_currency, quote_currency)
# Since in Poloniex the base currency is the cost currency, iow in poloniex
# for BTC_ETH we buy ETH with BTC and sell ETH for BTC, we need to turn it
# into the Rotkehlchen way which is following the base/quote approach.
pair = invert_pair(pair)
return Trade(
timestamp=timestamp,
location=Location.POLONIEX,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(poloniex_trade['globalTradeID']),
)
def query_online_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[Trade]:
"""Queries gemini for trades
"""
log.debug('Query gemini trade history',
start_ts=start_ts,
end_ts=end_ts)
trades = []
gemini_trades = []
for symbol in self.symbols:
gemini_trades = self._get_trades_for_symbol(
symbol=symbol,
start_ts=start_ts,
end_ts=end_ts,
)
for entry in gemini_trades:
try:
timestamp = deserialize_timestamp(entry['timestamp'])
if timestamp > end_ts:
break
trades.append(
Trade(
timestamp=timestamp,
location=Location.GEMINI,
pair=gemini_symbol_to_pair(symbol),
trade_type=deserialize_trade_type(entry['type']),
amount=deserialize_asset_amount(entry['amount']),
rate=deserialize_price(entry['price']),
fee=deserialize_fee(entry['fee_amount']),
fee_currency=Asset(entry['fee_currency']),
link=str(entry['tid']),
notes='',
))
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Gemini pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Gemini asset {e.asset_name}. '
f'Ignoring the trade.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a gemini trade. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a gemini trade.',
raw_trade=entry,
error=msg,
)
continue
return trades
def process_polo_loans(
msg_aggregator: MessagesAggregator,
data: List[Dict],
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[Loan]:
"""Takes in the list of loans from poloniex as returned by the return_lending_history
api call, processes it and returns it into our loan format
"""
new_data = []
for loan in reversed(data):
log.debug('processing poloniex loan', **make_sensitive(loan))
try:
close_time = deserialize_timestamp_from_poloniex_date(loan['close'])
open_time = deserialize_timestamp_from_poloniex_date(loan['open'])
if open_time < start_ts:
continue
if close_time > end_ts:
continue
our_loan = Loan(
location=Location.POLONIEX,
open_time=open_time,
close_time=close_time,
currency=asset_from_poloniex(loan['currency']),
fee=deserialize_fee(loan['fee']),
earned=deserialize_asset_amount(loan['earned']),
amount_lent=deserialize_asset_amount(loan['amount']),
)
except UnsupportedAsset as e:
msg_aggregator.add_warning(
f'Found poloniex loan with unsupported asset'
f' {e.asset_name}. Ignoring it.',
)
continue
except UnknownAsset as e:
msg_aggregator.add_warning(
f'Found poloniex loan with unknown asset'
f' {e.asset_name}. Ignoring it.',
)
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
msg_aggregator.add_error(
'Deserialization error while reading a poloniex loan. Check '
'logs for more details. Ignoring it.',
)
log.error(
'Deserialization error while reading a poloniex loan',
loan=loan,
error=msg,
)
continue
new_data.append(our_loan)
new_data.sort(key=lambda loan: loan.open_time)
return new_data
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
result = self.query_until_finished(
endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict={'type': 'deposit'},
)
result.extend(
self.query_until_finished(
endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict={'type': 'withdrawal'},
))
log.debug('Kraken deposit/withdrawals query result',
num_results=len(result))
movements = []
for movement in result:
try:
asset = asset_from_kraken(movement['asset'])
movement_type = movement['type']
if movement_type not in ('deposit', 'withdrawal'):
# Other known types: 'transfer'
continue # Can be for moving funds from spot to stake etc.
movements.append(
AssetMovement(
location=Location.KRAKEN,
category=deserialize_asset_movement_category(
movement_type),
timestamp=deserialize_timestamp_from_kraken(
movement['time']),
address=None, # no data from kraken ledger endpoint
transaction_id=
None, # no data from kraken ledger endpoint
asset=asset,
amount=deserialize_asset_amount_force_positive(
movement['amount']),
fee_asset=asset,
fee=deserialize_fee(movement['fee']),
link=str(movement['refid']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown kraken asset {e.asset_name}. '
f'Ignoring its deposit/withdrawals query.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a kraken deposit/withdrawals. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a kraken deposit/withdrawal.',
raw_asset_movement=movement,
error=msg,
)
continue
return movements
def _deserialize_wallettx(
self,
entry: Dict[str, Any],
from_ts: Timestamp,
to_ts: Timestamp,
) -> Optional[AssetMovement]:
"""Deserializes a bitpanda fiatwallets/transactions or wallets/transactions
entry to a deposit/withdrawal
Returns None and logs error is there is a problem or simpy None if
it's not a type of entry we are interested in
"""
try:
transaction_type = entry['type']
if (transaction_type
not in ('fiat_wallet_transaction', 'wallet_transaction')
or entry['attributes']['status'] != 'finished'):
return None
time = Timestamp(
deserialize_int_from_str(
symbol=entry['attributes']['time']['unix'],
location='bitpanda wallet transaction',
))
if time < from_ts or time > to_ts:
# should we also stop querying from calling method?
# Probably yes but docs don't mention anything about results
# being ordered by time so let's be conservative
return None
try:
movement_category = deserialize_asset_movement_category(
entry['attributes']['type']) # noqa: E501
except DeserializationError:
return None # not a deposit/withdrawal
if transaction_type == 'fiat_wallet_transaction':
asset_id = entry['attributes']['fiat_id']
asset = self.fiat_map.get(asset_id)
else:
asset_id = entry['attributes']['cryptocoin_id']
asset = self.cryptocoin_map.get(asset_id)
if asset is None:
self.msg_aggregator.add_error(
f'While deserializing Bitpanda fiat transaction, could not find '
f'bitpanda asset with id {asset_id} in the mapping', )
return None
amount = deserialize_asset_amount(entry['attributes']['amount'])
fee = deserialize_fee(entry['attributes']['fee'])
tx_id = entry['id']
transaction_id = entry['attributes'].get('tx_id')
address = entry['attributes'].get('recipient')
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key {msg} for wallet transaction entry'
self.msg_aggregator.add_error(
f'Error processing bitpanda wallet transaction entry due to {msg}'
) # noqa: E501
log.error(
'Error processing bitpanda wallet transaction entry',
error=msg,
entry=entry,
)
return None
return AssetMovement(
location=Location.BITPANDA,
category=movement_category,
address=address,
transaction_id=transaction_id,
timestamp=time,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=tx_id,
)
def query_online_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> Tuple[List[Trade], Tuple[Timestamp, Timestamp]]:
"""Queries coinbase pro for trades"""
log.debug('Query coinbasepro trade history',
start_ts=start_ts,
end_ts=end_ts)
self.first_connection()
trades = []
# first get all orders, to see which product ids we need to query fills for
orders = []
for batch in self._paginated_query(
endpoint='orders',
query_options={'status': 'done'},
):
orders.extend(batch)
queried_product_ids = set()
for order_entry in orders:
product_id = order_entry.get('product_id', None)
if product_id is None:
msg = (
'Skipping coinbasepro trade since it lacks a product_id. '
'Check logs for details')
self.msg_aggregator.add_error(msg)
log.error(
'Error processing a coinbasepro order.',
raw_trade=order_entry,
error=msg,
)
continue
if product_id in queried_product_ids or product_id not in self.available_products:
continue # already queried this product id or delisted product id
# Now let's get all fills for this product id
queried_product_ids.add(product_id)
fills = []
for batch in self._paginated_query(
endpoint='fills',
query_options={'product_id': product_id},
):
fills.extend(batch)
try:
base_asset, quote_asset = coinbasepro_to_worldpair(product_id)
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring the trade.', )
continue
for fill_entry in fills:
try:
timestamp = coinbasepro_deserialize_timestamp(
fill_entry, 'created_at')
if timestamp < start_ts or timestamp > end_ts:
continue
# Fee currency seems to always be quote asset
# https://github.com/ccxt/ccxt/blob/ddf3a15cbff01541f0b37c35891aa143bb7f9d7b/python/ccxt/coinbasepro.py#L724 # noqa: E501
trades.append(
Trade(
timestamp=timestamp,
location=Location.COINBASEPRO,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=TradeType.deserialize(
fill_entry['side']),
amount=deserialize_asset_amount(
fill_entry['size']),
rate=deserialize_price(fill_entry['price']),
fee=deserialize_fee(fill_entry['fee']),
fee_currency=quote_asset,
link=str(fill_entry['trade_id']) + '_' +
fill_entry['order_id'],
notes='',
))
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring the trade.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a coinbasepro trade. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a coinbasepro fill.',
raw_trade=fill_entry,
error=msg,
)
continue
return trades, (start_ts, end_ts)
def _deserialize_asset_movement(
self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from coinbase and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data['status'] != 'completed':
return None
payout_date = raw_data.get('payout_at', None)
if payout_date:
timestamp = deserialize_timestamp_from_date(
payout_date, 'iso8601', 'coinbase')
else:
timestamp = deserialize_timestamp_from_date(
raw_data['created_at'],
'iso8601',
'coinbase',
)
# Only get address/transaction id for "send" type of transactions
address = None
transaction_id = None
# movement_category: Union[Literal['deposit'], Literal['withdrawal']]
if 'type' in raw_data:
# Then this should be a "send" which is the way Coinbase uses to send
# crypto outside of the exchange
# https://developers.coinbase.com/api/v2?python#transaction-resource
msg = 'Non "send" type found in coinbase deposit/withdrawal processing'
assert raw_data['type'] == 'send', msg
movement_category = AssetMovementCategory.WITHDRAWAL
# Can't see the fee being charged from the "send" resource
amount = deserialize_asset_amount_force_positive(
raw_data['amount']['amount'])
asset = asset_from_coinbase(raw_data['amount']['currency'],
time=timestamp)
# Fees dont appear in the docs but from an experiment of sending ETH
# to an address from coinbase there is the network fee in the response
fee = Fee(ZERO)
raw_network = raw_data.get('network', None)
if raw_network:
raw_fee = raw_network.get('transaction_fee', None)
if raw_fee:
# Since this is a withdrawal the fee should be the same as the moved asset
if asset != asset_from_coinbase(raw_fee['currency'],
time=timestamp):
# If not we set ZERO fee and ignore
log.error(
f'In a coinbase withdrawal of {asset.identifier} the fee'
f'is denoted in {raw_fee["currency"]}', )
else:
fee = deserialize_fee(raw_fee['amount'])
if 'network' in raw_data:
transaction_id = get_key_if_has_val(
raw_data['network'], 'hash')
if 'to' in raw_data:
address = deserialize_asset_movement_address(
raw_data['to'], 'address', asset)
else:
movement_category = deserialize_asset_movement_category(
raw_data['resource'])
amount = deserialize_asset_amount_force_positive(
raw_data['amount']['amount'])
fee = deserialize_fee(raw_data['fee']['amount'])
asset = asset_from_coinbase(raw_data['amount']['currency'],
time=timestamp)
return AssetMovement(
location=Location.COINBASE,
category=movement_category,
address=address,
transaction_id=transaction_id,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=str(raw_data['id']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a coinbase '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of coinbase '
f'asset_movement {raw_data}. Error was: {str(e)}', )
return None
def verify_otctrade_data(data: ExternalTrade, ) -> Tuple[Optional[Trade], str]:
"""
Takes in the trade data dictionary, validates it and returns a trade instance
If there is an error it returns an error message in the second part of the tuple
"""
for field in otc_fields:
if field not in data:
return None, f'{field} was not provided'
if data[field] in ('', None) and field not in otc_optional_fields:
return None, f'{field} was empty'
if field in otc_numerical_fields and not is_number(data[field]):
return None, f'{field} should be a number'
# Satisfy mypy typing
assert isinstance(data['otc_pair'], str)
assert isinstance(data['otc_fee_currency'], str)
assert isinstance(data['otc_fee'], str)
pair = TradePair(data['otc_pair'])
try:
first = get_pair_position_asset(pair, 'first')
second = get_pair_position_asset(pair, 'second')
fee_currency = Asset(data['otc_fee_currency'])
except UnknownAsset as e:
return None, f'Provided asset {e.asset_name} is not known to Rotkehlchen'
# Not catching DeserializationError here since we have asserts for the data
# being strings right above
try:
trade_type = deserialize_trade_type(str(data['otc_type']))
amount = deserialize_asset_amount(data['otc_amount'])
rate = deserialize_price(data['otc_rate'])
fee = deserialize_fee(data['otc_fee'])
except DeserializationError as e:
return None, f'Deserialization Error: {str(e)}'
try:
assert isinstance(data['otc_timestamp'], str)
timestamp = create_timestamp(data['otc_timestamp'],
formatstr='%d/%m/%Y %H:%M')
except ValueError as e:
return None, f'Could not process the given datetime: {e}'
log.debug(
'Creating OTC trade data',
sensitive_log=True,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
)
if data['otc_fee_currency'] not in (first, second):
return None, 'Trade fee currency should be one of the two in the currency pair'
if data['otc_type'] not in ('buy', 'sell'):
return None, 'Trade type can only be buy or sell'
trade = Trade(
timestamp=timestamp,
location=Location.EXTERNAL,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(data['otc_link']),
notes=str(data['otc_notes']),
)
return trade, ''
"""
if case == KucoinCase.DEPOSITS:
category = AssetMovementCategory.DEPOSIT
elif case == KucoinCase.WITHDRAWALS:
category = AssetMovementCategory.WITHDRAWAL
else:
raise AssertionError(f'Unexpected case: {case}')
try:
timestamp_ms = deserialize_timestamp(raw_result['createdAt'])
timestamp = Timestamp(int(timestamp_ms / 1000))
address = raw_result['address']
# The transaction id can have an @ which we should just get rid of
transaction_id = raw_result['walletTxId'].split('@')[0]
amount = deserialize_asset_amount(raw_result['amount'])
fee = deserialize_fee(raw_result['fee'])
fee_currency_symbol = raw_result['currency']
link_id = raw_result.get('id',
'') # NB: id only exists for withdrawals
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
fee_asset = asset_from_kucoin(fee_currency_symbol)
asset_movement = AssetMovement(
timestamp=timestamp,
location=Location.KUCOIN,
category=category,
address=address,
transaction_id=transaction_id,
asset=fee_asset,
def _deserialize_asset_movement(
self,
raw_data: Dict[str, Any],
movement_type: AssetMovementCategory,
) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from FTX and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data['status'] not in ('complete', 'confirmed'):
return None
timestamp = deserialize_timestamp_from_date(
raw_data['time'], 'iso8601', 'FTX')
amount = deserialize_asset_amount_force_positive(raw_data['size'])
asset = asset_from_ftx(raw_data['coin'])
fee = Fee(ZERO)
movement_category = movement_type
if raw_data.get('fee', None) is not None:
fee = deserialize_fee(raw_data['fee'])
address = raw_data.get('address', None)
if isinstance(address, dict):
address = raw_data['address'].get('address', None)
transaction_id = raw_data.get('txid', None)
return AssetMovement(
location=Location.FTX,
category=movement_category,
address=address,
transaction_id=transaction_id,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=str(raw_data['id']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found FTX deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found FTX deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of an FTX '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
'Error processing FTX trade',
trade=raw_data,
error=msg,
)
return None
def query_online_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[Trade]:
"""Queries coinbase pro for trades"""
log.debug('Query coinbasepro trade history',
start_ts=start_ts,
end_ts=end_ts)
trades = []
raw_trades = []
for batch in self._paginated_query(
endpoint='orders',
query_options={'status': 'done'},
):
raw_trades.extend(batch)
for entry in raw_trades:
timestamp = coinbasepro_deserialize_timestamp(entry, 'created_at')
if timestamp < start_ts or timestamp > end_ts:
continue
try:
pair = coinbasepro_to_worldpair(entry['product_id'])
# Fee currency seems to always be quote asset
# https://github.com/ccxt/ccxt/blob/ddf3a15cbff01541f0b37c35891aa143bb7f9d7b/python/ccxt/coinbasepro.py#L724 # noqa: E501
_, quote_asset = pair_get_assets(pair)
trades.append(
Trade(
timestamp=timestamp,
location=Location.COINBASEPRO,
pair=coinbasepro_to_worldpair(entry['product_id']),
trade_type=deserialize_trade_type(entry['side']),
amount=deserialize_asset_amount(entry['size']),
rate=deserialize_price(entry['price']),
fee=deserialize_fee(entry['fill_fees']),
fee_currency=quote_asset,
link=entry['id'],
notes='',
))
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring the trade.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a coinbasepro trade. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a coinbasepro trade.',
raw_trade=entry,
error=msg,
)
continue
return trades
def _deserialize_trade(
self,
entry: Dict[str, Any],
from_ts: Timestamp,
to_ts: Timestamp,
) -> Optional[Trade]:
"""Deserializes a bitpanda trades result entry to a Trade
Returns None and logs error is there is a problem or simpy None if
it's not a type of trade we are interested in
"""
try:
if entry['type'] != 'trade' or entry['attributes'][
'status'] != 'finished':
return None
time = Timestamp(
deserialize_int_from_str(
symbol=entry['attributes']['time']['unix'],
location='bitpanda trade',
))
if time < from_ts or time > to_ts:
# should we also stop querying from calling method?
# Probably yes but docs don't mention anything about results
# being ordered by time so let's be conservative
return None
cryptocoin_id = entry['attributes']['cryptocoin_id']
crypto_asset = self.cryptocoin_map.get(cryptocoin_id)
if crypto_asset is None:
self.msg_aggregator.add_error(
f'While deserializing a trade, could not find bitpanda cryptocoin '
f'with id {cryptocoin_id} in the mapping. Skipping trade.',
)
return None
fiat_id = entry['attributes']['fiat_id']
fiat_asset = self.fiat_map.get(fiat_id)
if fiat_asset is None:
self.msg_aggregator.add_error(
f'While deserializing a trade, could not find bitpanda fiat '
f'with id {fiat_id} in the mapping. Skipping trade.', )
return None
trade_type = TradeType.deserialize(entry['attributes']['type'])
if trade_type in (TradeType.BUY, TradeType.SELL):
# you buy crypto with fiat and sell it for fiat
base_asset = crypto_asset
quote_asset = fiat_asset
amount = deserialize_asset_amount(
entry['attributes']['amount_cryptocoin'])
price = deserialize_price(entry['attributes']['price'])
else:
self.msg_aggregator.add_error(
'Found bitpanda trade with unknown trade type {trade_type}'
) # noqa: E501
return None
trade_id = entry['id']
fee = Fee(ZERO)
fee_asset = A_BEST
if entry['attributes']['bfc_used'] is True:
fee = deserialize_fee(
entry['attributes']['best_fee_collection']['attributes']
['wallet_transaction']['attributes']['fee'], # noqa: E501
)
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key {msg} for trade entry'
self.msg_aggregator.add_error(
f'Error processing bitpanda trade due to {msg}')
log.error(
'Error processing bitpanda trade entry',
error=msg,
entry=entry,
)
return None
return Trade(
timestamp=time,
location=Location.BITPANDA,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=trade_type,
amount=amount,
rate=price,
fee=fee,
fee_currency=fee_asset,
link=trade_id,
)
def trade_from_binance(
binance_trade: Dict,
binance_symbols_to_pair: Dict[str, BinancePair],
) -> Trade:
"""Turn a binance trade returned from trade history to our common trade
history format
From the official binance api docs (01/09/18):
https://github.com/binance-exchange/binance-official-api-docs/blob/62ff32d27bb32d9cc74d63d547c286bb3c9707ef/rest-api.md#terminology
base asset refers to the asset that is the quantity of a symbol.
quote asset refers to the asset that is the price of a symbol.
Throws:
- UnsupportedAsset due to asset_from_binance
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entries missing an expected entry
"""
amount = deserialize_asset_amount(binance_trade['qty'])
rate = deserialize_price(binance_trade['price'])
if binance_trade['symbol'] not in binance_symbols_to_pair:
raise DeserializationError(
f'Error reading a binance trade. Could not find '
f'{binance_trade["symbol"]} in binance_symbols_to_pair',
)
binance_pair = binance_symbols_to_pair[binance_trade['symbol']]
timestamp = deserialize_timestamp_from_binance(binance_trade['time'])
base_asset = asset_from_binance(binance_pair.binance_base_asset)
quote_asset = asset_from_binance(binance_pair.binance_quote_asset)
if binance_trade['isBuyer']:
order_type = TradeType.BUY
# e.g. in RDNETH we buy RDN by paying ETH
else:
order_type = TradeType.SELL
fee_currency = asset_from_binance(binance_trade['commissionAsset'])
fee = deserialize_fee(binance_trade['commission'])
log.debug(
'Processing binance Trade',
sensitive_log=True,
amount=amount,
rate=rate,
timestamp=timestamp,
pair=binance_trade['symbol'],
base_asset=base_asset,
quote=quote_asset,
order_type=order_type,
commision_asset=binance_trade['commissionAsset'],
fee=fee,
)
return Trade(
timestamp=timestamp,
location=Location.BINANCE,
pair=trade_pair_from_assets(base_asset, quote_asset),
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(binance_trade['id']),
)
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List:
resp = self._api_query_list('get', 'user/walletHistory')
log.debug('Bitmex deposit/withdrawals query', results_num=len(resp))
movements = list()
for movement in resp:
try:
transaction_type = movement['transactType']
if transaction_type == 'Deposit':
transaction_type = AssetMovementCategory.DEPOSIT
elif transaction_type == 'Withdrawal':
transaction_type = AssetMovementCategory.WITHDRAWAL
else:
continue
timestamp = iso8601ts_to_timestamp(movement['timestamp'])
if timestamp < start_ts:
continue
if timestamp > end_ts:
continue
asset = bitmex_to_world(movement['currency'])
amount = deserialize_asset_amount(movement['amount'])
fee = deserialize_fee(movement['fee'])
# bitmex has negative numbers for withdrawals
if amount < 0:
amount *= -1
if asset == A_BTC:
# bitmex stores amounts in satoshis
amount = satoshis_to_btc(amount)
fee = satoshis_to_btc(fee)
movements.append(
AssetMovement(
location=Location.BITMEX,
category=transaction_type,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=str(movement['transactID']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found bitmex deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
f'Unexpected data encountered during deserialization of a bitmex '
f'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of bitmex '
f'asset_movement {movement}. Error was: {str(e)}', )
continue
return movements
def trade_from_kraken(kraken_trade: Dict[str, Any]) -> Trade:
"""Turn a kraken trade returned from kraken trade history to our common trade
history format
- Can raise UnknownAsset due to kraken_to_world_pair
- Can raise UnprocessableTradePair due to kraken_to_world_pair
- Can raise DeserializationError due to dict entries not being as expected
- Can raise KeyError due to dict entries missing an expected entry
"""
currency_pair = kraken_to_world_pair(kraken_trade['pair'])
quote_currency = get_pair_position_asset(currency_pair, 'second')
timestamp = deserialize_timestamp_from_kraken(kraken_trade['time'])
amount = deserialize_asset_amount(kraken_trade['vol'])
cost = deserialize_price(kraken_trade['cost'])
fee = deserialize_fee(kraken_trade['fee'])
order_type = deserialize_trade_type(kraken_trade['type'])
rate = deserialize_price(kraken_trade['price'])
# pylint does not seem to see that Price is essentially FVal
if not cost.is_close(amount * rate): # pylint: disable=no-member
log.warning(
f'cost ({cost}) != amount ({amount}) * rate ({rate}) for kraken trade'
)
log.debug(
'Processing kraken Trade',
sensitive_log=True,
timestamp=timestamp,
order_type=order_type,
kraken_pair=kraken_trade['pair'],
pair=currency_pair,
quote_currency=quote_currency,
amount=amount,
cost=cost,
fee=fee,
rate=rate,
)
# Kraken trades can have the same ordertxid and postxid for different trades ..
# Also note postxid is optional and can be missing
# The only thing that could differentiate them is timestamps in the milliseconds range
# For example here are parts of two different kraken_trade:
# {'ordertxid': 'AM4ZOZ-GLEMD-ZICOGR', 'postxid': 'AKH2SE-M7IF5-CFI7AT',
# 'pair': 'XXBTZEUR', 'time': FVal(1561161486.2955)
# {'ordertxid': 'AM4ZOZ-GLEMD-ZICOGR', 'postxid': 'AKH2SE-M7IF5-CFI7AT',
# 'pair': 'XXBTZEUR', 'time': FVal(1561161486.3005)
#
# In order to counter this for the unique exchange trade link we are going
# to use a concatenation of the above
exchange_uuid = (
str(kraken_trade['ordertxid']) +
str(kraken_trade.get('postxid', '')) + # postxid is optional
str(kraken_trade['time']))
return Trade(
timestamp=timestamp,
location=Location.KRAKEN,
pair=currency_pair,
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=quote_currency,
link=exchange_uuid,
)
def _deserialize_asset_movement(
self,
raw_movement: Dict[str, Any],
) -> AssetMovement:
"""Process a deposit/withdrawal user transaction from Bitstamp and
deserialize it.
Can raise DeserializationError.
From Bitstamp documentation, deposits/withdrawals can have a fee
(the amount is expected to be in the currency involved)
https://www.bitstamp.net/fee-schedule/
Bitstamp support confirmed the following withdrawal JSON:
{
"fee": "0.00050000",
"btc_usd": "0.00",
"datetime": "2020-12-04 09:30:00.000000",
"usd": "0.0",
"btc": "-0.50000000",
"type": "1",
"id": 123456789,
"eur": "0.0"
}
NB: any asset key not related with the pair is discarded (e.g. 'eur').
"""
type_ = raw_movement['type']
category: AssetMovementCategory
if type_ == 0:
category = AssetMovementCategory.DEPOSIT
elif type_ == 1:
category = AssetMovementCategory.WITHDRAWAL
else:
raise AssertionError(f'Unexpected Bitstamp asset movement case: {type_}.')
timestamp = deserialize_timestamp_from_bitstamp_date(raw_movement['datetime'])
trade_pair_data = self._get_trade_pair_data_from_transaction(raw_movement)
base_asset_amount = deserialize_asset_amount(
raw_movement[trade_pair_data.base_asset_symbol],
)
quote_asset_amount = deserialize_asset_amount(
raw_movement[trade_pair_data.quote_asset_symbol],
)
amount: FVal
fee_asset: Asset
if base_asset_amount != ZERO and quote_asset_amount == ZERO:
amount = base_asset_amount
fee_asset = trade_pair_data.base_asset
elif base_asset_amount == ZERO and quote_asset_amount != ZERO:
amount = quote_asset_amount
fee_asset = trade_pair_data.quote_asset
else:
raise DeserializationError(
'Could not deserialize Bitstamp asset movement from user transaction. '
f'Unexpected asset amount combination found in: {raw_movement}.',
)
asset_movement = AssetMovement(
timestamp=timestamp,
location=Location.BITSTAMP,
category=category,
address=None, # requires query "crypto_transactions" endpoint
transaction_id=None, # requires query "crypto_transactions" endpoint
asset=fee_asset,
amount=abs(amount),
fee_asset=fee_asset,
fee=deserialize_fee(raw_movement['fee']),
link=str(raw_movement['id']),
)
return asset_movement
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
"""Queries coinbase pro for asset movements"""
log.debug('Query coinbasepro asset movements',
start_ts=start_ts,
end_ts=end_ts)
movements = []
raw_movements = []
for batch in self._paginated_query(
endpoint='transfers',
query_options={'type': 'withdraw'},
):
raw_movements.extend(batch)
for batch in self._paginated_query(
endpoint='transfers',
query_options={'type': 'deposit'},
):
raw_movements.extend(batch)
account_to_currency = self.create_or_return_account_to_currency_map()
for entry in raw_movements:
try:
# Check if the transaction has not been completed. If so it should be skipped
if entry.get('completed_at', None) is None:
log.warning(
f'Skipping coinbase pro deposit/withdrawal '
f'due not having been completed: {entry}', )
continue
timestamp = coinbasepro_deserialize_timestamp(
entry, 'completed_at')
if timestamp < start_ts or timestamp > end_ts:
continue
category = deserialize_asset_movement_category(entry['type'])
asset = account_to_currency.get(entry['account_id'], None)
if asset is None:
log.warning(
f'Skipping coinbase pro asset_movement {entry} due to inability to '
f'match account id to an asset', )
continue
address = None
transaction_id = None
fee = Fee(ZERO)
if category == AssetMovementCategory.DEPOSIT:
try:
address = entry['details']['crypto_address']
transaction_id = entry['details'][
'crypto_transaction_hash']
except KeyError:
pass
else: # withdrawal
try:
address = entry['details']['sent_to_address']
transaction_id = entry['details'][
'crypto_transaction_hash']
fee = deserialize_fee(entry['details']['fee'])
except KeyError:
pass
if transaction_id and (
asset == A_ETH or asset.asset_type
== AssetType.ETHEREUM_TOKEN): # noqa: E501
transaction_id = '0x' + transaction_id
movements.append(
AssetMovement(
location=Location.COINBASEPRO,
category=category,
address=address,
transaction_id=transaction_id,
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
entry['amount']),
fee_asset=asset,
fee=fee,
link=str(entry['id']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring its deposit/withdrawal.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a Coinbasepro deposit/withdrawal. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a coinbasepro deposit/withdrawal.',
raw_asset_movement=entry,
error=msg,
)
continue
return movements
def query_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
end_at_least_ts: Timestamp,
) -> List[AssetMovement]:
with self.lock:
cache = self.check_trades_cache_list(
start_ts,
end_at_least_ts,
special_name='deposits_withdrawals',
)
result: List[Any]
if cache is not None:
result = cache
else:
result = self.query_until_finished(
endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict=dict(type='deposit'),
)
result.extend(
self.query_until_finished(
endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict=dict(type='withdrawal'),
))
with self.lock:
self.update_trades_cache(
result,
start_ts,
end_ts,
special_name='deposits_withdrawals',
)
log.debug('Kraken deposit/withdrawals query result',
num_results=len(result))
movements = list()
for movement in result:
try:
movements.append(
AssetMovement(
exchange=Exchange.KRAKEN,
category=deserialize_asset_movement_category(
movement['type']),
timestamp=deserialize_timestamp_from_kraken(
movement['time']),
asset=asset_from_kraken(movement['asset']),
amount=deserialize_asset_amount(movement['amount']),
fee=deserialize_fee(movement['fee']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown kraken asset {e.asset_name}. '
f'Ignoring its deposit/withdrawals query.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a kraken deposit/withdrawals. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a kraken deposit/withdrawal.',
raw_asset_movement=movement,
error=msg,
)
continue
return movements