def _consume_cointracking_entry(self, csv_row: Dict[str, Any]) -> None: """Consumes a cointracking entry row from the CSV and adds it into the database Can raise: - DeserializationError if something is wrong with the format of the expected values - UnsupportedCointrackingEntry if importing of this entry is not supported. - IndexError if the CSV file is corrupt - KeyError if the an expected CSV key is missing - UnknownAsset if one of the assets founds in the entry are not supported """ row_type = csv_row['Type'] timestamp = deserialize_timestamp_from_date( date=csv_row['Date'], formatstr='%d.%m.%Y %H:%M:%S', location='cointracking.info', ) notes = csv_row['Comment'] location = exchange_row_to_location(csv_row['Exchange']) fee = Fee(ZERO) fee_currency = A_USD # whatever (used only if there is no fee) if csv_row['Fee'] != '': fee = deserialize_fee(csv_row['Fee']) fee_currency = Asset(csv_row['Cur.Fee']) if row_type in ('Gift/Tip', 'Trade', 'Income'): base_asset = Asset(csv_row['Cur.Buy']) quote_asset = None if csv_row['Cur.Sell'] == '' else Asset( csv_row['Cur.Sell']) if quote_asset is None and row_type not in ('Gift/Tip', 'Income'): raise DeserializationError( 'Got a trade entry with an empty quote asset') if quote_asset is None: # Really makes no difference as this is just a gift and the amount is zero quote_asset = A_USD pair = TradePair( f'{base_asset.identifier}_{quote_asset.identifier}') base_amount_bought = deserialize_asset_amount(csv_row['Buy']) if csv_row['Sell'] != '-': quote_amount_sold = deserialize_asset_amount(csv_row['Sell']) else: quote_amount_sold = AssetAmount(ZERO) rate = Price(quote_amount_sold / base_amount_bought) trade = Trade( timestamp=timestamp, location=location, pair=pair, trade_type=TradeType. BUY, # It's always a buy during cointracking import amount=base_amount_bought, rate=rate, fee=fee, fee_currency=fee_currency, link='', notes=notes, ) self.db.add_trades([trade]) elif row_type == 'Deposit' or row_type == 'Withdrawal': category = deserialize_asset_movement_category(row_type.lower()) if category == AssetMovementCategory.DEPOSIT: amount = deserialize_asset_amount(csv_row['Buy']) asset = Asset(csv_row['Cur.Buy']) else: amount = deserialize_asset_amount_force_positive( csv_row['Sell']) asset = Asset(csv_row['Cur.Sell']) asset_movement = AssetMovement( location=location, category=category, address=None, transaction_id=None, timestamp=timestamp, asset=asset, amount=amount, fee=fee, fee_asset=fee_currency, link='', ) self.db.add_asset_movements([asset_movement]) else: raise UnsupportedCointrackingEntry( f'Unknown entrype type "{row_type}" encountered during cointracking ' f'data import. Ignoring entry', )
def _deserialize_asset_movement( self, movement_type: AssetMovementCategory, movement_data: Dict[str, Any], ) -> Optional[AssetMovement]: """Processes a single deposit/withdrawal from polo and deserializes it Can log error/warning and return None if something went wrong at deserialization """ try: if movement_type == AssetMovementCategory.DEPOSIT: fee = Fee(ZERO) uid_key = 'depositNumber' transaction_id = get_key_if_has_val(movement_data, 'txid') else: fee = deserialize_fee(movement_data['fee']) uid_key = 'withdrawalNumber' split = movement_data['status'].split(':') if len(split) != 2: transaction_id = None else: transaction_id = split[1].lstrip() if transaction_id == '': transaction_id = None asset = asset_from_poloniex(movement_data['currency']) return AssetMovement( location=Location.POLONIEX, category=movement_type, address=deserialize_asset_movement_address(movement_data, 'address', asset), transaction_id=transaction_id, timestamp=deserialize_timestamp(movement_data['timestamp']), asset=asset, amount=deserialize_asset_amount_force_positive(movement_data['amount']), fee_asset=asset, fee=fee, link=str(movement_data[uid_key]), ) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found {str(movement_type)} of unsupported poloniex asset ' f'{e.asset_name}. Ignoring it.', ) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found {str(movement_type)} of unknown poloniex asset ' f'{e.asset_name}. Ignoring it.', ) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Unexpected data encountered during deserialization of a poloniex ' 'asset movement. Check logs for details and open a bug report.', ) log.error( f'Unexpected data encountered during deserialization of poloniex ' f'{str(movement_type)}: {movement_data}. Error was: {str(e)}', ) return None
def trade_from_poloniex(poloniex_trade: Dict[str, Any], pair: TradePair) -> Trade: """Turn a poloniex trade returned from poloniex trade history to our common trade history format Throws: - UnsupportedAsset due to asset_from_poloniex() - DeserializationError due to the data being in unexpected format - UnprocessableTradePair due to the pair data being in an unexpected format """ try: trade_type = deserialize_trade_type(poloniex_trade['type']) amount = deserialize_asset_amount(poloniex_trade['amount']) rate = deserialize_price(poloniex_trade['rate']) perc_fee = deserialize_fee(poloniex_trade['fee']) base_currency = asset_from_poloniex(get_pair_position_str(pair, 'first')) quote_currency = asset_from_poloniex(get_pair_position_str(pair, 'second')) timestamp = deserialize_timestamp_from_poloniex_date(poloniex_trade['date']) except KeyError as e: raise DeserializationError( f'Poloniex trade deserialization error. Missing key entry for {str(e)} in trade dict', ) cost = rate * amount if trade_type == TradeType.BUY: fee = Fee(amount * perc_fee) fee_currency = quote_currency elif trade_type == TradeType.SELL: fee = Fee(cost * perc_fee) fee_currency = base_currency else: raise DeserializationError(f'Got unexpected trade type "{trade_type}" for poloniex trade') if poloniex_trade['category'] == 'settlement': if trade_type == TradeType.BUY: trade_type = TradeType.SETTLEMENT_BUY else: trade_type = TradeType.SETTLEMENT_SELL log.debug( 'Processing poloniex Trade', sensitive_log=True, timestamp=timestamp, order_type=trade_type, pair=pair, base_currency=base_currency, quote_currency=quote_currency, amount=amount, fee=fee, rate=rate, ) # Use the converted assets in our pair pair = trade_pair_from_assets(base_currency, quote_currency) # Since in Poloniex the base currency is the cost currency, iow in poloniex # for BTC_ETH we buy ETH with BTC and sell ETH for BTC, we need to turn it # into the Rotkehlchen way which is following the base/quote approach. pair = invert_pair(pair) return Trade( timestamp=timestamp, location=Location.POLONIEX, pair=pair, trade_type=trade_type, amount=amount, rate=rate, fee=fee, fee_currency=fee_currency, link=str(poloniex_trade['globalTradeID']), )
def query_online_trade_history( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[Trade]: """Queries gemini for trades """ log.debug('Query gemini trade history', start_ts=start_ts, end_ts=end_ts) trades = [] gemini_trades = [] for symbol in self.symbols: gemini_trades = self._get_trades_for_symbol( symbol=symbol, start_ts=start_ts, end_ts=end_ts, ) for entry in gemini_trades: try: timestamp = deserialize_timestamp(entry['timestamp']) if timestamp > end_ts: break trades.append( Trade( timestamp=timestamp, location=Location.GEMINI, pair=gemini_symbol_to_pair(symbol), trade_type=deserialize_trade_type(entry['type']), amount=deserialize_asset_amount(entry['amount']), rate=deserialize_price(entry['price']), fee=deserialize_fee(entry['fee_amount']), fee_currency=Asset(entry['fee_currency']), link=str(entry['tid']), notes='', )) except UnprocessableTradePair as e: self.msg_aggregator.add_warning( f'Found unprocessable Gemini pair {e.pair}. Ignoring the trade.', ) continue except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown Gemini asset {e.asset_name}. ' f'Ignoring the trade.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Failed to deserialize a gemini trade. ' 'Check logs for details. Ignoring it.', ) log.error( 'Error processing a gemini trade.', raw_trade=entry, error=msg, ) continue return trades
def process_polo_loans( msg_aggregator: MessagesAggregator, data: List[Dict], start_ts: Timestamp, end_ts: Timestamp, ) -> List[Loan]: """Takes in the list of loans from poloniex as returned by the return_lending_history api call, processes it and returns it into our loan format """ new_data = [] for loan in reversed(data): log.debug('processing poloniex loan', **make_sensitive(loan)) try: close_time = deserialize_timestamp_from_poloniex_date(loan['close']) open_time = deserialize_timestamp_from_poloniex_date(loan['open']) if open_time < start_ts: continue if close_time > end_ts: continue our_loan = Loan( location=Location.POLONIEX, open_time=open_time, close_time=close_time, currency=asset_from_poloniex(loan['currency']), fee=deserialize_fee(loan['fee']), earned=deserialize_asset_amount(loan['earned']), amount_lent=deserialize_asset_amount(loan['amount']), ) except UnsupportedAsset as e: msg_aggregator.add_warning( f'Found poloniex loan with unsupported asset' f' {e.asset_name}. Ignoring it.', ) continue except UnknownAsset as e: msg_aggregator.add_warning( f'Found poloniex loan with unknown asset' f' {e.asset_name}. Ignoring it.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' msg_aggregator.add_error( 'Deserialization error while reading a poloniex loan. Check ' 'logs for more details. Ignoring it.', ) log.error( 'Deserialization error while reading a poloniex loan', loan=loan, error=msg, ) continue new_data.append(our_loan) new_data.sort(key=lambda loan: loan.open_time) return new_data
def query_online_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[AssetMovement]: result = self.query_until_finished( endpoint='Ledgers', keyname='ledger', start_ts=start_ts, end_ts=end_ts, extra_dict={'type': 'deposit'}, ) result.extend( self.query_until_finished( endpoint='Ledgers', keyname='ledger', start_ts=start_ts, end_ts=end_ts, extra_dict={'type': 'withdrawal'}, )) log.debug('Kraken deposit/withdrawals query result', num_results=len(result)) movements = [] for movement in result: try: asset = asset_from_kraken(movement['asset']) movement_type = movement['type'] if movement_type not in ('deposit', 'withdrawal'): # Other known types: 'transfer' continue # Can be for moving funds from spot to stake etc. movements.append( AssetMovement( location=Location.KRAKEN, category=deserialize_asset_movement_category( movement_type), timestamp=deserialize_timestamp_from_kraken( movement['time']), address=None, # no data from kraken ledger endpoint transaction_id= None, # no data from kraken ledger endpoint asset=asset, amount=deserialize_asset_amount_force_positive( movement['amount']), fee_asset=asset, fee=deserialize_fee(movement['fee']), link=str(movement['refid']), )) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown kraken asset {e.asset_name}. ' f'Ignoring its deposit/withdrawals query.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Failed to deserialize a kraken deposit/withdrawals. ' 'Check logs for details. Ignoring it.', ) log.error( 'Error processing a kraken deposit/withdrawal.', raw_asset_movement=movement, error=msg, ) continue return movements
def _deserialize_wallettx( self, entry: Dict[str, Any], from_ts: Timestamp, to_ts: Timestamp, ) -> Optional[AssetMovement]: """Deserializes a bitpanda fiatwallets/transactions or wallets/transactions entry to a deposit/withdrawal Returns None and logs error is there is a problem or simpy None if it's not a type of entry we are interested in """ try: transaction_type = entry['type'] if (transaction_type not in ('fiat_wallet_transaction', 'wallet_transaction') or entry['attributes']['status'] != 'finished'): return None time = Timestamp( deserialize_int_from_str( symbol=entry['attributes']['time']['unix'], location='bitpanda wallet transaction', )) if time < from_ts or time > to_ts: # should we also stop querying from calling method? # Probably yes but docs don't mention anything about results # being ordered by time so let's be conservative return None try: movement_category = deserialize_asset_movement_category( entry['attributes']['type']) # noqa: E501 except DeserializationError: return None # not a deposit/withdrawal if transaction_type == 'fiat_wallet_transaction': asset_id = entry['attributes']['fiat_id'] asset = self.fiat_map.get(asset_id) else: asset_id = entry['attributes']['cryptocoin_id'] asset = self.cryptocoin_map.get(asset_id) if asset is None: self.msg_aggregator.add_error( f'While deserializing Bitpanda fiat transaction, could not find ' f'bitpanda asset with id {asset_id} in the mapping', ) return None amount = deserialize_asset_amount(entry['attributes']['amount']) fee = deserialize_fee(entry['attributes']['fee']) tx_id = entry['id'] transaction_id = entry['attributes'].get('tx_id') address = entry['attributes'].get('recipient') except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key {msg} for wallet transaction entry' self.msg_aggregator.add_error( f'Error processing bitpanda wallet transaction entry due to {msg}' ) # noqa: E501 log.error( 'Error processing bitpanda wallet transaction entry', error=msg, entry=entry, ) return None return AssetMovement( location=Location.BITPANDA, category=movement_category, address=address, transaction_id=transaction_id, timestamp=time, asset=asset, amount=amount, fee_asset=asset, fee=fee, link=tx_id, )
def query_online_trade_history( self, start_ts: Timestamp, end_ts: Timestamp, ) -> Tuple[List[Trade], Tuple[Timestamp, Timestamp]]: """Queries coinbase pro for trades""" log.debug('Query coinbasepro trade history', start_ts=start_ts, end_ts=end_ts) self.first_connection() trades = [] # first get all orders, to see which product ids we need to query fills for orders = [] for batch in self._paginated_query( endpoint='orders', query_options={'status': 'done'}, ): orders.extend(batch) queried_product_ids = set() for order_entry in orders: product_id = order_entry.get('product_id', None) if product_id is None: msg = ( 'Skipping coinbasepro trade since it lacks a product_id. ' 'Check logs for details') self.msg_aggregator.add_error(msg) log.error( 'Error processing a coinbasepro order.', raw_trade=order_entry, error=msg, ) continue if product_id in queried_product_ids or product_id not in self.available_products: continue # already queried this product id or delisted product id # Now let's get all fills for this product id queried_product_ids.add(product_id) fills = [] for batch in self._paginated_query( endpoint='fills', query_options={'product_id': product_id}, ): fills.extend(batch) try: base_asset, quote_asset = coinbasepro_to_worldpair(product_id) except UnprocessableTradePair as e: self.msg_aggregator.add_warning( f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.', ) continue except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown Coinbasepro asset {e.asset_name}. ' f'Ignoring the trade.', ) continue for fill_entry in fills: try: timestamp = coinbasepro_deserialize_timestamp( fill_entry, 'created_at') if timestamp < start_ts or timestamp > end_ts: continue # Fee currency seems to always be quote asset # https://github.com/ccxt/ccxt/blob/ddf3a15cbff01541f0b37c35891aa143bb7f9d7b/python/ccxt/coinbasepro.py#L724 # noqa: E501 trades.append( Trade( timestamp=timestamp, location=Location.COINBASEPRO, base_asset=base_asset, quote_asset=quote_asset, trade_type=TradeType.deserialize( fill_entry['side']), amount=deserialize_asset_amount( fill_entry['size']), rate=deserialize_price(fill_entry['price']), fee=deserialize_fee(fill_entry['fee']), fee_currency=quote_asset, link=str(fill_entry['trade_id']) + '_' + fill_entry['order_id'], notes='', )) except UnprocessableTradePair as e: self.msg_aggregator.add_warning( f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.', ) continue except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown Coinbasepro asset {e.asset_name}. ' f'Ignoring the trade.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Failed to deserialize a coinbasepro trade. ' 'Check logs for details. Ignoring it.', ) log.error( 'Error processing a coinbasepro fill.', raw_trade=fill_entry, error=msg, ) continue return trades, (start_ts, end_ts)
def _deserialize_asset_movement( self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]: """Processes a single deposit/withdrawal from coinbase and deserializes it Can log error/warning and return None if something went wrong at deserialization """ try: if raw_data['status'] != 'completed': return None payout_date = raw_data.get('payout_at', None) if payout_date: timestamp = deserialize_timestamp_from_date( payout_date, 'iso8601', 'coinbase') else: timestamp = deserialize_timestamp_from_date( raw_data['created_at'], 'iso8601', 'coinbase', ) # Only get address/transaction id for "send" type of transactions address = None transaction_id = None # movement_category: Union[Literal['deposit'], Literal['withdrawal']] if 'type' in raw_data: # Then this should be a "send" which is the way Coinbase uses to send # crypto outside of the exchange # https://developers.coinbase.com/api/v2?python#transaction-resource msg = 'Non "send" type found in coinbase deposit/withdrawal processing' assert raw_data['type'] == 'send', msg movement_category = AssetMovementCategory.WITHDRAWAL # Can't see the fee being charged from the "send" resource amount = deserialize_asset_amount_force_positive( raw_data['amount']['amount']) asset = asset_from_coinbase(raw_data['amount']['currency'], time=timestamp) # Fees dont appear in the docs but from an experiment of sending ETH # to an address from coinbase there is the network fee in the response fee = Fee(ZERO) raw_network = raw_data.get('network', None) if raw_network: raw_fee = raw_network.get('transaction_fee', None) if raw_fee: # Since this is a withdrawal the fee should be the same as the moved asset if asset != asset_from_coinbase(raw_fee['currency'], time=timestamp): # If not we set ZERO fee and ignore log.error( f'In a coinbase withdrawal of {asset.identifier} the fee' f'is denoted in {raw_fee["currency"]}', ) else: fee = deserialize_fee(raw_fee['amount']) if 'network' in raw_data: transaction_id = get_key_if_has_val( raw_data['network'], 'hash') if 'to' in raw_data: address = deserialize_asset_movement_address( raw_data['to'], 'address', asset) else: movement_category = deserialize_asset_movement_category( raw_data['resource']) amount = deserialize_asset_amount_force_positive( raw_data['amount']['amount']) fee = deserialize_fee(raw_data['fee']['amount']) asset = asset_from_coinbase(raw_data['amount']['currency'], time=timestamp) return AssetMovement( location=Location.COINBASE, category=movement_category, address=address, transaction_id=transaction_id, timestamp=timestamp, asset=asset, amount=amount, fee_asset=asset, fee=fee, link=str(raw_data['id']), ) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found coinbase deposit/withdrawal with unknown asset ' f'{e.asset_name}. Ignoring it.', ) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found coinbase deposit/withdrawal with unsupported asset ' f'{e.asset_name}. Ignoring it.', ) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Unexpected data encountered during deserialization of a coinbase ' 'asset movement. Check logs for details and open a bug report.', ) log.error( f'Unexpected data encountered during deserialization of coinbase ' f'asset_movement {raw_data}. Error was: {str(e)}', ) return None
def verify_otctrade_data(data: ExternalTrade, ) -> Tuple[Optional[Trade], str]: """ Takes in the trade data dictionary, validates it and returns a trade instance If there is an error it returns an error message in the second part of the tuple """ for field in otc_fields: if field not in data: return None, f'{field} was not provided' if data[field] in ('', None) and field not in otc_optional_fields: return None, f'{field} was empty' if field in otc_numerical_fields and not is_number(data[field]): return None, f'{field} should be a number' # Satisfy mypy typing assert isinstance(data['otc_pair'], str) assert isinstance(data['otc_fee_currency'], str) assert isinstance(data['otc_fee'], str) pair = TradePair(data['otc_pair']) try: first = get_pair_position_asset(pair, 'first') second = get_pair_position_asset(pair, 'second') fee_currency = Asset(data['otc_fee_currency']) except UnknownAsset as e: return None, f'Provided asset {e.asset_name} is not known to Rotkehlchen' # Not catching DeserializationError here since we have asserts for the data # being strings right above try: trade_type = deserialize_trade_type(str(data['otc_type'])) amount = deserialize_asset_amount(data['otc_amount']) rate = deserialize_price(data['otc_rate']) fee = deserialize_fee(data['otc_fee']) except DeserializationError as e: return None, f'Deserialization Error: {str(e)}' try: assert isinstance(data['otc_timestamp'], str) timestamp = create_timestamp(data['otc_timestamp'], formatstr='%d/%m/%Y %H:%M') except ValueError as e: return None, f'Could not process the given datetime: {e}' log.debug( 'Creating OTC trade data', sensitive_log=True, pair=pair, trade_type=trade_type, amount=amount, rate=rate, fee=fee, fee_currency=fee_currency, ) if data['otc_fee_currency'] not in (first, second): return None, 'Trade fee currency should be one of the two in the currency pair' if data['otc_type'] not in ('buy', 'sell'): return None, 'Trade type can only be buy or sell' trade = Trade( timestamp=timestamp, location=Location.EXTERNAL, pair=pair, trade_type=trade_type, amount=amount, rate=rate, fee=fee, fee_currency=fee_currency, link=str(data['otc_link']), notes=str(data['otc_notes']), ) return trade, ''
""" if case == KucoinCase.DEPOSITS: category = AssetMovementCategory.DEPOSIT elif case == KucoinCase.WITHDRAWALS: category = AssetMovementCategory.WITHDRAWAL else: raise AssertionError(f'Unexpected case: {case}') try: timestamp_ms = deserialize_timestamp(raw_result['createdAt']) timestamp = Timestamp(int(timestamp_ms / 1000)) address = raw_result['address'] # The transaction id can have an @ which we should just get rid of transaction_id = raw_result['walletTxId'].split('@')[0] amount = deserialize_asset_amount(raw_result['amount']) fee = deserialize_fee(raw_result['fee']) fee_currency_symbol = raw_result['currency'] link_id = raw_result.get('id', '') # NB: id only exists for withdrawals except KeyError as e: raise DeserializationError(f'Missing key: {str(e)}.') from e fee_asset = asset_from_kucoin(fee_currency_symbol) asset_movement = AssetMovement( timestamp=timestamp, location=Location.KUCOIN, category=category, address=address, transaction_id=transaction_id, asset=fee_asset,
def _deserialize_asset_movement( self, raw_data: Dict[str, Any], movement_type: AssetMovementCategory, ) -> Optional[AssetMovement]: """Processes a single deposit/withdrawal from FTX and deserializes it Can log error/warning and return None if something went wrong at deserialization """ try: if raw_data['status'] not in ('complete', 'confirmed'): return None timestamp = deserialize_timestamp_from_date( raw_data['time'], 'iso8601', 'FTX') amount = deserialize_asset_amount_force_positive(raw_data['size']) asset = asset_from_ftx(raw_data['coin']) fee = Fee(ZERO) movement_category = movement_type if raw_data.get('fee', None) is not None: fee = deserialize_fee(raw_data['fee']) address = raw_data.get('address', None) if isinstance(address, dict): address = raw_data['address'].get('address', None) transaction_id = raw_data.get('txid', None) return AssetMovement( location=Location.FTX, category=movement_category, address=address, transaction_id=transaction_id, timestamp=timestamp, asset=asset, amount=amount, fee_asset=asset, fee=fee, link=str(raw_data['id']), ) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found FTX deposit/withdrawal with unknown asset ' f'{e.asset_name}. Ignoring it.', ) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found FTX deposit/withdrawal with unsupported asset ' f'{e.asset_name}. Ignoring it.', ) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Unexpected data encountered during deserialization of an FTX ' 'asset movement. Check logs for details and open a bug report.', ) log.error( 'Error processing FTX trade', trade=raw_data, error=msg, ) return None
def query_online_trade_history( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[Trade]: """Queries coinbase pro for trades""" log.debug('Query coinbasepro trade history', start_ts=start_ts, end_ts=end_ts) trades = [] raw_trades = [] for batch in self._paginated_query( endpoint='orders', query_options={'status': 'done'}, ): raw_trades.extend(batch) for entry in raw_trades: timestamp = coinbasepro_deserialize_timestamp(entry, 'created_at') if timestamp < start_ts or timestamp > end_ts: continue try: pair = coinbasepro_to_worldpair(entry['product_id']) # Fee currency seems to always be quote asset # https://github.com/ccxt/ccxt/blob/ddf3a15cbff01541f0b37c35891aa143bb7f9d7b/python/ccxt/coinbasepro.py#L724 # noqa: E501 _, quote_asset = pair_get_assets(pair) trades.append( Trade( timestamp=timestamp, location=Location.COINBASEPRO, pair=coinbasepro_to_worldpair(entry['product_id']), trade_type=deserialize_trade_type(entry['side']), amount=deserialize_asset_amount(entry['size']), rate=deserialize_price(entry['price']), fee=deserialize_fee(entry['fill_fees']), fee_currency=quote_asset, link=entry['id'], notes='', )) except UnprocessableTradePair as e: self.msg_aggregator.add_warning( f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.', ) continue except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown Coinbasepro asset {e.asset_name}. ' f'Ignoring the trade.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Failed to deserialize a coinbasepro trade. ' 'Check logs for details. Ignoring it.', ) log.error( 'Error processing a coinbasepro trade.', raw_trade=entry, error=msg, ) continue return trades
def _deserialize_trade( self, entry: Dict[str, Any], from_ts: Timestamp, to_ts: Timestamp, ) -> Optional[Trade]: """Deserializes a bitpanda trades result entry to a Trade Returns None and logs error is there is a problem or simpy None if it's not a type of trade we are interested in """ try: if entry['type'] != 'trade' or entry['attributes'][ 'status'] != 'finished': return None time = Timestamp( deserialize_int_from_str( symbol=entry['attributes']['time']['unix'], location='bitpanda trade', )) if time < from_ts or time > to_ts: # should we also stop querying from calling method? # Probably yes but docs don't mention anything about results # being ordered by time so let's be conservative return None cryptocoin_id = entry['attributes']['cryptocoin_id'] crypto_asset = self.cryptocoin_map.get(cryptocoin_id) if crypto_asset is None: self.msg_aggregator.add_error( f'While deserializing a trade, could not find bitpanda cryptocoin ' f'with id {cryptocoin_id} in the mapping. Skipping trade.', ) return None fiat_id = entry['attributes']['fiat_id'] fiat_asset = self.fiat_map.get(fiat_id) if fiat_asset is None: self.msg_aggregator.add_error( f'While deserializing a trade, could not find bitpanda fiat ' f'with id {fiat_id} in the mapping. Skipping trade.', ) return None trade_type = TradeType.deserialize(entry['attributes']['type']) if trade_type in (TradeType.BUY, TradeType.SELL): # you buy crypto with fiat and sell it for fiat base_asset = crypto_asset quote_asset = fiat_asset amount = deserialize_asset_amount( entry['attributes']['amount_cryptocoin']) price = deserialize_price(entry['attributes']['price']) else: self.msg_aggregator.add_error( 'Found bitpanda trade with unknown trade type {trade_type}' ) # noqa: E501 return None trade_id = entry['id'] fee = Fee(ZERO) fee_asset = A_BEST if entry['attributes']['bfc_used'] is True: fee = deserialize_fee( entry['attributes']['best_fee_collection']['attributes'] ['wallet_transaction']['attributes']['fee'], # noqa: E501 ) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key {msg} for trade entry' self.msg_aggregator.add_error( f'Error processing bitpanda trade due to {msg}') log.error( 'Error processing bitpanda trade entry', error=msg, entry=entry, ) return None return Trade( timestamp=time, location=Location.BITPANDA, base_asset=base_asset, quote_asset=quote_asset, trade_type=trade_type, amount=amount, rate=price, fee=fee, fee_currency=fee_asset, link=trade_id, )
def trade_from_binance( binance_trade: Dict, binance_symbols_to_pair: Dict[str, BinancePair], ) -> Trade: """Turn a binance trade returned from trade history to our common trade history format From the official binance api docs (01/09/18): https://github.com/binance-exchange/binance-official-api-docs/blob/62ff32d27bb32d9cc74d63d547c286bb3c9707ef/rest-api.md#terminology base asset refers to the asset that is the quantity of a symbol. quote asset refers to the asset that is the price of a symbol. Throws: - UnsupportedAsset due to asset_from_binance - DeserializationError due to unexpected format of dict entries - KeyError due to dict entries missing an expected entry """ amount = deserialize_asset_amount(binance_trade['qty']) rate = deserialize_price(binance_trade['price']) if binance_trade['symbol'] not in binance_symbols_to_pair: raise DeserializationError( f'Error reading a binance trade. Could not find ' f'{binance_trade["symbol"]} in binance_symbols_to_pair', ) binance_pair = binance_symbols_to_pair[binance_trade['symbol']] timestamp = deserialize_timestamp_from_binance(binance_trade['time']) base_asset = asset_from_binance(binance_pair.binance_base_asset) quote_asset = asset_from_binance(binance_pair.binance_quote_asset) if binance_trade['isBuyer']: order_type = TradeType.BUY # e.g. in RDNETH we buy RDN by paying ETH else: order_type = TradeType.SELL fee_currency = asset_from_binance(binance_trade['commissionAsset']) fee = deserialize_fee(binance_trade['commission']) log.debug( 'Processing binance Trade', sensitive_log=True, amount=amount, rate=rate, timestamp=timestamp, pair=binance_trade['symbol'], base_asset=base_asset, quote=quote_asset, order_type=order_type, commision_asset=binance_trade['commissionAsset'], fee=fee, ) return Trade( timestamp=timestamp, location=Location.BINANCE, pair=trade_pair_from_assets(base_asset, quote_asset), trade_type=order_type, amount=amount, rate=rate, fee=fee, fee_currency=fee_currency, link=str(binance_trade['id']), )
def query_online_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List: resp = self._api_query_list('get', 'user/walletHistory') log.debug('Bitmex deposit/withdrawals query', results_num=len(resp)) movements = list() for movement in resp: try: transaction_type = movement['transactType'] if transaction_type == 'Deposit': transaction_type = AssetMovementCategory.DEPOSIT elif transaction_type == 'Withdrawal': transaction_type = AssetMovementCategory.WITHDRAWAL else: continue timestamp = iso8601ts_to_timestamp(movement['timestamp']) if timestamp < start_ts: continue if timestamp > end_ts: continue asset = bitmex_to_world(movement['currency']) amount = deserialize_asset_amount(movement['amount']) fee = deserialize_fee(movement['fee']) # bitmex has negative numbers for withdrawals if amount < 0: amount *= -1 if asset == A_BTC: # bitmex stores amounts in satoshis amount = satoshis_to_btc(amount) fee = satoshis_to_btc(fee) movements.append( AssetMovement( location=Location.BITMEX, category=transaction_type, timestamp=timestamp, asset=asset, amount=amount, fee_asset=asset, fee=fee, link=str(movement['transactID']), )) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found bitmex deposit/withdrawal with unknown asset ' f'{e.asset_name}. Ignoring it.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( f'Unexpected data encountered during deserialization of a bitmex ' f'asset movement. Check logs for details and open a bug report.', ) log.error( f'Unexpected data encountered during deserialization of bitmex ' f'asset_movement {movement}. Error was: {str(e)}', ) continue return movements
def trade_from_kraken(kraken_trade: Dict[str, Any]) -> Trade: """Turn a kraken trade returned from kraken trade history to our common trade history format - Can raise UnknownAsset due to kraken_to_world_pair - Can raise UnprocessableTradePair due to kraken_to_world_pair - Can raise DeserializationError due to dict entries not being as expected - Can raise KeyError due to dict entries missing an expected entry """ currency_pair = kraken_to_world_pair(kraken_trade['pair']) quote_currency = get_pair_position_asset(currency_pair, 'second') timestamp = deserialize_timestamp_from_kraken(kraken_trade['time']) amount = deserialize_asset_amount(kraken_trade['vol']) cost = deserialize_price(kraken_trade['cost']) fee = deserialize_fee(kraken_trade['fee']) order_type = deserialize_trade_type(kraken_trade['type']) rate = deserialize_price(kraken_trade['price']) # pylint does not seem to see that Price is essentially FVal if not cost.is_close(amount * rate): # pylint: disable=no-member log.warning( f'cost ({cost}) != amount ({amount}) * rate ({rate}) for kraken trade' ) log.debug( 'Processing kraken Trade', sensitive_log=True, timestamp=timestamp, order_type=order_type, kraken_pair=kraken_trade['pair'], pair=currency_pair, quote_currency=quote_currency, amount=amount, cost=cost, fee=fee, rate=rate, ) # Kraken trades can have the same ordertxid and postxid for different trades .. # Also note postxid is optional and can be missing # The only thing that could differentiate them is timestamps in the milliseconds range # For example here are parts of two different kraken_trade: # {'ordertxid': 'AM4ZOZ-GLEMD-ZICOGR', 'postxid': 'AKH2SE-M7IF5-CFI7AT', # 'pair': 'XXBTZEUR', 'time': FVal(1561161486.2955) # {'ordertxid': 'AM4ZOZ-GLEMD-ZICOGR', 'postxid': 'AKH2SE-M7IF5-CFI7AT', # 'pair': 'XXBTZEUR', 'time': FVal(1561161486.3005) # # In order to counter this for the unique exchange trade link we are going # to use a concatenation of the above exchange_uuid = ( str(kraken_trade['ordertxid']) + str(kraken_trade.get('postxid', '')) + # postxid is optional str(kraken_trade['time'])) return Trade( timestamp=timestamp, location=Location.KRAKEN, pair=currency_pair, trade_type=order_type, amount=amount, rate=rate, fee=fee, fee_currency=quote_currency, link=exchange_uuid, )
def _deserialize_asset_movement( self, raw_movement: Dict[str, Any], ) -> AssetMovement: """Process a deposit/withdrawal user transaction from Bitstamp and deserialize it. Can raise DeserializationError. From Bitstamp documentation, deposits/withdrawals can have a fee (the amount is expected to be in the currency involved) https://www.bitstamp.net/fee-schedule/ Bitstamp support confirmed the following withdrawal JSON: { "fee": "0.00050000", "btc_usd": "0.00", "datetime": "2020-12-04 09:30:00.000000", "usd": "0.0", "btc": "-0.50000000", "type": "1", "id": 123456789, "eur": "0.0" } NB: any asset key not related with the pair is discarded (e.g. 'eur'). """ type_ = raw_movement['type'] category: AssetMovementCategory if type_ == 0: category = AssetMovementCategory.DEPOSIT elif type_ == 1: category = AssetMovementCategory.WITHDRAWAL else: raise AssertionError(f'Unexpected Bitstamp asset movement case: {type_}.') timestamp = deserialize_timestamp_from_bitstamp_date(raw_movement['datetime']) trade_pair_data = self._get_trade_pair_data_from_transaction(raw_movement) base_asset_amount = deserialize_asset_amount( raw_movement[trade_pair_data.base_asset_symbol], ) quote_asset_amount = deserialize_asset_amount( raw_movement[trade_pair_data.quote_asset_symbol], ) amount: FVal fee_asset: Asset if base_asset_amount != ZERO and quote_asset_amount == ZERO: amount = base_asset_amount fee_asset = trade_pair_data.base_asset elif base_asset_amount == ZERO and quote_asset_amount != ZERO: amount = quote_asset_amount fee_asset = trade_pair_data.quote_asset else: raise DeserializationError( 'Could not deserialize Bitstamp asset movement from user transaction. ' f'Unexpected asset amount combination found in: {raw_movement}.', ) asset_movement = AssetMovement( timestamp=timestamp, location=Location.BITSTAMP, category=category, address=None, # requires query "crypto_transactions" endpoint transaction_id=None, # requires query "crypto_transactions" endpoint asset=fee_asset, amount=abs(amount), fee_asset=fee_asset, fee=deserialize_fee(raw_movement['fee']), link=str(raw_movement['id']), ) return asset_movement
def query_online_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[AssetMovement]: """Queries coinbase pro for asset movements""" log.debug('Query coinbasepro asset movements', start_ts=start_ts, end_ts=end_ts) movements = [] raw_movements = [] for batch in self._paginated_query( endpoint='transfers', query_options={'type': 'withdraw'}, ): raw_movements.extend(batch) for batch in self._paginated_query( endpoint='transfers', query_options={'type': 'deposit'}, ): raw_movements.extend(batch) account_to_currency = self.create_or_return_account_to_currency_map() for entry in raw_movements: try: # Check if the transaction has not been completed. If so it should be skipped if entry.get('completed_at', None) is None: log.warning( f'Skipping coinbase pro deposit/withdrawal ' f'due not having been completed: {entry}', ) continue timestamp = coinbasepro_deserialize_timestamp( entry, 'completed_at') if timestamp < start_ts or timestamp > end_ts: continue category = deserialize_asset_movement_category(entry['type']) asset = account_to_currency.get(entry['account_id'], None) if asset is None: log.warning( f'Skipping coinbase pro asset_movement {entry} due to inability to ' f'match account id to an asset', ) continue address = None transaction_id = None fee = Fee(ZERO) if category == AssetMovementCategory.DEPOSIT: try: address = entry['details']['crypto_address'] transaction_id = entry['details'][ 'crypto_transaction_hash'] except KeyError: pass else: # withdrawal try: address = entry['details']['sent_to_address'] transaction_id = entry['details'][ 'crypto_transaction_hash'] fee = deserialize_fee(entry['details']['fee']) except KeyError: pass if transaction_id and ( asset == A_ETH or asset.asset_type == AssetType.ETHEREUM_TOKEN): # noqa: E501 transaction_id = '0x' + transaction_id movements.append( AssetMovement( location=Location.COINBASEPRO, category=category, address=address, transaction_id=transaction_id, timestamp=timestamp, asset=asset, amount=deserialize_asset_amount_force_positive( entry['amount']), fee_asset=asset, fee=fee, link=str(entry['id']), )) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown Coinbasepro asset {e.asset_name}. ' f'Ignoring its deposit/withdrawal.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Failed to deserialize a Coinbasepro deposit/withdrawal. ' 'Check logs for details. Ignoring it.', ) log.error( 'Error processing a coinbasepro deposit/withdrawal.', raw_asset_movement=entry, error=msg, ) continue return movements
def query_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, end_at_least_ts: Timestamp, ) -> List[AssetMovement]: with self.lock: cache = self.check_trades_cache_list( start_ts, end_at_least_ts, special_name='deposits_withdrawals', ) result: List[Any] if cache is not None: result = cache else: result = self.query_until_finished( endpoint='Ledgers', keyname='ledger', start_ts=start_ts, end_ts=end_ts, extra_dict=dict(type='deposit'), ) result.extend( self.query_until_finished( endpoint='Ledgers', keyname='ledger', start_ts=start_ts, end_ts=end_ts, extra_dict=dict(type='withdrawal'), )) with self.lock: self.update_trades_cache( result, start_ts, end_ts, special_name='deposits_withdrawals', ) log.debug('Kraken deposit/withdrawals query result', num_results=len(result)) movements = list() for movement in result: try: movements.append( AssetMovement( exchange=Exchange.KRAKEN, category=deserialize_asset_movement_category( movement['type']), timestamp=deserialize_timestamp_from_kraken( movement['time']), asset=asset_from_kraken(movement['asset']), amount=deserialize_asset_amount(movement['amount']), fee=deserialize_fee(movement['fee']), )) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown kraken asset {e.asset_name}. ' f'Ignoring its deposit/withdrawals query.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Failed to deserialize a kraken deposit/withdrawals. ' 'Check logs for details. Ignoring it.', ) log.error( 'Error processing a kraken deposit/withdrawal.', raw_asset_movement=movement, error=msg, ) continue return movements