def query_online_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[AssetMovement]: result = self._get_paginated_query( endpoint='transfers', start_ts=start_ts, end_ts=end_ts, ) movements = [] for entry in result: try: timestamp = deserialize_timestamp(entry['timestampms']) timestamp = Timestamp(int(timestamp / 1000)) asset = Asset(entry['currency']) movement = AssetMovement( location=Location.GEMINI, category=deserialize_asset_movement_category( entry['type']), address=deserialize_asset_movement_address( entry, 'destination', asset), transaction_id=get_key_if_has_val(entry, 'txHash'), timestamp=timestamp, asset=asset, amount=deserialize_asset_amount_force_positive( entry['amount']), fee_asset=asset, # Gemini does not include withdrawal fees neither in the API nor in their UI fee=Fee(ZERO), link=str(entry['eid']), ) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found gemini deposit/withdrawal with unknown asset ' f'{e.asset_name}. Ignoring it.', ) continue except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found gemini deposit/withdrawal with unsupported asset ' f'{e.asset_name}. Ignoring it.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Error processing a gemini deposit/withdrawal. Check logs ' 'for details. Ignoring it.', ) log.error( 'Error processing a gemini deposit_withdrawal', asset_movement=entry, error=msg, ) continue movements.append(movement) return movements
def asset_movements_from_dictlist( given_data: List[Dict[str, Any]], start_ts: Timestamp, end_ts: Timestamp, ) -> List[AssetMovement]: """ Gets a list of dict asset movements, most probably read from the json files and a time period. Returns it as a list of the AssetMovement tuples that are inside the time period May raise: - DeserializationError: If the given_data dict contains data in an unexpected format - KeyError: If the given_data dict contains data in an unexpected format """ returned_movements = list() for movement in given_data: timestamp = deserialize_timestamp(movement['timestamp']) if timestamp < start_ts: continue if timestamp > end_ts: break category = deserialize_asset_movement_category(movement['category']) amount = deserialize_asset_amount(movement['amount']) fee = deserialize_fee(movement['fee']) returned_movements.append(AssetMovement( exchange=deserialize_exchange_name(movement['exchange']), category=category, timestamp=timestamp, asset=Asset(movement['asset']), amount=amount, fee=fee, )) return returned_movements
def query_online_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[AssetMovement]: result = self.query_until_finished( endpoint='Ledgers', keyname='ledger', start_ts=start_ts, end_ts=end_ts, extra_dict=dict(type='deposit'), ) result.extend(self.query_until_finished( endpoint='Ledgers', keyname='ledger', start_ts=start_ts, end_ts=end_ts, extra_dict=dict(type='withdrawal'), )) log.debug('Kraken deposit/withdrawals query result', num_results=len(result)) movements = list() for movement in result: try: asset = asset_from_kraken(movement['asset']) movements.append(AssetMovement( location=Location.KRAKEN, category=deserialize_asset_movement_category(movement['type']), timestamp=deserialize_timestamp_from_kraken(movement['time']), asset=asset, amount=deserialize_asset_amount(movement['amount']), fee_asset=asset, fee=deserialize_fee(movement['fee']), link=str(movement['refid']), )) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown kraken asset {e.asset_name}. ' f'Ignoring its deposit/withdrawals query.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Failed to deserialize a kraken deposit/withdrawals. ' 'Check logs for details. Ignoring it.', ) log.error( 'Error processing a kraken deposit/withdrawal.', raw_asset_movement=movement, error=msg, ) continue return movements
def _asset_movement_from_independentreserve( raw_tx: Dict) -> Optional[AssetMovement]: """Convert IndependentReserve raw data to an AssetMovement https://www.independentreserve.com/products/api#GetTransactions May raise: - DeserializationError - UnknownAsset - KeyError """ log.debug(f'Processing raw IndependentReserve transaction: {raw_tx}') movement_type = deserialize_asset_movement_category(raw_tx['Type']) asset = independentreserve_asset(raw_tx['CurrencyCode']) bitcoin_tx_id = raw_tx.get('BitcoinTransactionId') eth_tx_id = raw_tx.get('EthereumTransactionId') if asset == A_BTC and bitcoin_tx_id is not None: transaction_id = raw_tx['BitcoinTransactionId'] elif eth_tx_id is not None: transaction_id = eth_tx_id else: transaction_id = None timestamp = deserialize_timestamp_from_date( date=raw_tx['CreatedTimestampUtc'], formatstr='iso8601', location='IndependentReserve', ) comment = raw_tx.get('Comment') address = None if comment is not None and comment.startswith('Withdrawing to'): address = comment.rsplit()[-1] raw_amount = raw_tx.get( 'Credit' ) if movement_type == AssetMovementCategory.DEPOSIT else raw_tx.get( 'Debit') # noqa: E501 if raw_amount is None: # skip return None # Can end up being None for some things like this: 'Comment': 'Initial balance after Bitcoin fork' # noqa: E501 amount = deserialize_asset_amount(raw_amount) return AssetMovement( location=Location.INDEPENDENTRESERVE, category=movement_type, address=address, transaction_id=transaction_id, timestamp=timestamp, asset=asset, amount=amount, fee_asset=asset, # whatever -- no fee fee=Fee(ZERO), # we can't get fee from this exchange link=raw_tx['CreatedTimestampUtc'] + str(amount) + str(movement_type) + asset.identifier, )
def _read_asset_movements(self, filepath: str) -> List[AssetMovement]: """Reads a csv account type report and extracts the AssetMovements""" with open(filepath, newline='') as csvfile: reader = csv.DictReader(csvfile) movements = [] for row in reader: try: if row['type'] in ('withdrawal', 'deposit'): timestamp = deserialize_timestamp_from_date( row['time'], 'iso8601', 'coinbasepro', ) amount = deserialize_asset_amount(row['amount']) if row['type'] == 'withdrawal': # For withdrawals the withdraw amount is negative amount = AssetAmount(amount * FVal('-1')) asset = Asset(row['amount/balance unit']) movements.append( AssetMovement( location=Location.COINBASEPRO, category=deserialize_asset_movement_category( row['type']), timestamp=timestamp, asset=asset, amount=amount, fee_asset=asset, # I don't see any fee in deposit withdrawals in coinbasepro fee=Fee(ZERO), link=str(row['transfer id']), )) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown Coinbasepro asset {e.asset_name}. ' f'Ignoring its deposit/withdrawal.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Failed to deserialize a Coinbasepro deposit/withdrawal. ' 'Check logs for details. Ignoring it.', ) log.error( 'Error processing a coinbasepro deposit/withdrawal.', raw_asset_movement=row, error=msg, ) continue return movements
def query_online_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[AssetMovement]: """Queries coinbase pro for asset movements""" log.debug('Query coinbasepro asset movements', start_ts=start_ts, end_ts=end_ts) movements = [] raw_movements = [] for batch in self._paginated_query( endpoint='transfers', query_options={'type': 'withdraw'}, ): raw_movements.extend(batch) for batch in self._paginated_query( endpoint='transfers', query_options={'type': 'deposit'}, ): raw_movements.extend(batch) account_to_currency = self.create_or_return_account_to_currency_map() for entry in raw_movements: try: # Check if the transaction has not been completed. If so it should be skipped if entry.get('completed_at', None) is None: log.warning( f'Skipping coinbase pro deposit/withdrawal ' f'due to not having been completed: {entry}', ) continue timestamp = coinbasepro_deserialize_timestamp( entry, 'completed_at') if timestamp < start_ts or timestamp > end_ts: continue category = deserialize_asset_movement_category(entry['type']) asset = account_to_currency.get(entry['account_id'], None) if asset is None: log.warning( f'Skipping coinbase pro asset_movement {entry} due to ' f'inability to match account id to an asset', ) continue address = None transaction_id = None fee = Fee(ZERO) if category == AssetMovementCategory.DEPOSIT: try: address = entry['details']['crypto_address'] transaction_id = entry['details'][ 'crypto_transaction_hash'] except KeyError: pass else: # withdrawal try: address = entry['details']['sent_to_address'] transaction_id = entry['details'][ 'crypto_transaction_hash'] fee = deserialize_fee(entry['details']['fee']) except KeyError: pass if transaction_id and ( asset == A_ETH or asset.asset_type == AssetType.ETHEREUM_TOKEN): # noqa: E501 transaction_id = '0x' + transaction_id movements.append( AssetMovement( location=Location.COINBASEPRO, category=category, address=address, transaction_id=transaction_id, timestamp=timestamp, asset=asset, amount=deserialize_asset_amount_force_positive( entry['amount']), fee_asset=asset, fee=fee, link=str(entry['id']), )) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown Coinbasepro asset {e.asset_name}. ' f'Ignoring its deposit/withdrawal.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Failed to deserialize a Coinbasepro deposit/withdrawal. ' 'Check logs for details. Ignoring it.', ) log.error( 'Error processing a coinbasepro deposit/withdrawal.', raw_asset_movement=entry, error=msg, ) continue return movements
def _consume_cointracking_entry(self, csv_row: Dict[str, Any]) -> None: """Consumes a cointracking entry row from the CSV and adds it into the database Can raise: - DeserializationError if something is wrong with the format of the expected values - UnsupportedCointrackingEntry if importing of this entry is not supported. - IndexError if the CSV file is corrupt - KeyError if the an expected CSV key is missing - UnknownAsset if one of the assets founds in the entry are not supported """ row_type = csv_row['Type'] timestamp = deserialize_timestamp_from_date( date=csv_row['Date'], formatstr='%d.%m.%Y %H:%M:%S', location='cointracking.info', ) notes = csv_row['Comment'] location = exchange_row_to_location(csv_row['Exchange']) fee = Fee(ZERO) fee_currency = A_USD # whatever (used only if there is no fee) if csv_row['Fee'] != '': fee = deserialize_fee(csv_row['Fee']) fee_currency = Asset(csv_row['Cur.Fee']) if row_type in ('Gift/Tip', 'Trade', 'Income'): base_asset = Asset(csv_row['Cur.Buy']) quote_asset = None if csv_row['Cur.Sell'] == '' else Asset( csv_row['Cur.Sell']) if quote_asset is None and row_type not in ('Gift/Tip', 'Income'): raise DeserializationError( 'Got a trade entry with an empty quote asset') if quote_asset is None: # Really makes no difference as this is just a gift and the amount is zero quote_asset = A_USD pair = TradePair( f'{base_asset.identifier}_{quote_asset.identifier}') base_amount_bought = deserialize_asset_amount(csv_row['Buy']) if csv_row['Sell'] != '-': quote_amount_sold = deserialize_asset_amount(csv_row['Sell']) else: quote_amount_sold = AssetAmount(ZERO) rate = Price(quote_amount_sold / base_amount_bought) trade = Trade( timestamp=timestamp, location=location, pair=pair, trade_type=TradeType. BUY, # It's always a buy during cointracking import amount=base_amount_bought, rate=rate, fee=fee, fee_currency=fee_currency, link='', notes=notes, ) self.db.add_trades([trade]) elif row_type == 'Deposit' or row_type == 'Withdrawal': category = deserialize_asset_movement_category(row_type.lower()) if category == AssetMovementCategory.DEPOSIT: amount = deserialize_asset_amount(csv_row['Buy']) asset = Asset(csv_row['Cur.Buy']) else: amount = deserialize_asset_amount_force_positive( csv_row['Sell']) asset = Asset(csv_row['Cur.Sell']) asset_movement = AssetMovement( location=location, category=category, address=None, transaction_id=None, timestamp=timestamp, asset=asset, amount=amount, fee=fee, fee_asset=fee_currency, link='', ) self.db.add_asset_movements([asset_movement]) else: raise UnsupportedCointrackingEntry( f'Unknown entrype type "{row_type}" encountered during cointracking ' f'data import. Ignoring entry', )
def _deserialize_asset_movement( self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]: """Processes a single deposit/withdrawal from coinbase and deserializes it Can log error/warning and return None if something went wrong at deserialization """ try: if raw_data['status'] != 'completed': return None payout_date = raw_data.get('payout_at', None) if payout_date: timestamp = deserialize_timestamp_from_date( payout_date, 'iso8601', 'coinbase') else: timestamp = deserialize_timestamp_from_date( raw_data['created_at'], 'iso8601', 'coinbase', ) # Only get address/transaction id for "send" type of transactions address = None transaction_id = None # movement_category: Union[Literal['deposit'], Literal['withdrawal']] if 'type' in raw_data: # Then this should be a "send" which is the way Coinbase uses to send # crypto outside of the exchange # https://developers.coinbase.com/api/v2?python#transaction-resource msg = 'Non "send" type found in coinbase deposit/withdrawal processing' assert raw_data['type'] == 'send', msg movement_category = AssetMovementCategory.WITHDRAWAL # Can't see the fee being charged from the "send" resource amount = deserialize_asset_amount_force_positive( raw_data['amount']['amount']) asset = asset_from_coinbase(raw_data['amount']['currency'], time=timestamp) # Fees dont appear in the docs but from an experiment of sending ETH # to an address from coinbase there is the network fee in the response fee = Fee(ZERO) raw_network = raw_data.get('network', None) if raw_network: raw_fee = raw_network.get('transaction_fee', None) if raw_fee: # Since this is a withdrawal the fee should be the same as the moved asset if asset != asset_from_coinbase(raw_fee['currency'], time=timestamp): # If not we set ZERO fee and ignore log.error( f'In a coinbase withdrawal of {asset.identifier} the fee' f'is denoted in {raw_fee["currency"]}', ) else: fee = deserialize_fee(raw_fee['amount']) if 'network' in raw_data: transaction_id = get_key_if_has_val( raw_data['network'], 'hash') if 'to' in raw_data: address = deserialize_asset_movement_address( raw_data['to'], 'address', asset) else: movement_category = deserialize_asset_movement_category( raw_data['resource']) amount = deserialize_asset_amount_force_positive( raw_data['amount']['amount']) fee = deserialize_fee(raw_data['fee']['amount']) asset = asset_from_coinbase(raw_data['amount']['currency'], time=timestamp) return AssetMovement( location=Location.COINBASE, category=movement_category, address=address, transaction_id=transaction_id, timestamp=timestamp, asset=asset, amount=amount, fee_asset=asset, fee=fee, link=str(raw_data['id']), ) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found coinbase deposit/withdrawal with unknown asset ' f'{e.asset_name}. Ignoring it.', ) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found coinbase deposit/withdrawal with unsupported asset ' f'{e.asset_name}. Ignoring it.', ) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Unexpected data encountered during deserialization of a coinbase ' 'asset movement. Check logs for details and open a bug report.', ) log.error( f'Unexpected data encountered during deserialization of coinbase ' f'asset_movement {raw_data}. Error was: {str(e)}', ) return None
def _consume_cointracking_entry(self, csv_row: List[str]) -> None: """Consumes a cointracking entry row from the CSV and adds it into the database Can raise: - DeserializationError if something is wrong with the format of the expected values - UnsupportedCointrackingEntry if importing of this entry is not supported. - IndexError if the CSV file is corrupt - UnknownAsset if one of the assets founds in the entry are not supported """ row_type = csv_row[1] # Type timestamp = deserialize_timestamp_from_date( date=csv_row[9], formatstr='%d.%m.%Y %H:%M', location='cointracking.info', ) notes = csv_row[8] location = exchange_row_to_location(csv_row[6]) if row_type == 'Gift/Tip' or row_type == 'Trade': base_asset = Asset(csv_row[3]) quote_asset = None if csv_row[5] == '' else Asset(csv_row[5]) if not quote_asset and row_type != 'Gift/Tip': raise DeserializationError( 'Got a trade entry with an empty quote asset') if quote_asset is None: # Really makes no difference as this is just a gift and the amount is zero quote_asset = A_USD pair = TradePair( f'{base_asset.identifier}_{quote_asset.identifier}') base_amount_bought = deserialize_asset_amount(csv_row[2]) if csv_row[4] != '-': quote_amount_sold = deserialize_asset_amount(csv_row[4]) else: quote_amount_sold = AssetAmount(ZERO) rate = Price(quote_amount_sold / base_amount_bought) trade = Trade( timestamp=timestamp, location=location, pair=pair, trade_type=TradeType. BUY, # It's always a buy during cointracking import amount=base_amount_bought, rate=rate, fee=Fee( ZERO), # There are no fees when import from cointracking fee_currency=base_asset, link='', notes=notes, ) self.db.add_trades([trade]) elif row_type == 'Deposit' or row_type == 'Withdrawal': category = deserialize_asset_movement_category(row_type.lower()) if category == AssetMovementCategory.DEPOSIT: amount = deserialize_asset_amount(csv_row[2]) asset = Asset(csv_row[3]) else: amount = deserialize_asset_amount(csv_row[4]) asset = Asset(csv_row[5]) asset_movement = AssetMovement( location=location, category=category, timestamp=timestamp, asset=asset, amount=amount, fee_asset=asset, fee=Fee(ZERO), link='', ) self.db.add_asset_movements([asset_movement]) else: raise UnsupportedCointrackingEntry( f'Unknown entrype type "{row_type}" encountered during cointracking ' f'data import. Ignoring entry', )
def query_online_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[AssetMovement]: self.query_kraken_ledgers(start_ts=start_ts, end_ts=end_ts) filter_query = HistoryEventFilterQuery.make( from_ts=Timestamp(start_ts), to_ts=Timestamp(end_ts), event_types=[ HistoryEventType.DEPOSIT, HistoryEventType.WITHDRAWAL, ], location=Location.KRAKEN, location_label=self.name, ) events = self.history_events_db.get_history_events( filter_query=filter_query, has_premium=True, ) log.debug('Kraken deposit/withdrawals query result', num_results=len(events)) movements = [] get_attr = operator.attrgetter('event_identifier') # Create a list of lists where each sublist has the events for the same event identifier grouped_events = [ list(g) for k, g in itertools.groupby(sorted(events, key=get_attr), get_attr) ] # noqa: E501 for movement_events in grouped_events: if len(movement_events) == 2: if movement_events[0].event_subtype == HistoryEventSubType.FEE: fee = Fee(movement_events[0].balance.amount) movement = movement_events[1] elif movement_events[ 1].event_subtype == HistoryEventSubType.FEE: fee = Fee(movement_events[1].balance.amount) movement = movement_events[0] else: self.msg_aggregator.add_error( f'Failed to process deposit/withdrawal. {grouped_events}. Ignoring ...', ) continue else: movement = movement_events[0] fee = Fee(ZERO) amount = movement.balance.amount if movement.event_type == HistoryEventType.WITHDRAWAL: amount = amount * -1 try: asset = movement.asset movement_type = movement.event_type movements.append( AssetMovement( location=Location.KRAKEN, category=deserialize_asset_movement_category( movement_type), timestamp=ts_ms_to_sec(movement.timestamp), address=None, # no data from kraken ledger endpoint transaction_id= None, # no data from kraken ledger endpoint asset=asset, amount=amount, fee_asset=asset, fee=fee, link=movement.event_identifier, )) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown kraken asset {e.asset_name}. ' f'Ignoring its deposit/withdrawals query.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Failed to deserialize a kraken deposit/withdrawal. ' 'Check logs for details. Ignoring it.', ) log.error( 'Error processing a kraken deposit/withdrawal.', raw_asset_movement=movement_events, error=msg, ) continue return movements
def _deserialize_wallettx( self, entry: Dict[str, Any], from_ts: Timestamp, to_ts: Timestamp, ) -> Optional[AssetMovement]: """Deserializes a bitpanda fiatwallets/transactions or wallets/transactions entry to a deposit/withdrawal Returns None and logs error is there is a problem or simpy None if it's not a type of entry we are interested in """ try: transaction_type = entry['type'] if (transaction_type not in ('fiat_wallet_transaction', 'wallet_transaction') or entry['attributes']['status'] != 'finished'): return None time = Timestamp( deserialize_int_from_str( symbol=entry['attributes']['time']['unix'], location='bitpanda wallet transaction', )) if time < from_ts or time > to_ts: # should we also stop querying from calling method? # Probably yes but docs don't mention anything about results # being ordered by time so let's be conservative return None try: movement_category = deserialize_asset_movement_category( entry['attributes']['type']) # noqa: E501 except DeserializationError: return None # not a deposit/withdrawal if transaction_type == 'fiat_wallet_transaction': asset_id = entry['attributes']['fiat_id'] asset = self.fiat_map.get(asset_id) else: asset_id = entry['attributes']['cryptocoin_id'] asset = self.cryptocoin_map.get(asset_id) if asset is None: self.msg_aggregator.add_error( f'While deserializing Bitpanda fiat transaction, could not find ' f'bitpanda asset with id {asset_id} in the mapping', ) return None amount = deserialize_asset_amount(entry['attributes']['amount']) fee = deserialize_fee(entry['attributes']['fee']) tx_id = entry['id'] transaction_id = entry['attributes'].get('tx_id') address = entry['attributes'].get('recipient') except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key {msg} for wallet transaction entry' self.msg_aggregator.add_error( f'Error processing bitpanda wallet transaction entry due to {msg}' ) # noqa: E501 log.error( 'Error processing bitpanda wallet transaction entry', error=msg, entry=entry, ) return None return AssetMovement( location=Location.BITPANDA, category=movement_category, address=address, transaction_id=transaction_id, timestamp=time, asset=asset, amount=amount, fee_asset=asset, fee=fee, link=tx_id, )