def query_online_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[AssetMovement]: self.query_kraken_ledgers(start_ts=start_ts, end_ts=end_ts) filter_query = HistoryEventFilterQuery.make( from_ts=Timestamp(start_ts), to_ts=Timestamp(end_ts), event_types=[ HistoryEventType.DEPOSIT, HistoryEventType.WITHDRAWAL, ], location=Location.KRAKEN, location_label=self.name, ) events = self.history_events_db.get_history_events( filter_query=filter_query, has_premium=True, ) log.debug('Kraken deposit/withdrawals query result', num_results=len(events)) movements = [] get_attr = operator.attrgetter('event_identifier') # Create a list of lists where each sublist has the events for the same event identifier grouped_events = [ list(g) for k, g in itertools.groupby(sorted(events, key=get_attr), get_attr) ] # noqa: E501 for movement_events in grouped_events: if len(movement_events) == 2: if movement_events[0].event_subtype == HistoryEventSubType.FEE: fee = Fee(movement_events[0].balance.amount) movement = movement_events[1] elif movement_events[ 1].event_subtype == HistoryEventSubType.FEE: fee = Fee(movement_events[1].balance.amount) movement = movement_events[0] else: self.msg_aggregator.add_error( f'Failed to process deposit/withdrawal. {grouped_events}. Ignoring ...', ) continue else: movement = movement_events[0] fee = Fee(ZERO) amount = movement.balance.amount if movement.event_type == HistoryEventType.WITHDRAWAL: amount = amount * -1 try: asset = movement.asset movement_type = movement.event_type movements.append( AssetMovement( location=Location.KRAKEN, category=deserialize_asset_movement_category( movement_type), timestamp=ts_ms_to_sec(movement.timestamp), address=None, # no data from kraken ledger endpoint transaction_id= None, # no data from kraken ledger endpoint asset=asset, amount=amount, fee_asset=asset, fee=fee, link=movement.event_identifier, )) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown kraken asset {e.asset_name}. ' f'Ignoring its deposit/withdrawals query.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Failed to deserialize a kraken deposit/withdrawal. ' 'Check logs for details. Ignoring it.', ) log.error( 'Error processing a kraken deposit/withdrawal.', raw_asset_movement=movement_events, error=msg, ) continue return movements
def _deserialize_asset_movement( self, raw_movement: Dict[str, Any], ) -> AssetMovement: """Process a deposit/withdrawal user transaction from Bitstamp and deserialize it. Can raise DeserializationError. From Bitstamp documentation, deposits/withdrawals can have a fee (the amount is expected to be in the currency involved) https://www.bitstamp.net/fee-schedule/ Bitstamp support confirmed the following withdrawal JSON: { "fee": "0.00050000", "btc_usd": "0.00", "datetime": "2020-12-04 09:30:00.000000", "usd": "0.0", "btc": "-0.50000000", "type": "1", "id": 123456789, "eur": "0.0" } NB: any asset key not related with the pair is discarded (e.g. 'eur'). """ type_ = raw_movement['type'] category: AssetMovementCategory if type_ == 0: category = AssetMovementCategory.DEPOSIT elif type_ == 1: category = AssetMovementCategory.WITHDRAWAL else: raise AssertionError(f'Unexpected Bitstamp asset movement case: {type_}.') timestamp = deserialize_timestamp_from_bitstamp_date(raw_movement['datetime']) trade_pair_data = self._get_trade_pair_data_from_transaction(raw_movement) base_asset_amount = deserialize_asset_amount( raw_movement[trade_pair_data.base_asset_symbol], ) quote_asset_amount = deserialize_asset_amount( raw_movement[trade_pair_data.quote_asset_symbol], ) amount: FVal fee_asset: Asset if base_asset_amount != ZERO and quote_asset_amount == ZERO: amount = base_asset_amount fee_asset = trade_pair_data.base_asset elif base_asset_amount == ZERO and quote_asset_amount != ZERO: amount = quote_asset_amount fee_asset = trade_pair_data.quote_asset else: raise DeserializationError( 'Could not deserialize Bitstamp asset movement from user transaction. ' f'Unexpected asset amount combination found in: {raw_movement}.', ) asset_movement = AssetMovement( timestamp=timestamp, location=Location.BITSTAMP, category=category, address=None, # requires query "crypto_transactions" endpoint transaction_id=None, # requires query "crypto_transactions" endpoint asset=fee_asset, amount=abs(amount), fee_asset=fee_asset, fee=deserialize_fee(raw_movement['fee']), link=str(raw_movement['id']), ) return asset_movement
def _deserialize_asset_movement( self, movement_type: AssetMovementCategory, movement_data: Dict[str, Any], ) -> Optional[AssetMovement]: """Processes a single deposit/withdrawal from polo and deserializes it Can log error/warning and return None if something went wrong at deserialization """ try: if movement_type == AssetMovementCategory.DEPOSIT: fee = Fee(ZERO) uid_key = 'depositNumber' transaction_id = get_key_if_has_val(movement_data, 'txid') else: fee = deserialize_fee(movement_data['fee']) uid_key = 'withdrawalNumber' split = movement_data['status'].split(':') if len(split) != 2: transaction_id = None else: transaction_id = split[1].lstrip() if transaction_id == '': transaction_id = None asset = asset_from_poloniex(movement_data['currency']) return AssetMovement( location=Location.POLONIEX, category=movement_type, address=deserialize_asset_movement_address(movement_data, 'address', asset), transaction_id=transaction_id, timestamp=deserialize_timestamp(movement_data['timestamp']), asset=asset, amount=deserialize_asset_amount_force_positive(movement_data['amount']), fee_asset=asset, fee=fee, link=str(movement_data[uid_key]), ) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found {str(movement_type)} of unsupported poloniex asset ' f'{e.asset_name}. Ignoring it.', ) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found {str(movement_type)} of unknown poloniex asset ' f'{e.asset_name}. Ignoring it.', ) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Unexpected data encountered during deserialization of a poloniex ' 'asset movement. Check logs for details and open a bug report.', ) log.error( f'Unexpected data encountered during deserialization of poloniex ' f'{str(movement_type)}: {movement_data}. Error was: {str(e)}', ) return None
def assert_cryptocom_import_results(rotki: Rotkehlchen): """A utility function to help assert on correctness of importing data from crypto.com""" trades = rotki.data.db.get_trades() asset_movements = rotki.data.db.get_asset_movements() warnings = rotki.msg_aggregator.consume_warnings() errors = rotki.msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 0 def get_trade_note(desc: str): return f'{desc}\nSource: crypto.com (CSV import)' expected_trades = [Trade( timestamp=Timestamp(1595833195), location=Location.CRYPTOCOM, pair=TradePair('ETH_EUR'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('1.0')), rate=Price(FVal('281.14')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('Buy ETH'), ), Trade( timestamp=Timestamp(1596014214), location=Location.CRYPTOCOM, pair=TradePair('MCO_EUR'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('50.0')), rate=Price(FVal('3.521')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('Buy MCO'), ), Trade( timestamp=Timestamp(1596014223), location=Location.CRYPTOCOM, pair=TradePair('MCO_USD'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('12.32402069')), rate=Price(FVal('4.057117499045678736198226879')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('Sign-up Bonus Unlocked'), ), Trade( timestamp=Timestamp(1596209827), location=Location.CRYPTOCOM, pair=TradePair('ETH_MCO'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('0.14445954600007045')), rate=Price(FVal('85.28339137929999991192917299')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('MCO -> ETH'), ), Trade( timestamp=Timestamp(1596429934), location=Location.CRYPTOCOM, pair=TradePair('ETH_EUR'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('0.00061475')), rate=Price(FVal('309.0687271248474989833265555')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('Crypto Earn'), ), Trade( timestamp=Timestamp(1596465565), location=Location.CRYPTOCOM, pair=TradePair('CRO_MCO'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('1382.306147552291')), rate=Price(FVal('27.6439')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('MCO/CRO Overall Swap'), ), Trade( timestamp=Timestamp(1596730165), location=Location.CRYPTOCOM, pair=TradePair('CRO_MCO'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('1301.64')), rate=Price(FVal('26.0328')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('MCO/CRO Overall Swap'), ), Trade( timestamp=Timestamp(1599934176), location=Location.CRYPTOCOM, pair=TradePair('CRO_EUR'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('138.256')), rate=Price(FVal('0.1429232727693553986807082514')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('Card Rebate: Deliveries'), ), Trade( timestamp=Timestamp(1602515376), location=Location.CRYPTOCOM, pair=TradePair('CRO_EUR'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('52.151')), rate=Price(FVal('0.06692105616383194953116910510')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('Card Cashback'), ), Trade( timestamp=Timestamp(1602526176), location=Location.CRYPTOCOM, pair=TradePair('CRO_EUR'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('482.2566417')), rate=Price(FVal('0.08756748243245604635910191136')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('Referral Bonus Reward'), ), Trade( timestamp=Timestamp(1606833565), location=Location.CRYPTOCOM, pair=TradePair('CRO_DAI'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('0.007231228760408149')), rate=Price(FVal('14.26830000900286970270179629')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('Convert Dust'), )] assert expected_trades == trades expected_movements = [AssetMovement( location=Location.CRYPTOCOM, category=AssetMovementCategory.DEPOSIT, timestamp=Timestamp(1596992965), address=None, transaction_id=None, asset=A_DAI, amount=AssetAmount(FVal('115')), fee_asset=A_DAI, fee=Fee(ZERO), link='', ), AssetMovement( location=Location.CRYPTOCOM, category=AssetMovementCategory.WITHDRAWAL, address=None, transaction_id=None, timestamp=Timestamp(1596993025), asset=A_DAI, amount=AssetAmount(FVal('115')), fee_asset=A_DAI, fee=Fee(ZERO), link='', )] assert expected_movements == asset_movements
def test_coinbase_query_deposit_withdrawals(function_scope_coinbase): """Test that coinbase deposit/withdrawals history query works fine for the happy path""" coinbase = function_scope_coinbase with patch.object(coinbase.session, 'get', side_effect=mock_normal_coinbase_query): movements = coinbase.query_online_deposits_withdrawals( start_ts=0, end_ts=1576726126, ) warnings = coinbase.msg_aggregator.consume_warnings() errors = coinbase.msg_aggregator.consume_errors() assert len(warnings) == 0 assert len(errors) == 0 assert len(movements) == 3 expected_movements = [ AssetMovement( location=Location.COINBASE, category=AssetMovementCategory.DEPOSIT, timestamp=1519001640, asset=A_USD, amount=FVal('55'), fee_asset=A_USD, fee=FVal('0.05'), link='1130eaec-07d7-54c4-a72c-2e92826897df', ), AssetMovement( location=Location.COINBASE, category=AssetMovementCategory.WITHDRAWAL, timestamp=1485895742, asset=A_USD, amount=FVal('10.0'), fee_asset=A_USD, fee=FVal('0.01'), link='146eaec-07d7-54c4-a72c-2e92826897df', ), AssetMovement( location=Location.COINBASE, category=AssetMovementCategory.WITHDRAWAL, timestamp=1566726126, asset=A_ETH, amount=FVal('0.05770427'), fee_asset=A_ETH, fee=FVal('0.00021'), link='id1', ) ] assert expected_movements == movements # and now try to query within a specific range with patch.object(coinbase.session, 'get', side_effect=mock_normal_coinbase_query): movements = coinbase.query_online_deposits_withdrawals( start_ts=0, end_ts=1519001650, ) warnings = coinbase.msg_aggregator.consume_warnings() errors = coinbase.msg_aggregator.consume_errors() assert len(warnings) == 0 assert len(errors) == 0 assert len(movements) == 2 assert movements[0].category == AssetMovementCategory.DEPOSIT assert movements[0].timestamp == 1519001640 assert movements[1].category == AssetMovementCategory.WITHDRAWAL assert movements[1].timestamp == 1485895742
def _deserialize_asset_movement( self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]: """Processes a single deposit/withdrawal from coinbase and deserializes it Can log error/warning and return None if something went wrong at deserialization """ try: if raw_data['status'] != 'completed': return None payout_date = raw_data.get('payout_at', None) if payout_date: timestamp = deserialize_timestamp_from_date( payout_date, 'iso8601', 'coinbase') else: timestamp = deserialize_timestamp_from_date( raw_data['created_at'], 'iso8601', 'coinbase', ) # Only get address/transaction id for "send" type of transactions address = None transaction_id = None # movement_category: Union[Literal['deposit'], Literal['withdrawal']] if 'type' in raw_data: # Then this should be a "send" which is the way Coinbase uses to send # crypto outside of the exchange # https://developers.coinbase.com/api/v2?python#transaction-resource msg = 'Non "send" type found in coinbase deposit/withdrawal processing' assert raw_data['type'] == 'send', msg movement_category = AssetMovementCategory.WITHDRAWAL # Can't see the fee being charged from the "send" resource amount = deserialize_asset_amount_force_positive( raw_data['amount']['amount']) asset = asset_from_coinbase(raw_data['amount']['currency'], time=timestamp) # Fees dont appear in the docs but from an experiment of sending ETH # to an address from coinbase there is the network fee in the response fee = Fee(ZERO) raw_network = raw_data.get('network', None) if raw_network: raw_fee = raw_network.get('transaction_fee', None) if raw_fee: # Since this is a withdrawal the fee should be the same as the moved asset if asset != asset_from_coinbase(raw_fee['currency'], time=timestamp): # If not we set ZERO fee and ignore log.error( f'In a coinbase withdrawal of {asset.identifier} the fee' f'is denoted in {raw_fee["currency"]}', ) else: fee = deserialize_fee(raw_fee['amount']) if 'network' in raw_data: transaction_id = get_key_if_has_val( raw_data['network'], 'hash') if 'to' in raw_data: address = deserialize_asset_movement_address( raw_data['to'], 'address', asset) else: movement_category = deserialize_asset_movement_category( raw_data['resource']) amount = deserialize_asset_amount_force_positive( raw_data['amount']['amount']) fee = deserialize_fee(raw_data['fee']['amount']) asset = asset_from_coinbase(raw_data['amount']['currency'], time=timestamp) return AssetMovement( location=Location.COINBASE, category=movement_category, address=address, transaction_id=transaction_id, timestamp=timestamp, asset=asset, amount=amount, fee_asset=asset, fee=fee, link=str(raw_data['id']), ) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found coinbase deposit/withdrawal with unknown asset ' f'{e.asset_name}. Ignoring it.', ) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found coinbase deposit/withdrawal with unsupported asset ' f'{e.asset_name}. Ignoring it.', ) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Unexpected data encountered during deserialization of a coinbase ' 'asset movement. Check logs for details and open a bug report.', ) log.error( f'Unexpected data encountered during deserialization of coinbase ' f'asset_movement {raw_data}. Error was: {str(e)}', ) return None
def test_query_asset_movements_sandbox( sandbox_kuckoin, inquirer, # pylint: disable=unused-argument ): """Unfortunately the sandbox environment does not support deposits and withdrawals, therefore they must be mocked. Below a list of the movements and their timestamps in ascending mode: Deposits: - deposit 1 - deposit: 1612556651 - deposit 2 - deposit: 1612556652 - deposit 3 - deposit: 1612556653 -> skipped, inner deposit Withdrawals: - withdraw 1: 1612556651 -> skipped, inner withdraw - withdraw 2: 1612556652 - withdraw 3: 1612556656 -> never requested By requesting trades from 1612556651 to 1612556654 and patching the time step as 2s (via MONTHS_IN_SECONDS) we should get back 3 movements. """ deposits_response_1 = (""" { "code":"200000", "data":{ "currentPage":1, "pageSize":2, "totalNum":2, "totalPage":1, "items":[ { "address":"0x5f047b29041bcfdbf0e4478cdfa753a336ba6989", "memo":"5c247c8a03aa677cea2a251d", "amount":1, "fee":0.0001, "currency":"KCS", "isInner":false, "walletTxId":"5bbb57386d99522d9f954c5a", "status":"SUCCESS", "remark":"movement 2 - deposit", "createdAt":1612556652000, "updatedAt":1612556652000 }, { "address":"0x5f047b29041bcfdbf0e4478cdfa753a336ba6989", "memo":"5c247c8a03aa677cea2a251d", "amount":1000, "fee":0.01, "currency":"LINK", "isInner":false, "walletTxId":"5bbb57386d99522d9f954c5b@test", "status":"SUCCESS", "remark":"movement 1 - deposit", "createdAt":1612556651000, "updatedAt":1612556651000 } ] } } """) deposits_response_2 = (""" { "code":"200000", "data":{ "currentPage":1, "pageSize":1, "totalNum":1, "totalPage":1, "items":[ { "address":"1DrT5xUaJ3CBZPDeFR2qdjppM6dzs4rsMt", "memo":"", "currency":"BCHSV", "amount":1, "fee":0.1, "walletTxId":"b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada6", "isInner":true, "status":"SUCCESS", "remark":"movement 4 - deposit", "createdAt":1612556653000, "updatedAt":1612556653000 } ] } } """) withdrawals_response_1 = (""" { "code":"200000", "data":{ "currentPage":1, "pageSize":2, "totalNum":2, "totalPage":1, "items":[ { "id":"5c2dc64e03aa675aa263f1a4", "address":"1DrT5xUaJ3CBZPDeFR2qdjppM6dzs4rsMt", "memo":"", "currency":"BCHSV", "amount":2.5, "fee":0.25, "walletTxId":"b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada4", "isInner":false, "status":"SUCCESS", "remark":"movement 4 - withdraw", "createdAt":1612556652000, "updatedAt":1612556652000 }, { "id":"5c2dc64e03aa675aa263f1a3", "address":"0x5bedb060b8eb8d823e2414d82acce78d38be7fe9", "memo":"", "currency":"ETH", "amount":1, "fee":0.01, "walletTxId":"3e2414d82acce78d38be7fe9", "isInner":true, "status":"SUCCESS", "remark":"movement 3 - withdraw", "createdAt":1612556651000, "updatedAt":1612556651000 } ] } } """) withdrawals_response_2 = (""" { "code":"200000", "data":{ "currentPage":0, "pageSize":0, "totalNum":0, "totalPage":0, "items":[] } } """) withdrawals_response_3 = (""" { "code":"200000", "data":{ "currentPage":1, "pageSize":1, "totalNum":1, "totalPage":1, "items":[ { "id":"5c2dc64e03aa675aa263f1a5", "address":"0x5bedb060b8eb8d823e2414d82acce78d38be7f00", "memo":"", "currency":"KCS", "amount":2.5, "fee":0.25, "walletTxId":"b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada5", "isInner":false, "status":"SUCCESS", "remark":"movement 5 - withdraw", "createdAt":1612556655000, "updatedAt":1612556655000 } ] } } """) expected_asset_movements = [ AssetMovement( location=Location.KUCOIN, category=AssetMovementCategory.DEPOSIT, timestamp=Timestamp(1612556652), address='0x5f047b29041bcfdbf0e4478cdfa753a336ba6989', transaction_id='5bbb57386d99522d9f954c5a', asset=A_KCS, amount=AssetAmount(FVal('1')), fee_asset=A_KCS, fee=Fee(FVal('0.0001')), link='', ), AssetMovement( location=Location.KUCOIN, category=AssetMovementCategory.DEPOSIT, timestamp=Timestamp(1612556651), address='0x5f047b29041bcfdbf0e4478cdfa753a336ba6989', transaction_id='5bbb57386d99522d9f954c5b', asset=A_LINK, amount=AssetAmount(FVal('1000')), fee_asset=A_LINK, fee=Fee(FVal('0.01')), link='', ), AssetMovement( location=Location.KUCOIN, category=AssetMovementCategory.WITHDRAWAL, timestamp=Timestamp(1612556652), address='1DrT5xUaJ3CBZPDeFR2qdjppM6dzs4rsMt', transaction_id= 'b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada4', asset=A_BSV, amount=AssetAmount(FVal('2.5')), fee_asset=A_BSV, fee=Fee(FVal('0.25')), link='5c2dc64e03aa675aa263f1a4', ), ] def get_endpoints_response(): results = [ f'{deposits_response_1}', f'{deposits_response_2}', f'{withdrawals_response_1}', f'{withdrawals_response_2}', # if pagination works as expected and the requesting loop is broken, # the response below won't be processed f'{withdrawals_response_3}', ] for result_ in results: yield result_ def mock_api_query_response(case, options): # pylint: disable=unused-argument return MockResponse(HTTPStatus.OK, next(get_response))
def test_coinbase_query_deposit_withdrawals(function_scope_coinbase): """Test that coinbase deposit/withdrawals history query works fine for the happy path""" coinbase = function_scope_coinbase with patch.object(coinbase.session, 'get', side_effect=mock_normal_coinbase_query): movements = coinbase.query_online_deposits_withdrawals( start_ts=0, end_ts=1566726126, ) warnings = coinbase.msg_aggregator.consume_warnings() errors = coinbase.msg_aggregator.consume_errors() assert len(warnings) == 0 assert len(errors) == 0 assert len(movements) == 5 expected_movements = [ AssetMovement( location=Location.COINBASE, category=AssetMovementCategory.DEPOSIT, timestamp=1519001640, address=None, transaction_id=None, asset=A_USD, amount=FVal('55.00'), fee_asset=A_USD, fee=FVal('0.05'), link='1130eaec-07d7-54c4-a72c-2e92826897df', ), AssetMovement( location=Location.COINBASE, category=AssetMovementCategory.WITHDRAWAL, address=None, transaction_id=None, timestamp=1485895742, asset=A_USD, amount=FVal('10.00'), fee_asset=A_USD, fee=FVal('0.01'), link='146eaec-07d7-54c4-a72c-2e92826897df', ), AssetMovement( location=Location.COINBASE, category=AssetMovementCategory.WITHDRAWAL, address='0x6dcD6449dbCa615e40d696328209686eA95327b2', transaction_id= '0x558bfa4d2a4ef598ddb92233459c00eda9e6c14cda75e6773b90208cb6938169', timestamp=1566726126, asset=A_ETH, amount=FVal('0.05770427'), fee_asset=A_ETH, fee=FVal('0.00021'), link='https://etherscan.io/tx/bbb', ), AssetMovement( location=Location.COINBASE, category=AssetMovementCategory.WITHDRAWAL, address='0x6dcD6449dbCa615e40d696328209686eA95327b2', transaction_id=None, timestamp=1566726126, asset=A_ETH, amount=FVal('0.05770427'), fee_asset=A_ETH, fee=ZERO, link='id2', ), AssetMovement( location=Location.COINBASE, category=AssetMovementCategory.DEPOSIT, address=None, transaction_id='ccc', timestamp=1502554304, asset=A_BTC, amount=FVal('0.10181673'), fee_asset=A_BTC, fee=ZERO, link='https://blockchain.info/tx/ccc', ) ] assert expected_movements == movements # and now try to query within a specific range with patch.object(coinbase.session, 'get', side_effect=mock_normal_coinbase_query): movements = coinbase.query_online_deposits_withdrawals( start_ts=0, end_ts=1519001640, ) warnings = coinbase.msg_aggregator.consume_warnings() errors = coinbase.msg_aggregator.consume_errors() assert len(warnings) == 0 assert len(errors) == 0 assert len(movements) == 3 assert movements[0].category == AssetMovementCategory.DEPOSIT assert movements[0].timestamp == 1519001640 assert movements[1].category == AssetMovementCategory.WITHDRAWAL assert movements[1].timestamp == 1485895742 assert movements[2].category == AssetMovementCategory.DEPOSIT assert movements[2].timestamp == 1502554304
def _consume_blockfi_entry(self, csv_row: Dict[str, Any]) -> None: """ Process entry for BlockFi transaction history. Trades for this file are ignored and istead should be extracted from the file containing only trades. This method can raise: - UnsupportedBlockFiEntry - UnknownAsset - DeserializationError """ if len(csv_row['Confirmed At']) != 0: timestamp = deserialize_timestamp_from_date( date=csv_row['Confirmed At'], formatstr='%Y-%m-%d %H:%M:%S', location='BlockFi', ) else: log.debug(f'Ignoring unconfirmed BlockFi entry {csv_row}') return asset = symbol_to_asset_or_token(csv_row['Cryptocurrency']) amount = deserialize_asset_amount_force_positive(csv_row['Amount']) entry_type = csv_row['Transaction Type'] # BlockFI doesn't provide information about fees fee = Fee(ZERO) fee_asset = A_USD # Can be whatever if entry_type in ('Deposit', 'Wire Deposit', 'ACH Deposit'): asset_movement = AssetMovement( location=Location.BLOCKFI, category=AssetMovementCategory.DEPOSIT, address=None, transaction_id=None, timestamp=timestamp, asset=asset, amount=amount, fee=fee, fee_asset=fee_asset, link='', ) self.db.add_asset_movements([asset_movement]) elif entry_type in ('Withdrawal', 'Wire Withdrawal', 'ACH Withdrawal'): asset_movement = AssetMovement( location=Location.BLOCKFI, category=AssetMovementCategory.WITHDRAWAL, address=None, transaction_id=None, timestamp=timestamp, asset=asset, amount=amount, fee=fee, fee_asset=fee_asset, link='', ) self.db.add_asset_movements([asset_movement]) elif entry_type == 'Withdrawal Fee': action = LedgerAction( identifier=0, # whatever is not used at insertion timestamp=timestamp, action_type=LedgerActionType.EXPENSE, location=Location.BLOCKFI, amount=amount, asset=asset, rate=None, rate_asset=None, link=None, notes=f'{entry_type} from BlockFi', ) self.db_ledger.add_ledger_action(action) elif entry_type in ('Interest Payment', 'Bonus Payment', 'Referral Bonus'): action = LedgerAction( identifier=0, # whatever is not used at insertion timestamp=timestamp, action_type=LedgerActionType.INCOME, location=Location.BLOCKFI, amount=amount, asset=asset, rate=None, rate_asset=None, link=None, notes=f'{entry_type} from BlockFi', ) self.db_ledger.add_ledger_action(action) elif entry_type == 'Trade': pass else: raise UnsupportedCSVEntry( f'Unsuported entry {entry_type}. Data: {csv_row}')
def _consume_cointracking_entry(self, csv_row: List[str]): """Consumes a cointracking entry row from the CSV and adds it into the database Can raise: - DeserializationError if something is wrong with the format of the expected values - UnsupportedCointrackingEntry if importing of this entry is not supported. - IndexError if the CSV file is corrupt - UnknownAsset if one of the assets founds in the entry are not supported """ row_type = csv_row[1] # Type timestamp = deserialize_timestamp_from_date( date=csv_row[9], formatstr='%d.%m.%Y %H:%M', location='cointracking.info', ) notes = csv_row[8] location = exchange_row_to_location(csv_row[6]) if row_type == 'Gift/Tip' or row_type == 'Trade': base_asset = Asset(csv_row[3]) quote_asset = None if csv_row[5] == '' else Asset(csv_row[5]) if not quote_asset and row_type != 'Gift/Tip': raise DeserializationError( 'Got a trade entry with an empty quote asset') if quote_asset is None: # Really makes no difference as this is just a gift and the amount is zero quote_asset = A_USD pair = TradePair( f'{base_asset.identifier}_{quote_asset.identifier}') base_amount_bought = deserialize_asset_amount(csv_row[2]) if csv_row[4] != '-': quote_amount_sold = deserialize_asset_amount(csv_row[4]) else: quote_amount_sold = ZERO rate = quote_amount_sold / base_amount_bought trade = Trade( timestamp=timestamp, location=location, pair=pair, trade_type=TradeType. BUY, # It's always a buy during cointracking import amount=base_amount_bought, rate=rate, fee=Fee( ZERO), # There are no fees when import from cointracking fee_currency=base_asset, link='', notes=notes, ) self.db.add_trades([trade]) elif row_type == 'Deposit' or row_type == 'Withdrawal': category = deserialize_asset_movement_category(row_type.lower()) if category == AssetMovementCategory.DEPOSIT: amount = deserialize_asset_amount(csv_row[2]) asset = Asset(csv_row[3]) else: amount = deserialize_asset_amount(csv_row[4]) asset = Asset(csv_row[5]) asset_movement = AssetMovement( location=location, category=category, timestamp=timestamp, asset=asset, amount=amount, fee_asset=asset, fee=Fee(ZERO), link='', ) self.db.add_asset_movements([asset_movement]) else: raise UnsupportedCointrackingEntry( f'Unknown entrype type "{row_type}" encountered during cointracking ' f'data import. Ignoring entry', )
def test_bitmex_api_withdrawals_deposit_and_query_after_subquery(sandbox_bitmex): """Test the happy case of bitmex withdrawals deposit query This test also tests an important case where a subquery for a an in-between time range is done first and then an encompassing range is requested. And we test that the full query, queries the remaining timestamp ranges. """ # This is an initial subquery of a small range where no deposit happened. result = sandbox_bitmex.query_deposits_withdrawals( start_ts=1536492800, end_ts=1536492976, only_cache=False, ) assert len(result) == 0 # Now after the subquery we test that the exchange engine logic properly # queries the required start/end timestamp ranges now = ts_now() result = sandbox_bitmex.query_deposits_withdrawals( start_ts=0, end_ts=now, only_cache=False, ) expected_result = [ AssetMovement( location=Location.BITMEX, category=AssetMovementCategory.DEPOSIT, address=None, transaction_id=None, timestamp=1536486278, asset=A_BTC, amount=FVal('0.46966992'), fee_asset=A_BTC, fee=FVal(0), link='166b9aac-70ac-cedc-69a0-dbd12c0661bf', ), AssetMovement( location=Location.BITMEX, category=AssetMovementCategory.DEPOSIT, address=None, transaction_id=None, timestamp=1537014656, asset=A_BTC, amount=FVal(0.16960386), fee_asset=A_BTC, fee=FVal(0E-8), link='b6c6fd2c-4d0c-b101-a41c-fa5aa1ce7ef1', ), AssetMovement( location=Location.BITMEX, category=AssetMovementCategory.DEPOSIT, address=None, transaction_id=None, timestamp=1536563759, asset=A_BTC, amount=FVal('0.38474377'), fee_asset=A_BTC, fee=FVal(0), link='72500751-d052-5bbb-18d7-08363edef812', ), AssetMovement( location=Location.BITMEX, category=AssetMovementCategory.WITHDRAWAL, address='mv4rnyY3Su5gjcDNzbMLKBQkBicCtHUtFB', transaction_id=None, timestamp=1536536707, asset=A_BTC, amount=FVal('0.00700000'), fee_asset=A_BTC, fee=FVal('0.00300000'), link='bf19ca4e-e084-11f9-12cd-6ae41e26f9db', ), ] assert result == expected_result # also make sure that asset movements contain Asset and not strings for movement in result: assert isinstance(movement.asset, Asset)
def assert_nexo_results(rotki: Rotkehlchen): """A utility function to help assert on correctness of importing data from nexo""" ledger_db = DBLedgerActions(rotki.data.db, rotki.msg_aggregator) ledger_actions = ledger_db.get_ledger_actions(None, None, None) asset_movements = rotki.data.db.get_asset_movements() warnings = rotki.msg_aggregator.consume_warnings() errors = rotki.msg_aggregator.consume_errors() assert len(errors) == 0 assert len(warnings) == 0 expected_actions = [LedgerAction( identifier=3, timestamp=Timestamp(1565888464), action_type=LedgerActionType.INCOME, location=Location.NEXO, amount=AssetAmount(FVal('22.5653042')), asset=symbol_to_asset_or_token('NEXO'), rate=None, rate_asset=None, link='NXT0000000009', notes='Dividend from Nexo', ), LedgerAction( identifier=2, timestamp=Timestamp(1597492915), action_type=LedgerActionType.INCOME, location=Location.NEXO, amount=AssetAmount(FVal('10.3585507')), asset=symbol_to_asset_or_token('NEXO'), rate=None, rate_asset=None, link='NXT0000000007', notes='Dividend from Nexo', ), LedgerAction( identifier=1, timestamp=Timestamp(1614993620), action_type=LedgerActionType.INCOME, location=Location.NEXO, amount=AssetAmount(FVal('1')), asset=symbol_to_asset_or_token('USDC'), rate=None, rate_asset=None, link='NXT0000000002', notes='Interest from Nexo', )] expected_movements = [AssetMovement( location=Location.NEXO, category=AssetMovementCategory.DEPOSIT, timestamp=Timestamp(1556116964), address=None, transaction_id=None, asset=A_BTC, amount=AssetAmount(FVal('1')), fee_asset=A_USD, fee=Fee(ZERO), link='NXT0000000013', ), AssetMovement( location=Location.NEXO, category=AssetMovementCategory.WITHDRAWAL, timestamp=Timestamp(1556122699), address=None, transaction_id=None, asset=A_BTC, amount=AssetAmount(FVal('0.9995')), fee_asset=A_USD, fee=Fee(ZERO), link='NXT0000000012', ), AssetMovement( location=Location.NEXO, category=AssetMovementCategory.DEPOSIT, timestamp=Timestamp(1558720210), address=None, transaction_id=None, asset=symbol_to_asset_or_token('NEXO'), amount=AssetAmount(FVal('1.00001')), fee_asset=A_USD, fee=Fee(ZERO), link='NXT0000000011', ), AssetMovement( location=Location.NEXO, category=AssetMovementCategory.DEPOSIT, timestamp=Timestamp(1565912821), address=None, transaction_id=None, asset=A_EUR, amount=AssetAmount(FVal('10000')), fee_asset=A_USD, fee=Fee(ZERO), link='NXT0000000010', ), AssetMovement( location=Location.NEXO, category=AssetMovementCategory.WITHDRAWAL, timestamp=Timestamp(1608131364), address=None, transaction_id=None, asset=A_EUR, amount=AssetAmount(FVal('2000.79')), fee_asset=A_USD, fee=Fee(ZERO), link='NXT0000000005', ), AssetMovement( location=Location.NEXO, category=AssetMovementCategory.DEPOSIT, timestamp=Timestamp(1614366540), address=None, transaction_id=None, asset=A_EUR, amount=AssetAmount(FVal('10')), fee_asset=A_USD, fee=Fee(ZERO), link='NXT0000000003', ), AssetMovement( location=Location.NEXO, category=AssetMovementCategory.DEPOSIT, timestamp=Timestamp(1615024314), address=None, transaction_id=None, asset=symbol_to_asset_or_token('USDC'), amount=AssetAmount(FVal('1')), fee_asset=A_USD, fee=Fee(ZERO), link='NXT0000000001', )] assert ledger_actions == expected_actions assert asset_movements == expected_movements
def assert_blockfi_transactions_import_results(rotki: Rotkehlchen): """A utility function to help assert on correctness of importing data from blockfi""" ledger_db = DBLedgerActions(rotki.data.db, rotki.msg_aggregator) ledger_actions = ledger_db.get_ledger_actions(None, None, None) asset_movements = rotki.data.db.get_asset_movements() warnings = rotki.msg_aggregator.consume_warnings() errors = rotki.msg_aggregator.consume_errors() assert len(errors) == 0 assert len(warnings) == 0 expected_actions = [LedgerAction( identifier=3, timestamp=Timestamp(1600293599), action_type=LedgerActionType.INCOME, location=Location.BLOCKFI, amount=AssetAmount(FVal('0.48385358')), asset=A_ETH, rate=None, rate_asset=None, link=None, notes='Bonus Payment from BlockFi', ), LedgerAction( identifier=2, timestamp=Timestamp(1606953599), action_type=LedgerActionType.INCOME, location=Location.BLOCKFI, amount=AssetAmount(FVal('0.00052383')), asset=A_BTC, rate=None, rate_asset=None, link=None, notes='Referral Bonus from BlockFi', ), LedgerAction( identifier=1, timestamp=Timestamp(1612051199), action_type=LedgerActionType.INCOME, location=Location.BLOCKFI, amount=AssetAmount(FVal('0.56469042')), asset=A_ETH, rate=None, rate_asset=None, link=None, notes='Interest Payment from BlockFi', )] assert expected_actions == ledger_actions expected_movements = [AssetMovement( location=Location.BLOCKFI, category=AssetMovementCategory.DEPOSIT, timestamp=Timestamp(1595247055), address=None, transaction_id=None, asset=A_BTC, amount=AssetAmount(FVal('1.11415058')), fee_asset=A_USD, fee=Fee(ZERO), link='', ), AssetMovement( location=Location.BLOCKFI, category=AssetMovementCategory.WITHDRAWAL, address=None, transaction_id=None, timestamp=Timestamp(1605977971), asset=A_ETH, amount=AssetAmount(FVal('3')), fee_asset=A_USD, fee=Fee(ZERO), link='', )] assert expected_movements == asset_movements
def test_add_asset_movements(data_dir, username): """Test that adding and retrieving asset movements from the DB works fine. Also duplicates should be ignored and an error returned """ msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) movement1 = AssetMovement( location=Location.BITMEX, category=AssetMovementCategory.DEPOSIT, address=None, transaction_id=None, timestamp=1451606400, asset=A_BTC, amount=FVal('1.0'), fee_asset=A_EUR, fee=Fee(FVal('0')), link='', ) movement2 = AssetMovement( location=Location.POLONIEX, category=AssetMovementCategory.WITHDRAWAL, address='0xfoo', transaction_id='0xboo', timestamp=1451608501, asset=A_ETH, amount=FVal('1.0'), fee_asset=A_EUR, fee=Fee(FVal('0.01')), link='', ) movement3 = AssetMovement( location=Location.BITTREX, category=AssetMovementCategory.WITHDRAWAL, address='0xcoo', transaction_id='0xdoo', timestamp=1461708501, asset=A_ETH, amount=FVal('1.0'), fee_asset=A_EUR, fee=Fee(FVal('0.01')), link='', ) # Add and retrieve the first 2 margins. All should be fine. data.db.add_asset_movements([movement1, movement2]) errors = msg_aggregator.consume_errors() warnings = msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 0 returned_movements = data.db.get_asset_movements() assert returned_movements == [movement1, movement2] # Add the last 2 movements. Since movement2 already exists in the DB it should be # ignored and a warning should be shown data.db.add_asset_movements([movement2, movement3]) errors = msg_aggregator.consume_errors() warnings = msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 1 returned_movements = data.db.get_asset_movements() assert returned_movements == [movement1, movement2, movement3]
def assert_poloniex_asset_movements( to_check_list: List[Any], deserialized: bool, movements_to_check: Optional[Tuple[int, ...]] = None, ) -> None: expected = [ AssetMovement( location=Location.POLONIEX, category=AssetMovementCategory.WITHDRAWAL, address='0xB7E033598Cb94EF5A35349316D3A2e4f95f308Da', transaction_id= '0xbd4da74e1a0b81c21d056c6f58a5b306de85d21ddf89992693b812bb117eace4', timestamp=Timestamp(1468994442), asset=A_ETH, amount=FVal('10.0'), fee_asset=A_ETH, fee=Fee(FVal('0.1')), link='2', ), AssetMovement( location=Location.POLONIEX, category=AssetMovementCategory.WITHDRAWAL, address='131rdg5Rzn6BFufnnQaHhVa5ZtRU1J2EZR', transaction_id= '2d27ae26fa9c70d6709e27ac94d4ce2fde19b3986926e9f3bfcf3e2d68354ec5', timestamp=Timestamp(1458994442), asset=A_BTC, amount=FVal('5.0'), fee_asset=A_BTC, fee=Fee(FVal('0.5')), link='1', ), AssetMovement( location=Location.POLONIEX, category=AssetMovementCategory.DEPOSIT, address='131rdg5Rzn6BFufnnQaHhVa5ZtRU1J2EZR', transaction_id= 'b05bdec7430a56b5a5ed34af4a31a54859dda9b7c88a5586bc5d6540cdfbfc7a', timestamp=Timestamp(1448994442), asset=A_BTC, amount=FVal('50.0'), fee_asset=A_BTC, fee=Fee(FVal('0')), link='1', ), AssetMovement( location=Location.POLONIEX, category=AssetMovementCategory.DEPOSIT, address='0xB7E033598Cb94EF5A35349316D3A2e4f95f308Da', transaction_id= '0xf7e7eeb44edcad14c0f90a5fffb1cbb4b80e8f9652124a0838f6906ca939ccd2', timestamp=Timestamp(1438994442), asset=A_ETH, amount=FVal('100.0'), fee_asset=A_ETH, fee=Fee(FVal('0')), link='2', ) ] assert_asset_movements(expected, to_check_list, deserialized, movements_to_check)
def _consume_nexo(self, csv_row: Dict[str, Any]) -> None: """ Consume CSV file from NEXO. This method can raise: - UnsupportedNexoEntry - UnknownAsset - DeserializationError """ ignored_entries = ('ExchangeToWithdraw', 'DepositToExchange') if 'rejected' not in csv_row['Details']: timestamp = deserialize_timestamp_from_date( date=csv_row['Date / Time'], formatstr='%Y-%m-%d %H:%M', location='NEXO', ) else: log.debug(f'Ignoring rejected nexo entry {csv_row}') return asset = symbol_to_asset_or_token(csv_row['Currency']) amount = deserialize_asset_amount_force_positive(csv_row['Amount']) entry_type = csv_row['Type'] transaction = csv_row['Transaction'] if entry_type in ('Deposit', 'ExchangeDepositedOn', 'LockingTermDeposit'): asset_movement = AssetMovement( location=Location.NEXO, category=AssetMovementCategory.DEPOSIT, address=None, transaction_id=None, timestamp=timestamp, asset=asset, amount=amount, fee=Fee(ZERO), fee_asset=A_USD, link=transaction, ) self.db.add_asset_movements([asset_movement]) elif entry_type in ('Withdrawal', 'WithdrawExchanged'): asset_movement = AssetMovement( location=Location.NEXO, category=AssetMovementCategory.WITHDRAWAL, address=None, transaction_id=None, timestamp=timestamp, asset=asset, amount=amount, fee=Fee(ZERO), fee_asset=A_USD, link=transaction, ) self.db.add_asset_movements([asset_movement]) elif entry_type == 'Withdrawal Fee': action = LedgerAction( identifier=0, # whatever is not used at insertion timestamp=timestamp, action_type=LedgerActionType.EXPENSE, location=Location.NEXO, amount=amount, asset=asset, rate=None, rate_asset=None, link=None, notes=f'{entry_type} from Nexo', ) self.db_ledger.add_ledger_action(action) elif entry_type in ('Interest', 'Bonus', 'Dividend'): action = LedgerAction( identifier=0, # whatever is not used at insertion timestamp=timestamp, action_type=LedgerActionType.INCOME, location=Location.NEXO, amount=amount, asset=asset, rate=None, rate_asset=None, link=transaction, notes=f'{entry_type} from Nexo', ) self.db_ledger.add_ledger_action(action) elif entry_type in ignored_entries: pass else: raise UnsupportedCSVEntry( f'Unsuported entry {entry_type}. Data: {csv_row}')
'close': '2018-03-03 23:05:04', 'currency': 'DASH', # cryptocompare hourly DASH/EUR: 475.565 'fee': '0.0001', 'earned': '0.0025', 'amount': '2', }, ] asset_movements_list = [ AssetMovement( # before query period -- 8.915 * 0.001 = 8.915e-3 location=Location.KRAKEN, category=AssetMovementCategory.WITHDRAWAL, address=None, transaction_id=None, timestamp=Timestamp(1479510304), # 18/11/2016, asset=A_ETH, # cryptocompare hourly ETH/EUR: 8.915 amount=FVal('95'), fee_asset=A_ETH, fee=Fee(FVal('0.001')), link='krakenid1', ), AssetMovement( # 0.0087*52.885 = 0.4600995 location=Location.KRAKEN, category=AssetMovementCategory.WITHDRAWAL, address=None, transaction_id=None, timestamp=Timestamp(1493291104), # 27/04/2017, asset=A_ETH, # cryptocompare hourly ETH/EUR: 52.885 amount=FVal('125'), fee_asset=A_ETH,
def test_gemini_query_deposits_withdrawals(sandbox_gemini): """Test that querying the asset movements endpoint works correctly Since Gemini sandbox does not support transfers, this uses a mocked call. """ transfers_patch = mock_gemini_transfers(sandbox_gemini, requests.post) with transfers_patch: movements = sandbox_gemini.query_deposits_withdrawals( start_ts=0, end_ts=Timestamp(1584881354), ) assert len(movements) == 6 expected_movements = [AssetMovement( location=Location.GEMINI, category=AssetMovementCategory.DEPOSIT, timestamp=Timestamp(1507913541), asset=A_USD, amount=FVal('36'), fee_asset=A_USD, fee=ZERO, link='320013281', ), AssetMovement( location=Location.GEMINI, category=AssetMovementCategory.DEPOSIT, timestamp=Timestamp(1499990797), asset=A_ETH, amount=FVal('100'), fee_asset=A_ETH, fee=ZERO, link='309356152', ), AssetMovement( location=Location.GEMINI, category=AssetMovementCategory.DEPOSIT, timestamp=Timestamp(1495550176), asset=A_BTC, amount=FVal('1500'), fee_asset=A_BTC, fee=ZERO, link='298112782', ), AssetMovement( location=Location.GEMINI, category=AssetMovementCategory.DEPOSIT, timestamp=Timestamp(1458862076), asset=A_USD, amount=FVal('500'), fee_asset=A_USD, fee=ZERO, link='265799530', ), AssetMovement( location=Location.GEMINI, category=AssetMovementCategory.WITHDRAWAL, timestamp=Timestamp(1450403787), asset=A_BTC, amount=FVal('5'), fee_asset=A_BTC, fee=ZERO, link='82897811', ), AssetMovement( location=Location.GEMINI, category=AssetMovementCategory.WITHDRAWAL, timestamp=Timestamp(1535451930), asset=A_USD, amount=FVal('1'), fee_asset=A_USD, fee=ZERO, link='341167014', )] # The deposits should be returned with the oldest first (so given list is reversed) assert movements == expected_movements[::-1]
def test_query_asset_movements_over_limit( rotkehlchen_api_server_with_exchanges, start_with_valid_premium, ): """Test that using the asset movements query endpoint works fine""" start_ts = 0 end_ts = 1598453214 server = rotkehlchen_api_server_with_exchanges rotki = server.rest_api.rotkehlchen # Make sure online kraken is not queried by setting query ranges rotki.data.db.update_used_query_range( name='kraken_asset_movements', start_ts=start_ts, end_ts=end_ts, ) polo_entries_num = 4 # Set a ton of kraken asset movements in the DB kraken_entries_num = FREE_ASSET_MOVEMENTS_LIMIT + 50 movements = [ AssetMovement(location=Location.KRAKEN, category=AssetMovementCategory.DEPOSIT, address=None, transaction_id=None, timestamp=x, asset=A_BTC, amount=FVal(x * 100), fee_asset=A_BTC, fee=FVal(x), link='') for x in range(kraken_entries_num) ] rotki.data.db.add_asset_movements(movements) all_movements_num = kraken_entries_num + polo_entries_num setup = prepare_rotki_for_history_processing_test( server.rest_api.rotkehlchen) # Check that querying movements with/without limits works even if we query two times for _ in range(2): # query asset movements of polo which has less movements than the limit with setup.polo_patch: response = requests.get( api_url_for( server, "assetmovementsresource", ), json={'location': 'poloniex'}, ) result = assert_proper_response_with_result(response) assert result['entries_found'] == all_movements_num assert result[ 'entries_limit'] == -1 if start_with_valid_premium else FREE_ASSET_MOVEMENTS_LIMIT # noqa: E501 assert_poloniex_asset_movements( [x['entry'] for x in result['entries']], deserialized=True) # now query kraken which has a ton of DB entries response = requests.get( api_url_for(server, "assetmovementsresource"), json={'location': 'kraken'}, ) result = assert_proper_response_with_result(response) if start_with_valid_premium: assert len(result['entries']) == kraken_entries_num assert result['entries_limit'] == -1 assert result['entries_found'] == all_movements_num else: assert len(result['entries'] ) == FREE_ASSET_MOVEMENTS_LIMIT - polo_entries_num assert result['entries_limit'] == FREE_ASSET_MOVEMENTS_LIMIT assert result['entries_found'] == all_movements_num
amount = deserialize_asset_amount(raw_result['amount']) fee = deserialize_fee(raw_result['fee']) fee_currency_symbol = raw_result['currency'] link_id = raw_result.get('id', '') # NB: id only exists for withdrawals except KeyError as e: raise DeserializationError(f'Missing key: {str(e)}.') from e fee_asset = asset_from_kucoin(fee_currency_symbol) asset_movement = AssetMovement( timestamp=timestamp, location=Location.KUCOIN, category=category, address=address, transaction_id=transaction_id, asset=fee_asset, amount=amount, fee_asset=fee_asset, fee=fee, link=link_id, ) return asset_movement @staticmethod def _deserialize_trade( raw_result: Dict[str, Any], case: Literal[KucoinCase.TRADES, KucoinCase.OLD_TRADES], ) -> Trade: """Process a trade result and deserialize it For the old v1 trades look here:
def assert_cointracking_import_results(rotki: Rotkehlchen): """A utility function to help assert on correctness of importing data from cointracking.info""" trades = rotki.data.db.get_trades() asset_movements = rotki.data.db.get_asset_movements() warnings = rotki.msg_aggregator.consume_warnings() errors = rotki.msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 3 expected_trades = [Trade( timestamp=Timestamp(1566687719), location=Location.COINBASE, pair=TradePair('ETH_EUR'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('0.05772716')), rate=Price(FVal('190.3783245183029963712055123')), fee=Fee(FVal("0.02")), fee_currency=A_EUR, link='', notes='', ), Trade( timestamp=Timestamp(1567418410), location=Location.EXTERNAL, pair=TradePair('BTC_USD'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('0.00100000')), rate=Price(ZERO), fee=Fee(ZERO), fee_currency=A_USD, link='', notes='Just a small gift from someone', ), Trade( timestamp=Timestamp(1567504805), location=Location.EXTERNAL, pair=TradePair('ETH_USD'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('2')), rate=Price(ZERO), fee=Fee(ZERO), fee_currency=A_USD, link='', notes='Sign up bonus', )] assert expected_trades == trades expected_movements = [AssetMovement( location=Location.POLONIEX, category=AssetMovementCategory.DEPOSIT, timestamp=Timestamp(1565848624), address=None, transaction_id=None, asset=A_XMR, amount=AssetAmount(FVal('5')), fee_asset=A_USD, fee=Fee(ZERO), link='', ), AssetMovement( location=Location.COINBASE, category=AssetMovementCategory.WITHDRAWAL, address=None, transaction_id=None, timestamp=Timestamp(1566726155), asset=A_ETH, amount=AssetAmount(FVal('0.05770427')), fee_asset=A_ETH, fee=Fee(FVal("0.0001")), link='', )] assert expected_movements == asset_movements
def test_cointracking_data_import(rotkehlchen_server): dir_path = os.path.dirname(os.path.realpath(__file__)) filepath = os.path.join(dir_path, 'data', 'cointracking_trades_list.csv') # Check that for unknown source we get an error response = rotkehlchen_server.import_data_from(source='other_source', filepath=filepath) assert response['result'] is False assert 'unknown location' in response['message'] # Check that the test cointracking data are imported succesfully rotkehlchen_server.import_data_from(source='cointracking_info', filepath=filepath) trades = rotkehlchen_server.rotkehlchen.data.db.get_trades() asset_movements = rotkehlchen_server.rotkehlchen.data.db.get_asset_movements( ) warnings = rotkehlchen_server.rotkehlchen.msg_aggregator.consume_warnings() errors = rotkehlchen_server.rotkehlchen.msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 3 expected_trades = [ Trade( timestamp=1566687660, location=Location.COINBASE, pair='ETH_EUR', trade_type=TradeType.BUY, amount=FVal('0.05772716'), rate=FVal('190.3783245183029963712055123'), fee=ZERO, fee_currency=A_ETH, link='', notes='', ), Trade( timestamp=1567418400, location=Location.EXTERNAL, pair='BTC_USD', trade_type=TradeType.BUY, amount=FVal('0.00100000'), rate=ZERO, fee=ZERO, fee_currency=A_BTC, link='', notes='Just a small gift from someone', ) ] assert expected_trades == trades expected_movements = [ AssetMovement( location=Location.POLONIEX, category=AssetMovementCategory.DEPOSIT, timestamp=1565848620, asset=A_XMR, amount=FVal('5'), fee_asset=A_XMR, fee=ZERO, link='', ), AssetMovement( location=Location.COINBASE, category=AssetMovementCategory.WITHDRAWAL, timestamp=1566726120, asset=A_ETH, amount=FVal('0.05770427'), fee_asset=A_ETH, fee=ZERO, link='', ) ] assert expected_movements == asset_movements
def query_online_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List: resp = self._api_query_list('get', 'user/walletHistory') log.debug('Bitmex deposit/withdrawals query', results_num=len(resp)) movements = list() for movement in resp: try: transaction_type = movement['transactType'] if transaction_type == 'Deposit': transaction_type = AssetMovementCategory.DEPOSIT elif transaction_type == 'Withdrawal': transaction_type = AssetMovementCategory.WITHDRAWAL else: continue timestamp = iso8601ts_to_timestamp(movement['timestamp']) if timestamp < start_ts: continue if timestamp > end_ts: continue asset = bitmex_to_world(movement['currency']) amount = deserialize_asset_amount(movement['amount']) fee = deserialize_fee(movement['fee']) # bitmex has negative numbers for withdrawals if amount < 0: amount *= -1 if asset == A_BTC: # bitmex stores amounts in satoshis amount = satoshis_to_btc(amount) fee = satoshis_to_btc(fee) movements.append( AssetMovement( location=Location.BITMEX, category=transaction_type, timestamp=timestamp, asset=asset, amount=amount, fee_asset=asset, fee=fee, link=str(movement['transactID']), )) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found bitmex deposit/withdrawal with unknown asset ' f'{e.asset_name}. Ignoring it.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( f'Unexpected data encountered during deserialization of a bitmex ' f'asset movement. Check logs for details and open a bug report.', ) log.error( f'Unexpected data encountered during deserialization of bitmex ' f'asset_movement {movement}. Error was: {str(e)}', ) continue return movements
def _consume_cryptocom_entry(self, csv_row: Dict[str, Any]) -> None: """Consumes a cryptocom entry row from the CSV and adds it into the database Can raise: - DeserializationError if something is wrong with the format of the expected values - UnsupportedCryptocomEntry if importing of this entry is not supported. - KeyError if the an expected CSV key is missing - UnknownAsset if one of the assets founds in the entry are not supported """ row_type = csv_row['Transaction Kind'] timestamp = deserialize_timestamp_from_date( date=csv_row['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='crypto.com', ) description = csv_row['Transaction Description'] notes = f'{description}\nSource: crypto.com (CSV import)' # No fees info until (Nov 2020) on crypto.com # fees are not displayed in the export data fee = Fee(ZERO) fee_currency = A_USD # whatever (used only if there is no fee) if row_type in ( 'crypto_purchase', 'crypto_exchange', 'referral_gift', 'referral_bonus', 'crypto_earn_interest_paid', 'referral_card_cashback', 'card_cashback_reverted', 'reimbursement', ): # variable mapping to raw data currency = csv_row['Currency'] to_currency = csv_row['To Currency'] native_currency = csv_row['Native Currency'] amount = csv_row['Amount'] to_amount = csv_row['To Amount'] native_amount = csv_row['Native Amount'] trade_type = TradeType.BUY if to_currency != native_currency else TradeType.SELL if row_type == 'crypto_exchange': # trades crypto to crypto base_asset = Asset(to_currency) quote_asset = Asset(currency) if quote_asset is None: raise DeserializationError('Got a trade entry with an empty quote asset') base_amount_bought = deserialize_asset_amount(to_amount) quote_amount_sold = deserialize_asset_amount(amount) else: base_asset = Asset(currency) quote_asset = Asset(native_currency) base_amount_bought = deserialize_asset_amount(amount) quote_amount_sold = deserialize_asset_amount(native_amount) rate = Price(abs(quote_amount_sold / base_amount_bought)) pair = TradePair(f'{base_asset.identifier}_{quote_asset.identifier}') trade = Trade( timestamp=timestamp, location=Location.CRYPTOCOM, pair=pair, trade_type=trade_type, amount=base_amount_bought, rate=rate, fee=fee, fee_currency=fee_currency, link='', notes=notes, ) self.db.add_trades([trade]) elif row_type in ('crypto_withdrawal', 'crypto_deposit'): if row_type == 'crypto_withdrawal': category = AssetMovementCategory.WITHDRAWAL amount = deserialize_asset_amount_force_positive(csv_row['Amount']) else: category = AssetMovementCategory.DEPOSIT amount = deserialize_asset_amount(csv_row['Amount']) asset = Asset(csv_row['Currency']) asset_movement = AssetMovement( location=Location.CRYPTOCOM, category=category, address=None, transaction_id=None, timestamp=timestamp, asset=asset, amount=amount, fee=fee, fee_asset=asset, link='', ) self.db.add_asset_movements([asset_movement]) elif row_type in ( 'crypto_earn_program_created', 'crypto_earn_program_withdrawn', 'lockup_lock', 'lockup_unlock', 'dynamic_coin_swap_bonus_exchange_deposit', 'crypto_wallet_swap_debited', 'crypto_wallet_swap_credited', 'lockup_swap_debited', 'lockup_swap_credited', 'lockup_swap_rebate', 'dynamic_coin_swap_bonus_exchange_deposit', # we don't handle cryto.com exchange yet 'crypto_to_exchange_transfer', 'exchange_to_crypto_transfer', # supercharger actions 'supercharger_deposit', 'supercharger_withdrawal', # already handled using _import_cryptocom_double_entries 'dynamic_coin_swap_debited', 'dynamic_coin_swap_credited', 'dust_conversion_debited', 'dust_conversion_credited', ): # those types are ignored because it doesn't affect the wallet balance # or are not handled here return else: raise UnsupportedCryptocomEntry( f'Unknown entrype type "{row_type}" encountered during ' f'cryptocom data import. Ignoring entry', )
def test_gemini_query_deposits_withdrawals(sandbox_gemini): """Test that querying the asset movements endpoint works correctly Since Gemini sandbox does not support transfers, this uses a mocked call. """ transfers_patch = mock_gemini_transfers(sandbox_gemini, requests.post) with transfers_patch: movements = sandbox_gemini.query_deposits_withdrawals( start_ts=0, end_ts=Timestamp(1584881354), only_cache=False, ) assert len(movements) == 6 expected_movements = [ AssetMovement( location=Location.GEMINI, category=AssetMovementCategory.DEPOSIT, timestamp=Timestamp(1507913541), address=None, transaction_id=None, asset=A_USD, amount=FVal('36'), fee_asset=A_USD, fee=ZERO, link='320013281', ), AssetMovement( location=Location.GEMINI, category=AssetMovementCategory.DEPOSIT, address=None, transaction_id= '605c5fa8bf99458d24d61e09941bc443ddc44839d9aaa508b14b296c0c8269b2', timestamp=Timestamp(1499990797), asset=A_ETH, amount=FVal('100'), fee_asset=A_ETH, fee=ZERO, link='309356152', ), AssetMovement( location=Location.GEMINI, category=AssetMovementCategory.DEPOSIT, address=None, transaction_id= '163eeee4741f8962b748289832dd7f27f754d892f5d23bf3ea6fba6e350d9ce3', timestamp=Timestamp(1495550176), asset=A_BTC, amount=FVal('1500'), fee_asset=A_BTC, fee=ZERO, link='298112782', ), AssetMovement( location=Location.GEMINI, category=AssetMovementCategory.DEPOSIT, address=None, transaction_id=None, timestamp=Timestamp(1458862076), asset=A_USD, amount=FVal('500'), fee_asset=A_USD, fee=ZERO, link='265799530', ), AssetMovement( location=Location.GEMINI, category=AssetMovementCategory.WITHDRAWAL, address='mqjvCtt4TJfQaC7nUgLMvHwuDPXMTEUGqx', transaction_id= 'c458b86955b80db0718cfcadbff3df3734a906367982c6eb191e61117b810bbb', timestamp=Timestamp(1450403787), asset=A_BTC, amount=FVal('5'), fee_asset=A_BTC, fee=ZERO, link='82897811', ), AssetMovement( location=Location.GEMINI, category=AssetMovementCategory.WITHDRAWAL, address='0xd24400ae8BfEBb18cA49Be86258a3C749cf46853', transaction_id= '7bffd85893ee8e72e31061a84d25c45f2c4537c2f765a1e79feb06a7294445c3', timestamp=Timestamp(1535451930), asset=A_USD, amount=FVal('1'), fee_asset=A_USD, fee=ZERO, link='341167014', ) ] # The deposits should be returned with the oldest first (so given list is reversed) assert movements == expected_movements[::-1]
def query_online_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[AssetMovement]: """Queries coinbase pro for asset movements""" log.debug('Query coinbasepro asset movements', start_ts=start_ts, end_ts=end_ts) movements = [] raw_movements = [] for batch in self._paginated_query( endpoint='transfers', query_options={'type': 'withdraw'}, ): raw_movements.extend(batch) for batch in self._paginated_query( endpoint='transfers', query_options={'type': 'deposit'}, ): raw_movements.extend(batch) account_to_currency = self.create_or_return_account_to_currency_map() for entry in raw_movements: try: # Check if the transaction has not been completed. If so it should be skipped if entry.get('completed_at', None) is None: log.warning( f'Skipping coinbase pro deposit/withdrawal ' f'due to not having been completed: {entry}', ) continue timestamp = coinbasepro_deserialize_timestamp( entry, 'completed_at') if timestamp < start_ts or timestamp > end_ts: continue category = deserialize_asset_movement_category(entry['type']) asset = account_to_currency.get(entry['account_id'], None) if asset is None: log.warning( f'Skipping coinbase pro asset_movement {entry} due to ' f'inability to match account id to an asset', ) continue address = None transaction_id = None fee = Fee(ZERO) if category == AssetMovementCategory.DEPOSIT: try: address = entry['details']['crypto_address'] transaction_id = entry['details'][ 'crypto_transaction_hash'] except KeyError: pass else: # withdrawal try: address = entry['details']['sent_to_address'] transaction_id = entry['details'][ 'crypto_transaction_hash'] fee = deserialize_fee(entry['details']['fee']) except KeyError: pass if transaction_id and ( asset == A_ETH or asset.asset_type == AssetType.ETHEREUM_TOKEN): # noqa: E501 transaction_id = '0x' + transaction_id movements.append( AssetMovement( location=Location.COINBASEPRO, category=category, address=address, transaction_id=transaction_id, timestamp=timestamp, asset=asset, amount=deserialize_asset_amount_force_positive( entry['amount']), fee_asset=asset, fee=fee, link=str(entry['id']), )) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown Coinbasepro asset {e.asset_name}. ' f'Ignoring its deposit/withdrawal.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Failed to deserialize a Coinbasepro deposit/withdrawal. ' 'Check logs for details. Ignoring it.', ) log.error( 'Error processing a coinbasepro deposit/withdrawal.', raw_asset_movement=entry, error=msg, ) continue return movements
def _consume_cointracking_entry(self, csv_row: Dict[str, Any]) -> None: """Consumes a cointracking entry row from the CSV and adds it into the database Can raise: - DeserializationError if something is wrong with the format of the expected values - UnsupportedCointrackingEntry if importing of this entry is not supported. - IndexError if the CSV file is corrupt - UnknownAsset if one of the assets founds in the entry are not supported """ row_type = csv_row['Type'] timestamp = deserialize_timestamp_from_date( date=csv_row['Date'], formatstr='%d.%m.%Y %H:%M:%S', location='cointracking.info', ) notes = csv_row['Comment'] location = exchange_row_to_location(csv_row['Exchange']) fee = Fee(ZERO) fee_currency = A_USD # whatever (used only if there is no fee) if csv_row['Fee'] != '': fee = deserialize_fee(csv_row['Fee']) fee_currency = Asset(csv_row['Cur.Fee']) if row_type in ('Gift/Tip', 'Trade', 'Income'): base_asset = Asset(csv_row['Cur.Buy']) quote_asset = None if csv_row['Cur.Sell'] == '' else Asset(csv_row['Cur.Sell']) if quote_asset is None and row_type not in ('Gift/Tip', 'Income'): raise DeserializationError('Got a trade entry with an empty quote asset') if quote_asset is None: # Really makes no difference as this is just a gift and the amount is zero quote_asset = A_USD pair = TradePair(f'{base_asset.identifier}_{quote_asset.identifier}') base_amount_bought = deserialize_asset_amount(csv_row['Buy']) if csv_row['Sell'] != '-': quote_amount_sold = deserialize_asset_amount(csv_row['Sell']) else: quote_amount_sold = AssetAmount(ZERO) rate = Price(quote_amount_sold / base_amount_bought) trade = Trade( timestamp=timestamp, location=location, pair=pair, trade_type=TradeType.BUY, # It's always a buy during cointracking import amount=base_amount_bought, rate=rate, fee=fee, fee_currency=fee_currency, link='', notes=notes, ) self.db.add_trades([trade]) elif row_type == 'Deposit' or row_type == 'Withdrawal': category = deserialize_asset_movement_category(row_type.lower()) if category == AssetMovementCategory.DEPOSIT: amount = deserialize_asset_amount(csv_row['Buy']) asset = Asset(csv_row['Cur.Buy']) else: amount = deserialize_asset_amount(csv_row['Sell']) asset = Asset(csv_row['Cur.Sell']) asset_movement = AssetMovement( location=location, category=category, timestamp=timestamp, asset=asset, amount=amount, fee=fee, fee_asset=fee_currency, link='', ) self.db.add_asset_movements([asset_movement]) else: raise UnsupportedCointrackingEntry( f'Unknown entrype type "{row_type}" encountered during cointracking ' f'data import. Ignoring entry', )
def query_online_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[AssetMovement]: result = self.query_until_finished( endpoint='Ledgers', keyname='ledger', start_ts=start_ts, end_ts=end_ts, extra_dict={'type': 'deposit'}, ) result.extend(self.query_until_finished( endpoint='Ledgers', keyname='ledger', start_ts=start_ts, end_ts=end_ts, extra_dict={'type': 'withdrawal'}, )) log.debug('Kraken deposit/withdrawals query result', num_results=len(result)) movements = [] for movement in result: try: asset = asset_from_kraken(movement['asset']) movement_type = movement['type'] if movement_type not in ('deposit', 'withdrawal'): # Other known types: 'transfer' continue # Can be for moving funds from spot to stake etc. movements.append(AssetMovement( location=Location.KRAKEN, category=deserialize_asset_movement_category(movement_type), timestamp=deserialize_timestamp_from_kraken(movement['time']), address=None, # no data from kraken ledger endpoint transaction_id=None, # no data from kraken ledger endpoint asset=asset, amount=deserialize_asset_amount_force_positive(movement['amount']), fee_asset=asset, fee=deserialize_fee(movement['fee']), link=str(movement['refid']), )) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown kraken asset {e.asset_name}. ' f'Ignoring its deposit/withdrawals query.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Failed to deserialize a kraken deposit/withdrawals. ' 'Check logs for details. Ignoring it.', ) log.error( 'Error processing a kraken deposit/withdrawal.', raw_asset_movement=movement, error=msg, ) continue return movements
def test_add_asset_movements(data_dir, username, caplog): """Test that adding and retrieving asset movements from the DB works fine. Also duplicates should be ignored and an error returned """ msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) movement1 = AssetMovement( location=Location.BITMEX, category=AssetMovementCategory.DEPOSIT, address=None, transaction_id=None, timestamp=1451606400, asset=A_BTC, amount=FVal('1.0'), fee_asset=A_EUR, fee=Fee(FVal('0')), link='', ) movement2 = AssetMovement( location=Location.POLONIEX, category=AssetMovementCategory.WITHDRAWAL, address='0xfoo', transaction_id='0xboo', timestamp=1451608501, asset=A_ETH, amount=FVal('1.0'), fee_asset=A_EUR, fee=Fee(FVal('0.01')), link='', ) movement3 = AssetMovement( location=Location.BITTREX, category=AssetMovementCategory.WITHDRAWAL, address='0xcoo', transaction_id='0xdoo', timestamp=1461708501, asset=A_ETH, amount=FVal('1.0'), fee_asset=A_EUR, fee=Fee(FVal('0.01')), link='', ) # Add and retrieve the first 2 margins. All should be fine. data.db.add_asset_movements([movement1, movement2]) errors = msg_aggregator.consume_errors() warnings = msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 0 returned_movements = data.db.get_asset_movements( filter_query=AssetMovementsFilterQuery.make(), has_premium=True, ) assert returned_movements == [movement1, movement2] # Add the last 2 movements. Since movement2 already exists in the DB it should be # ignored and a warning should be logged data.db.add_asset_movements([movement2, movement3]) assert ('Did not add "withdrawal of ETH with id 94405f38c7b86dd2e7943164d' '67ff44a32d56cef25840b3f5568e23c037fae0a') in caplog.text returned_movements = data.db.get_asset_movements( filter_query=AssetMovementsFilterQuery.make(), has_premium=True, ) assert returned_movements == [movement1, movement2, movement3]
def _deserialize_asset_movement( self, raw_data: Dict[str, Any], movement_type: AssetMovementCategory, ) -> Optional[AssetMovement]: """Processes a single deposit/withdrawal from FTX and deserializes it Can log error/warning and return None if something went wrong at deserialization """ try: if raw_data['status'] not in ('complete', 'confirmed'): return None timestamp = deserialize_timestamp_from_date( raw_data['time'], 'iso8601', 'FTX') amount = deserialize_asset_amount_force_positive(raw_data['size']) asset = asset_from_ftx(raw_data['coin']) fee = Fee(ZERO) movement_category = movement_type if raw_data.get('fee', None) is not None: fee = deserialize_fee(raw_data['fee']) address = raw_data.get('address', None) if isinstance(address, dict): address = raw_data['address'].get('address', None) transaction_id = raw_data.get('txid', None) return AssetMovement( location=Location.FTX, category=movement_category, address=address, transaction_id=transaction_id, timestamp=timestamp, asset=asset, amount=amount, fee_asset=asset, fee=fee, link=str(raw_data['id']), ) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found FTX deposit/withdrawal with unknown asset ' f'{e.asset_name}. Ignoring it.', ) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found FTX deposit/withdrawal with unsupported asset ' f'{e.asset_name}. Ignoring it.', ) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Unexpected data encountered during deserialization of an FTX ' 'asset movement. Check logs for details and open a bug report.', ) log.error( 'Error processing FTX trade', trade=raw_data, error=msg, ) return None