def test_int_overflow_at_tuple_insertion(database, caplog): """Test that if somehow an int that will overflow makes it there we handle it Related: https://github.com/rotki/rotki/issues/2175 """ caplog.set_level(logging.INFO) database.add_asset_movements([AssetMovement( location=Location.BITTREX, category=AssetMovementCategory.DEPOSIT, timestamp=177778, address='0xfoo', transaction_id=99999999999999999999999999999999999999999, asset=A_BTC, amount=FVal(1), fee_asset=A_BTC, fee=Fee(FVal('0.0001')), link='a link', )]) errors = database.msg_aggregator.consume_errors() assert len(errors) == 1 assert 'Failed to add "asset_movement" to the DB with overflow error' in errors[0] assert 'Overflow error while trying to add "asset_movement" tuples to the DB. Tuples:' in caplog.text # noqa: E501
def test_deserialize_v2_trade_sell(mock_kucoin): raw_result = { 'symbol': 'BCHSV-USDT', 'tradeId': '601da995e0ee8b00063a075c', 'orderId': '601da9950c92050006bd45c5', 'counterOrderId': '601da9950c92050006bd457d', 'side': 'sell', 'liquidity': 'taker', 'forceTaker': True, 'price': '37624.4', 'size': '0.0013', 'funds': '48.91172', 'fee': '0.034238204', 'feeRate': '0.0007', 'feeCurrency': 'USDT', 'stop': '', 'tradeType': 'TRADE', 'type': 'market', 'createdAt': 1612556794259, } expected_trade = Trade( timestamp=Timestamp(1612556794), location=Location.KUCOIN, pair=TradePair('BSV_USDT'), trade_type=TradeType.SELL, amount=AssetAmount(FVal('0.0013')), rate=Price(FVal('37624.4')), fee=Fee(FVal('0.034238204')), fee_currency=Asset('USDT'), link='601da995e0ee8b00063a075c', notes='', ) trade = mock_kucoin._deserialize_trade( raw_result=raw_result, case=KucoinCase.TRADES, ) assert trade == expected_trade
def test_deserialize_v2_trade_buy(mock_kucoin): raw_result = { 'symbol': 'KCS-USDT', 'tradeId': '601da9faf1297d0007efd712', 'orderId': '601da9fa0c92050006bd83be', 'counterOrderId': '601bad620c9205000642300f', 'side': 'buy', 'liquidity': 'taker', 'forceTaker': True, 'price': 1000, 'size': '0.2', 'funds': 200, 'fee': '0.14', 'feeRate': '0.0007', 'feeCurrency': 'USDT', 'stop': '', 'tradeType': 'TRADE', 'type': 'market', 'createdAt': 1612556794259, } expected_trade = Trade( timestamp=Timestamp(1612556794), location=Location.KUCOIN, pair=TradePair('KCS_USDT'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('0.2')), rate=Price(FVal('1000')), fee=Fee(FVal('0.14')), fee_currency=Asset('USDT'), link='601da9faf1297d0007efd712', notes='', ) trade = mock_kucoin._deserialize_trade( raw_result=raw_result, case=KucoinCase.TRADES, ) assert trade == expected_trade
def _consume_blockfi_entry(self, csv_row: Dict[str, Any]) -> None: """ Process entry for BlockFi transaction history. Trades for this file are ignored and istead should be extracted from the file containing only trades. This method can raise: - UnsupportedBlockFiEntry - UnknownAsset - DeserializationError """ if len(csv_row['Confirmed At']) != 0: timestamp = deserialize_timestamp_from_date( date=csv_row['Confirmed At'], formatstr='%Y-%m-%d %H:%M:%S', location='BlockFi', ) else: log.debug(f'Ignoring unconfirmed BlockFi entry {csv_row}') return asset = symbol_to_asset_or_token(csv_row['Cryptocurrency']) amount = deserialize_asset_amount_force_positive(csv_row['Amount']) entry_type = csv_row['Transaction Type'] # BlockFI doesn't provide information about fees fee = Fee(ZERO) fee_asset = A_USD # Can be whatever if entry_type in ('Deposit', 'Wire Deposit', 'ACH Deposit'): asset_movement = AssetMovement( location=Location.BLOCKFI, category=AssetMovementCategory.DEPOSIT, address=None, transaction_id=None, timestamp=timestamp, asset=asset, amount=amount, fee=fee, fee_asset=fee_asset, link='', ) self.db.add_asset_movements([asset_movement]) elif entry_type in ('Withdrawal', 'Wire Withdrawal', 'ACH Withdrawal'): asset_movement = AssetMovement( location=Location.BLOCKFI, category=AssetMovementCategory.WITHDRAWAL, address=None, transaction_id=None, timestamp=timestamp, asset=asset, amount=amount, fee=fee, fee_asset=fee_asset, link='', ) self.db.add_asset_movements([asset_movement]) elif entry_type == 'Withdrawal Fee': action = LedgerAction( identifier=0, # whatever is not used at insertion timestamp=timestamp, action_type=LedgerActionType.EXPENSE, location=Location.BLOCKFI, amount=amount, asset=asset, rate=None, rate_asset=None, link=None, notes=f'{entry_type} from BlockFi', ) self.db_ledger.add_ledger_action(action) elif entry_type in ('Interest Payment', 'Bonus Payment', 'Referral Bonus'): action = LedgerAction( identifier=0, # whatever is not used at insertion timestamp=timestamp, action_type=LedgerActionType.INCOME, location=Location.BLOCKFI, amount=amount, asset=asset, rate=None, rate_asset=None, link=None, notes=f'{entry_type} from BlockFi', ) self.db_ledger.add_ledger_action(action) elif entry_type == 'Trade': pass else: raise UnsupportedCSVEntry( f'Unsuported entry {entry_type}. Data: {csv_row}')
def query_online_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[AssetMovement]: """Queries coinbase pro for asset movements""" log.debug('Query coinbasepro asset movements', start_ts=start_ts, end_ts=end_ts) movements = [] raw_movements = [] for batch in self._paginated_query( endpoint='transfers', query_options={'type': 'withdraw'}, ): raw_movements.extend(batch) for batch in self._paginated_query( endpoint='transfers', query_options={'type': 'deposit'}, ): raw_movements.extend(batch) account_to_currency = self.create_or_return_account_to_currency_map() for entry in raw_movements: try: # Check if the transaction has not been completed. If so it should be skipped if entry.get('completed_at', None) is None: log.warning( f'Skipping coinbase pro deposit/withdrawal ' f'due not having been completed: {entry}', ) continue timestamp = coinbasepro_deserialize_timestamp( entry, 'completed_at') if timestamp < start_ts or timestamp > end_ts: continue category = deserialize_asset_movement_category(entry['type']) asset = account_to_currency.get(entry['account_id'], None) if asset is None: log.warning( f'Skipping coinbase pro asset_movement {entry} due to inability to ' f'match account id to an asset', ) continue address = None transaction_id = None fee = Fee(ZERO) if category == AssetMovementCategory.DEPOSIT: try: address = entry['details']['crypto_address'] transaction_id = entry['details'][ 'crypto_transaction_hash'] except KeyError: pass else: # withdrawal try: address = entry['details']['sent_to_address'] transaction_id = entry['details'][ 'crypto_transaction_hash'] fee = deserialize_fee(entry['details']['fee']) except KeyError: pass if transaction_id and ( asset == A_ETH or asset.asset_type == AssetType.ETHEREUM_TOKEN): # noqa: E501 transaction_id = '0x' + transaction_id movements.append( AssetMovement( location=Location.COINBASEPRO, category=category, address=address, transaction_id=transaction_id, timestamp=timestamp, asset=asset, amount=deserialize_asset_amount_force_positive( entry['amount']), fee_asset=asset, fee=fee, link=str(entry['id']), )) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown Coinbasepro asset {e.asset_name}. ' f'Ignoring its deposit/withdrawal.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Failed to deserialize a Coinbasepro deposit/withdrawal. ' 'Check logs for details. Ignoring it.', ) log.error( 'Error processing a coinbasepro deposit/withdrawal.', raw_asset_movement=entry, error=msg, ) continue return movements
def _consume_cointracking_entry(self, csv_row: Dict[str, Any]) -> None: """Consumes a cointracking entry row from the CSV and adds it into the database Can raise: - DeserializationError if something is wrong with the format of the expected values - UnsupportedCointrackingEntry if importing of this entry is not supported. - IndexError if the CSV file is corrupt - KeyError if the an expected CSV key is missing - UnknownAsset if one of the assets founds in the entry are not supported """ row_type = csv_row['Type'] timestamp = deserialize_timestamp_from_date( date=csv_row['Date'], formatstr='%d.%m.%Y %H:%M:%S', location='cointracking.info', ) notes = csv_row['Comment'] location = exchange_row_to_location(csv_row['Exchange']) fee = Fee(ZERO) fee_currency = A_USD # whatever (used only if there is no fee) if csv_row['Fee'] != '': fee = deserialize_fee(csv_row['Fee']) fee_currency = Asset(csv_row['Cur.Fee']) if row_type in ('Gift/Tip', 'Trade', 'Income'): base_asset = Asset(csv_row['Cur.Buy']) quote_asset = None if csv_row['Cur.Sell'] == '' else Asset( csv_row['Cur.Sell']) if quote_asset is None and row_type not in ('Gift/Tip', 'Income'): raise DeserializationError( 'Got a trade entry with an empty quote asset') if quote_asset is None: # Really makes no difference as this is just a gift and the amount is zero quote_asset = A_USD pair = TradePair( f'{base_asset.identifier}_{quote_asset.identifier}') base_amount_bought = deserialize_asset_amount(csv_row['Buy']) if csv_row['Sell'] != '-': quote_amount_sold = deserialize_asset_amount(csv_row['Sell']) else: quote_amount_sold = AssetAmount(ZERO) rate = Price(quote_amount_sold / base_amount_bought) trade = Trade( timestamp=timestamp, location=location, pair=pair, trade_type=TradeType. BUY, # It's always a buy during cointracking import amount=base_amount_bought, rate=rate, fee=fee, fee_currency=fee_currency, link='', notes=notes, ) self.db.add_trades([trade]) elif row_type == 'Deposit' or row_type == 'Withdrawal': category = deserialize_asset_movement_category(row_type.lower()) if category == AssetMovementCategory.DEPOSIT: amount = deserialize_asset_amount(csv_row['Buy']) asset = Asset(csv_row['Cur.Buy']) else: amount = deserialize_asset_amount_force_positive( csv_row['Sell']) asset = Asset(csv_row['Cur.Sell']) asset_movement = AssetMovement( location=location, category=category, address=None, transaction_id=None, timestamp=timestamp, asset=asset, amount=amount, fee=fee, fee_asset=fee_currency, link='', ) self.db.add_asset_movements([asset_movement]) else: raise UnsupportedCointrackingEntry( f'Unknown entrype type "{row_type}" encountered during cointracking ' f'data import. Ignoring entry', )
def _consume_cryptocom_entry(self, csv_row: Dict[str, Any]) -> None: """Consumes a cryptocom entry row from the CSV and adds it into the database Can raise: - DeserializationError if something is wrong with the format of the expected values - UnsupportedCryptocomEntry if importing of this entry is not supported. - KeyError if the an expected CSV key is missing - UnknownAsset if one of the assets founds in the entry are not supported """ row_type = csv_row['Transaction Kind'] timestamp = deserialize_timestamp_from_date( date=csv_row['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='crypto.com', ) description = csv_row['Transaction Description'] notes = f'{description}\nSource: crypto.com (CSV import)' # No fees info until (Nov 2020) on crypto.com # fees are not displayed in the export data fee = Fee(ZERO) fee_currency = A_USD # whatever (used only if there is no fee) if row_type in ( 'crypto_purchase', 'crypto_exchange', 'referral_gift', 'referral_bonus', 'crypto_earn_interest_paid', 'referral_card_cashback', 'card_cashback_reverted', 'reimbursement', ): # variable mapping to raw data currency = csv_row['Currency'] to_currency = csv_row['To Currency'] native_currency = csv_row['Native Currency'] amount = csv_row['Amount'] to_amount = csv_row['To Amount'] native_amount = csv_row['Native Amount'] trade_type = TradeType.BUY if to_currency != native_currency else TradeType.SELL if row_type == 'crypto_exchange': # trades crypto to crypto base_asset = Asset(to_currency) quote_asset = Asset(currency) if quote_asset is None: raise DeserializationError( 'Got a trade entry with an empty quote asset') base_amount_bought = deserialize_asset_amount(to_amount) quote_amount_sold = deserialize_asset_amount(amount) else: base_asset = Asset(currency) quote_asset = Asset(native_currency) base_amount_bought = deserialize_asset_amount(amount) quote_amount_sold = deserialize_asset_amount(native_amount) rate = Price(abs(quote_amount_sold / base_amount_bought)) pair = TradePair( f'{base_asset.identifier}_{quote_asset.identifier}') trade = Trade( timestamp=timestamp, location=Location.CRYPTOCOM, pair=pair, trade_type=trade_type, amount=base_amount_bought, rate=rate, fee=fee, fee_currency=fee_currency, link='', notes=notes, ) self.db.add_trades([trade]) elif row_type == 'crypto_withdrawal' or row_type == 'crypto_deposit': if row_type == 'crypto_withdrawal': category = AssetMovementCategory.WITHDRAWAL amount = deserialize_asset_amount_force_positive( csv_row['Amount']) else: category = AssetMovementCategory.DEPOSIT amount = deserialize_asset_amount(csv_row['Amount']) asset = Asset(csv_row['Currency']) asset_movement = AssetMovement( location=Location.CRYPTOCOM, category=category, address=None, transaction_id=None, timestamp=timestamp, asset=asset, amount=amount, fee=fee, fee_asset=asset, link='', ) self.db.add_asset_movements([asset_movement]) elif row_type in ( 'crypto_earn_program_created', 'lockup_lock', 'lockup_unlock', 'dynamic_coin_swap_bonus_exchange_deposit', 'crypto_wallet_swap_debited', 'crypto_wallet_swap_credited', 'lockup_swap_debited', 'lockup_swap_credited', 'lockup_swap_rebate', 'dynamic_coin_swap_bonus_exchange_deposit', # we don't handle cryto.com exchange yet 'crypto_to_exchange_transfer', 'exchange_to_crypto_transfer', # supercharger actions 'supercharger_deposit', 'supercharger_withdrawal', # already handled using _import_cryptocom_double_entries 'dynamic_coin_swap_debited', 'dynamic_coin_swap_credited', 'dust_conversion_debited', 'dust_conversion_credited', ): # those types are ignored because it doesn't affect the wallet balance # or are not handled here return else: raise UnsupportedCryptocomEntry( f'Unknown entrype type "{row_type}" encountered during ' f'cryptocom data import. Ignoring entry', )
def _deserialize_asset_movement( self, movement_type: AssetMovementCategory, movement_data: Dict[str, Any], ) -> Optional[AssetMovement]: """Processes a single deposit/withdrawal from polo and deserializes it Can log error/warning and return None if something went wrong at deserialization """ try: if movement_type == AssetMovementCategory.DEPOSIT: fee = Fee(ZERO) uid_key = 'depositNumber' transaction_id = get_key_if_has_val(movement_data, 'txid') else: fee = deserialize_fee(movement_data['fee']) uid_key = 'withdrawalNumber' split = movement_data['status'].split(':') if len(split) != 2: transaction_id = None else: transaction_id = split[1].lstrip() if transaction_id == '': transaction_id = None asset = asset_from_poloniex(movement_data['currency']) return AssetMovement( location=Location.POLONIEX, category=movement_type, address=deserialize_asset_movement_address(movement_data, 'address', asset), transaction_id=transaction_id, timestamp=deserialize_timestamp(movement_data['timestamp']), asset=asset, amount=deserialize_asset_amount_force_positive(movement_data['amount']), fee_asset=asset, fee=fee, link=str(movement_data[uid_key]), ) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found {str(movement_type)} of unsupported poloniex asset ' f'{e.asset_name}. Ignoring it.', ) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found {str(movement_type)} of unknown poloniex asset ' f'{e.asset_name}. Ignoring it.', ) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Unexpected data encountered during deserialization of a poloniex ' 'asset movement. Check logs for details and open a bug report.', ) log.error( f'Unexpected data encountered during deserialization of poloniex ' f'{str(movement_type)}: {movement_data}. Error was: {str(e)}', ) return None
def test_add_trades(data_dir, username, caplog): """Test that adding and retrieving trades from the DB works fine. Also duplicates should be ignored and an error returned """ msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) trade1 = Trade( timestamp=1451606400, location=Location.KRAKEN, base_asset=A_ETH, quote_asset=A_EUR, trade_type=TradeType.BUY, amount=FVal('1.1'), rate=FVal('10'), fee=Fee(FVal('0.01')), fee_currency=A_EUR, link='', notes='', ) trade2 = Trade( timestamp=1451607500, location=Location.BINANCE, base_asset=A_BTC, quote_asset=A_ETH, trade_type=TradeType.BUY, amount=FVal('0.00120'), rate=FVal('10'), fee=Fee(FVal('0.001')), fee_currency=A_ETH, link='', notes='', ) trade3 = Trade( timestamp=1451608600, location=Location.COINBASE, base_asset=A_BTC, quote_asset=A_ETH, trade_type=TradeType.SELL, amount=FVal('0.00120'), rate=FVal('1'), fee=Fee(FVal('0.001')), fee_currency=A_ETH, link='', notes='', ) # Add and retrieve the first 2 trades. All should be fine. data.db.add_trades([trade1, trade2]) errors = msg_aggregator.consume_errors() warnings = msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 0 returned_trades = data.db.get_trades(filter_query=TradesFilterQuery.make(), has_premium=True) assert returned_trades == [trade1, trade2] # Add the last 2 trades. Since trade2 already exists in the DB it should be # ignored and a warning should be logged data.db.add_trades([trade2, trade3]) assert 'Did not add "buy trade with id a1ed19c8284940b4e59bdac941db2fd3c0ed004ddb10fdd3b9ef0a3a9b2c97bc' in caplog.text # noqa: E501 returned_trades = data.db.get_trades(filter_query=TradesFilterQuery.make(), has_premium=True) assert returned_trades == [trade1, trade2, trade3]
def assert_cryptocom_import_results(rotki: Rotkehlchen): """A utility function to help assert on correctness of importing data from crypto.com""" trades = rotki.data.db.get_trades() asset_movements = rotki.data.db.get_asset_movements() warnings = rotki.msg_aggregator.consume_warnings() errors = rotki.msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 0 def get_trade_note(desc: str): return f'{desc}\nSource: crypto.com (CSV import)' expected_trades = [Trade( timestamp=Timestamp(1595833195), location=Location.CRYPTOCOM, pair=TradePair('ETH_EUR'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('1.0')), rate=Price(FVal('281.14')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('Buy ETH'), ), Trade( timestamp=Timestamp(1596014214), location=Location.CRYPTOCOM, pair=TradePair('MCO_EUR'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('50.0')), rate=Price(FVal('3.521')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('Buy MCO'), ), Trade( timestamp=Timestamp(1596014223), location=Location.CRYPTOCOM, pair=TradePair('MCO_USD'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('12.32402069')), rate=Price(FVal('4.057117499045678736198226879')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('Sign-up Bonus Unlocked'), ), Trade( timestamp=Timestamp(1596209827), location=Location.CRYPTOCOM, pair=TradePair('ETH_MCO'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('0.14445954600007045')), rate=Price(FVal('85.28339137929999991192917299')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('MCO -> ETH'), ), Trade( timestamp=Timestamp(1596429934), location=Location.CRYPTOCOM, pair=TradePair('ETH_EUR'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('0.00061475')), rate=Price(FVal('309.0687271248474989833265555')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('Crypto Earn'), ), Trade( timestamp=Timestamp(1596465565), location=Location.CRYPTOCOM, pair=TradePair('CRO_MCO'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('1382.306147552291')), rate=Price(FVal('27.6439')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('MCO/CRO Overall Swap'), ), Trade( timestamp=Timestamp(1596730165), location=Location.CRYPTOCOM, pair=TradePair('CRO_MCO'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('1301.64')), rate=Price(FVal('26.0328')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('MCO/CRO Overall Swap'), ), Trade( timestamp=Timestamp(1599934176), location=Location.CRYPTOCOM, pair=TradePair('CRO_EUR'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('138.256')), rate=Price(FVal('0.1429232727693553986807082514')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('Card Rebate: Deliveries'), ), Trade( timestamp=Timestamp(1602515376), location=Location.CRYPTOCOM, pair=TradePair('CRO_EUR'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('52.151')), rate=Price(FVal('0.06692105616383194953116910510')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('Card Cashback'), ), Trade( timestamp=Timestamp(1602526176), location=Location.CRYPTOCOM, pair=TradePair('CRO_EUR'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('482.2566417')), rate=Price(FVal('0.08756748243245604635910191136')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('Referral Bonus Reward'), ), Trade( timestamp=Timestamp(1606833565), location=Location.CRYPTOCOM, pair=TradePair('CRO_DAI'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('0.007231228760408149')), rate=Price(FVal('14.26830000900286970270179629')), fee=Fee(ZERO), fee_currency=A_USD, link='', notes=get_trade_note('Convert Dust'), )] assert expected_trades == trades expected_movements = [AssetMovement( location=Location.CRYPTOCOM, category=AssetMovementCategory.DEPOSIT, timestamp=Timestamp(1596992965), address=None, transaction_id=None, asset=A_DAI, amount=AssetAmount(FVal('115')), fee_asset=A_DAI, fee=Fee(ZERO), link='', ), AssetMovement( location=Location.CRYPTOCOM, category=AssetMovementCategory.WITHDRAWAL, address=None, transaction_id=None, timestamp=Timestamp(1596993025), asset=A_DAI, amount=AssetAmount(FVal('115')), fee_asset=A_DAI, fee=Fee(ZERO), link='', )] assert expected_movements == asset_movements
def trade_from_conversion(trade_a: Dict[str, Any], trade_b: Dict[str, Any]) -> Optional[Trade]: """Turn information from a conversion into a trade Mary raise: - UnknownAsset due to Asset instantiation - DeserializationError due to unexpected format of dict entries - KeyError due to dict entires missing an expected entry """ # Check that the status is complete if trade_a['status'] != 'completed': return None # Trade b will represent the asset we are converting to if trade_b['amount']['amount'].startswith('-'): trade_a, trade_b = trade_b, trade_a timestamp = deserialize_timestamp_from_date(trade_a['updated_at'], 'iso8601', 'coinbase') tx_amount = AssetAmount( abs(deserialize_asset_amount(trade_a['amount']['amount']))) tx_asset = asset_from_coinbase(trade_a['amount']['currency'], time=timestamp) native_amount = deserialize_asset_amount(trade_b['amount']['amount']) native_asset = asset_from_coinbase(trade_b['amount']['currency'], time=timestamp) amount = tx_amount # The rate is how much you get/give in quotecurrency if you buy/sell 1 unit of base currency rate = Price(native_amount / tx_amount) # Obtain fee amount in the native currency using data from both trades amount_after_fee = deserialize_asset_amount( trade_b['native_amount']['amount']) amount_before_fee = deserialize_asset_amount( trade_a['native_amount']['amount']) # amount_after_fee + amount_before_fee is a negative amount and the fee needs to be positive conversion_native_fee_amount = abs(amount_after_fee + amount_before_fee) if ZERO not in (tx_amount, conversion_native_fee_amount, amount_before_fee, amount_after_fee): # To get the asset in which the fee is nominated we pay attention to the creation # date of each event. As per our hypothesis the fee is nominated in the asset # for which the first transaction part was initialized time_created_a = deserialize_timestamp_from_date( date=trade_a['created_at'], formatstr='iso8601', location='coinbase', ) time_created_b = deserialize_timestamp_from_date( date=trade_b['created_at'], formatstr='iso8601', location='coinbase', ) if time_created_a < time_created_b: # We have the fee amount in the native currency. To get it in the # converted asset we have to get the rate asset_native_rate = tx_amount / abs(amount_before_fee) fee_amount = Fee(conversion_native_fee_amount * asset_native_rate) fee_asset = asset_from_coinbase(trade_a['amount']['currency'], time=timestamp) else: trade_b_amount = abs( deserialize_asset_amount(trade_b['amount']['amount'])) asset_native_rate = trade_b_amount / abs(amount_after_fee) fee_amount = Fee(conversion_native_fee_amount * asset_native_rate) fee_asset = asset_from_coinbase(trade_b['amount']['currency'], time=timestamp) else: fee_amount = Fee(ZERO) fee_asset = asset_from_coinbase(trade_a['amount']['currency'], time=timestamp) return Trade( timestamp=timestamp, location=Location.COINBASE, # in coinbase you are buying/selling tx_asset for native_asset base_asset=tx_asset, quote_asset=native_asset, trade_type=TradeType.SELL, amount=amount, rate=rate, fee=fee_amount, fee_currency=fee_asset, link=str(trade_a['trade']['id']), )
def assert_poloniex_asset_movements( to_check_list: List[Any], deserialized: bool, movements_to_check: Optional[Tuple[int, ...]] = None, ) -> None: expected = [ AssetMovement( location=Location.POLONIEX, category=AssetMovementCategory.WITHDRAWAL, address='0xB7E033598Cb94EF5A35349316D3A2e4f95f308Da', transaction_id= '0xbd4da74e1a0b81c21d056c6f58a5b306de85d21ddf89992693b812bb117eace4', timestamp=Timestamp(1468994442), asset=A_ETH, amount=FVal('10.0'), fee_asset=A_ETH, fee=Fee(FVal('0.1')), link='2', ), AssetMovement( location=Location.POLONIEX, category=AssetMovementCategory.WITHDRAWAL, address='131rdg5Rzn6BFufnnQaHhVa5ZtRU1J2EZR', transaction_id= '2d27ae26fa9c70d6709e27ac94d4ce2fde19b3986926e9f3bfcf3e2d68354ec5', timestamp=Timestamp(1458994442), asset=A_BTC, amount=FVal('5.0'), fee_asset=A_BTC, fee=Fee(FVal('0.5')), link='1', ), AssetMovement( location=Location.POLONIEX, category=AssetMovementCategory.DEPOSIT, address='131rdg5Rzn6BFufnnQaHhVa5ZtRU1J2EZR', transaction_id= 'b05bdec7430a56b5a5ed34af4a31a54859dda9b7c88a5586bc5d6540cdfbfc7a', timestamp=Timestamp(1448994442), asset=A_BTC, amount=FVal('50.0'), fee_asset=A_BTC, fee=Fee(FVal('0')), link='1', ), AssetMovement( location=Location.POLONIEX, category=AssetMovementCategory.DEPOSIT, address='0xB7E033598Cb94EF5A35349316D3A2e4f95f308Da', transaction_id= '0xf7e7eeb44edcad14c0f90a5fffb1cbb4b80e8f9652124a0838f6906ca939ccd2', timestamp=Timestamp(1438994442), asset=A_ETH, amount=FVal('100.0'), fee_asset=A_ETH, fee=Fee(FVal('0')), link='2', ) ] assert_asset_movements(expected, to_check_list, deserialized, movements_to_check)
def test_query_asset_movements_sandbox( sandbox_kuckoin, inquirer, # pylint: disable=unused-argument ): """Unfortunately the sandbox environment does not support deposits and withdrawals, therefore they must be mocked. Below a list of the movements and their timestamps in ascending mode: Deposits: - deposit 1 - deposit: 1612556651 - deposit 2 - deposit: 1612556652 - deposit 3 - deposit: 1612556653 -> skipped, inner deposit Withdrawals: - withdraw 1: 1612556651 -> skipped, inner withdraw - withdraw 2: 1612556652 - withdraw 3: 1612556656 -> never requested By requesting trades from 1612556651 to 1612556654 and patching the time step as 2s (via MONTHS_IN_SECONDS) we should get back 3 movements. """ deposits_response_1 = (""" { "code":"200000", "data":{ "currentPage":1, "pageSize":2, "totalNum":2, "totalPage":1, "items":[ { "address":"0x5f047b29041bcfdbf0e4478cdfa753a336ba6989", "memo":"5c247c8a03aa677cea2a251d", "amount":1, "fee":0.0001, "currency":"KCS", "isInner":false, "walletTxId":"5bbb57386d99522d9f954c5a", "status":"SUCCESS", "remark":"movement 2 - deposit", "createdAt":1612556652000, "updatedAt":1612556652000 }, { "address":"0x5f047b29041bcfdbf0e4478cdfa753a336ba6989", "memo":"5c247c8a03aa677cea2a251d", "amount":1000, "fee":0.01, "currency":"LINK", "isInner":false, "walletTxId":"5bbb57386d99522d9f954c5b@test", "status":"SUCCESS", "remark":"movement 1 - deposit", "createdAt":1612556651000, "updatedAt":1612556651000 } ] } } """) deposits_response_2 = (""" { "code":"200000", "data":{ "currentPage":1, "pageSize":1, "totalNum":1, "totalPage":1, "items":[ { "address":"1DrT5xUaJ3CBZPDeFR2qdjppM6dzs4rsMt", "memo":"", "currency":"BCHSV", "amount":1, "fee":0.1, "walletTxId":"b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada6", "isInner":true, "status":"SUCCESS", "remark":"movement 4 - deposit", "createdAt":1612556653000, "updatedAt":1612556653000 } ] } } """) withdrawals_response_1 = (""" { "code":"200000", "data":{ "currentPage":1, "pageSize":2, "totalNum":2, "totalPage":1, "items":[ { "id":"5c2dc64e03aa675aa263f1a4", "address":"1DrT5xUaJ3CBZPDeFR2qdjppM6dzs4rsMt", "memo":"", "currency":"BCHSV", "amount":2.5, "fee":0.25, "walletTxId":"b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada4", "isInner":false, "status":"SUCCESS", "remark":"movement 4 - withdraw", "createdAt":1612556652000, "updatedAt":1612556652000 }, { "id":"5c2dc64e03aa675aa263f1a3", "address":"0x5bedb060b8eb8d823e2414d82acce78d38be7fe9", "memo":"", "currency":"ETH", "amount":1, "fee":0.01, "walletTxId":"3e2414d82acce78d38be7fe9", "isInner":true, "status":"SUCCESS", "remark":"movement 3 - withdraw", "createdAt":1612556651000, "updatedAt":1612556651000 } ] } } """) withdrawals_response_2 = (""" { "code":"200000", "data":{ "currentPage":0, "pageSize":0, "totalNum":0, "totalPage":0, "items":[] } } """) withdrawals_response_3 = (""" { "code":"200000", "data":{ "currentPage":1, "pageSize":1, "totalNum":1, "totalPage":1, "items":[ { "id":"5c2dc64e03aa675aa263f1a5", "address":"0x5bedb060b8eb8d823e2414d82acce78d38be7f00", "memo":"", "currency":"KCS", "amount":2.5, "fee":0.25, "walletTxId":"b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada5", "isInner":false, "status":"SUCCESS", "remark":"movement 5 - withdraw", "createdAt":1612556655000, "updatedAt":1612556655000 } ] } } """) expected_asset_movements = [ AssetMovement( location=Location.KUCOIN, category=AssetMovementCategory.DEPOSIT, timestamp=Timestamp(1612556652), address='0x5f047b29041bcfdbf0e4478cdfa753a336ba6989', transaction_id='5bbb57386d99522d9f954c5a', asset=Asset('KCS'), amount=AssetAmount(FVal('1')), fee_asset=Asset('KCS'), fee=Fee(FVal('0.0001')), link='', ), AssetMovement( location=Location.KUCOIN, category=AssetMovementCategory.DEPOSIT, timestamp=Timestamp(1612556651), address='0x5f047b29041bcfdbf0e4478cdfa753a336ba6989', transaction_id='5bbb57386d99522d9f954c5b', asset=Asset('LINK'), amount=AssetAmount(FVal('1000')), fee_asset=Asset('LINK'), fee=Fee(FVal('0.01')), link='', ), AssetMovement( location=Location.KUCOIN, category=AssetMovementCategory.WITHDRAWAL, timestamp=Timestamp(1612556652), address='1DrT5xUaJ3CBZPDeFR2qdjppM6dzs4rsMt', transaction_id= 'b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada4', asset=Asset('BSV'), amount=AssetAmount(FVal('2.5')), fee_asset=Asset('BSV'), fee=Fee(FVal('0.25')), link='5c2dc64e03aa675aa263f1a4', ), ] def get_endpoints_response(): results = [ f'{deposits_response_1}', f'{deposits_response_2}', f'{withdrawals_response_1}', f'{withdrawals_response_2}', # if pagination works as expected and the requesting loop is broken, # the response below won't be processed f'{withdrawals_response_3}', ] for result_ in results: yield result_ def mock_api_query_response(case, options): # pylint: disable=unused-argument return MockResponse(HTTPStatus.OK, next(get_response))
def test_query_trades_sandbox(sandbox_kuckoin, inquirer): # pylint: disable=unused-argument """The sandbox account has 6 trades. Below a list of the trades and their timestamps in ascending mode. - trade 1: 1612556651 -> skipped - trade 2: 1612556693 - trade 3: 1612556765 - trade 4: 1612556765 - trade 5: 1612556765 - trade 6: 1612556794 -> skipped By requesting trades from 1612556693 to 1612556765, the first and last trade should be skipped. """ expected_trades = [ Trade( timestamp=Timestamp(1612556765), location=Location.KUCOIN, pair=TradePair('ETH_BTC'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('0.02934995')), rate=Price(FVal('0.046058')), fee=Fee(FVal('9.4625999797E-7')), fee_currency=Asset('BTC'), link='601da9ddf73c300006194ec6', notes='', ), Trade( timestamp=Timestamp(1612556765), location=Location.KUCOIN, pair=TradePair('ETH_BTC'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('0.02')), rate=Price(FVal('0.04561')), fee=Fee(FVal('6.3854E-7')), fee_currency=Asset('BTC'), link='601da9ddf73c300006194ec5', notes='', ), Trade( timestamp=Timestamp(1612556765), location=Location.KUCOIN, pair=TradePair('ETH_BTC'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('0.06')), rate=Price(FVal('0.0456')), fee=Fee(FVal('0.0000019152')), fee_currency=Asset('BTC'), link='601da9ddf73c300006194ec4', notes='', ), Trade( timestamp=Timestamp(1612556693), location=Location.KUCOIN, pair=TradePair('BTC_USDT'), trade_type=TradeType.SELL, amount=AssetAmount(FVal('0.0013')), rate=Price(FVal('37624.4')), fee=Fee(FVal('0.034238204')), fee_currency=Asset('USDT'), link='601da995e0ee8b00063a075c', notes='', ), ] trades = sandbox_kuckoin.query_online_trade_history( start_ts=Timestamp(1612556693), end_ts=Timestamp(1612556765), ) assert trades == expected_trades
def _consume_nexo(self, csv_row: Dict[str, Any]) -> None: """ Consume CSV file from NEXO. This method can raise: - UnsupportedNexoEntry - UnknownAsset - DeserializationError """ ignored_entries = ('ExchangeToWithdraw', 'DepositToExchange') if 'rejected' not in csv_row['Details']: timestamp = deserialize_timestamp_from_date( date=csv_row['Date / Time'], formatstr='%Y-%m-%d %H:%M', location='NEXO', ) else: log.debug(f'Ignoring rejected nexo entry {csv_row}') return asset = symbol_to_asset_or_token(csv_row['Currency']) amount = deserialize_asset_amount_force_positive(csv_row['Amount']) entry_type = csv_row['Type'] transaction = csv_row['Transaction'] if entry_type in ('Deposit', 'ExchangeDepositedOn', 'LockingTermDeposit'): asset_movement = AssetMovement( location=Location.NEXO, category=AssetMovementCategory.DEPOSIT, address=None, transaction_id=None, timestamp=timestamp, asset=asset, amount=amount, fee=Fee(ZERO), fee_asset=A_USD, link=transaction, ) self.db.add_asset_movements([asset_movement]) elif entry_type in ('Withdrawal', 'WithdrawExchanged'): asset_movement = AssetMovement( location=Location.NEXO, category=AssetMovementCategory.WITHDRAWAL, address=None, transaction_id=None, timestamp=timestamp, asset=asset, amount=amount, fee=Fee(ZERO), fee_asset=A_USD, link=transaction, ) self.db.add_asset_movements([asset_movement]) elif entry_type == 'Withdrawal Fee': action = LedgerAction( identifier=0, # whatever is not used at insertion timestamp=timestamp, action_type=LedgerActionType.EXPENSE, location=Location.NEXO, amount=amount, asset=asset, rate=None, rate_asset=None, link=None, notes=f'{entry_type} from Nexo', ) self.db_ledger.add_ledger_action(action) elif entry_type in ('Interest', 'Bonus', 'Dividend'): action = LedgerAction( identifier=0, # whatever is not used at insertion timestamp=timestamp, action_type=LedgerActionType.INCOME, location=Location.NEXO, amount=amount, asset=asset, rate=None, rate_asset=None, link=transaction, notes=f'{entry_type} from Nexo', ) self.db_ledger.add_ledger_action(action) elif entry_type in ignored_entries: pass else: raise UnsupportedCSVEntry( f'Unsuported entry {entry_type}. Data: {csv_row}')
def query_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, end_at_least_ts: Timestamp, ) -> List[AssetMovement]: with self.lock: cache = self.check_trades_cache_dict( start_ts, end_at_least_ts, special_name='deposits_withdrawals', ) if cache is None: result = self.returnDepositsWithdrawals(start_ts, end_ts) with self.lock: self.update_trades_cache( result, start_ts, end_ts, special_name='deposits_withdrawals', ) else: result = cache log.debug( 'Poloniex deposits/withdrawal query', results_num=len(result['withdrawals']) + len(result['deposits']), ) movements = list() for withdrawal in result['withdrawals']: try: movements.append(AssetMovement( exchange='poloniex', category='withdrawal', timestamp=withdrawal['timestamp'], asset=asset_from_poloniex(withdrawal['currency']), amount=FVal(withdrawal['amount']), fee=Fee(FVal(withdrawal['fee'])), )) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found withdrawal of unsupported poloniex asset {e.asset_name}. Ignoring it.', ) continue except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found withdrawal of unknown poloniex asset {e.asset_name}. Ignoring it.', ) continue for deposit in result['deposits']: try: movements.append(AssetMovement( exchange='poloniex', category='deposit', timestamp=deposit['timestamp'], asset=asset_from_poloniex(deposit['currency']), amount=FVal(deposit['amount']), fee=Fee(ZERO), )) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found deposit of unsupported poloniex asset {e.asset_name}. Ignoring it.', ) continue except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found deposit of unknown poloniex asset {e.asset_name}. Ignoring it.', ) continue return movements
def assert_cointracking_import_results(rotki: Rotkehlchen): """A utility function to help assert on correctness of importing data from cointracking.info""" trades = rotki.data.db.get_trades() asset_movements = rotki.data.db.get_asset_movements() warnings = rotki.msg_aggregator.consume_warnings() errors = rotki.msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 3 expected_trades = [Trade( timestamp=Timestamp(1566687719), location=Location.COINBASE, pair=TradePair('ETH_EUR'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('0.05772716')), rate=Price(FVal('190.3783245183029963712055123')), fee=Fee(FVal("0.02")), fee_currency=A_EUR, link='', notes='', ), Trade( timestamp=Timestamp(1567418410), location=Location.EXTERNAL, pair=TradePair('BTC_USD'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('0.00100000')), rate=Price(ZERO), fee=Fee(ZERO), fee_currency=A_USD, link='', notes='Just a small gift from someone', ), Trade( timestamp=Timestamp(1567504805), location=Location.EXTERNAL, pair=TradePair('ETH_USD'), trade_type=TradeType.BUY, amount=AssetAmount(FVal('2')), rate=Price(ZERO), fee=Fee(ZERO), fee_currency=A_USD, link='', notes='Sign up bonus', )] assert expected_trades == trades expected_movements = [AssetMovement( location=Location.POLONIEX, category=AssetMovementCategory.DEPOSIT, timestamp=Timestamp(1565848624), address=None, transaction_id=None, asset=A_XMR, amount=AssetAmount(FVal('5')), fee_asset=A_USD, fee=Fee(ZERO), link='', ), AssetMovement( location=Location.COINBASE, category=AssetMovementCategory.WITHDRAWAL, address=None, transaction_id=None, timestamp=Timestamp(1566726155), asset=A_ETH, amount=AssetAmount(FVal('0.05770427')), fee_asset=A_ETH, fee=Fee(FVal("0.0001")), link='', )] assert expected_movements == asset_movements
def test_measure_trades_api_query(rotkehlchen_api_server, start_with_valid_premium): """Measures the response time of the combined trades view API query. This is required since it's quite a complicated query and takes a lot of time to process so we can use this test to measure any potential optimizations. """ trades = [ Trade( timestamp=x, location=Location.EXTERNAL, base_asset=A_WETH, quote_asset=A_EUR, trade_type=TradeType.BUY, amount=AssetAmount(FVal('1')), rate=Price(FVal('320')), fee=Fee(ZERO), fee_currency=A_EUR, link='', notes='', ) for x in range(1, 10000) ] rotki = rotkehlchen_api_server.rest_api.rotkehlchen rotki.data.db.add_trades(trades) swaps = [ AMMSwap( tx_hash='0x' + str(x), log_index=x + i, address='0xfoo', from_address='0xfrom', to_address='0xto', timestamp=11 + x, location=Location.UNISWAP, token0=A_WETH, token1=A_EUR, amount0_in=FVal(5), amount1_in=ZERO, amount0_out=ZERO, amount1_out=FVal(4.95), ) for x in range(2000) for i in range(2) ] rotki.data.db.add_amm_swaps(swaps) start = time.time() requests.get( api_url_for( rotkehlchen_api_server, 'tradesresource', ), json={'only_cache': True}, ) end = time.time() test_warnings.warn( UserWarning( f'Premium: {start_with_valid_premium}. Full Query Time: {end - start}', )) start = time.time() requests.get( api_url_for( rotkehlchen_api_server, 'tradesresource', ), json={ 'only_cache': True, 'offset': 200, 'limit': 10 }, ) end = time.time() test_warnings.warn( UserWarning( f'Premium: {start_with_valid_premium}. First Page Query Time: {end - start}', )) start = time.time() requests.get( api_url_for( rotkehlchen_api_server, 'tradesresource', ), json={ 'only_cache': True, 'offset': 210, 'limit': 10 }, ) end = time.time() test_warnings.warn( UserWarning( f'Premium: {start_with_valid_premium}. Second Page Query Time: {end - start}', )) start = time.time() requests.get( api_url_for( rotkehlchen_api_server, 'tradesresource', ), json={ 'only_cache': True, 'offset': 220, 'limit': 10 }, ) end = time.time() test_warnings.warn( UserWarning( f'Premium: {start_with_valid_premium}. Third Page Query Time: {end - start}', ))
def test_query_owned_assets(data_dir, username): """Test the get_owned_assets with also an unknown asset in the DB""" msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) balances = deepcopy(asset_balances) balances.extend([ DBAssetBalance( category=BalanceType.ASSET, time=Timestamp(1488326400), asset=A_BTC, amount='1', usd_value='1222.66', ), DBAssetBalance( category=BalanceType.ASSET, time=Timestamp(1489326500), asset=A_XMR, amount='2', usd_value='33.8', ), ]) data.db.add_multiple_balances(balances) data.db.conn.commit() # also make sure that assets from trades are included data.db.add_trades([ Trade( timestamp=Timestamp(1), location=Location.EXTERNAL, base_asset=A_ETH, quote_asset=A_BTC, trade_type=TradeType.BUY, amount=AssetAmount(FVal(1)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), Trade( timestamp=Timestamp(99), location=Location.EXTERNAL, base_asset=A_ETH, quote_asset=A_BTC, trade_type=TradeType.BUY, amount=AssetAmount(FVal(2)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), Trade( timestamp=Timestamp(1), location=Location.EXTERNAL, base_asset=A_SDC, quote_asset=A_SDT2, trade_type=TradeType.BUY, amount=AssetAmount(FVal(1)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), Trade( timestamp=Timestamp(1), location=Location.EXTERNAL, base_asset=A_SUSHI, quote_asset=A_1INCH, trade_type=TradeType.BUY, amount=AssetAmount(FVal(1)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), Trade( timestamp=Timestamp(3), location=Location.EXTERNAL, base_asset=A_SUSHI, quote_asset=A_1INCH, trade_type=TradeType.BUY, amount=AssetAmount(FVal(2)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), ]) assets_list = data.db.query_owned_assets() assert set(assets_list) == { A_USD, A_ETH, A_BTC, A_XMR, A_SDC, A_SDT2, A_SUSHI, A_1INCH } # noqa: E501 assert all(isinstance(x, Asset) for x in assets_list) warnings = data.db.msg_aggregator.consume_warnings() assert len(warnings) == 0
def test_query_trades_associated_locations( rotkehlchen_api_server_with_exchanges): """Test that querying the trades endpoint works as expected when we have associated locations including associated exchanges and imported locations. """ rotki = rotkehlchen_api_server_with_exchanges.rest_api.rotkehlchen setup = mock_history_processing_and_exchanges(rotki) trades = [ Trade( timestamp=Timestamp(1596429934), location=Location.EXTERNAL, base_asset=A_WETH, quote_asset=A_EUR, trade_type=TradeType.BUY, amount=AssetAmount(FVal('1')), rate=Price(FVal('320')), fee=Fee(ZERO), fee_currency=A_EUR, link='', notes='', ), Trade( timestamp=Timestamp(1596429934), location=Location.KRAKEN, base_asset=A_WETH, quote_asset=A_EUR, trade_type=TradeType.BUY, amount=AssetAmount(FVal('1')), rate=Price(FVal('320')), fee=Fee(ZERO), fee_currency=A_EUR, link='', notes='', ), Trade( timestamp=Timestamp(1596429934), location=Location.BISQ, base_asset=A_WETH, quote_asset=A_EUR, trade_type=TradeType.BUY, amount=AssetAmount(FVal('1')), rate=Price(FVal('320')), fee=Fee(ZERO), fee_currency=A_EUR, link='', notes='', ), Trade( timestamp=Timestamp(1596429934), location=Location.BINANCE, base_asset=A_WETH, quote_asset=A_EUR, trade_type=TradeType.BUY, amount=AssetAmount(FVal('1')), rate=Price(FVal('320')), fee=Fee(ZERO), fee_currency=A_EUR, link='', notes='', ) ] # Add multiple entries for same exchange + connected exchange rotki.data.db.add_trades(trades) # Simply get all trades without any filtering with setup.binance_patch, setup.polo_patch: response = requests.get( api_url_for( rotkehlchen_api_server_with_exchanges, 'tradesresource', ), ) result = assert_proper_response_with_result(response) result = result['entries'] assert len( result ) == 9 # 3 polo, (2 + 1) binance trades, 1 kraken, 1 external, 1 BISQ expected_locations = ( Location.KRAKEN, Location.POLONIEX, Location.BINANCE, Location.BISQ, Location.EXTERNAL, ) returned_locations = {x['entry']['location'] for x in result} assert returned_locations == set(map(str, expected_locations)) response = requests.get( api_url_for( rotkehlchen_api_server_with_exchanges, 'tradesresource', ), json={ 'location': 'kraken', 'only_cache': True }, ) result = assert_proper_response_with_result(response) result = result['entries'] assert len(result) == 1 response = requests.get( api_url_for( rotkehlchen_api_server_with_exchanges, 'tradesresource', ), json={ 'location': 'binance', 'only_cache': True }, ) result = assert_proper_response_with_result(response) result = result['entries'] assert len(result) == 3 response = requests.get( api_url_for( rotkehlchen_api_server_with_exchanges, 'tradesresource', ), json={'location': 'nexo'}, ) result = assert_proper_response_with_result(response) result = result['entries'] assert len(result) == 0
def test_add_asset_movements(data_dir, username, caplog): """Test that adding and retrieving asset movements from the DB works fine. Also duplicates should be ignored and an error returned """ msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) movement1 = AssetMovement( location=Location.BITMEX, category=AssetMovementCategory.DEPOSIT, address=None, transaction_id=None, timestamp=1451606400, asset=A_BTC, amount=FVal('1.0'), fee_asset=A_EUR, fee=Fee(FVal('0')), link='', ) movement2 = AssetMovement( location=Location.POLONIEX, category=AssetMovementCategory.WITHDRAWAL, address='0xfoo', transaction_id='0xboo', timestamp=1451608501, asset=A_ETH, amount=FVal('1.0'), fee_asset=A_EUR, fee=Fee(FVal('0.01')), link='', ) movement3 = AssetMovement( location=Location.BITTREX, category=AssetMovementCategory.WITHDRAWAL, address='0xcoo', transaction_id='0xdoo', timestamp=1461708501, asset=A_ETH, amount=FVal('1.0'), fee_asset=A_EUR, fee=Fee(FVal('0.01')), link='', ) # Add and retrieve the first 2 margins. All should be fine. data.db.add_asset_movements([movement1, movement2]) errors = msg_aggregator.consume_errors() warnings = msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 0 returned_movements = data.db.get_asset_movements( filter_query=AssetMovementsFilterQuery.make(), has_premium=True, ) assert returned_movements == [movement1, movement2] # Add the last 2 movements. Since movement2 already exists in the DB it should be # ignored and a warning should be logged data.db.add_asset_movements([movement2, movement3]) assert ('Did not add "withdrawal of ETH with id 94405f38c7b86dd2e7943164d' '67ff44a32d56cef25840b3f5568e23c037fae0a') in caplog.text returned_movements = data.db.get_asset_movements( filter_query=AssetMovementsFilterQuery.make(), has_premium=True, ) assert returned_movements == [movement1, movement2, movement3]
def _deserialize_asset_movement( self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]: """Processes a single deposit/withdrawal from coinbase and deserializes it Can log error/warning and return None if something went wrong at deserialization """ try: if raw_data['status'] != 'completed': return None payout_date = raw_data.get('payout_at', None) if payout_date: timestamp = deserialize_timestamp_from_date( payout_date, 'iso8601', 'coinbase') else: timestamp = deserialize_timestamp_from_date( raw_data['created_at'], 'iso8601', 'coinbase', ) # Only get address/transaction id for "send" type of transactions address = None transaction_id = None # movement_category: Union[Literal['deposit'], Literal['withdrawal']] if 'type' in raw_data: # Then this should be a "send" which is the way Coinbase uses to send # crypto outside of the exchange # https://developers.coinbase.com/api/v2?python#transaction-resource msg = 'Non "send" type found in coinbase deposit/withdrawal processing' assert raw_data['type'] == 'send', msg movement_category = AssetMovementCategory.WITHDRAWAL # Can't see the fee being charged from the "send" resource amount = deserialize_asset_amount_force_positive( raw_data['amount']['amount']) asset = asset_from_coinbase(raw_data['amount']['currency'], time=timestamp) # Fees dont appear in the docs but from an experiment of sending ETH # to an address from coinbase there is the network fee in the response fee = Fee(ZERO) raw_network = raw_data.get('network', None) if raw_network: raw_fee = raw_network.get('transaction_fee', None) if raw_fee: # Since this is a withdrawal the fee should be the same as the moved asset if asset != asset_from_coinbase(raw_fee['currency'], time=timestamp): # If not we set ZERO fee and ignore log.error( f'In a coinbase withdrawal of {asset.identifier} the fee' f'is denoted in {raw_fee["currency"]}', ) else: fee = deserialize_fee(raw_fee['amount']) if 'network' in raw_data: transaction_id = get_key_if_has_val( raw_data['network'], 'hash') if 'to' in raw_data: address = deserialize_asset_movement_address( raw_data['to'], 'address', asset) else: movement_category = deserialize_asset_movement_category( raw_data['resource']) amount = deserialize_asset_amount_force_positive( raw_data['amount']['amount']) fee = deserialize_fee(raw_data['fee']['amount']) asset = asset_from_coinbase(raw_data['amount']['currency'], time=timestamp) return AssetMovement( location=Location.COINBASE, category=movement_category, address=address, transaction_id=transaction_id, timestamp=timestamp, asset=asset, amount=amount, fee_asset=asset, fee=fee, link=str(raw_data['id']), ) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found coinbase deposit/withdrawal with unknown asset ' f'{e.asset_name}. Ignoring it.', ) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found coinbase deposit/withdrawal with unsupported asset ' f'{e.asset_name}. Ignoring it.', ) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Unexpected data encountered during deserialization of a coinbase ' 'asset movement. Check logs for details and open a bug report.', ) log.error( f'Unexpected data encountered during deserialization of coinbase ' f'asset_movement {raw_data}. Error was: {str(e)}', ) return None
def trade_from_poloniex(poloniex_trade: Dict[str, Any], pair: TradePair) -> Trade: """Turn a poloniex trade returned from poloniex trade history to our common trade history format Throws: - UnsupportedAsset due to asset_from_poloniex() - DeserializationError due to the data being in unexpected format - UnprocessableTradePair due to the pair data being in an unexpected format """ try: trade_type = deserialize_trade_type(poloniex_trade['type']) amount = deserialize_asset_amount(poloniex_trade['amount']) rate = deserialize_price(poloniex_trade['rate']) perc_fee = deserialize_fee(poloniex_trade['fee']) base_currency = asset_from_poloniex(get_pair_position_str(pair, 'first')) quote_currency = asset_from_poloniex(get_pair_position_str(pair, 'second')) timestamp = deserialize_timestamp_from_poloniex_date(poloniex_trade['date']) except KeyError as e: raise DeserializationError( f'Poloniex trade deserialization error. Missing key entry for {str(e)} in trade dict', ) cost = rate * amount if trade_type == TradeType.BUY: fee = Fee(amount * perc_fee) fee_currency = quote_currency elif trade_type == TradeType.SELL: fee = Fee(cost * perc_fee) fee_currency = base_currency else: raise DeserializationError(f'Got unexpected trade type "{trade_type}" for poloniex trade') if poloniex_trade['category'] == 'settlement': if trade_type == TradeType.BUY: trade_type = TradeType.SETTLEMENT_BUY else: trade_type = TradeType.SETTLEMENT_SELL log.debug( 'Processing poloniex Trade', sensitive_log=True, timestamp=timestamp, order_type=trade_type, pair=pair, base_currency=base_currency, quote_currency=quote_currency, amount=amount, fee=fee, rate=rate, ) # Use the converted assets in our pair pair = trade_pair_from_assets(base_currency, quote_currency) # Since in Poloniex the base currency is the cost currency, iow in poloniex # for BTC_ETH we buy ETH with BTC and sell ETH for BTC, we need to turn it # into the Rotkehlchen way which is following the base/quote approach. pair = invert_pair(pair) return Trade( timestamp=timestamp, location=Location.POLONIEX, pair=pair, trade_type=trade_type, amount=amount, rate=rate, fee=fee, fee_currency=fee_currency, link=str(poloniex_trade['globalTradeID']), )
def test_query_owned_assets(data_dir, username): """Test the get_owned_assets with also an unknown asset in the DB""" msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) balances = deepcopy(asset_balances) balances.extend([ DBAssetBalance( category=BalanceType.ASSET, time=Timestamp(1488326400), asset=A_BTC, amount='1', usd_value='1222.66', ), DBAssetBalance( category=BalanceType.ASSET, time=Timestamp(1489326500), asset=A_XMR, amount='2', usd_value='33.8', ), ]) data.db.add_multiple_balances(balances) cursor = data.db.conn.cursor() cursor.execute( 'INSERT INTO timed_balances(' ' time, currency, amount, usd_value, category) ' ' VALUES(?, ?, ?, ?, ?)', (1469326500, 'ADSADX', '10.1', '100.5', 'A'), ) data.db.conn.commit() # also make sure that assets from trades are included data.db.add_trades([ Trade( timestamp=Timestamp(1), location=Location.EXTERNAL, pair=TradePair('ETH_BTC'), trade_type=TradeType.BUY, amount=AssetAmount(FVal(1)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), Trade( timestamp=Timestamp(99), location=Location.EXTERNAL, pair=TradePair('ETH_BTC'), trade_type=TradeType.BUY, amount=AssetAmount(FVal(2)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), Trade( timestamp=Timestamp(1), location=Location.EXTERNAL, pair=TradePair('SDC_SDT-2'), trade_type=TradeType.BUY, amount=AssetAmount(FVal(1)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), Trade( timestamp=Timestamp(1), location=Location.EXTERNAL, pair=TradePair('SUSHI_1INCH'), trade_type=TradeType.BUY, amount=AssetAmount(FVal(1)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), Trade( timestamp=Timestamp(3), location=Location.EXTERNAL, pair=TradePair('SUSHI_1INCH'), trade_type=TradeType.BUY, amount=AssetAmount(FVal(2)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), Trade( timestamp=Timestamp(1), location=Location.EXTERNAL, pair=TradePair('UNKNOWNTOKEN_BTC'), trade_type=TradeType.BUY, amount=AssetAmount(FVal(1)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), ]) assets_list = data.db.query_owned_assets() assert set(assets_list) == {A_USD, A_ETH, A_DAI, A_BTC, A_XMR, Asset('SDC'), Asset('SDT-2'), Asset('SUSHI'), Asset('1INCH')} # noqa: E501 assert all(isinstance(x, Asset) for x in assets_list) warnings = data.db.msg_aggregator.consume_warnings() assert len(warnings) == 1 assert 'Unknown/unsupported asset ADSADX' in warnings[0]
def _import_cryptocom_double_entries(self, data: Any, double_type: str) -> None: """Look for events that have double entries and handle them as trades. This method looks for `*_debited` and `*_credited` entries using the same timestamp to handle them as one trade. Known double_type: 'dynamic_coin_swap' or 'dust_conversion' """ double_rows: Dict[Any, Dict[str, Any]] = {} debited_row = None credited_row = None for row in data: if row['Transaction Kind'] == f'{double_type}_debited': timestamp = deserialize_timestamp_from_date( date=row['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='crypto.com', ) if timestamp not in double_rows: double_rows[timestamp] = {} double_rows[timestamp]['debited'] = row elif row['Transaction Kind'] == f'{double_type}_credited': timestamp = deserialize_timestamp_from_date( date=row['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='crypto.com', ) if timestamp not in double_rows: double_rows[timestamp] = {} double_rows[timestamp]['credited'] = row for timestamp in double_rows: credited_row = double_rows[timestamp]['credited'] debited_row = double_rows[timestamp]['debited'] if credited_row is not None and debited_row is not None: description = credited_row['Transaction Description'] notes = f'{description}\nSource: crypto.com (CSV import)' # No fees here fee = Fee(ZERO) fee_currency = A_USD base_asset = Asset(credited_row['Currency']) quote_asset = Asset(debited_row['Currency']) pair = TradePair( f'{base_asset.identifier}_{quote_asset.identifier}') base_amount_bought = deserialize_asset_amount( credited_row['Amount']) quote_amount_sold = deserialize_asset_amount( debited_row['Amount']) rate = Price(abs(base_amount_bought / quote_amount_sold)) trade = Trade( timestamp=timestamp, location=Location.CRYPTOCOM, pair=pair, trade_type=TradeType.BUY, amount=base_amount_bought, rate=rate, fee=fee, fee_currency=fee_currency, link='', notes=notes, ) self.db.add_trades([trade])
def test_add_trades(data_dir, username): """Test that adding and retrieving trades from the DB works fine. Also duplicates should be ignored and an error returned """ msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) trade1 = Trade( timestamp=1451606400, location=Location.KRAKEN, pair='ETH_EUR', trade_type=TradeType.BUY, amount=FVal('1.1'), rate=FVal('10'), fee=Fee(FVal('0.01')), fee_currency=A_EUR, link='', notes='', ) trade2 = Trade( timestamp=1451607500, location=Location.BINANCE, pair='BTC_ETH', trade_type=TradeType.BUY, amount=FVal('0.00120'), rate=FVal('10'), fee=Fee(FVal('0.001')), fee_currency=A_ETH, link='', notes='', ) trade3 = Trade( timestamp=1451608600, location=Location.COINBASE, pair='BTC_ETH', trade_type=TradeType.SELL, amount=FVal('0.00120'), rate=FVal('1'), fee=Fee(FVal('0.001')), fee_currency=A_ETH, link='', notes='', ) # Add and retrieve the first 2 trades. All should be fine. data.db.add_trades([trade1, trade2]) errors = msg_aggregator.consume_errors() warnings = msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 0 returned_trades = data.db.get_trades() assert returned_trades == [trade1, trade2] # Add the last 2 trades. Since trade2 already exists in the DB it should be # ignored and a warning should be shown data.db.add_trades([trade2, trade3]) errors = msg_aggregator.consume_errors() warnings = msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 1 returned_trades = data.db.get_trades() assert returned_trades == [trade1, trade2, trade3]
def query_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, end_at_least_ts: Timestamp, ) -> List[AssetMovement]: with self.lock: cache = self.check_trades_cache_list( start_ts, end_at_least_ts, special_name='deposits_withdrawals', ) result: List[Any] if cache is not None: result = cache else: result = self.query_until_finished( endpoint='Ledgers', keyname='ledger', start_ts=start_ts, end_ts=end_ts, extra_dict=dict(type='deposit'), ) result.extend( self.query_until_finished( endpoint='Ledgers', keyname='ledger', start_ts=start_ts, end_ts=end_ts, extra_dict=dict(type='withdrawal'), )) with self.lock: self.update_trades_cache( result, start_ts, end_ts, special_name='deposits_withdrawals', ) log.debug('Kraken deposit/withdrawals query result', num_results=len(result)) movements = list() for movement in result: try: movements.append( AssetMovement( exchange='kraken', category=movement['type'], # Kraken timestamps have floating point timestamp=Timestamp( convert_to_int(movement['time'], accept_only_exact=False)), asset=asset_from_kraken(movement['asset']), amount=FVal(movement['amount']), fee=Fee(FVal(movement['fee'])), )) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown kraken asset {e.asset_name}. ' f'Ignoring its deposit/withdrawals query.', ) continue return movements
def test_add_margin_positions(data_dir, username): """Test that adding and retrieving margin positions from the DB works fine. Also duplicates should be ignored and an error returned """ msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) margin1 = MarginPosition( location=Location.BITMEX, open_time=1451606400, close_time=1451616500, profit_loss=FVal('1.0'), pl_currency=A_BTC, fee=Fee(FVal('0.01')), fee_currency=A_EUR, link='', notes='', ) margin2 = MarginPosition( location=Location.BITMEX, open_time=1451626500, close_time=1451636500, profit_loss=FVal('0.5'), pl_currency=A_BTC, fee=Fee(FVal('0.01')), fee_currency=A_EUR, link='', notes='', ) margin3 = MarginPosition( location=Location.POLONIEX, open_time=1452636501, close_time=1459836501, profit_loss=FVal('2.5'), pl_currency=A_BTC, fee=Fee(FVal('0.01')), fee_currency=A_EUR, link='', notes='', ) # Add and retrieve the first 2 margins. All should be fine. data.db.add_margin_positions([margin1, margin2]) errors = msg_aggregator.consume_errors() warnings = msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 0 returned_margins = data.db.get_margin_positions() assert returned_margins == [margin1, margin2] # Add the last 2 margins. Since margin2 already exists in the DB it should be # ignored and a warning should be shown data.db.add_margin_positions([margin2, margin3]) errors = msg_aggregator.consume_errors() warnings = msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 1 returned_margins = data.db.get_margin_positions() assert returned_margins == [margin1, margin2, margin3]
'fee': '0.0001', 'earned': '0.0025', 'amount': '2', }, ] asset_movements_list = [ AssetMovement( # before query period -- 8.915 * 0.001 = 8.915e-3 location=Location.KRAKEN, category=AssetMovementCategory.WITHDRAWAL, timestamp=Timestamp(1479510304), # 18/11/2016, asset=A_ETH, # cryptocompare hourly ETH/EUR: 8.915 amount=FVal('95'), fee_asset=A_ETH, fee=Fee(FVal('0.001')), link='krakenid1', ), AssetMovement( # 0.0087*52.885 = 0.4600995 location=Location.KRAKEN, category=AssetMovementCategory.WITHDRAWAL, timestamp=Timestamp(1493291104), # 27/04/2017, asset=A_ETH, # cryptocompare hourly ETH/EUR: 52.885 amount=FVal('125'), fee_asset=A_ETH, fee=Fee(FVal('0.0087')), link='krakenid2', ), AssetMovement( # deposits have no effect location=Location.KRAKEN, category=AssetMovementCategory.DEPOSIT,
def _import_cryptocom_associated_entries(self, data: Any, tx_kind: str) -> None: """Look for events that have associated entries and handle them as trades. This method looks for `*_debited` and `*_credited` entries using the same timestamp to handle them as one trade. Known kind: 'dynamic_coin_swap' or 'dust_conversion' May raise: - UnknownAsset if an unknown asset is encountered in the imported files - KeyError if a row contains unexpected data entries """ multiple_rows: Dict[Any, Dict[str, Any]] = {} investments_deposits: Dict[str, List[Any]] = defaultdict(list) investments_withdrawals: Dict[str, List[Any]] = defaultdict(list) debited_row = None credited_row = None for row in data: if row['Transaction Kind'] == f'{tx_kind}_debited': timestamp = deserialize_timestamp_from_date( date=row['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='cryptocom', ) if timestamp not in multiple_rows: multiple_rows[timestamp] = {} if 'debited' not in multiple_rows[timestamp]: multiple_rows[timestamp]['debited'] = [] multiple_rows[timestamp]['debited'].append(row) elif row['Transaction Kind'] == f'{tx_kind}_credited': # They only is one credited row timestamp = deserialize_timestamp_from_date( date=row['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='cryptocom', ) if timestamp not in multiple_rows: multiple_rows[timestamp] = {} multiple_rows[timestamp]['credited'] = row elif row['Transaction Kind'] == f'{tx_kind}_deposit': asset = row['Currency'] investments_deposits[asset].append(row) elif row['Transaction Kind'] == f'{tx_kind}_withdrawal': asset = row['Currency'] investments_withdrawals[asset].append(row) for timestamp in multiple_rows: # When we convert multiple assets dust to CRO # in one time, it will create multiple debited rows with # the same timestamp debited_rows = multiple_rows[timestamp]['debited'] credited_row = multiple_rows[timestamp]['credited'] total_debited_usd = functools.reduce( lambda acc, row: acc + deserialize_asset_amount(row[ 'Native Amount (in USD)']), debited_rows, ZERO, ) # If the value of the transaction is too small (< 0,01$), # crypto.com will display 0 as native amount # if we have multiple debited rows, we can't import them # since we can't compute their dedicated rates, so we skip them if len(debited_rows) > 1 and total_debited_usd == 0: return if credited_row is not None and len(debited_rows) != 0: for debited_row in debited_rows: description = credited_row['Transaction Description'] notes = f'{description}\nSource: crypto.com (CSV import)' # No fees here fee = Fee(ZERO) fee_currency = A_USD base_asset = symbol_to_asset_or_token( credited_row['Currency']) quote_asset = symbol_to_asset_or_token( debited_row['Currency']) part_of_total = ( FVal(1) if len(debited_rows) == 1 else deserialize_asset_amount( debited_row["Native Amount (in USD)"], ) / total_debited_usd) quote_amount_sold = deserialize_asset_amount( debited_row['Amount'], ) * part_of_total base_amount_bought = deserialize_asset_amount( credited_row['Amount'], ) * part_of_total rate = Price(abs(base_amount_bought / quote_amount_sold)) trade = Trade( timestamp=timestamp, location=Location.CRYPTOCOM, base_asset=base_asset, quote_asset=quote_asset, trade_type=TradeType.BUY, amount=AssetAmount(base_amount_bought), rate=rate, fee=fee, fee_currency=fee_currency, link='', notes=notes, ) self.db.add_trades([trade]) # Compute investments profit if len(investments_withdrawals) != 0: for asset in investments_withdrawals: asset_object = symbol_to_asset_or_token(asset) if asset not in investments_deposits: log.error( f'Investment withdrawal without deposit at crypto.com. Ignoring ' f'staking info for asset {asset_object}', ) continue # Sort by date in ascending order withdrawals_rows = sorted( investments_withdrawals[asset], key=lambda x: deserialize_timestamp_from_date( date=x['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='cryptocom', ), ) investments_rows = sorted( investments_deposits[asset], key=lambda x: deserialize_timestamp_from_date( date=x['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='cryptocom', ), ) last_date = Timestamp(0) for withdrawal in withdrawals_rows: withdrawal_date = deserialize_timestamp_from_date( date=withdrawal['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='cryptocom', ) amount_deposited = ZERO for deposit in investments_rows: deposit_date = deserialize_timestamp_from_date( date=deposit['Timestamp (UTC)'], formatstr='%Y-%m-%d %H:%M:%S', location='cryptocom', ) if last_date < deposit_date <= withdrawal_date: # Amount is negative amount_deposited += deserialize_asset_amount( deposit['Amount']) amount_withdrawal = deserialize_asset_amount( withdrawal['Amount']) # Compute profit profit = amount_withdrawal + amount_deposited if profit >= ZERO: last_date = withdrawal_date action = LedgerAction( identifier=0, # whatever is not used at insertion timestamp=withdrawal_date, action_type=LedgerActionType.INCOME, location=Location.CRYPTOCOM, amount=AssetAmount(profit), asset=asset_object, rate=None, rate_asset=None, link=None, notes=f'Stake profit for asset {asset}', ) self.db_ledger.add_ledger_action(action)