def test_int_overflow_at_tuple_insertion(database, caplog):
"""Test that if somehow an int that will overflow makes it there we handle it
Related: https://github.com/rotki/rotki/issues/2175
"""
caplog.set_level(logging.INFO)
database.add_asset_movements([AssetMovement(
location=Location.BITTREX,
category=AssetMovementCategory.DEPOSIT,
timestamp=177778,
address='0xfoo',
transaction_id=99999999999999999999999999999999999999999,
asset=A_BTC,
amount=FVal(1),
fee_asset=A_BTC,
fee=Fee(FVal('0.0001')),
link='a link',
)])
errors = database.msg_aggregator.consume_errors()
assert len(errors) == 1
assert 'Failed to add "asset_movement" to the DB with overflow error' in errors[0]
assert 'Overflow error while trying to add "asset_movement" tuples to the DB. Tuples:' in caplog.text # noqa: E501
def test_deserialize_v2_trade_sell(mock_kucoin):
raw_result = {
'symbol': 'BCHSV-USDT',
'tradeId': '601da995e0ee8b00063a075c',
'orderId': '601da9950c92050006bd45c5',
'counterOrderId': '601da9950c92050006bd457d',
'side': 'sell',
'liquidity': 'taker',
'forceTaker': True,
'price': '37624.4',
'size': '0.0013',
'funds': '48.91172',
'fee': '0.034238204',
'feeRate': '0.0007',
'feeCurrency': 'USDT',
'stop': '',
'tradeType': 'TRADE',
'type': 'market',
'createdAt': 1612556794259,
}
expected_trade = Trade(
timestamp=Timestamp(1612556794),
location=Location.KUCOIN,
pair=TradePair('BSV_USDT'),
trade_type=TradeType.SELL,
amount=AssetAmount(FVal('0.0013')),
rate=Price(FVal('37624.4')),
fee=Fee(FVal('0.034238204')),
fee_currency=Asset('USDT'),
link='601da995e0ee8b00063a075c',
notes='',
)
trade = mock_kucoin._deserialize_trade(
raw_result=raw_result,
case=KucoinCase.TRADES,
)
assert trade == expected_trade
def test_deserialize_v2_trade_buy(mock_kucoin):
raw_result = {
'symbol': 'KCS-USDT',
'tradeId': '601da9faf1297d0007efd712',
'orderId': '601da9fa0c92050006bd83be',
'counterOrderId': '601bad620c9205000642300f',
'side': 'buy',
'liquidity': 'taker',
'forceTaker': True,
'price': 1000,
'size': '0.2',
'funds': 200,
'fee': '0.14',
'feeRate': '0.0007',
'feeCurrency': 'USDT',
'stop': '',
'tradeType': 'TRADE',
'type': 'market',
'createdAt': 1612556794259,
}
expected_trade = Trade(
timestamp=Timestamp(1612556794),
location=Location.KUCOIN,
pair=TradePair('KCS_USDT'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.2')),
rate=Price(FVal('1000')),
fee=Fee(FVal('0.14')),
fee_currency=Asset('USDT'),
link='601da9faf1297d0007efd712',
notes='',
)
trade = mock_kucoin._deserialize_trade(
raw_result=raw_result,
case=KucoinCase.TRADES,
)
assert trade == expected_trade
def _consume_blockfi_entry(self, csv_row: Dict[str, Any]) -> None:
"""
Process entry for BlockFi transaction history. Trades for this file are ignored
and istead should be extracted from the file containing only trades.
This method can raise:
- UnsupportedBlockFiEntry
- UnknownAsset
- DeserializationError
"""
if len(csv_row['Confirmed At']) != 0:
timestamp = deserialize_timestamp_from_date(
date=csv_row['Confirmed At'],
formatstr='%Y-%m-%d %H:%M:%S',
location='BlockFi',
)
else:
log.debug(f'Ignoring unconfirmed BlockFi entry {csv_row}')
return
asset = symbol_to_asset_or_token(csv_row['Cryptocurrency'])
amount = deserialize_asset_amount_force_positive(csv_row['Amount'])
entry_type = csv_row['Transaction Type']
# BlockFI doesn't provide information about fees
fee = Fee(ZERO)
fee_asset = A_USD # Can be whatever
if entry_type in ('Deposit', 'Wire Deposit', 'ACH Deposit'):
asset_movement = AssetMovement(
location=Location.BLOCKFI,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif entry_type in ('Withdrawal', 'Wire Withdrawal', 'ACH Withdrawal'):
asset_movement = AssetMovement(
location=Location.BLOCKFI,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif entry_type == 'Withdrawal Fee':
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.EXPENSE,
location=Location.BLOCKFI,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from BlockFi',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ('Interest Payment', 'Bonus Payment',
'Referral Bonus'):
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.INCOME,
location=Location.BLOCKFI,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from BlockFi',
)
self.db_ledger.add_ledger_action(action)
elif entry_type == 'Trade':
pass
else:
raise UnsupportedCSVEntry(
f'Unsuported entry {entry_type}. Data: {csv_row}')
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
"""Queries coinbase pro for asset movements"""
log.debug('Query coinbasepro asset movements',
start_ts=start_ts,
end_ts=end_ts)
movements = []
raw_movements = []
for batch in self._paginated_query(
endpoint='transfers',
query_options={'type': 'withdraw'},
):
raw_movements.extend(batch)
for batch in self._paginated_query(
endpoint='transfers',
query_options={'type': 'deposit'},
):
raw_movements.extend(batch)
account_to_currency = self.create_or_return_account_to_currency_map()
for entry in raw_movements:
try:
# Check if the transaction has not been completed. If so it should be skipped
if entry.get('completed_at', None) is None:
log.warning(
f'Skipping coinbase pro deposit/withdrawal '
f'due not having been completed: {entry}', )
continue
timestamp = coinbasepro_deserialize_timestamp(
entry, 'completed_at')
if timestamp < start_ts or timestamp > end_ts:
continue
category = deserialize_asset_movement_category(entry['type'])
asset = account_to_currency.get(entry['account_id'], None)
if asset is None:
log.warning(
f'Skipping coinbase pro asset_movement {entry} due to inability to '
f'match account id to an asset', )
continue
address = None
transaction_id = None
fee = Fee(ZERO)
if category == AssetMovementCategory.DEPOSIT:
try:
address = entry['details']['crypto_address']
transaction_id = entry['details'][
'crypto_transaction_hash']
except KeyError:
pass
else: # withdrawal
try:
address = entry['details']['sent_to_address']
transaction_id = entry['details'][
'crypto_transaction_hash']
fee = deserialize_fee(entry['details']['fee'])
except KeyError:
pass
if transaction_id and (
asset == A_ETH or asset.asset_type
== AssetType.ETHEREUM_TOKEN): # noqa: E501
transaction_id = '0x' + transaction_id
movements.append(
AssetMovement(
location=Location.COINBASEPRO,
category=category,
address=address,
transaction_id=transaction_id,
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
entry['amount']),
fee_asset=asset,
fee=fee,
link=str(entry['id']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring its deposit/withdrawal.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a Coinbasepro deposit/withdrawal. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a coinbasepro deposit/withdrawal.',
raw_asset_movement=entry,
error=msg,
)
continue
return movements
def _consume_cointracking_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cointracking entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCointrackingEntry if importing of this entry is not supported.
- IndexError if the CSV file is corrupt
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Type']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date'],
formatstr='%d.%m.%Y %H:%M:%S',
location='cointracking.info',
)
notes = csv_row['Comment']
location = exchange_row_to_location(csv_row['Exchange'])
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if csv_row['Fee'] != '':
fee = deserialize_fee(csv_row['Fee'])
fee_currency = Asset(csv_row['Cur.Fee'])
if row_type in ('Gift/Tip', 'Trade', 'Income'):
base_asset = Asset(csv_row['Cur.Buy'])
quote_asset = None if csv_row['Cur.Sell'] == '' else Asset(
csv_row['Cur.Sell'])
if quote_asset is None and row_type not in ('Gift/Tip', 'Income'):
raise DeserializationError(
'Got a trade entry with an empty quote asset')
if quote_asset is None:
# Really makes no difference as this is just a gift and the amount is zero
quote_asset = A_USD
pair = TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}')
base_amount_bought = deserialize_asset_amount(csv_row['Buy'])
if csv_row['Sell'] != '-':
quote_amount_sold = deserialize_asset_amount(csv_row['Sell'])
else:
quote_amount_sold = AssetAmount(ZERO)
rate = Price(quote_amount_sold / base_amount_bought)
trade = Trade(
timestamp=timestamp,
location=location,
pair=pair,
trade_type=TradeType.
BUY, # It's always a buy during cointracking import
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type == 'Deposit' or row_type == 'Withdrawal':
category = deserialize_asset_movement_category(row_type.lower())
if category == AssetMovementCategory.DEPOSIT:
amount = deserialize_asset_amount(csv_row['Buy'])
asset = Asset(csv_row['Cur.Buy'])
else:
amount = deserialize_asset_amount_force_positive(
csv_row['Sell'])
asset = Asset(csv_row['Cur.Sell'])
asset_movement = AssetMovement(
location=location,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_currency,
link='',
)
self.db.add_asset_movements([asset_movement])
else:
raise UnsupportedCointrackingEntry(
f'Unknown entrype type "{row_type}" encountered during cointracking '
f'data import. Ignoring entry', )
def _consume_cryptocom_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cryptocom entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCryptocomEntry if importing of this entry is not supported.
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Transaction Kind']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='crypto.com',
)
description = csv_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees info until (Nov 2020) on crypto.com
# fees are not displayed in the export data
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if row_type in (
'crypto_purchase',
'crypto_exchange',
'referral_gift',
'referral_bonus',
'crypto_earn_interest_paid',
'referral_card_cashback',
'card_cashback_reverted',
'reimbursement',
):
# variable mapping to raw data
currency = csv_row['Currency']
to_currency = csv_row['To Currency']
native_currency = csv_row['Native Currency']
amount = csv_row['Amount']
to_amount = csv_row['To Amount']
native_amount = csv_row['Native Amount']
trade_type = TradeType.BUY if to_currency != native_currency else TradeType.SELL
if row_type == 'crypto_exchange':
# trades crypto to crypto
base_asset = Asset(to_currency)
quote_asset = Asset(currency)
if quote_asset is None:
raise DeserializationError(
'Got a trade entry with an empty quote asset')
base_amount_bought = deserialize_asset_amount(to_amount)
quote_amount_sold = deserialize_asset_amount(amount)
else:
base_asset = Asset(currency)
quote_asset = Asset(native_currency)
base_amount_bought = deserialize_asset_amount(amount)
quote_amount_sold = deserialize_asset_amount(native_amount)
rate = Price(abs(quote_amount_sold / base_amount_bought))
pair = TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}')
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
pair=pair,
trade_type=trade_type,
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type == 'crypto_withdrawal' or row_type == 'crypto_deposit':
if row_type == 'crypto_withdrawal':
category = AssetMovementCategory.WITHDRAWAL
amount = deserialize_asset_amount_force_positive(
csv_row['Amount'])
else:
category = AssetMovementCategory.DEPOSIT
amount = deserialize_asset_amount(csv_row['Amount'])
asset = Asset(csv_row['Currency'])
asset_movement = AssetMovement(
location=Location.CRYPTOCOM,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif row_type in (
'crypto_earn_program_created',
'lockup_lock',
'lockup_unlock',
'dynamic_coin_swap_bonus_exchange_deposit',
'crypto_wallet_swap_debited',
'crypto_wallet_swap_credited',
'lockup_swap_debited',
'lockup_swap_credited',
'lockup_swap_rebate',
'dynamic_coin_swap_bonus_exchange_deposit',
# we don't handle cryto.com exchange yet
'crypto_to_exchange_transfer',
'exchange_to_crypto_transfer',
# supercharger actions
'supercharger_deposit',
'supercharger_withdrawal',
# already handled using _import_cryptocom_double_entries
'dynamic_coin_swap_debited',
'dynamic_coin_swap_credited',
'dust_conversion_debited',
'dust_conversion_credited',
):
# those types are ignored because it doesn't affect the wallet balance
# or are not handled here
return
else:
raise UnsupportedCryptocomEntry(
f'Unknown entrype type "{row_type}" encountered during '
f'cryptocom data import. Ignoring entry', )
def _deserialize_asset_movement(
self,
movement_type: AssetMovementCategory,
movement_data: Dict[str, Any],
) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from polo and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if movement_type == AssetMovementCategory.DEPOSIT:
fee = Fee(ZERO)
uid_key = 'depositNumber'
transaction_id = get_key_if_has_val(movement_data, 'txid')
else:
fee = deserialize_fee(movement_data['fee'])
uid_key = 'withdrawalNumber'
split = movement_data['status'].split(':')
if len(split) != 2:
transaction_id = None
else:
transaction_id = split[1].lstrip()
if transaction_id == '':
transaction_id = None
asset = asset_from_poloniex(movement_data['currency'])
return AssetMovement(
location=Location.POLONIEX,
category=movement_type,
address=deserialize_asset_movement_address(movement_data, 'address', asset),
transaction_id=transaction_id,
timestamp=deserialize_timestamp(movement_data['timestamp']),
asset=asset,
amount=deserialize_asset_amount_force_positive(movement_data['amount']),
fee_asset=asset,
fee=fee,
link=str(movement_data[uid_key]),
)
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(movement_type)} of unsupported poloniex asset '
f'{e.asset_name}. Ignoring it.',
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(movement_type)} of unknown poloniex asset '
f'{e.asset_name}. Ignoring it.',
)
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a poloniex '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of poloniex '
f'{str(movement_type)}: {movement_data}. Error was: {str(e)}',
)
return None
def test_add_trades(data_dir, username, caplog):
"""Test that adding and retrieving trades from the DB works fine.
Also duplicates should be ignored and an error returned
"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
trade1 = Trade(
timestamp=1451606400,
location=Location.KRAKEN,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=FVal('1.1'),
rate=FVal('10'),
fee=Fee(FVal('0.01')),
fee_currency=A_EUR,
link='',
notes='',
)
trade2 = Trade(
timestamp=1451607500,
location=Location.BINANCE,
base_asset=A_BTC,
quote_asset=A_ETH,
trade_type=TradeType.BUY,
amount=FVal('0.00120'),
rate=FVal('10'),
fee=Fee(FVal('0.001')),
fee_currency=A_ETH,
link='',
notes='',
)
trade3 = Trade(
timestamp=1451608600,
location=Location.COINBASE,
base_asset=A_BTC,
quote_asset=A_ETH,
trade_type=TradeType.SELL,
amount=FVal('0.00120'),
rate=FVal('1'),
fee=Fee(FVal('0.001')),
fee_currency=A_ETH,
link='',
notes='',
)
# Add and retrieve the first 2 trades. All should be fine.
data.db.add_trades([trade1, trade2])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
returned_trades = data.db.get_trades(filter_query=TradesFilterQuery.make(),
has_premium=True)
assert returned_trades == [trade1, trade2]
# Add the last 2 trades. Since trade2 already exists in the DB it should be
# ignored and a warning should be logged
data.db.add_trades([trade2, trade3])
assert 'Did not add "buy trade with id a1ed19c8284940b4e59bdac941db2fd3c0ed004ddb10fdd3b9ef0a3a9b2c97bc' in caplog.text # noqa: E501
returned_trades = data.db.get_trades(filter_query=TradesFilterQuery.make(),
has_premium=True)
assert returned_trades == [trade1, trade2, trade3]
def assert_cryptocom_import_results(rotki: Rotkehlchen):
"""A utility function to help assert on correctness of importing data from crypto.com"""
trades = rotki.data.db.get_trades()
asset_movements = rotki.data.db.get_asset_movements()
warnings = rotki.msg_aggregator.consume_warnings()
errors = rotki.msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
def get_trade_note(desc: str):
return f'{desc}\nSource: crypto.com (CSV import)'
expected_trades = [Trade(
timestamp=Timestamp(1595833195),
location=Location.CRYPTOCOM,
pair=TradePair('ETH_EUR'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1.0')),
rate=Price(FVal('281.14')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('Buy ETH'),
), Trade(
timestamp=Timestamp(1596014214),
location=Location.CRYPTOCOM,
pair=TradePair('MCO_EUR'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('50.0')),
rate=Price(FVal('3.521')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('Buy MCO'),
), Trade(
timestamp=Timestamp(1596014223),
location=Location.CRYPTOCOM,
pair=TradePair('MCO_USD'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('12.32402069')),
rate=Price(FVal('4.057117499045678736198226879')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('Sign-up Bonus Unlocked'),
), Trade(
timestamp=Timestamp(1596209827),
location=Location.CRYPTOCOM,
pair=TradePair('ETH_MCO'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.14445954600007045')),
rate=Price(FVal('85.28339137929999991192917299')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('MCO -> ETH'),
), Trade(
timestamp=Timestamp(1596429934),
location=Location.CRYPTOCOM,
pair=TradePair('ETH_EUR'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.00061475')),
rate=Price(FVal('309.0687271248474989833265555')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('Crypto Earn'),
), Trade(
timestamp=Timestamp(1596465565),
location=Location.CRYPTOCOM,
pair=TradePair('CRO_MCO'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1382.306147552291')),
rate=Price(FVal('27.6439')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('MCO/CRO Overall Swap'),
), Trade(
timestamp=Timestamp(1596730165),
location=Location.CRYPTOCOM,
pair=TradePair('CRO_MCO'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1301.64')),
rate=Price(FVal('26.0328')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('MCO/CRO Overall Swap'),
), Trade(
timestamp=Timestamp(1599934176),
location=Location.CRYPTOCOM,
pair=TradePair('CRO_EUR'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('138.256')),
rate=Price(FVal('0.1429232727693553986807082514')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('Card Rebate: Deliveries'),
), Trade(
timestamp=Timestamp(1602515376),
location=Location.CRYPTOCOM,
pair=TradePair('CRO_EUR'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('52.151')),
rate=Price(FVal('0.06692105616383194953116910510')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('Card Cashback'),
), Trade(
timestamp=Timestamp(1602526176),
location=Location.CRYPTOCOM,
pair=TradePair('CRO_EUR'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('482.2566417')),
rate=Price(FVal('0.08756748243245604635910191136')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('Referral Bonus Reward'),
), Trade(
timestamp=Timestamp(1606833565),
location=Location.CRYPTOCOM,
pair=TradePair('CRO_DAI'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.007231228760408149')),
rate=Price(FVal('14.26830000900286970270179629')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('Convert Dust'),
)]
assert expected_trades == trades
expected_movements = [AssetMovement(
location=Location.CRYPTOCOM,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1596992965),
address=None,
transaction_id=None,
asset=A_DAI,
amount=AssetAmount(FVal('115')),
fee_asset=A_DAI,
fee=Fee(ZERO),
link='',
), AssetMovement(
location=Location.CRYPTOCOM,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=Timestamp(1596993025),
asset=A_DAI,
amount=AssetAmount(FVal('115')),
fee_asset=A_DAI,
fee=Fee(ZERO),
link='',
)]
assert expected_movements == asset_movements
def trade_from_conversion(trade_a: Dict[str, Any],
trade_b: Dict[str, Any]) -> Optional[Trade]:
"""Turn information from a conversion into a trade
Mary raise:
- UnknownAsset due to Asset instantiation
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entires missing an expected entry
"""
# Check that the status is complete
if trade_a['status'] != 'completed':
return None
# Trade b will represent the asset we are converting to
if trade_b['amount']['amount'].startswith('-'):
trade_a, trade_b = trade_b, trade_a
timestamp = deserialize_timestamp_from_date(trade_a['updated_at'],
'iso8601', 'coinbase')
tx_amount = AssetAmount(
abs(deserialize_asset_amount(trade_a['amount']['amount'])))
tx_asset = asset_from_coinbase(trade_a['amount']['currency'],
time=timestamp)
native_amount = deserialize_asset_amount(trade_b['amount']['amount'])
native_asset = asset_from_coinbase(trade_b['amount']['currency'],
time=timestamp)
amount = tx_amount
# The rate is how much you get/give in quotecurrency if you buy/sell 1 unit of base currency
rate = Price(native_amount / tx_amount)
# Obtain fee amount in the native currency using data from both trades
amount_after_fee = deserialize_asset_amount(
trade_b['native_amount']['amount'])
amount_before_fee = deserialize_asset_amount(
trade_a['native_amount']['amount'])
# amount_after_fee + amount_before_fee is a negative amount and the fee needs to be positive
conversion_native_fee_amount = abs(amount_after_fee + amount_before_fee)
if ZERO not in (tx_amount, conversion_native_fee_amount, amount_before_fee,
amount_after_fee):
# To get the asset in which the fee is nominated we pay attention to the creation
# date of each event. As per our hypothesis the fee is nominated in the asset
# for which the first transaction part was initialized
time_created_a = deserialize_timestamp_from_date(
date=trade_a['created_at'],
formatstr='iso8601',
location='coinbase',
)
time_created_b = deserialize_timestamp_from_date(
date=trade_b['created_at'],
formatstr='iso8601',
location='coinbase',
)
if time_created_a < time_created_b:
# We have the fee amount in the native currency. To get it in the
# converted asset we have to get the rate
asset_native_rate = tx_amount / abs(amount_before_fee)
fee_amount = Fee(conversion_native_fee_amount * asset_native_rate)
fee_asset = asset_from_coinbase(trade_a['amount']['currency'],
time=timestamp)
else:
trade_b_amount = abs(
deserialize_asset_amount(trade_b['amount']['amount']))
asset_native_rate = trade_b_amount / abs(amount_after_fee)
fee_amount = Fee(conversion_native_fee_amount * asset_native_rate)
fee_asset = asset_from_coinbase(trade_b['amount']['currency'],
time=timestamp)
else:
fee_amount = Fee(ZERO)
fee_asset = asset_from_coinbase(trade_a['amount']['currency'],
time=timestamp)
return Trade(
timestamp=timestamp,
location=Location.COINBASE,
# in coinbase you are buying/selling tx_asset for native_asset
base_asset=tx_asset,
quote_asset=native_asset,
trade_type=TradeType.SELL,
amount=amount,
rate=rate,
fee=fee_amount,
fee_currency=fee_asset,
link=str(trade_a['trade']['id']),
)
def assert_poloniex_asset_movements(
to_check_list: List[Any],
deserialized: bool,
movements_to_check: Optional[Tuple[int, ...]] = None,
) -> None:
expected = [
AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.WITHDRAWAL,
address='0xB7E033598Cb94EF5A35349316D3A2e4f95f308Da',
transaction_id=
'0xbd4da74e1a0b81c21d056c6f58a5b306de85d21ddf89992693b812bb117eace4',
timestamp=Timestamp(1468994442),
asset=A_ETH,
amount=FVal('10.0'),
fee_asset=A_ETH,
fee=Fee(FVal('0.1')),
link='2',
),
AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.WITHDRAWAL,
address='131rdg5Rzn6BFufnnQaHhVa5ZtRU1J2EZR',
transaction_id=
'2d27ae26fa9c70d6709e27ac94d4ce2fde19b3986926e9f3bfcf3e2d68354ec5',
timestamp=Timestamp(1458994442),
asset=A_BTC,
amount=FVal('5.0'),
fee_asset=A_BTC,
fee=Fee(FVal('0.5')),
link='1',
),
AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.DEPOSIT,
address='131rdg5Rzn6BFufnnQaHhVa5ZtRU1J2EZR',
transaction_id=
'b05bdec7430a56b5a5ed34af4a31a54859dda9b7c88a5586bc5d6540cdfbfc7a',
timestamp=Timestamp(1448994442),
asset=A_BTC,
amount=FVal('50.0'),
fee_asset=A_BTC,
fee=Fee(FVal('0')),
link='1',
),
AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.DEPOSIT,
address='0xB7E033598Cb94EF5A35349316D3A2e4f95f308Da',
transaction_id=
'0xf7e7eeb44edcad14c0f90a5fffb1cbb4b80e8f9652124a0838f6906ca939ccd2',
timestamp=Timestamp(1438994442),
asset=A_ETH,
amount=FVal('100.0'),
fee_asset=A_ETH,
fee=Fee(FVal('0')),
link='2',
)
]
assert_asset_movements(expected, to_check_list, deserialized,
movements_to_check)
def test_query_asset_movements_sandbox(
sandbox_kuckoin,
inquirer, # pylint: disable=unused-argument
):
"""Unfortunately the sandbox environment does not support deposits and
withdrawals, therefore they must be mocked.
Below a list of the movements and their timestamps in ascending mode:
Deposits:
- deposit 1 - deposit: 1612556651
- deposit 2 - deposit: 1612556652
- deposit 3 - deposit: 1612556653 -> skipped, inner deposit
Withdrawals:
- withdraw 1: 1612556651 -> skipped, inner withdraw
- withdraw 2: 1612556652
- withdraw 3: 1612556656 -> never requested
By requesting trades from 1612556651 to 1612556654 and patching the time
step as 2s (via MONTHS_IN_SECONDS) we should get back 3 movements.
"""
deposits_response_1 = ("""
{
"code":"200000",
"data":{
"currentPage":1,
"pageSize":2,
"totalNum":2,
"totalPage":1,
"items":[
{
"address":"0x5f047b29041bcfdbf0e4478cdfa753a336ba6989",
"memo":"5c247c8a03aa677cea2a251d",
"amount":1,
"fee":0.0001,
"currency":"KCS",
"isInner":false,
"walletTxId":"5bbb57386d99522d9f954c5a",
"status":"SUCCESS",
"remark":"movement 2 - deposit",
"createdAt":1612556652000,
"updatedAt":1612556652000
},
{
"address":"0x5f047b29041bcfdbf0e4478cdfa753a336ba6989",
"memo":"5c247c8a03aa677cea2a251d",
"amount":1000,
"fee":0.01,
"currency":"LINK",
"isInner":false,
"walletTxId":"5bbb57386d99522d9f954c5b@test",
"status":"SUCCESS",
"remark":"movement 1 - deposit",
"createdAt":1612556651000,
"updatedAt":1612556651000
}
]
}
}
""")
deposits_response_2 = ("""
{
"code":"200000",
"data":{
"currentPage":1,
"pageSize":1,
"totalNum":1,
"totalPage":1,
"items":[
{
"address":"1DrT5xUaJ3CBZPDeFR2qdjppM6dzs4rsMt",
"memo":"",
"currency":"BCHSV",
"amount":1,
"fee":0.1,
"walletTxId":"b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada6",
"isInner":true,
"status":"SUCCESS",
"remark":"movement 4 - deposit",
"createdAt":1612556653000,
"updatedAt":1612556653000
}
]
}
}
""")
withdrawals_response_1 = ("""
{
"code":"200000",
"data":{
"currentPage":1,
"pageSize":2,
"totalNum":2,
"totalPage":1,
"items":[
{
"id":"5c2dc64e03aa675aa263f1a4",
"address":"1DrT5xUaJ3CBZPDeFR2qdjppM6dzs4rsMt",
"memo":"",
"currency":"BCHSV",
"amount":2.5,
"fee":0.25,
"walletTxId":"b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada4",
"isInner":false,
"status":"SUCCESS",
"remark":"movement 4 - withdraw",
"createdAt":1612556652000,
"updatedAt":1612556652000
},
{
"id":"5c2dc64e03aa675aa263f1a3",
"address":"0x5bedb060b8eb8d823e2414d82acce78d38be7fe9",
"memo":"",
"currency":"ETH",
"amount":1,
"fee":0.01,
"walletTxId":"3e2414d82acce78d38be7fe9",
"isInner":true,
"status":"SUCCESS",
"remark":"movement 3 - withdraw",
"createdAt":1612556651000,
"updatedAt":1612556651000
}
]
}
}
""")
withdrawals_response_2 = ("""
{
"code":"200000",
"data":{
"currentPage":0,
"pageSize":0,
"totalNum":0,
"totalPage":0,
"items":[]
}
}
""")
withdrawals_response_3 = ("""
{
"code":"200000",
"data":{
"currentPage":1,
"pageSize":1,
"totalNum":1,
"totalPage":1,
"items":[
{
"id":"5c2dc64e03aa675aa263f1a5",
"address":"0x5bedb060b8eb8d823e2414d82acce78d38be7f00",
"memo":"",
"currency":"KCS",
"amount":2.5,
"fee":0.25,
"walletTxId":"b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada5",
"isInner":false,
"status":"SUCCESS",
"remark":"movement 5 - withdraw",
"createdAt":1612556655000,
"updatedAt":1612556655000
}
]
}
}
""")
expected_asset_movements = [
AssetMovement(
location=Location.KUCOIN,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1612556652),
address='0x5f047b29041bcfdbf0e4478cdfa753a336ba6989',
transaction_id='5bbb57386d99522d9f954c5a',
asset=Asset('KCS'),
amount=AssetAmount(FVal('1')),
fee_asset=Asset('KCS'),
fee=Fee(FVal('0.0001')),
link='',
),
AssetMovement(
location=Location.KUCOIN,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1612556651),
address='0x5f047b29041bcfdbf0e4478cdfa753a336ba6989',
transaction_id='5bbb57386d99522d9f954c5b',
asset=Asset('LINK'),
amount=AssetAmount(FVal('1000')),
fee_asset=Asset('LINK'),
fee=Fee(FVal('0.01')),
link='',
),
AssetMovement(
location=Location.KUCOIN,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=Timestamp(1612556652),
address='1DrT5xUaJ3CBZPDeFR2qdjppM6dzs4rsMt',
transaction_id=
'b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada4',
asset=Asset('BSV'),
amount=AssetAmount(FVal('2.5')),
fee_asset=Asset('BSV'),
fee=Fee(FVal('0.25')),
link='5c2dc64e03aa675aa263f1a4',
),
]
def get_endpoints_response():
results = [
f'{deposits_response_1}',
f'{deposits_response_2}',
f'{withdrawals_response_1}',
f'{withdrawals_response_2}',
# if pagination works as expected and the requesting loop is broken,
# the response below won't be processed
f'{withdrawals_response_3}',
]
for result_ in results:
yield result_
def mock_api_query_response(case, options): # pylint: disable=unused-argument
return MockResponse(HTTPStatus.OK, next(get_response))
def test_query_trades_sandbox(sandbox_kuckoin, inquirer): # pylint: disable=unused-argument
"""The sandbox account has 6 trades. Below a list of the trades and their
timestamps in ascending mode.
- trade 1: 1612556651 -> skipped
- trade 2: 1612556693
- trade 3: 1612556765
- trade 4: 1612556765
- trade 5: 1612556765
- trade 6: 1612556794 -> skipped
By requesting trades from 1612556693 to 1612556765, the first and last trade
should be skipped.
"""
expected_trades = [
Trade(
timestamp=Timestamp(1612556765),
location=Location.KUCOIN,
pair=TradePair('ETH_BTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.02934995')),
rate=Price(FVal('0.046058')),
fee=Fee(FVal('9.4625999797E-7')),
fee_currency=Asset('BTC'),
link='601da9ddf73c300006194ec6',
notes='',
),
Trade(
timestamp=Timestamp(1612556765),
location=Location.KUCOIN,
pair=TradePair('ETH_BTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.02')),
rate=Price(FVal('0.04561')),
fee=Fee(FVal('6.3854E-7')),
fee_currency=Asset('BTC'),
link='601da9ddf73c300006194ec5',
notes='',
),
Trade(
timestamp=Timestamp(1612556765),
location=Location.KUCOIN,
pair=TradePair('ETH_BTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.06')),
rate=Price(FVal('0.0456')),
fee=Fee(FVal('0.0000019152')),
fee_currency=Asset('BTC'),
link='601da9ddf73c300006194ec4',
notes='',
),
Trade(
timestamp=Timestamp(1612556693),
location=Location.KUCOIN,
pair=TradePair('BTC_USDT'),
trade_type=TradeType.SELL,
amount=AssetAmount(FVal('0.0013')),
rate=Price(FVal('37624.4')),
fee=Fee(FVal('0.034238204')),
fee_currency=Asset('USDT'),
link='601da995e0ee8b00063a075c',
notes='',
),
]
trades = sandbox_kuckoin.query_online_trade_history(
start_ts=Timestamp(1612556693),
end_ts=Timestamp(1612556765),
)
assert trades == expected_trades
def _consume_nexo(self, csv_row: Dict[str, Any]) -> None:
"""
Consume CSV file from NEXO.
This method can raise:
- UnsupportedNexoEntry
- UnknownAsset
- DeserializationError
"""
ignored_entries = ('ExchangeToWithdraw', 'DepositToExchange')
if 'rejected' not in csv_row['Details']:
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date / Time'],
formatstr='%Y-%m-%d %H:%M',
location='NEXO',
)
else:
log.debug(f'Ignoring rejected nexo entry {csv_row}')
return
asset = symbol_to_asset_or_token(csv_row['Currency'])
amount = deserialize_asset_amount_force_positive(csv_row['Amount'])
entry_type = csv_row['Type']
transaction = csv_row['Transaction']
if entry_type in ('Deposit', 'ExchangeDepositedOn',
'LockingTermDeposit'):
asset_movement = AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=Fee(ZERO),
fee_asset=A_USD,
link=transaction,
)
self.db.add_asset_movements([asset_movement])
elif entry_type in ('Withdrawal', 'WithdrawExchanged'):
asset_movement = AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=Fee(ZERO),
fee_asset=A_USD,
link=transaction,
)
self.db.add_asset_movements([asset_movement])
elif entry_type == 'Withdrawal Fee':
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.EXPENSE,
location=Location.NEXO,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from Nexo',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ('Interest', 'Bonus', 'Dividend'):
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.INCOME,
location=Location.NEXO,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=transaction,
notes=f'{entry_type} from Nexo',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ignored_entries:
pass
else:
raise UnsupportedCSVEntry(
f'Unsuported entry {entry_type}. Data: {csv_row}')
def query_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
end_at_least_ts: Timestamp,
) -> List[AssetMovement]:
with self.lock:
cache = self.check_trades_cache_dict(
start_ts,
end_at_least_ts,
special_name='deposits_withdrawals',
)
if cache is None:
result = self.returnDepositsWithdrawals(start_ts, end_ts)
with self.lock:
self.update_trades_cache(
result,
start_ts,
end_ts,
special_name='deposits_withdrawals',
)
else:
result = cache
log.debug(
'Poloniex deposits/withdrawal query',
results_num=len(result['withdrawals']) + len(result['deposits']),
)
movements = list()
for withdrawal in result['withdrawals']:
try:
movements.append(AssetMovement(
exchange='poloniex',
category='withdrawal',
timestamp=withdrawal['timestamp'],
asset=asset_from_poloniex(withdrawal['currency']),
amount=FVal(withdrawal['amount']),
fee=Fee(FVal(withdrawal['fee'])),
))
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found withdrawal of unsupported poloniex asset {e.asset_name}. Ignoring it.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found withdrawal of unknown poloniex asset {e.asset_name}. Ignoring it.',
)
continue
for deposit in result['deposits']:
try:
movements.append(AssetMovement(
exchange='poloniex',
category='deposit',
timestamp=deposit['timestamp'],
asset=asset_from_poloniex(deposit['currency']),
amount=FVal(deposit['amount']),
fee=Fee(ZERO),
))
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found deposit of unsupported poloniex asset {e.asset_name}. Ignoring it.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found deposit of unknown poloniex asset {e.asset_name}. Ignoring it.',
)
continue
return movements
def assert_cointracking_import_results(rotki: Rotkehlchen):
"""A utility function to help assert on correctness of importing data from cointracking.info"""
trades = rotki.data.db.get_trades()
asset_movements = rotki.data.db.get_asset_movements()
warnings = rotki.msg_aggregator.consume_warnings()
errors = rotki.msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 3
expected_trades = [Trade(
timestamp=Timestamp(1566687719),
location=Location.COINBASE,
pair=TradePair('ETH_EUR'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.05772716')),
rate=Price(FVal('190.3783245183029963712055123')),
fee=Fee(FVal("0.02")),
fee_currency=A_EUR,
link='',
notes='',
), Trade(
timestamp=Timestamp(1567418410),
location=Location.EXTERNAL,
pair=TradePair('BTC_USD'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.00100000')),
rate=Price(ZERO),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='Just a small gift from someone',
), Trade(
timestamp=Timestamp(1567504805),
location=Location.EXTERNAL,
pair=TradePair('ETH_USD'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('2')),
rate=Price(ZERO),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='Sign up bonus',
)]
assert expected_trades == trades
expected_movements = [AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1565848624),
address=None,
transaction_id=None,
asset=A_XMR,
amount=AssetAmount(FVal('5')),
fee_asset=A_USD,
fee=Fee(ZERO),
link='',
), AssetMovement(
location=Location.COINBASE,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=Timestamp(1566726155),
asset=A_ETH,
amount=AssetAmount(FVal('0.05770427')),
fee_asset=A_ETH,
fee=Fee(FVal("0.0001")),
link='',
)]
assert expected_movements == asset_movements
def test_measure_trades_api_query(rotkehlchen_api_server,
start_with_valid_premium):
"""Measures the response time of the combined trades view API query.
This is required since it's quite a complicated query and takes a lot of time to process
so we can use this test to measure any potential optimizations.
"""
trades = [
Trade(
timestamp=x,
location=Location.EXTERNAL,
base_asset=A_WETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1')),
rate=Price(FVal('320')),
fee=Fee(ZERO),
fee_currency=A_EUR,
link='',
notes='',
) for x in range(1, 10000)
]
rotki = rotkehlchen_api_server.rest_api.rotkehlchen
rotki.data.db.add_trades(trades)
swaps = [
AMMSwap(
tx_hash='0x' + str(x),
log_index=x + i,
address='0xfoo',
from_address='0xfrom',
to_address='0xto',
timestamp=11 + x,
location=Location.UNISWAP,
token0=A_WETH,
token1=A_EUR,
amount0_in=FVal(5),
amount1_in=ZERO,
amount0_out=ZERO,
amount1_out=FVal(4.95),
) for x in range(2000) for i in range(2)
]
rotki.data.db.add_amm_swaps(swaps)
start = time.time()
requests.get(
api_url_for(
rotkehlchen_api_server,
'tradesresource',
),
json={'only_cache': True},
)
end = time.time()
test_warnings.warn(
UserWarning(
f'Premium: {start_with_valid_premium}. Full Query Time: {end - start}',
))
start = time.time()
requests.get(
api_url_for(
rotkehlchen_api_server,
'tradesresource',
),
json={
'only_cache': True,
'offset': 200,
'limit': 10
},
)
end = time.time()
test_warnings.warn(
UserWarning(
f'Premium: {start_with_valid_premium}. First Page Query Time: {end - start}',
))
start = time.time()
requests.get(
api_url_for(
rotkehlchen_api_server,
'tradesresource',
),
json={
'only_cache': True,
'offset': 210,
'limit': 10
},
)
end = time.time()
test_warnings.warn(
UserWarning(
f'Premium: {start_with_valid_premium}. Second Page Query Time: {end - start}',
))
start = time.time()
requests.get(
api_url_for(
rotkehlchen_api_server,
'tradesresource',
),
json={
'only_cache': True,
'offset': 220,
'limit': 10
},
)
end = time.time()
test_warnings.warn(
UserWarning(
f'Premium: {start_with_valid_premium}. Third Page Query Time: {end - start}',
))
def test_query_owned_assets(data_dir, username):
"""Test the get_owned_assets with also an unknown asset in the DB"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
balances = deepcopy(asset_balances)
balances.extend([
DBAssetBalance(
category=BalanceType.ASSET,
time=Timestamp(1488326400),
asset=A_BTC,
amount='1',
usd_value='1222.66',
),
DBAssetBalance(
category=BalanceType.ASSET,
time=Timestamp(1489326500),
asset=A_XMR,
amount='2',
usd_value='33.8',
),
])
data.db.add_multiple_balances(balances)
data.db.conn.commit()
# also make sure that assets from trades are included
data.db.add_trades([
Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
base_asset=A_ETH,
quote_asset=A_BTC,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
),
Trade(
timestamp=Timestamp(99),
location=Location.EXTERNAL,
base_asset=A_ETH,
quote_asset=A_BTC,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(2)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
),
Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
base_asset=A_SDC,
quote_asset=A_SDT2,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
),
Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
base_asset=A_SUSHI,
quote_asset=A_1INCH,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
),
Trade(
timestamp=Timestamp(3),
location=Location.EXTERNAL,
base_asset=A_SUSHI,
quote_asset=A_1INCH,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(2)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
),
])
assets_list = data.db.query_owned_assets()
assert set(assets_list) == {
A_USD, A_ETH, A_BTC, A_XMR, A_SDC, A_SDT2, A_SUSHI, A_1INCH
} # noqa: E501
assert all(isinstance(x, Asset) for x in assets_list)
warnings = data.db.msg_aggregator.consume_warnings()
assert len(warnings) == 0
def test_query_trades_associated_locations(
rotkehlchen_api_server_with_exchanges):
"""Test that querying the trades endpoint works as expected when we have associated
locations including associated exchanges and imported locations.
"""
rotki = rotkehlchen_api_server_with_exchanges.rest_api.rotkehlchen
setup = mock_history_processing_and_exchanges(rotki)
trades = [
Trade(
timestamp=Timestamp(1596429934),
location=Location.EXTERNAL,
base_asset=A_WETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1')),
rate=Price(FVal('320')),
fee=Fee(ZERO),
fee_currency=A_EUR,
link='',
notes='',
),
Trade(
timestamp=Timestamp(1596429934),
location=Location.KRAKEN,
base_asset=A_WETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1')),
rate=Price(FVal('320')),
fee=Fee(ZERO),
fee_currency=A_EUR,
link='',
notes='',
),
Trade(
timestamp=Timestamp(1596429934),
location=Location.BISQ,
base_asset=A_WETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1')),
rate=Price(FVal('320')),
fee=Fee(ZERO),
fee_currency=A_EUR,
link='',
notes='',
),
Trade(
timestamp=Timestamp(1596429934),
location=Location.BINANCE,
base_asset=A_WETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1')),
rate=Price(FVal('320')),
fee=Fee(ZERO),
fee_currency=A_EUR,
link='',
notes='',
)
]
# Add multiple entries for same exchange + connected exchange
rotki.data.db.add_trades(trades)
# Simply get all trades without any filtering
with setup.binance_patch, setup.polo_patch:
response = requests.get(
api_url_for(
rotkehlchen_api_server_with_exchanges,
'tradesresource',
), )
result = assert_proper_response_with_result(response)
result = result['entries']
assert len(
result
) == 9 # 3 polo, (2 + 1) binance trades, 1 kraken, 1 external, 1 BISQ
expected_locations = (
Location.KRAKEN,
Location.POLONIEX,
Location.BINANCE,
Location.BISQ,
Location.EXTERNAL,
)
returned_locations = {x['entry']['location'] for x in result}
assert returned_locations == set(map(str, expected_locations))
response = requests.get(
api_url_for(
rotkehlchen_api_server_with_exchanges,
'tradesresource',
),
json={
'location': 'kraken',
'only_cache': True
},
)
result = assert_proper_response_with_result(response)
result = result['entries']
assert len(result) == 1
response = requests.get(
api_url_for(
rotkehlchen_api_server_with_exchanges,
'tradesresource',
),
json={
'location': 'binance',
'only_cache': True
},
)
result = assert_proper_response_with_result(response)
result = result['entries']
assert len(result) == 3
response = requests.get(
api_url_for(
rotkehlchen_api_server_with_exchanges,
'tradesresource',
),
json={'location': 'nexo'},
)
result = assert_proper_response_with_result(response)
result = result['entries']
assert len(result) == 0
def test_add_asset_movements(data_dir, username, caplog):
"""Test that adding and retrieving asset movements from the DB works fine.
Also duplicates should be ignored and an error returned
"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
movement1 = AssetMovement(
location=Location.BITMEX,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=1451606400,
asset=A_BTC,
amount=FVal('1.0'),
fee_asset=A_EUR,
fee=Fee(FVal('0')),
link='',
)
movement2 = AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.WITHDRAWAL,
address='0xfoo',
transaction_id='0xboo',
timestamp=1451608501,
asset=A_ETH,
amount=FVal('1.0'),
fee_asset=A_EUR,
fee=Fee(FVal('0.01')),
link='',
)
movement3 = AssetMovement(
location=Location.BITTREX,
category=AssetMovementCategory.WITHDRAWAL,
address='0xcoo',
transaction_id='0xdoo',
timestamp=1461708501,
asset=A_ETH,
amount=FVal('1.0'),
fee_asset=A_EUR,
fee=Fee(FVal('0.01')),
link='',
)
# Add and retrieve the first 2 margins. All should be fine.
data.db.add_asset_movements([movement1, movement2])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
returned_movements = data.db.get_asset_movements(
filter_query=AssetMovementsFilterQuery.make(),
has_premium=True,
)
assert returned_movements == [movement1, movement2]
# Add the last 2 movements. Since movement2 already exists in the DB it should be
# ignored and a warning should be logged
data.db.add_asset_movements([movement2, movement3])
assert ('Did not add "withdrawal of ETH with id 94405f38c7b86dd2e7943164d'
'67ff44a32d56cef25840b3f5568e23c037fae0a') in caplog.text
returned_movements = data.db.get_asset_movements(
filter_query=AssetMovementsFilterQuery.make(),
has_premium=True,
)
assert returned_movements == [movement1, movement2, movement3]
def _deserialize_asset_movement(
self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from coinbase and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data['status'] != 'completed':
return None
payout_date = raw_data.get('payout_at', None)
if payout_date:
timestamp = deserialize_timestamp_from_date(
payout_date, 'iso8601', 'coinbase')
else:
timestamp = deserialize_timestamp_from_date(
raw_data['created_at'],
'iso8601',
'coinbase',
)
# Only get address/transaction id for "send" type of transactions
address = None
transaction_id = None
# movement_category: Union[Literal['deposit'], Literal['withdrawal']]
if 'type' in raw_data:
# Then this should be a "send" which is the way Coinbase uses to send
# crypto outside of the exchange
# https://developers.coinbase.com/api/v2?python#transaction-resource
msg = 'Non "send" type found in coinbase deposit/withdrawal processing'
assert raw_data['type'] == 'send', msg
movement_category = AssetMovementCategory.WITHDRAWAL
# Can't see the fee being charged from the "send" resource
amount = deserialize_asset_amount_force_positive(
raw_data['amount']['amount'])
asset = asset_from_coinbase(raw_data['amount']['currency'],
time=timestamp)
# Fees dont appear in the docs but from an experiment of sending ETH
# to an address from coinbase there is the network fee in the response
fee = Fee(ZERO)
raw_network = raw_data.get('network', None)
if raw_network:
raw_fee = raw_network.get('transaction_fee', None)
if raw_fee:
# Since this is a withdrawal the fee should be the same as the moved asset
if asset != asset_from_coinbase(raw_fee['currency'],
time=timestamp):
# If not we set ZERO fee and ignore
log.error(
f'In a coinbase withdrawal of {asset.identifier} the fee'
f'is denoted in {raw_fee["currency"]}', )
else:
fee = deserialize_fee(raw_fee['amount'])
if 'network' in raw_data:
transaction_id = get_key_if_has_val(
raw_data['network'], 'hash')
if 'to' in raw_data:
address = deserialize_asset_movement_address(
raw_data['to'], 'address', asset)
else:
movement_category = deserialize_asset_movement_category(
raw_data['resource'])
amount = deserialize_asset_amount_force_positive(
raw_data['amount']['amount'])
fee = deserialize_fee(raw_data['fee']['amount'])
asset = asset_from_coinbase(raw_data['amount']['currency'],
time=timestamp)
return AssetMovement(
location=Location.COINBASE,
category=movement_category,
address=address,
transaction_id=transaction_id,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=str(raw_data['id']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a coinbase '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of coinbase '
f'asset_movement {raw_data}. Error was: {str(e)}', )
return None
def trade_from_poloniex(poloniex_trade: Dict[str, Any], pair: TradePair) -> Trade:
"""Turn a poloniex trade returned from poloniex trade history to our common trade
history format
Throws:
- UnsupportedAsset due to asset_from_poloniex()
- DeserializationError due to the data being in unexpected format
- UnprocessableTradePair due to the pair data being in an unexpected format
"""
try:
trade_type = deserialize_trade_type(poloniex_trade['type'])
amount = deserialize_asset_amount(poloniex_trade['amount'])
rate = deserialize_price(poloniex_trade['rate'])
perc_fee = deserialize_fee(poloniex_trade['fee'])
base_currency = asset_from_poloniex(get_pair_position_str(pair, 'first'))
quote_currency = asset_from_poloniex(get_pair_position_str(pair, 'second'))
timestamp = deserialize_timestamp_from_poloniex_date(poloniex_trade['date'])
except KeyError as e:
raise DeserializationError(
f'Poloniex trade deserialization error. Missing key entry for {str(e)} in trade dict',
)
cost = rate * amount
if trade_type == TradeType.BUY:
fee = Fee(amount * perc_fee)
fee_currency = quote_currency
elif trade_type == TradeType.SELL:
fee = Fee(cost * perc_fee)
fee_currency = base_currency
else:
raise DeserializationError(f'Got unexpected trade type "{trade_type}" for poloniex trade')
if poloniex_trade['category'] == 'settlement':
if trade_type == TradeType.BUY:
trade_type = TradeType.SETTLEMENT_BUY
else:
trade_type = TradeType.SETTLEMENT_SELL
log.debug(
'Processing poloniex Trade',
sensitive_log=True,
timestamp=timestamp,
order_type=trade_type,
pair=pair,
base_currency=base_currency,
quote_currency=quote_currency,
amount=amount,
fee=fee,
rate=rate,
)
# Use the converted assets in our pair
pair = trade_pair_from_assets(base_currency, quote_currency)
# Since in Poloniex the base currency is the cost currency, iow in poloniex
# for BTC_ETH we buy ETH with BTC and sell ETH for BTC, we need to turn it
# into the Rotkehlchen way which is following the base/quote approach.
pair = invert_pair(pair)
return Trade(
timestamp=timestamp,
location=Location.POLONIEX,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(poloniex_trade['globalTradeID']),
)
def test_query_owned_assets(data_dir, username):
"""Test the get_owned_assets with also an unknown asset in the DB"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
balances = deepcopy(asset_balances)
balances.extend([
DBAssetBalance(
category=BalanceType.ASSET,
time=Timestamp(1488326400),
asset=A_BTC,
amount='1',
usd_value='1222.66',
),
DBAssetBalance(
category=BalanceType.ASSET,
time=Timestamp(1489326500),
asset=A_XMR,
amount='2',
usd_value='33.8',
),
])
data.db.add_multiple_balances(balances)
cursor = data.db.conn.cursor()
cursor.execute(
'INSERT INTO timed_balances('
' time, currency, amount, usd_value, category) '
' VALUES(?, ?, ?, ?, ?)',
(1469326500, 'ADSADX', '10.1', '100.5', 'A'),
)
data.db.conn.commit()
# also make sure that assets from trades are included
data.db.add_trades([
Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
pair=TradePair('ETH_BTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
), Trade(
timestamp=Timestamp(99),
location=Location.EXTERNAL,
pair=TradePair('ETH_BTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(2)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
), Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
pair=TradePair('SDC_SDT-2'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
), Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
pair=TradePair('SUSHI_1INCH'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
), Trade(
timestamp=Timestamp(3),
location=Location.EXTERNAL,
pair=TradePair('SUSHI_1INCH'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(2)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
), Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
pair=TradePair('UNKNOWNTOKEN_BTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
),
])
assets_list = data.db.query_owned_assets()
assert set(assets_list) == {A_USD, A_ETH, A_DAI, A_BTC, A_XMR, Asset('SDC'), Asset('SDT-2'), Asset('SUSHI'), Asset('1INCH')} # noqa: E501
assert all(isinstance(x, Asset) for x in assets_list)
warnings = data.db.msg_aggregator.consume_warnings()
assert len(warnings) == 1
assert 'Unknown/unsupported asset ADSADX' in warnings[0]
def _import_cryptocom_double_entries(self, data: Any,
double_type: str) -> None:
"""Look for events that have double entries and handle them as trades.
This method looks for `*_debited` and `*_credited` entries using the
same timestamp to handle them as one trade.
Known double_type: 'dynamic_coin_swap' or 'dust_conversion'
"""
double_rows: Dict[Any, Dict[str, Any]] = {}
debited_row = None
credited_row = None
for row in data:
if row['Transaction Kind'] == f'{double_type}_debited':
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='crypto.com',
)
if timestamp not in double_rows:
double_rows[timestamp] = {}
double_rows[timestamp]['debited'] = row
elif row['Transaction Kind'] == f'{double_type}_credited':
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='crypto.com',
)
if timestamp not in double_rows:
double_rows[timestamp] = {}
double_rows[timestamp]['credited'] = row
for timestamp in double_rows:
credited_row = double_rows[timestamp]['credited']
debited_row = double_rows[timestamp]['debited']
if credited_row is not None and debited_row is not None:
description = credited_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees here
fee = Fee(ZERO)
fee_currency = A_USD
base_asset = Asset(credited_row['Currency'])
quote_asset = Asset(debited_row['Currency'])
pair = TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}')
base_amount_bought = deserialize_asset_amount(
credited_row['Amount'])
quote_amount_sold = deserialize_asset_amount(
debited_row['Amount'])
rate = Price(abs(base_amount_bought / quote_amount_sold))
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
pair=pair,
trade_type=TradeType.BUY,
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
def test_add_trades(data_dir, username):
"""Test that adding and retrieving trades from the DB works fine.
Also duplicates should be ignored and an error returned
"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
trade1 = Trade(
timestamp=1451606400,
location=Location.KRAKEN,
pair='ETH_EUR',
trade_type=TradeType.BUY,
amount=FVal('1.1'),
rate=FVal('10'),
fee=Fee(FVal('0.01')),
fee_currency=A_EUR,
link='',
notes='',
)
trade2 = Trade(
timestamp=1451607500,
location=Location.BINANCE,
pair='BTC_ETH',
trade_type=TradeType.BUY,
amount=FVal('0.00120'),
rate=FVal('10'),
fee=Fee(FVal('0.001')),
fee_currency=A_ETH,
link='',
notes='',
)
trade3 = Trade(
timestamp=1451608600,
location=Location.COINBASE,
pair='BTC_ETH',
trade_type=TradeType.SELL,
amount=FVal('0.00120'),
rate=FVal('1'),
fee=Fee(FVal('0.001')),
fee_currency=A_ETH,
link='',
notes='',
)
# Add and retrieve the first 2 trades. All should be fine.
data.db.add_trades([trade1, trade2])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
returned_trades = data.db.get_trades()
assert returned_trades == [trade1, trade2]
# Add the last 2 trades. Since trade2 already exists in the DB it should be
# ignored and a warning should be shown
data.db.add_trades([trade2, trade3])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 1
returned_trades = data.db.get_trades()
assert returned_trades == [trade1, trade2, trade3]
def query_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
end_at_least_ts: Timestamp,
) -> List[AssetMovement]:
with self.lock:
cache = self.check_trades_cache_list(
start_ts,
end_at_least_ts,
special_name='deposits_withdrawals',
)
result: List[Any]
if cache is not None:
result = cache
else:
result = self.query_until_finished(
endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict=dict(type='deposit'),
)
result.extend(
self.query_until_finished(
endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict=dict(type='withdrawal'),
))
with self.lock:
self.update_trades_cache(
result,
start_ts,
end_ts,
special_name='deposits_withdrawals',
)
log.debug('Kraken deposit/withdrawals query result',
num_results=len(result))
movements = list()
for movement in result:
try:
movements.append(
AssetMovement(
exchange='kraken',
category=movement['type'],
# Kraken timestamps have floating point
timestamp=Timestamp(
convert_to_int(movement['time'],
accept_only_exact=False)),
asset=asset_from_kraken(movement['asset']),
amount=FVal(movement['amount']),
fee=Fee(FVal(movement['fee'])),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown kraken asset {e.asset_name}. '
f'Ignoring its deposit/withdrawals query.', )
continue
return movements
def test_add_margin_positions(data_dir, username):
"""Test that adding and retrieving margin positions from the DB works fine.
Also duplicates should be ignored and an error returned
"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
margin1 = MarginPosition(
location=Location.BITMEX,
open_time=1451606400,
close_time=1451616500,
profit_loss=FVal('1.0'),
pl_currency=A_BTC,
fee=Fee(FVal('0.01')),
fee_currency=A_EUR,
link='',
notes='',
)
margin2 = MarginPosition(
location=Location.BITMEX,
open_time=1451626500,
close_time=1451636500,
profit_loss=FVal('0.5'),
pl_currency=A_BTC,
fee=Fee(FVal('0.01')),
fee_currency=A_EUR,
link='',
notes='',
)
margin3 = MarginPosition(
location=Location.POLONIEX,
open_time=1452636501,
close_time=1459836501,
profit_loss=FVal('2.5'),
pl_currency=A_BTC,
fee=Fee(FVal('0.01')),
fee_currency=A_EUR,
link='',
notes='',
)
# Add and retrieve the first 2 margins. All should be fine.
data.db.add_margin_positions([margin1, margin2])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
returned_margins = data.db.get_margin_positions()
assert returned_margins == [margin1, margin2]
# Add the last 2 margins. Since margin2 already exists in the DB it should be
# ignored and a warning should be shown
data.db.add_margin_positions([margin2, margin3])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 1
returned_margins = data.db.get_margin_positions()
assert returned_margins == [margin1, margin2, margin3]
'fee': '0.0001',
'earned': '0.0025',
'amount': '2',
},
]
asset_movements_list = [
AssetMovement(
# before query period -- 8.915 * 0.001 = 8.915e-3
location=Location.KRAKEN,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=Timestamp(1479510304), # 18/11/2016,
asset=A_ETH, # cryptocompare hourly ETH/EUR: 8.915
amount=FVal('95'),
fee_asset=A_ETH,
fee=Fee(FVal('0.001')),
link='krakenid1',
),
AssetMovement( # 0.0087*52.885 = 0.4600995
location=Location.KRAKEN,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=Timestamp(1493291104), # 27/04/2017,
asset=A_ETH, # cryptocompare hourly ETH/EUR: 52.885
amount=FVal('125'),
fee_asset=A_ETH,
fee=Fee(FVal('0.0087')),
link='krakenid2',
),
AssetMovement( # deposits have no effect
location=Location.KRAKEN,
category=AssetMovementCategory.DEPOSIT,
def _import_cryptocom_associated_entries(self, data: Any,
tx_kind: str) -> None:
"""Look for events that have associated entries and handle them as trades.
This method looks for `*_debited` and `*_credited` entries using the
same timestamp to handle them as one trade.
Known kind: 'dynamic_coin_swap' or 'dust_conversion'
May raise:
- UnknownAsset if an unknown asset is encountered in the imported files
- KeyError if a row contains unexpected data entries
"""
multiple_rows: Dict[Any, Dict[str, Any]] = {}
investments_deposits: Dict[str, List[Any]] = defaultdict(list)
investments_withdrawals: Dict[str, List[Any]] = defaultdict(list)
debited_row = None
credited_row = None
for row in data:
if row['Transaction Kind'] == f'{tx_kind}_debited':
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
if timestamp not in multiple_rows:
multiple_rows[timestamp] = {}
if 'debited' not in multiple_rows[timestamp]:
multiple_rows[timestamp]['debited'] = []
multiple_rows[timestamp]['debited'].append(row)
elif row['Transaction Kind'] == f'{tx_kind}_credited':
# They only is one credited row
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
if timestamp not in multiple_rows:
multiple_rows[timestamp] = {}
multiple_rows[timestamp]['credited'] = row
elif row['Transaction Kind'] == f'{tx_kind}_deposit':
asset = row['Currency']
investments_deposits[asset].append(row)
elif row['Transaction Kind'] == f'{tx_kind}_withdrawal':
asset = row['Currency']
investments_withdrawals[asset].append(row)
for timestamp in multiple_rows:
# When we convert multiple assets dust to CRO
# in one time, it will create multiple debited rows with
# the same timestamp
debited_rows = multiple_rows[timestamp]['debited']
credited_row = multiple_rows[timestamp]['credited']
total_debited_usd = functools.reduce(
lambda acc, row: acc + deserialize_asset_amount(row[
'Native Amount (in USD)']),
debited_rows,
ZERO,
)
# If the value of the transaction is too small (< 0,01$),
# crypto.com will display 0 as native amount
# if we have multiple debited rows, we can't import them
# since we can't compute their dedicated rates, so we skip them
if len(debited_rows) > 1 and total_debited_usd == 0:
return
if credited_row is not None and len(debited_rows) != 0:
for debited_row in debited_rows:
description = credited_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees here
fee = Fee(ZERO)
fee_currency = A_USD
base_asset = symbol_to_asset_or_token(
credited_row['Currency'])
quote_asset = symbol_to_asset_or_token(
debited_row['Currency'])
part_of_total = (
FVal(1) if len(debited_rows) == 1 else
deserialize_asset_amount(
debited_row["Native Amount (in USD)"], ) /
total_debited_usd)
quote_amount_sold = deserialize_asset_amount(
debited_row['Amount'], ) * part_of_total
base_amount_bought = deserialize_asset_amount(
credited_row['Amount'], ) * part_of_total
rate = Price(abs(base_amount_bought / quote_amount_sold))
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=TradeType.BUY,
amount=AssetAmount(base_amount_bought),
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
# Compute investments profit
if len(investments_withdrawals) != 0:
for asset in investments_withdrawals:
asset_object = symbol_to_asset_or_token(asset)
if asset not in investments_deposits:
log.error(
f'Investment withdrawal without deposit at crypto.com. Ignoring '
f'staking info for asset {asset_object}', )
continue
# Sort by date in ascending order
withdrawals_rows = sorted(
investments_withdrawals[asset],
key=lambda x: deserialize_timestamp_from_date(
date=x['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
),
)
investments_rows = sorted(
investments_deposits[asset],
key=lambda x: deserialize_timestamp_from_date(
date=x['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
),
)
last_date = Timestamp(0)
for withdrawal in withdrawals_rows:
withdrawal_date = deserialize_timestamp_from_date(
date=withdrawal['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
amount_deposited = ZERO
for deposit in investments_rows:
deposit_date = deserialize_timestamp_from_date(
date=deposit['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
if last_date < deposit_date <= withdrawal_date:
# Amount is negative
amount_deposited += deserialize_asset_amount(
deposit['Amount'])
amount_withdrawal = deserialize_asset_amount(
withdrawal['Amount'])
# Compute profit
profit = amount_withdrawal + amount_deposited
if profit >= ZERO:
last_date = withdrawal_date
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=withdrawal_date,
action_type=LedgerActionType.INCOME,
location=Location.CRYPTOCOM,
amount=AssetAmount(profit),
asset=asset_object,
rate=None,
rate_asset=None,
link=None,
notes=f'Stake profit for asset {asset}',
)
self.db_ledger.add_ledger_action(action)
