def test_add_margin_positions(data_dir, username, caplog):
"""Test that adding and retrieving margin positions from the DB works fine.
Also duplicates should be ignored and an error returned
"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
margin1 = MarginPosition(
location=Location.BITMEX,
open_time=1451606400,
close_time=1451616500,
profit_loss=FVal('1.0'),
pl_currency=A_BTC,
fee=Fee(FVal('0.01')),
fee_currency=A_EUR,
link='',
notes='',
)
margin2 = MarginPosition(
location=Location.BITMEX,
open_time=1451626500,
close_time=1451636500,
profit_loss=FVal('0.5'),
pl_currency=A_BTC,
fee=Fee(FVal('0.01')),
fee_currency=A_EUR,
link='',
notes='',
)
margin3 = MarginPosition(
location=Location.POLONIEX,
open_time=1452636501,
close_time=1459836501,
profit_loss=FVal('2.5'),
pl_currency=A_BTC,
fee=Fee(FVal('0.01')),
fee_currency=A_EUR,
link='',
notes='',
)
# Add and retrieve the first 2 margins. All should be fine.
data.db.add_margin_positions([margin1, margin2])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
returned_margins = data.db.get_margin_positions()
assert returned_margins == [margin1, margin2]
# Add the last 2 margins. Since margin2 already exists in the DB it should be
# ignored and a warning should be logged
data.db.add_margin_positions([margin2, margin3])
assert (
'Did not add "Margin position with id 0a57acc1f4c09da0f194c59c4cd240e6'
'8e2d36e56c05b3f7115def9b8ee3943f') in caplog.text
returned_margins = data.db.get_margin_positions()
assert returned_margins == [margin1, margin2, margin3]
def assert_cointracking_import_results(rotki: Rotkehlchen):
"""A utility function to help assert on correctness of importing data from cointracking.info"""
trades = rotki.data.db.get_trades()
asset_movements = rotki.data.db.get_asset_movements()
warnings = rotki.msg_aggregator.consume_warnings()
errors = rotki.msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 3
expected_trades = [
Trade(
timestamp=Timestamp(1566687660),
location=Location.COINBASE,
pair=TradePair('ETH_EUR'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.05772716')),
rate=Price(FVal('190.3783245183029963712055123')),
fee=Fee(ZERO),
fee_currency=A_ETH,
link='',
notes='',
),
Trade(
timestamp=Timestamp(1567418400),
location=Location.EXTERNAL,
pair=TradePair('BTC_USD'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.00100000')),
rate=Price(ZERO),
fee=Fee(ZERO),
fee_currency=A_BTC,
link='',
notes='Just a small gift from someone',
)
]
assert expected_trades == trades
expected_movements = [
AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1565848620),
asset=A_XMR,
amount=AssetAmount(FVal('5')),
fee_asset=A_XMR,
fee=Fee(ZERO),
link='',
),
AssetMovement(
location=Location.COINBASE,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=Timestamp(1566726120),
asset=A_ETH,
amount=AssetAmount(FVal('0.05770427')),
fee_asset=A_ETH,
fee=Fee(ZERO),
link='',
)
]
assert expected_movements == asset_movements
def _deserialize_asset_movement(
self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from binance and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if 'insertTime' in raw_data:
category = AssetMovementCategory.DEPOSIT
time_key = 'insertTime'
fee = Fee(ZERO)
else:
category = AssetMovementCategory.WITHDRAWAL
time_key = 'applyTime'
fee = Fee(deserialize_asset_amount(raw_data['transactionFee']))
timestamp = deserialize_timestamp_from_binance(raw_data[time_key])
asset = asset_from_binance(raw_data['asset'])
tx_id = get_key_if_has_val(raw_data, 'txId')
internal_id = get_key_if_has_val(raw_data, 'id')
link_str = str(internal_id) if internal_id else str(
tx_id) if tx_id else ''
return AssetMovement(
location=self.location,
category=category,
address=deserialize_asset_movement_address(
raw_data, 'address', asset),
transaction_id=tx_id,
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
raw_data['amount']),
fee_asset=asset,
fee=fee,
link=link_str,
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(self.location)} deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(self.location)} deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
f'Error processing a {str(self.location)} deposit/withdrawal. Check logs '
f'for details. Ignoring it.', )
log.error(
f'Error processing a {str(self.location)} deposit/withdrawal',
asset_movement=raw_data,
error=msg,
)
return None
def test_add_asset_movements(data_dir, username):
"""Test that adding and retrieving asset movements from the DB works fine.
Also duplicates should be ignored and an error returned
"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
movement1 = AssetMovement(
location=Location.BITMEX,
category=AssetMovementCategory.DEPOSIT,
timestamp=1451606400,
asset=A_BTC,
amount=FVal('1.0'),
fee_asset=A_EUR,
fee=Fee(FVal('0')),
link='',
)
movement2 = AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=1451608501,
asset=A_ETH,
amount=FVal('1.0'),
fee_asset=A_EUR,
fee=Fee(FVal('0.01')),
link='',
)
movement3 = AssetMovement(
location=Location.BITTREX,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=1461708501,
asset=A_ETH,
amount=FVal('1.0'),
fee_asset=A_EUR,
fee=Fee(FVal('0.01')),
link='',
)
# Add and retrieve the first 2 margins. All should be fine.
data.db.add_asset_movements([movement1, movement2])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
returned_movements = data.db.get_asset_movements()
assert returned_movements == [movement1, movement2]
# Add the last 2 movements. Since movement2 already exists in the DB it should be
# ignored and a warning should be shown
data.db.add_asset_movements([movement2, movement3])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 1
returned_movements = data.db.get_asset_movements()
assert returned_movements == [movement1, movement2, movement3]
def test_query_online_deposits_withdrawals(mock_bitstamp, start_ts, since_id):
"""Test `since_id` value will change depending on `start_ts` value.
Also tests `db_asset_movements` are sorted by `link` (as int) in ascending
mode.
"""
asset_btc = A_BTC
asset_usd = A_USD
movements = [
AssetMovement(
timestamp=1606901400,
location=Location.BITSTAMP,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
asset=asset_usd,
amount=FVal('10000'),
fee_asset=asset_usd,
fee=Fee(FVal('50')),
link='5',
),
AssetMovement(
timestamp=1606901400,
location=Location.BITSTAMP,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
asset=asset_btc,
amount=FVal('0.5'),
fee_asset=asset_btc,
fee=Fee(FVal('0.0005')),
link='2',
),
]
database = MagicMock()
database.get_asset_movements.return_value = movements
expected_call = call(
start_ts=start_ts,
end_ts=2,
options={
'since_id': since_id,
'limit': 1000,
'sort': 'asc',
'offset': 0,
},
case='asset_movements',
)
with patch.object(mock_bitstamp,
'db',
new_callable=MagicMock(return_value=database)):
with patch.object(mock_bitstamp,
'_api_query_paginated') as mock_api_query_paginated:
mock_bitstamp.query_online_deposits_withdrawals(
start_ts=Timestamp(start_ts),
end_ts=Timestamp(2),
)
assert mock_api_query_paginated.call_args == expected_call
def assert_kraken_asset_movements(
to_check_list: List[Any],
deserialized: bool,
movements_to_check: Optional[Tuple[int, ...]] = None,
):
expected = [
AssetMovement(
location=Location.KRAKEN,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=Timestamp(1458994442),
asset=A_BTC,
amount=FVal('5.0'),
fee_asset=A_BTC,
fee=Fee(FVal('0.1')),
link='1',
),
AssetMovement(
location=Location.KRAKEN,
address=None,
transaction_id=None,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1448994442),
asset=A_ETH,
amount=FVal('10.0'),
fee_asset=A_ETH,
fee=Fee(FVal('0.11')),
link='2',
),
AssetMovement(
location=Location.KRAKEN,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=Timestamp(1439994442),
asset=A_ETH,
amount=FVal('10.0'),
fee_asset=A_ETH,
fee=Fee(FVal('0.11')),
link='5',
),
AssetMovement(
location=Location.KRAKEN,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=Timestamp(1428994442),
asset=A_BTC,
amount=FVal('5.0'),
fee_asset=A_BTC,
fee=Fee(FVal('0.1')),
link='4',
)
]
assert_asset_movements(expected, to_check_list, deserialized,
movements_to_check)
def test_query_deposits_withdrawals(mock_ftx: Ftx):
"""Test happy path of deposits/withdrawls"""
with patch.object(mock_ftx.session,
'get',
side_effect=mock_normal_ftx_query):
movements = mock_ftx.query_online_deposits_withdrawals(
start_ts=Timestamp(0),
end_ts=TEST_END_TS,
)
warnings = mock_ftx.msg_aggregator.consume_warnings()
errors = mock_ftx.msg_aggregator.consume_errors()
assert len(warnings) == 0
assert len(errors) == 0
expected_movements = [
AssetMovement(
location=Location.FTX,
category=AssetMovementCategory.DEPOSIT,
address='0x541163adf0a2e830d9f940763e912807d1a359f5',
transaction_id=
'0xf787fa6b62edf1c97fb3f73f80a5eb7550bbf3dcf4269b9bfb9e8c1c0a3bc1a9',
timestamp=Timestamp(1612159566),
asset=A_ETH,
amount=FVal('20'),
fee_asset=A_ETH,
fee=Fee(ZERO),
link='3',
),
AssetMovement(
location=Location.FTX,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=Timestamp(1612159566),
address='0x903d12bf2c57a29f32365917c706ce0e1a84cce3',
transaction_id=
'0xbb27f24c2a348526fc23767d3d8bb303099e90f253ef9fdbb28ce38c1635d116',
asset=A_ETH,
amount=FVal('11.0'),
fee_asset=A_ETH,
fee=Fee(ZERO),
link='1',
),
AssetMovement(
location=Location.FTX,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=Timestamp(1612159566),
asset=A_USD,
amount=FVal('21.0'),
fee_asset=A_USD,
fee=Fee(ZERO),
link='2',
)
]
assert len(movements) == 3
assert movements == expected_movements
def test_deserialize_asset_movement_withdrawal(mock_bitstamp):
raw_movement = {
'id': 5,
'type': '1',
'datetime': '2020-12-02 09:30:00',
'btc': '0.00000000',
'usd': '-10000.00000000',
'btc_usd': '0.00',
'fee': '50.00000000',
'order_id': 2,
'eur': '0.00',
}
asset = A_USD
movement = AssetMovement(
timestamp=1606901400,
location=Location.BITSTAMP,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
asset=asset,
amount=FVal('10000'),
fee_asset=asset,
fee=Fee(FVal('50')),
link='5',
)
expected_movement = mock_bitstamp._deserialize_asset_movement(raw_movement)
assert movement == expected_movement
raw_movement = {
'id': 5,
'type': '1',
'datetime': '2018-03-21 06:46:06.559877',
'btc': '0',
'usd': '0',
'btc_usd': '0.00',
'fee': '0.1',
'order_id': 2,
'eur': '500',
}
asset = A_EUR
movement = AssetMovement(
timestamp=1521614766,
location=Location.BITSTAMP,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
asset=asset,
amount=FVal('500'),
fee_asset=asset,
fee=Fee(FVal('0.1')),
link='5',
)
expected_movement = mock_bitstamp._deserialize_asset_movement(raw_movement)
assert movement == expected_movement
def test_deserialize_asset_movement_deposit(mock_bitstamp):
raw_movement = {
'id': 2,
'type': '0',
'datetime': '2020-12-02 09:30:00',
'btc': '0.50000000',
'usd': '0.00000000',
'btc_usd': '0.00',
'fee': '0.00050000',
'order_id': 2,
'eur': '0.00',
}
asset = A_BTC
movement = AssetMovement(
timestamp=1606901400,
location=Location.BITSTAMP,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
asset=asset,
amount=FVal('0.5'),
fee_asset=asset,
fee=Fee(FVal('0.0005')),
link='2',
)
expected_movement = mock_bitstamp._deserialize_asset_movement(raw_movement)
assert movement == expected_movement
raw_movement = {
'id': 3,
'type': '0',
'datetime': '2018-03-21 06:46:06.559877',
'btc': '0',
'usd': '0.00000000',
'btc_usd': '0.00',
'fee': '0.1',
'order_id': 2,
'gbp': '1000.51',
}
asset = A_GBP
movement = AssetMovement(
timestamp=1521614766,
location=Location.BITSTAMP,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
asset=asset,
amount=FVal('1000.51'),
fee_asset=asset,
fee=Fee(FVal('0.1')),
link='3',
)
expected_movement = mock_bitstamp._deserialize_asset_movement(raw_movement)
assert movement == expected_movement
def trade_from_conversion(trade_a: Dict[str, Any], trade_b: Dict[str, Any]) -> Optional[Trade]:
"""Turn information from a conversion into a trade
Mary raise:
- UnknownAsset due to Asset instantiation
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entires missing an expected entry
"""
# Check that the status is complete
if trade_a['status'] != 'completed':
return None
# Trade b will represent the asset we are converting to
if trade_b['amount']['amount'].startswith('-'):
trade_a, trade_b = trade_b, trade_a
timestamp = deserialize_timestamp_from_date(trade_a['updated_at'], 'iso8601', 'coinbase')
trade_type = deserialize_trade_type('sell')
tx_amount = AssetAmount(abs(deserialize_asset_amount(trade_a['amount']['amount'])))
tx_asset = asset_from_coinbase(trade_a['amount']['currency'], time=timestamp)
native_amount = deserialize_asset_amount(trade_b['amount']['amount'])
native_asset = asset_from_coinbase(trade_b['amount']['currency'], time=timestamp)
amount = tx_amount
# The rate is how much you get/give in quotecurrency if you buy/sell 1 unit of base currency
rate = Price(native_amount / tx_amount)
# Obtain fee amount in the native currency using data from both trades
amount_after_fee = deserialize_asset_amount(trade_b['native_amount']['amount'])
amount_before_fee = deserialize_asset_amount(trade_a['native_amount']['amount'])
# amount_after_fee + amount_before_fee is a negative amount and the fee needs to be positive
conversion_native_fee_amount = abs(amount_after_fee + amount_before_fee)
if ZERO not in (tx_amount, conversion_native_fee_amount, amount_before_fee):
# We have the fee amount in the native currency. To get it in the
# converted asset we have to get the rate
asset_native_rate = tx_amount / abs(amount_before_fee)
fee_amount = Fee(conversion_native_fee_amount / asset_native_rate)
else:
fee_amount = Fee(ZERO)
fee_asset = asset_from_coinbase(trade_a['amount']['currency'], time=timestamp)
return Trade(
timestamp=timestamp,
location=Location.COINBASE,
# in coinbase you are buying/selling tx_asset for native_asset
base_asset=tx_asset,
quote_asset=native_asset,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee_amount,
fee_currency=fee_asset,
link=str(trade_a['trade']['id']),
)
def assert_poloniex_asset_movements(
to_check_list: List[Any],
deserialized: bool,
movements_to_check: Optional[Tuple[int, ...]] = None,
) -> None:
expected = [AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.WITHDRAWAL,
address='0xB7E033598Cb94EF5A35349316D3A2e4f95f308Da',
transaction_id='0xbd4da74e1a0b81c21d056c6f58a5b306de85d21ddf89992693b812bb117eace4',
timestamp=Timestamp(1468994442),
asset=A_ETH,
amount=FVal('10.0'),
fee_asset=A_ETH,
fee=Fee(FVal('0.1')),
link='2',
), AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.WITHDRAWAL,
address='131rdg5Rzn6BFufnnQaHhVa5ZtRU1J2EZR',
transaction_id='2d27ae26fa9c70d6709e27ac94d4ce2fde19b3986926e9f3bfcf3e2d68354ec5',
timestamp=Timestamp(1458994442),
asset=A_BTC,
amount=FVal('5.0'),
fee_asset=A_BTC,
fee=Fee(FVal('0.5')),
link='1',
), AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.DEPOSIT,
address='131rdg5Rzn6BFufnnQaHhVa5ZtRU1J2EZR',
transaction_id='b05bdec7430a56b5a5ed34af4a31a54859dda9b7c88a5586bc5d6540cdfbfc7a',
timestamp=Timestamp(1448994442),
asset=A_BTC,
amount=FVal('50.0'),
fee_asset=A_BTC,
fee=Fee(FVal('0')),
link='1',
), AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.DEPOSIT,
address='0xB7E033598Cb94EF5A35349316D3A2e4f95f308Da',
transaction_id='0xf7e7eeb44edcad14c0f90a5fffb1cbb4b80e8f9652124a0838f6906ca939ccd2',
timestamp=Timestamp(1438994442),
asset=A_ETH,
amount=FVal('100.0'),
fee_asset=A_ETH,
fee=Fee(FVal('0')),
link='2',
)]
assert_asset_movements(expected, to_check_list, deserialized, movements_to_check)
def deserialize_fee(fee: Optional[str]) -> Fee:
"""Deserializes a fee from a json entry. Fee in the JSON entry can also be null
in which case a ZERO fee is returned.
Can throw DeserializationError if the fee is not as expected
"""
if fee is None:
return Fee(ZERO)
try:
result = Fee(FVal(fee))
except ValueError as e:
raise DeserializationError(f'Failed to deserialize a fee entry due to: {str(e)}') from e
return result
def _trade_from_independentreserve(raw_trade: Dict) -> Trade:
"""Convert IndependentReserve raw data to a trade
https://www.independentreserve.com/products/api#GetClosedFilledOrders
May raise:
- DeserializationError
- UnknownAsset
- KeyError
"""
log.debug(f'Processing raw IndependentReserve trade: {raw_trade}')
trade_type = TradeType.BUY if 'Bid' in raw_trade['OrderType'] else TradeType.SELL
base_asset = independentreserve_asset(raw_trade['PrimaryCurrencyCode'])
quote_asset = independentreserve_asset(raw_trade['SecondaryCurrencyCode'])
amount = FVal(raw_trade['Volume']) - FVal(raw_trade['Outstanding'])
timestamp = deserialize_timestamp_from_date(
date=raw_trade['CreatedTimestampUtc'],
formatstr='iso8601',
location='IndependentReserve',
)
rate = Price(FVal(raw_trade['AvgPrice']))
fee_amount = FVal(raw_trade['FeePercent']) * amount
fee_asset = base_asset
return Trade(
timestamp=timestamp,
location=Location.INDEPENDENTRESERVE,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=trade_type,
amount=AssetAmount(amount),
rate=rate,
fee=Fee(fee_amount),
fee_currency=fee_asset,
link=str(raw_trade['OrderGuid']),
)
def mock_exchange_data_in_db(exchanges, rotki) -> None:
db = rotki.data.db
for exchange_name in exchanges:
db.add_trades([
Trade(
timestamp=Timestamp(1),
location=deserialize_location(exchange_name),
base_asset=A_BTC,
quote_asset=A_ETH,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_ETH,
link='foo',
notes='boo',
)
])
db.update_used_query_range(name=f'{exchange_name}_trades',
start_ts=0,
end_ts=9999)
db.update_used_query_range(name=f'{exchange_name}_margins',
start_ts=0,
end_ts=9999)
db.update_used_query_range(name=f'{exchange_name}_asset_movements',
start_ts=0,
end_ts=9999) # noqa: E501
def get_fee_in_profit_currency(self, trade: Trade) -> Fee:
fee_rate = self.query_historical_price(
from_asset=trade.fee_currency,
to_asset=self.profit_currency,
timestamp=trade.timestamp,
)
return Fee(fee_rate * trade.fee)
def test_deserialize_trade_buy(mock_bitfinex):
mock_bitfinex.currency_map = {
'WBT': 'WBTC',
'UST': 'USDt',
}
mock_bitfinex.pair_bfx_symbols_map = {'WBTUST': ('WBT', 'UST')}
raw_result = [
399251013,
'tWBT:UST',
1573485493000,
33963608932,
0.26334268,
187.37,
'LIMIT',
None,
-1,
-0.09868591,
'USD',
]
expected_trade = Trade(
timestamp=Timestamp(1573485493),
location=Location.BITFINEX,
base_asset=A_WBTC,
quote_asset=A_USDT,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.26334268')),
rate=Price(FVal('187.37')),
fee=Fee(FVal('0.09868591')),
fee_currency=A_USD,
link='399251013',
notes='',
)
trade = mock_bitfinex._deserialize_trade(raw_result=raw_result)
assert trade == expected_trade
def test_deserialize_v1_trade(mock_kucoin):
raw_result = {
'id': 'xxxx',
'symbol': 'NANO-ETH',
'dealPrice': '0.015743',
'dealValue': '0.00003441',
'amount': '0.002186',
'fee': '0.00000003',
'side': 'sell',
'createdAt': 1520471876,
}
expected_trade = Trade(
timestamp=Timestamp(1520471876),
location=Location.KUCOIN,
pair=TradePair('NANO_ETH'),
trade_type=TradeType.SELL,
amount=AssetAmount(FVal('0.002186')),
rate=Price(FVal('0.015743')),
fee=Fee(FVal('0.00000003')),
fee_currency=Asset('ETH'),
link='xxxx',
notes='',
)
trade = mock_kucoin._deserialize_trade(
raw_result=raw_result,
case=KucoinCase.OLD_TRADES,
)
assert trade == expected_trade
def test_deserialize_trade_sell(mock_bitfinex):
mock_bitfinex.currency_map = {'UST': 'USDt'}
mock_bitfinex.pair_bfx_symbols_map = {'ETHUST': ('ETH', 'UST')}
raw_result = [
399251013,
'tETHUST',
1573485493000,
33963608932,
-0.26334268,
187.37,
'LIMIT',
None,
-1,
-0.09868591,
'USD',
]
expected_trade = Trade(
timestamp=Timestamp(1573485493),
location=Location.BITFINEX,
pair=TradePair('ETH_USDT'),
trade_type=TradeType.SELL,
amount=AssetAmount(FVal('0.26334268')),
rate=Price(FVal('187.37')),
fee=Fee(FVal('0.09868591')),
fee_currency=Asset('USD'),
link='399251013',
notes='',
)
trade = mock_bitfinex._deserialize_trade(raw_result=raw_result)
assert trade == expected_trade
def test_get_associated_locations(
rotkehlchen_api_server_with_exchanges,
added_exchanges,
ethereum_accounts, # pylint: disable=unused-argument
start_with_valid_premium, # pylint: disable=unused-argument
):
rotki = rotkehlchen_api_server_with_exchanges.rest_api.rotkehlchen
mock_exchange_data_in_db(added_exchanges, rotki)
db = rotki.data.db
db.add_trades([Trade(
timestamp=Timestamp(1595833195),
location=Location.NEXO,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1.0')),
rate=Price(FVal('281.14')),
fee=Fee(ZERO),
fee_currency=A_EUR,
link='',
notes='',
)])
# get locations
response = requests.get(
api_url_for(
rotkehlchen_api_server_with_exchanges,
'associatedlocations',
),
)
result = assert_proper_response_with_result(response)
assert set(result) == {'nexo', 'binance', 'poloniex'}
def asset_movements_from_dictlist(
given_data: List[Dict[str, Any]],
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
""" Gets a list of dict asset movements, most probably read from the json files and
a time period. Returns it as a list of the AssetMovement tuples that are inside the time period
May raise:
- KeyError: If the given_data dict contains data in an unexpected format
"""
returned_movements = list()
for movement in given_data:
if movement['timestamp'] < start_ts:
continue
if movement['timestamp'] > end_ts:
break
returned_movements.append(AssetMovement(
exchange=movement['exchange'],
category=movement['category'],
timestamp=movement['timestamp'],
asset=Asset(movement['asset']),
amount=FVal(movement['amount']),
fee=Fee(FVal(movement['fee'])),
))
return returned_movements
def test_deserialize_asset_movement_deposit(mock_bitstamp):
raw_movement = {
'id': 2,
'type': 0,
'datetime': '2020-12-02 09:30:00',
'btc': '0.50000000',
'usd': '0.00000000',
'btc_usd': '0.00',
'fee': '0.00050000',
'order_id': 2,
'eur': "0.00",
}
asset = Asset('BTC')
movement = AssetMovement(
timestamp=1606901400,
location=Location.BITSTAMP,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
asset=asset,
amount=FVal('0.5'),
fee_asset=asset,
fee=Fee(FVal('0.0005')),
link='2',
)
expected_movement = mock_bitstamp._deserialize_asset_movement(raw_movement)
assert movement == expected_movement
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
result = self._get_paginated_query(
endpoint='transfers',
start_ts=start_ts,
end_ts=end_ts,
)
movements = []
for entry in result:
try:
timestamp = deserialize_timestamp(entry['timestampms'])
timestamp = Timestamp(int(timestamp / 1000))
asset = Asset(entry['currency'])
movement = AssetMovement(
location=Location.GEMINI,
category=deserialize_asset_movement_category(
entry['type']),
address=deserialize_asset_movement_address(
entry, 'destination', asset),
transaction_id=get_key_if_has_val(entry, 'txHash'),
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
entry['amount']),
fee_asset=asset,
# Gemini does not include withdrawal fees neither in the API nor in their UI
fee=Fee(ZERO),
link=str(entry['eid']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Error processing a gemini deposit/withdrawal. Check logs '
'for details. Ignoring it.', )
log.error(
'Error processing a gemini deposit_withdrawal',
asset_movement=entry,
error=msg,
)
continue
movements.append(movement)
return movements
def test_deserialize_asset_movement_withdrawal(mock_kucoin):
raw_result = {
'id': '5c2dc64e03aa675aa263f1ac',
'address': '0x5bedb060b8eb8d823e2414d82acce78d38be7fe9',
'memo': '',
'currency': 'ETH',
'amount': 1,
'fee': 0.01,
'walletTxId': '3e2414d82acce78d38be7fe9',
'isInner': False,
'status': 'SUCCESS',
'remark': 'test',
'createdAt': 1612556794259,
'updatedAt': 1612556795000,
}
expected_asset_movement = AssetMovement(
timestamp=Timestamp(1612556794),
location=Location.KUCOIN,
category=AssetMovementCategory.WITHDRAWAL,
address='0x5bedb060b8eb8d823e2414d82acce78d38be7fe9',
transaction_id='3e2414d82acce78d38be7fe9',
asset=Asset('ETH'),
amount=AssetAmount(FVal('1')),
fee_asset=Asset('ETH'),
fee=Fee(FVal('0.01')),
link='5c2dc64e03aa675aa263f1ac',
)
asset_movement = mock_kucoin._deserialize_asset_movement(
raw_result=raw_result,
case=KucoinCase.WITHDRAWALS,
)
assert asset_movement == expected_asset_movement
def test_int_overflow_at_tuple_insertion(database, caplog):
"""Test that if somehow an int that will overflow makes it there we handle it
Related: https://github.com/rotki/rotki/issues/2175
"""
caplog.set_level(logging.INFO)
database.add_asset_movements([
AssetMovement(
location=Location.BITTREX,
category=AssetMovementCategory.DEPOSIT,
timestamp=177778,
address='0xfoo',
transaction_id=99999999999999999999999999999999999999999,
asset=A_BTC,
amount=FVal(1),
fee_asset=A_BTC,
fee=Fee(FVal('0.0001')),
link='a link',
)
])
errors = database.msg_aggregator.consume_errors()
assert len(errors) == 1
assert 'Failed to add "asset_movement" to the DB with overflow error' in errors[
0]
assert 'Overflow error while trying to add "asset_movement" tuples to the DB. Tuples:' in caplog.text # noqa: E501
def _deserialize_asset_movement(self, raw_result: List[Any]) -> AssetMovement:
"""Process an asset movement (i.e. deposit or withdrawal) from Bitfinex
and deserialize it.
Bitfinex support confirmed the following in regards to
DESTINATION_ADDRESS and TRANSACTION_ID:
- Fiat movement: both attributes won't show any value.
- Cryptocurrency movement: address and tx id on the blockchain.
Timestamp is from MTS_STARTED (when the movement was created), and not
from MTS_UPDATED (when it was completed/cancelled).
Can raise DeserializationError.
Schema reference in:
https://docs.bitfinex.com/reference#rest-auth-movements
"""
if raw_result[9] != 'COMPLETED':
raise DeserializationError(
f'Unexpected bitfinex movement with status: {raw_result[5]}. '
f'Only completed movements are processed. Raw movement: {raw_result}',
)
try:
fee_asset = asset_from_bitfinex(
bitfinex_name=raw_result[1],
currency_map=self.currency_map,
)
except (UnknownAsset, UnsupportedAsset) as e:
asset_tag = 'Unknown' if isinstance(e, UnknownAsset) else 'Unsupported'
raise DeserializationError(
f'{asset_tag} {e.asset_name} found while processing movement asset '
f'due to: {str(e)}',
) from e
amount = deserialize_asset_amount(raw_result[12])
category = (
AssetMovementCategory.DEPOSIT
if amount > ZERO
else AssetMovementCategory.WITHDRAWAL
)
address = None
transaction_id = None
if fee_asset.is_fiat() is False:
address = str(raw_result[16])
transaction_id = str(raw_result[20])
asset_movement = AssetMovement(
timestamp=Timestamp(int(raw_result[5] / 1000)),
location=Location.BITFINEX,
category=category,
address=address,
transaction_id=transaction_id,
asset=fee_asset,
amount=abs(amount),
fee_asset=fee_asset,
fee=Fee(abs(deserialize_fee(raw_result[13]))),
link=str(raw_result[0]),
)
return asset_movement
def _deserialize_asset_movement(
self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from bittrex and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data['status'] != 'COMPLETED':
# Don't mind failed/in progress asset movements
return None
if 'source' in raw_data:
category = AssetMovementCategory.DEPOSIT
fee = Fee(ZERO)
else:
category = AssetMovementCategory.WITHDRAWAL
fee = deserialize_fee(raw_data.get('txCost', 0))
timestamp = deserialize_timestamp_from_date(
date=raw_data['completedAt'], # we only check completed orders
formatstr='iso8601',
location='bittrex',
)
asset = asset_from_bittrex(raw_data['currencySymbol'])
return AssetMovement(
location=Location.BITTREX,
category=category,
address=deserialize_asset_movement_address(
raw_data, 'cryptoAddress', asset),
transaction_id=get_key_if_has_val(raw_data, 'txId'),
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
raw_data['quantity']),
fee_asset=asset,
fee=fee,
link=str(raw_data.get('txId', '')),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found bittrex deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found bittrex deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a bittrex '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of bittrex '
f'asset_movement {raw_data}. Error was: {str(e)}', )
return None
def _deserialize_trade(self, raw_result: List[Any]) -> Trade:
"""Process a trade result from Bitfinex and deserialize it.
The base and quote assets are instantiated using the `fee_currency_symbol`
(from raw_result[10]) over the pair (from raw_result[1]).
Known pairs format: 'tETHUST', 'tETH:UST'.
Can raise DeserializationError.
Schema reference in:
https://docs.bitfinex.com/reference#rest-auth-trades
"""
amount = deserialize_asset_amount(raw_result[4])
trade_type = TradeType.BUY if amount >= ZERO else TradeType.SELL
bfx_pair = self._process_bfx_pair(raw_result[1])
if bfx_pair not in self.pair_bfx_symbols_map:
raise DeserializationError(
f'Could not deserialize bitfinex trade pair {raw_result[1]}. '
f'Raw trade: {raw_result}', )
bfx_base_asset_symbol, bfx_quote_asset_symbol = self.pair_bfx_symbols_map[
bfx_pair]
try:
base_asset = asset_from_bitfinex(
bitfinex_name=bfx_base_asset_symbol,
currency_map=self.currency_map,
)
quote_asset = asset_from_bitfinex(
bitfinex_name=bfx_quote_asset_symbol,
currency_map=self.currency_map,
)
fee_asset = asset_from_bitfinex(
bitfinex_name=raw_result[10],
currency_map=self.currency_map,
)
except (UnknownAsset, UnsupportedAsset) as e:
asset_tag = 'Unknown' if isinstance(
e, UnknownAsset) else 'Unsupported'
raise DeserializationError(
f'{asset_tag} {e.asset_name} found while processing trade pair due to: {str(e)}',
) from e
trade = Trade(
timestamp=Timestamp(int(raw_result[2] / 1000)),
location=Location.BITFINEX,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=trade_type,
amount=AssetAmount(abs(amount)),
rate=deserialize_price(raw_result[5]),
fee=Fee(abs(deserialize_fee(raw_result[9]))),
fee_currency=fee_asset,
link=str(raw_result[0]),
notes='',
)
return trade
def _deserialize_asset_movement(
self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from binance and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if 'insertTime' in raw_data:
category = AssetMovementCategory.DEPOSIT
time_key = 'insertTime'
else:
category = AssetMovementCategory.WITHDRAWAL
time_key = 'applyTime'
timestamp = deserialize_timestamp_from_binance(raw_data[time_key])
asset = asset_from_binance(raw_data['asset'])
location = Location.BINANCE if self.name == str(
Location.BINANCE) else Location.BINANCE_US # noqa: E501
return AssetMovement(
location=location,
category=category,
address=deserialize_asset_movement_address(
raw_data, 'address', asset),
transaction_id=get_key_if_has_val(raw_data, 'txId'),
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
raw_data['amount']),
fee_asset=asset,
# Binance does not include withdrawal fees neither in the API nor in their UI
fee=Fee(ZERO),
link=str(raw_data['txId']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found {self.name} deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found {self.name} deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
f'Error processing a {self.name} deposit/withdrawal. Check logs '
f'for details. Ignoring it.', )
log.error(
f'Error processing a {self.name} deposit_withdrawal',
asset_movement=raw_data,
error=msg,
)
return None
def get_fee_in_profit_currency(self, trade: Trade) -> Fee:
"""Get the profit_currency rate of the fee of the given trade
May raise:
- PriceQueryUnsupportedAsset if from/to asset is missing from all price oracles
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from the price oracle
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
if trade.fee_currency is None or trade.fee is None:
return Fee(ZERO)
fee_rate = PriceHistorian().query_historical_price(
from_asset=trade.fee_currency,
to_asset=self.profit_currency,
timestamp=trade.timestamp,
)
return Fee(fee_rate * trade.fee)
def test_assets_movements_not_accounted_for(accountant, expected):
# asset_movements_list partially copied from
# rotkehlchen/tests/integration/test_end_to_end_tax_report.py
asset_movements_list = [
AssetMovement(
# before query period -- 8.915 * 0.001 = 8.915e-3
location=Location.KRAKEN,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=Timestamp(1479510304), # 18/11/2016,
asset=A_ETH, # cryptocompare hourly ETH/EUR: 8.915
amount=FVal('95'),
fee_asset=A_ETH,
fee=Fee(FVal('0.001')),
link='krakenid1',
),
AssetMovement( # 0.00029*1964.685 = 0.56975865
location=Location.POLONIEX,
address='foo',
transaction_id='0xfoo',
category=AssetMovementCategory.WITHDRAWAL,
timestamp=Timestamp(1495969504), # 28/05/2017,
asset=A_BTC, # cryptocompare hourly BTC/EUR: 1964.685
amount=FVal('8.5'),
fee_asset=A_BTC,
fee=Fee(FVal('0.00029')),
link='poloniexid1',
)
]
history = []
result = accounting_history_process(
accountant,
1436979735,
1519693374,
history,
asset_movements_list=asset_movements_list,
)
assert FVal(result['overview']['asset_movement_fees']).is_close(expected)
assert FVal(
result['overview']['total_taxable_profit_loss']).is_close(-expected)
assert FVal(result['overview']['total_profit_loss']).is_close(-expected)