def test_asset_conversion_zero_fee():
"""Test a conversion with 0 fee"""
trade_a = {
"id": "77c5ad72-764e-414b-8bdb-b5aed20fb4b1",
"type": "trade",
"status": "completed",
"amount": {
"amount": "-6000.000000",
"currency": "1INCH",
},
"native_amount": {
"amount": "-1000.00",
"currency": "USD",
},
"description": None,
"created_at": "2020-06-08T02:32:16Z",
"updated_at": "2021-06-08T02:32:16Z",
"resource": "transaction",
"resource_path":
"/v2/accounts/sd5af/transactions/77c5ad72-764e-414b-8bdb-b5aed20fb4b1",
"instant_exchange": False,
"trade": {
"id":
"5dceef97-ef34-41e6-9171-3e60cd01639e",
"resource":
"trade",
"resource_path":
"/v2/accounts/sd5af/trades/5dceef97-ef34-41e6-9171-3e60cd01639e",
},
"details": {
"title": "Converted from USD Coin",
"subtitle": "Using USDC Wallet",
"header": "Converted 1,000.0000 USDC ($1,000.00)",
"health": "positive",
"payment_method_name": "USDC Wallet",
},
}
trade_b = {
"id": "8f530cd1-5ec0-4aae-afdc-198502a53b17",
"type": "trade",
"status": "completed",
"amount": {
"amount": "0.01694165",
"currency": "BTC",
},
"native_amount": {
"amount": "1000.00",
"currency": "USD",
},
"description": None,
"created_at": "2020-06-08T02:32:16Z",
"updated_at": "2020-06-08T02:32:16Z",
"resource": "transaction",
"resource_path":
"/v2/accounts/sd5af/transactions/8f530cd1-5ec0-4aae-afdc-198502a53b17",
"instant_exchange": False,
"trade": {
"id":
"5dceef97-ef34-41e6-9171-3e60cd01639e",
"resource":
"trade",
"resource_path":
"/v2/accounts/sd5af/trades/5dceef97-ef34-41e6-9171-3e60cd01639e",
},
"details": {
"title": "Converted to Bitcoin",
"subtitle": "Using USDC Wallet",
"header": "Converted 0.01694165 BTC ($910.00)",
"health": "positive",
"payment_method_name": "USDC Wallet",
},
}
trade = trade_from_conversion(trade_a, trade_b)
expected_trade = Trade(
timestamp=1623119536,
location=Location.COINBASE,
base_asset=A_1INCH,
quote_asset=A_BTC,
trade_type=TradeType.SELL,
amount=FVal('6000.0'),
rate=FVal('0.000002823608333333333333333333333'),
fee=FVal(ZERO),
fee_currency=A_1INCH,
link='5dceef97-ef34-41e6-9171-3e60cd01639e',
)
assert trade == expected_trade
def test_trade_from_binance(function_scope_binance):
binance = function_scope_binance
binance_trades_list = [
{
'symbol': 'RDNETH',
'id': 1,
'orderId': 1,
'price': FVal(0.0063213),
'qty': FVal(5.0),
'commission': FVal(0.005),
'commissionAsset': 'RDN',
'time': 1512561941000,
'isBuyer': True,
'isMaker': False,
'isBestMatch': True,
}, {
'symbol': 'ETHUSDT',
'id': 2,
'orderId': 2,
'price': FVal(481.0),
'qty': FVal(0.505),
'commission': FVal(0.242905),
'commissionAsset': 'USDT',
'time': 1531117990000,
'isBuyer': False,
'isMaker': True,
'isBestMatch': True,
}, {
'symbol': 'BTCUSDT',
'id': 3,
'orderId': 3,
'price': FVal(6376.39),
'qty': FVal(0.051942),
'commission': FVal(0.00005194),
'commissionAsset': 'BTC',
'time': 1531728338000,
'isBuyer': True,
'isMaker': False,
'isBestMatch': True,
}, {
'symbol': 'ADAUSDT',
'id': 4,
'orderId': 4,
'price': FVal(0.17442),
'qty': FVal(285.2),
'commission': FVal(0.00180015),
'commissionAsset': 'BNB',
'time': 1531871806000,
'isBuyer': False,
'isMaker': True,
'isBestMatch': True,
},
]
our_expected_list = [
Trade(
timestamp=1512561941,
location=Location.BINANCE,
base_asset=A_RDN,
quote_asset=A_ETH,
trade_type=TradeType.BUY,
amount=FVal(5.0),
rate=FVal(0.0063213),
fee=FVal(0.005),
fee_currency=A_RDN,
link='1',
),
Trade(
timestamp=1531117990,
location=Location.BINANCE,
base_asset=A_ETH,
quote_asset=A_USDT,
trade_type=TradeType.SELL,
amount=FVal(0.505),
rate=FVal(481.0),
fee=FVal(0.242905),
fee_currency=A_USDT,
link='2',
),
Trade(
timestamp=1531728338,
location=Location.BINANCE,
base_asset=A_BTC,
quote_asset=A_USDT,
trade_type=TradeType.BUY,
amount=FVal(0.051942),
rate=FVal(6376.39),
fee=FVal(0.00005194),
fee_currency=A_BTC,
link='3',
),
Trade(
timestamp=1531871806,
location=Location.BINANCE,
base_asset=A_ADA,
quote_asset=A_USDT,
trade_type=TradeType.SELL,
amount=FVal(285.2),
rate=FVal(0.17442),
fee=FVal(0.00180015),
fee_currency=A_BNB,
link='4',
),
]
for idx, binance_trade in enumerate(binance_trades_list):
our_trade = trade_from_binance(binance_trade, binance.symbols_to_pair, location=Location.BINANCE) # noqa: E501
assert our_trade == our_expected_list[idx]
assert isinstance(our_trade.fee_currency, Asset)
def test_add_trades(data_dir, username, caplog):
"""Test that adding and retrieving trades from the DB works fine.
Also duplicates should be ignored and an error returned
"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
trade1 = Trade(
timestamp=1451606400,
location=Location.KRAKEN,
pair='ETH_EUR',
trade_type=TradeType.BUY,
amount=FVal('1.1'),
rate=FVal('10'),
fee=Fee(FVal('0.01')),
fee_currency=A_EUR,
link='',
notes='',
)
trade2 = Trade(
timestamp=1451607500,
location=Location.BINANCE,
pair='BTC_ETH',
trade_type=TradeType.BUY,
amount=FVal('0.00120'),
rate=FVal('10'),
fee=Fee(FVal('0.001')),
fee_currency=A_ETH,
link='',
notes='',
)
trade3 = Trade(
timestamp=1451608600,
location=Location.COINBASE,
pair='BTC_ETH',
trade_type=TradeType.SELL,
amount=FVal('0.00120'),
rate=FVal('1'),
fee=Fee(FVal('0.001')),
fee_currency=A_ETH,
link='',
notes='',
)
# Add and retrieve the first 2 trades. All should be fine.
data.db.add_trades([trade1, trade2])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
returned_trades = data.db.get_trades()
assert returned_trades == [trade1, trade2]
# Add the last 2 trades. Since trade2 already exists in the DB it should be
# ignored and a warning should be logged
data.db.add_trades([trade2, trade3])
assert (
'Did not add "buy trade with id 38d56b6c435894fe1faaf19c5aec4f817de'
'dd6b0a26afc41be4748daf36a5a5c'
) in caplog.text
returned_trades = data.db.get_trades()
assert returned_trades == [trade1, trade2, trade3]
def test_measure_trades_api_query(rotkehlchen_api_server,
start_with_valid_premium):
"""Measures the response time of the combined trades view API query.
This is required since it's quite a complicated query and takes a lot of time to process
so we can use this test to measure any potential optimizations.
"""
trades = [
Trade(
timestamp=x,
location=Location.EXTERNAL,
base_asset=A_WETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1')),
rate=Price(FVal('320')),
fee=Fee(ZERO),
fee_currency=A_EUR,
link='',
notes='',
) for x in range(1, 10000)
]
rotki = rotkehlchen_api_server.rest_api.rotkehlchen
rotki.data.db.add_trades(trades)
swaps = [
AMMSwap(
tx_hash='0x' + str(x),
log_index=x + i,
address='0xfoo',
from_address='0xfrom',
to_address='0xto',
timestamp=11 + x,
location=Location.UNISWAP,
token0=A_WETH,
token1=A_EUR,
amount0_in=FVal(5),
amount1_in=ZERO,
amount0_out=ZERO,
amount1_out=FVal(4.95),
) for x in range(2000) for i in range(2)
]
rotki.data.db.add_amm_swaps(swaps)
start = time.time()
requests.get(
api_url_for(
rotkehlchen_api_server,
'tradesresource',
),
json={'only_cache': True},
)
end = time.time()
test_warnings.warn(
UserWarning(
f'Premium: {start_with_valid_premium}. Full Query Time: {end - start}',
))
start = time.time()
requests.get(
api_url_for(
rotkehlchen_api_server,
'tradesresource',
),
json={
'only_cache': True,
'offset': 200,
'limit': 10
},
)
end = time.time()
test_warnings.warn(
UserWarning(
f'Premium: {start_with_valid_premium}. First Page Query Time: {end - start}',
))
start = time.time()
requests.get(
api_url_for(
rotkehlchen_api_server,
'tradesresource',
),
json={
'only_cache': True,
'offset': 210,
'limit': 10
},
)
end = time.time()
test_warnings.warn(
UserWarning(
f'Premium: {start_with_valid_premium}. Second Page Query Time: {end - start}',
))
start = time.time()
requests.get(
api_url_for(
rotkehlchen_api_server,
'tradesresource',
),
json={
'only_cache': True,
'offset': 220,
'limit': 10
},
)
end = time.time()
test_warnings.warn(
UserWarning(
f'Premium: {start_with_valid_premium}. Third Page Query Time: {end - start}',
))
def test_query_trades_associated_locations(
rotkehlchen_api_server_with_exchanges):
"""Test that querying the trades endpoint works as expected when we have associated
locations including associated exchanges and imported locations.
"""
rotki = rotkehlchen_api_server_with_exchanges.rest_api.rotkehlchen
setup = mock_history_processing_and_exchanges(rotki)
trades = [
Trade(
timestamp=Timestamp(1596429934),
location=Location.EXTERNAL,
base_asset=A_WETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1')),
rate=Price(FVal('320')),
fee=Fee(ZERO),
fee_currency=A_EUR,
link='',
notes='',
),
Trade(
timestamp=Timestamp(1596429934),
location=Location.KRAKEN,
base_asset=A_WETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1')),
rate=Price(FVal('320')),
fee=Fee(ZERO),
fee_currency=A_EUR,
link='',
notes='',
),
Trade(
timestamp=Timestamp(1596429934),
location=Location.BISQ,
base_asset=A_WETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1')),
rate=Price(FVal('320')),
fee=Fee(ZERO),
fee_currency=A_EUR,
link='',
notes='',
),
Trade(
timestamp=Timestamp(1596429934),
location=Location.BINANCE,
base_asset=A_WETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1')),
rate=Price(FVal('320')),
fee=Fee(ZERO),
fee_currency=A_EUR,
link='',
notes='',
)
]
# Add multiple entries for same exchange + connected exchange
rotki.data.db.add_trades(trades)
# Simply get all trades without any filtering
with setup.binance_patch, setup.polo_patch:
response = requests.get(
api_url_for(
rotkehlchen_api_server_with_exchanges,
'tradesresource',
), )
result = assert_proper_response_with_result(response)
result = result['entries']
assert len(
result
) == 9 # 3 polo, (2 + 1) binance trades, 1 kraken, 1 external, 1 BISQ
expected_locations = (
Location.KRAKEN,
Location.POLONIEX,
Location.BINANCE,
Location.BISQ,
Location.EXTERNAL,
)
returned_locations = {x['entry']['location'] for x in result}
assert returned_locations == set(map(str, expected_locations))
response = requests.get(
api_url_for(
rotkehlchen_api_server_with_exchanges,
'tradesresource',
),
json={
'location': 'kraken',
'only_cache': True
},
)
result = assert_proper_response_with_result(response)
result = result['entries']
assert len(result) == 1
response = requests.get(
api_url_for(
rotkehlchen_api_server_with_exchanges,
'tradesresource',
),
json={
'location': 'binance',
'only_cache': True
},
)
result = assert_proper_response_with_result(response)
result = result['entries']
assert len(result) == 3
response = requests.get(
api_url_for(
rotkehlchen_api_server_with_exchanges,
'tradesresource',
),
json={'location': 'nexo'},
)
result = assert_proper_response_with_result(response)
result = result['entries']
assert len(result) == 0
def trade_from_kraken(kraken_trade: Dict[str, Any]) -> Trade:
"""Turn a kraken trade returned from kraken trade history to our common trade
history format
- Can raise UnknownAsset due to kraken_to_world_pair
- Can raise UnprocessableTradePair due to kraken_to_world_pair
- Can raise DeserializationError due to dict entries not being as expected
- Can raise KeyError due to dict entries missing an expected entry
"""
currency_pair = kraken_to_world_pair(kraken_trade['pair'])
quote_currency = get_pair_position_asset(currency_pair, 'second')
timestamp = deserialize_timestamp_from_kraken(kraken_trade['time'])
amount = deserialize_asset_amount(kraken_trade['vol'])
cost = deserialize_price(kraken_trade['cost'])
fee = deserialize_fee(kraken_trade['fee'])
order_type = deserialize_trade_type(kraken_trade['type'])
rate = deserialize_price(kraken_trade['price'])
# pylint does not seem to see that Price is essentially FVal
if not cost.is_close(amount * rate): # pylint: disable=no-member
log.warning(f'cost ({cost}) != amount ({amount}) * rate ({rate}) for kraken trade')
log.debug(
'Processing kraken Trade',
sensitive_log=True,
timestamp=timestamp,
order_type=order_type,
kraken_pair=kraken_trade['pair'],
pair=currency_pair,
quote_currency=quote_currency,
amount=amount,
cost=cost,
fee=fee,
rate=rate,
)
# Kraken trades can have the same ordertxid and postxid for different trades ..
# Also note postxid is optional and can be missing
# The only thing that could differentiate them is timestamps in the milliseconds range
# For example here are parts of two different kraken_trade:
# {'ordertxid': 'AM4ZOZ-GLEMD-ZICOGR', 'postxid': 'AKH2SE-M7IF5-CFI7AT',
# 'pair': 'XXBTZEUR', 'time': FVal(1561161486.2955)
# {'ordertxid': 'AM4ZOZ-GLEMD-ZICOGR', 'postxid': 'AKH2SE-M7IF5-CFI7AT',
# 'pair': 'XXBTZEUR', 'time': FVal(1561161486.3005)
#
# In order to counter this for the unique exchange trade link we are going
# to use a concatenation of the above
exchange_uuid = (
str(kraken_trade['ordertxid']) +
str(kraken_trade.get('postxid', '')) + # postxid is optional
str(kraken_trade['time'])
)
return Trade(
timestamp=timestamp,
location=Location.KRAKEN,
pair=currency_pair,
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=quote_currency,
link=exchange_uuid,
)
def trade_from_conversion(trade_a: Dict[str, Any],
trade_b: Dict[str, Any]) -> Optional[Trade]:
"""Turn information from a conversion into a trade
Mary raise:
- UnknownAsset due to Asset instantiation
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entires missing an expected entry
"""
# Check that the status is complete
if trade_a['status'] != 'completed':
return None
# Trade b will represent the asset we are converting to
if trade_b['amount']['amount'].startswith('-'):
trade_a, trade_b = trade_b, trade_a
timestamp = deserialize_timestamp_from_date(trade_a['updated_at'],
'iso8601', 'coinbase')
tx_amount = AssetAmount(
abs(deserialize_asset_amount(trade_a['amount']['amount'])))
tx_asset = asset_from_coinbase(trade_a['amount']['currency'],
time=timestamp)
native_amount = deserialize_asset_amount(trade_b['amount']['amount'])
native_asset = asset_from_coinbase(trade_b['amount']['currency'],
time=timestamp)
amount = tx_amount
# The rate is how much you get/give in quotecurrency if you buy/sell 1 unit of base currency
rate = Price(native_amount / tx_amount)
# Obtain fee amount in the native currency using data from both trades
amount_after_fee = deserialize_asset_amount(
trade_b['native_amount']['amount'])
amount_before_fee = deserialize_asset_amount(
trade_a['native_amount']['amount'])
# amount_after_fee + amount_before_fee is a negative amount and the fee needs to be positive
conversion_native_fee_amount = abs(amount_after_fee + amount_before_fee)
if ZERO not in (tx_amount, conversion_native_fee_amount, amount_before_fee,
amount_after_fee):
# To get the asset in which the fee is nominated we pay attention to the creation
# date of each event. As per our hypothesis the fee is nominated in the asset
# for which the first transaction part was initialized
time_created_a = deserialize_timestamp_from_date(
date=trade_a['created_at'],
formatstr='iso8601',
location='coinbase',
)
time_created_b = deserialize_timestamp_from_date(
date=trade_b['created_at'],
formatstr='iso8601',
location='coinbase',
)
if time_created_a < time_created_b:
# We have the fee amount in the native currency. To get it in the
# converted asset we have to get the rate
asset_native_rate = tx_amount / abs(amount_before_fee)
fee_amount = Fee(conversion_native_fee_amount * asset_native_rate)
fee_asset = asset_from_coinbase(trade_a['amount']['currency'],
time=timestamp)
else:
trade_b_amount = abs(
deserialize_asset_amount(trade_b['amount']['amount']))
asset_native_rate = trade_b_amount / abs(amount_after_fee)
fee_amount = Fee(conversion_native_fee_amount * asset_native_rate)
fee_asset = asset_from_coinbase(trade_b['amount']['currency'],
time=timestamp)
else:
fee_amount = Fee(ZERO)
fee_asset = asset_from_coinbase(trade_a['amount']['currency'],
time=timestamp)
return Trade(
timestamp=timestamp,
location=Location.COINBASE,
# in coinbase you are buying/selling tx_asset for native_asset
base_asset=tx_asset,
quote_asset=native_asset,
trade_type=TradeType.SELL,
amount=amount,
rate=rate,
fee=fee_amount,
fee_currency=fee_asset,
link=str(trade_a['trade']['id']),
)
def trade_from_binance(
binance_trade: Dict,
binance_symbols_to_pair: Dict[str, BinancePair],
name: str = 'binance',
) -> Trade:
"""Turn a binance trade returned from trade history to our common trade
history format
From the official binance api docs (01/09/18):
https://github.com/binance-exchange/binance-official-api-docs/blob/62ff32d27bb32d9cc74d63d547c286bb3c9707ef/rest-api.md#terminology
base asset refers to the asset that is the quantity of a symbol.
quote asset refers to the asset that is the price of a symbol.
Throws:
- UnsupportedAsset due to asset_from_binance
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entries missing an expected entry
"""
amount = deserialize_asset_amount(binance_trade['qty'])
rate = deserialize_price(binance_trade['price'])
if binance_trade['symbol'] not in binance_symbols_to_pair:
raise DeserializationError(
f'Error reading a {name} trade. Could not find '
f'{binance_trade["symbol"]} in binance_symbols_to_pair', )
binance_pair = binance_symbols_to_pair[binance_trade['symbol']]
timestamp = deserialize_timestamp_from_binance(binance_trade['time'])
base_asset = asset_from_binance(binance_pair.binance_base_asset)
quote_asset = asset_from_binance(binance_pair.binance_quote_asset)
if binance_trade['isBuyer']:
order_type = TradeType.BUY
# e.g. in RDNETH we buy RDN by paying ETH
else:
order_type = TradeType.SELL
fee_currency = asset_from_binance(binance_trade['commissionAsset'])
fee = deserialize_fee(binance_trade['commission'])
log.debug(
f'Processing {name} Trade',
sensitive_log=True,
amount=amount,
rate=rate,
timestamp=timestamp,
pair=binance_trade['symbol'],
base_asset=base_asset,
quote=quote_asset,
order_type=order_type,
commision_asset=binance_trade['commissionAsset'],
fee=fee,
)
return Trade(
timestamp=timestamp,
location=Location.BINANCE,
pair=trade_pair_from_assets(base_asset, quote_asset),
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(binance_trade['id']),
)
def _import_cryptocom_associated_entries(self, data: Any,
tx_kind: str) -> None:
"""Look for events that have associated entries and handle them as trades.
This method looks for `*_debited` and `*_credited` entries using the
same timestamp to handle them as one trade.
Known kind: 'dynamic_coin_swap' or 'dust_conversion'
May raise:
- UnknownAsset if an unknown asset is encountered in the imported files
- KeyError if a row contains unexpected data entries
"""
multiple_rows: Dict[Any, Dict[str, Any]] = {}
investments_deposits: Dict[str, List[Any]] = defaultdict(list)
investments_withdrawals: Dict[str, List[Any]] = defaultdict(list)
debited_row = None
credited_row = None
for row in data:
if row['Transaction Kind'] == f'{tx_kind}_debited':
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
if timestamp not in multiple_rows:
multiple_rows[timestamp] = {}
if 'debited' not in multiple_rows[timestamp]:
multiple_rows[timestamp]['debited'] = []
multiple_rows[timestamp]['debited'].append(row)
elif row['Transaction Kind'] == f'{tx_kind}_credited':
# They only is one credited row
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
if timestamp not in multiple_rows:
multiple_rows[timestamp] = {}
multiple_rows[timestamp]['credited'] = row
elif row['Transaction Kind'] == f'{tx_kind}_deposit':
asset = row['Currency']
investments_deposits[asset].append(row)
elif row['Transaction Kind'] == f'{tx_kind}_withdrawal':
asset = row['Currency']
investments_withdrawals[asset].append(row)
for timestamp in multiple_rows:
# When we convert multiple assets dust to CRO
# in one time, it will create multiple debited rows with
# the same timestamp
debited_rows = multiple_rows[timestamp]['debited']
credited_row = multiple_rows[timestamp]['credited']
total_debited_usd = functools.reduce(
lambda acc, row: acc + deserialize_asset_amount(row[
'Native Amount (in USD)']),
debited_rows,
ZERO,
)
# If the value of the transaction is too small (< 0,01$),
# crypto.com will display 0 as native amount
# if we have multiple debited rows, we can't import them
# since we can't compute their dedicated rates, so we skip them
if len(debited_rows) > 1 and total_debited_usd == 0:
return
if credited_row is not None and len(debited_rows) != 0:
for debited_row in debited_rows:
description = credited_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees here
fee = Fee(ZERO)
fee_currency = A_USD
base_asset = symbol_to_asset_or_token(
credited_row['Currency'])
quote_asset = symbol_to_asset_or_token(
debited_row['Currency'])
part_of_total = (
FVal(1) if len(debited_rows) == 1 else
deserialize_asset_amount(
debited_row["Native Amount (in USD)"], ) /
total_debited_usd)
quote_amount_sold = deserialize_asset_amount(
debited_row['Amount'], ) * part_of_total
base_amount_bought = deserialize_asset_amount(
credited_row['Amount'], ) * part_of_total
rate = Price(abs(base_amount_bought / quote_amount_sold))
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=TradeType.BUY,
amount=AssetAmount(base_amount_bought),
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
# Compute investments profit
if len(investments_withdrawals) != 0:
for asset in investments_withdrawals:
asset_object = symbol_to_asset_or_token(asset)
if asset not in investments_deposits:
log.error(
f'Investment withdrawal without deposit at crypto.com. Ignoring '
f'staking info for asset {asset_object}', )
continue
# Sort by date in ascending order
withdrawals_rows = sorted(
investments_withdrawals[asset],
key=lambda x: deserialize_timestamp_from_date(
date=x['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
),
)
investments_rows = sorted(
investments_deposits[asset],
key=lambda x: deserialize_timestamp_from_date(
date=x['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
),
)
last_date = Timestamp(0)
for withdrawal in withdrawals_rows:
withdrawal_date = deserialize_timestamp_from_date(
date=withdrawal['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
amount_deposited = ZERO
for deposit in investments_rows:
deposit_date = deserialize_timestamp_from_date(
date=deposit['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
if last_date < deposit_date <= withdrawal_date:
# Amount is negative
amount_deposited += deserialize_asset_amount(
deposit['Amount'])
amount_withdrawal = deserialize_asset_amount(
withdrawal['Amount'])
# Compute profit
profit = amount_withdrawal + amount_deposited
if profit >= ZERO:
last_date = withdrawal_date
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=withdrawal_date,
action_type=LedgerActionType.INCOME,
location=Location.CRYPTOCOM,
amount=AssetAmount(profit),
asset=asset_object,
rate=None,
rate_asset=None,
link=None,
notes=f'Stake profit for asset {asset}',
)
self.db_ledger.add_ledger_action(action)
def test_deserialize_trade_buy(mock_bitstamp):
raw_trade = {
'id': 2,
'type': 2,
'datetime': '2020-12-02 09:30:00',
'btc': '0.50000000',
'usd': '-10000.00000000',
'btc_usd': '0.00005000',
'fee': '20.00000000',
'order_id': 2,
}
expected_trade = Trade(
timestamp=1606901400,
location=Location.BITSTAMP,
pair=TradePair('BTC_USD'),
trade_type=TradeType.BUY,
amount=FVal('0.50000000'),
rate=FVal('0.00005000'),
fee=FVal('20.00000000'),
fee_currency=Asset('USD'),
link='2',
notes='',
)
trade = mock_bitstamp._deserialize_trade(raw_trade)
assert trade == expected_trade
raw_trade = {
'id': 2,
'type': 2,
'datetime': '2019-04-16 08:09:05.149343',
'btc': '0.00060000',
'usd': '0',
'btc_eur': '8364.0',
'eur': '-5.02',
'fee': '0.02',
'order_id': 2,
}
expected_trade = Trade(
timestamp=1555402145,
location=Location.BITSTAMP,
pair=TradePair('BTC_EUR'),
trade_type=TradeType.BUY,
amount=FVal('0.0006'),
rate=FVal('8364.0'),
fee=FVal('0.02'),
fee_currency=Asset('EUR'),
link='2',
notes='',
)
trade = mock_bitstamp._deserialize_trade(raw_trade)
assert trade == expected_trade
raw_trade = {
'id': 15,
'type': 2,
'datetime': '2019-04-15 16:19:14.826000',
'btc': '0',
'usd': '-7.70998',
'eur_usd': '1.12124',
'eur': '6.87630',
'fee': '0.02',
'order_id': 15,
}
expected_trade = Trade(
timestamp=1555345154,
location=Location.BITSTAMP,
pair=TradePair('EUR_USD'),
trade_type=TradeType.BUY,
amount=FVal('6.8763'),
rate=FVal('1.12124'),
fee=FVal('0.02'),
fee_currency=Asset('USD'),
link='15',
notes='',
)
trade = mock_bitstamp._deserialize_trade(raw_trade)
assert trade == expected_trade
def test_api_query_paginated_trades_pagination(mock_bitstamp):
"""Test pagination logic for trades works as expected.
First request: 2 results, 1 valid trade (id 2)
Second request: 2 results, no trades
Third request: 2 results, 1 valid trade (id 5) and 1 invalid trade (id 6)
Trades with id 2 and 5 are expected to be returned.
"""
# Not a trade
user_transaction_1 = """
{
"id": 1,
"type": -1,
"datetime": "2020-12-02 09:00:00"
}
"""
# First trade, buy BTC with USD, within timestamp range
user_transaction_2 = """
{
"id": 2,
"type": 2,
"datetime": "2020-12-02 09:30:00",
"btc": "0.50000000",
"usd": "-10000.00000000",
"btc_usd": "0.00005000",
"fee": "20.00000000",
"order_id": 2
}
"""
# Not a trade
user_transaction_3 = """
{
"id": 3,
"type": -1,
"datetime": "2020-12-02 18:00:00"
}
"""
# Not a trade
user_transaction_4 = """
{
"id": 4,
"type": -1,
"datetime": "2020-12-03 9:00:00"
}
"""
# Second trade, sell EUR for USD, within timestamp range
user_transaction_5 = """
{
"id": 5,
"type": 2,
"datetime": "2020-12-03 11:30:00",
"eur": "-1.00000000",
"usd": "1.22000000",
"eur_usd": "0.81967213",
"fee": "0.00610000",
"order_id": 3
}
"""
# Third trade, buy ETH with USDC, out of timestamp range
user_transaction_6 = """
{
"id": 6,
"type": 2,
"datetime": "2020-12-03 12:00:01",
"eth": "1.00000000",
"usdc": "-750.00000000",
"eth_usdc": "0.00133333",
"fee": "3.75000000",
"order_id": 1
}
"""
api_limit = 2
now = datetime.now()
now_ts = int(now.timestamp())
options = {
'since_id': USER_TRANSACTION_MIN_SINCE_ID,
'limit': api_limit,
'sort': USER_TRANSACTION_SORTING_MODE,
'offset': 0,
}
expected_calls = [
call(
endpoint='user_transactions',
method='post',
options={
'since_id': 1,
'limit': 2,
'sort': 'asc',
'offset': 0,
},
),
call(
endpoint='user_transactions',
method='post',
options={
'since_id': 3,
'limit': 2,
'sort': 'asc',
'offset': 0,
},
),
call(
endpoint='user_transactions',
method='post',
options={
'since_id': 3,
'limit': 2,
'sort': 'asc',
'offset': 2,
},
),
]
def get_paginated_response():
results = [
f'[{user_transaction_1},{user_transaction_2}]',
f'[{user_transaction_3},{user_transaction_4}]',
f'[{user_transaction_5},{user_transaction_6}]',
]
for result_ in results:
yield result_
def mock_api_query_response(endpoint, method, options): # pylint: disable=unused-argument
return MockResponse(HTTPStatus.OK, next(get_response))
get_response = get_paginated_response()
with patch(
'rotkehlchen.exchanges.bitstamp.API_MAX_LIMIT',
new_callable=MagicMock(return_value=api_limit),
):
with patch.object(
mock_bitstamp,
'_api_query',
side_effect=mock_api_query_response,
) as mock_api_query:
result = mock_bitstamp._api_query_paginated(
start_ts=Timestamp(0),
end_ts=Timestamp(now_ts),
options=options,
case='trades',
)
assert mock_api_query.call_args_list == expected_calls
expected_result = [
Trade(
timestamp=1606901400,
location=Location.BITSTAMP,
pair=TradePair('BTC_USD'),
trade_type=TradeType.BUY,
amount=FVal('0.50000000'),
rate=FVal('0.00005000'),
fee=FVal('20.00000000'),
fee_currency=Asset('USD'),
link='2',
notes='',
),
Trade(
timestamp=1606995000,
location=Location.BITSTAMP,
pair=TradePair('EUR_USD'),
trade_type=TradeType.SELL,
amount=FVal('1'),
rate=FVal('0.81967213'),
fee=FVal('0.00610000'),
fee_currency=Asset('USD'),
link='5',
notes='',
),
]
assert result == expected_result
def verify_otctrade_data(data: ExternalTrade, ) -> Tuple[Optional[Trade], str]:
"""
Takes in the trade data dictionary, validates it and returns a trade instance
If there is an error it returns an error message in the second part of the tuple
"""
for field in otc_fields:
if field not in data:
return None, f'{field} was not provided'
if data[field] in ('', None) and field not in otc_optional_fields:
return None, f'{field} was empty'
if field in otc_numerical_fields and not is_number(data[field]):
return None, f'{field} should be a number'
# Satisfy mypy typing
assert isinstance(data['otc_pair'], str)
assert isinstance(data['otc_fee_currency'], str)
assert isinstance(data['otc_fee'], str)
pair = TradePair(data['otc_pair'])
try:
first = get_pair_position_asset(pair, 'first')
second = get_pair_position_asset(pair, 'second')
fee_currency = Asset(data['otc_fee_currency'])
except UnknownAsset as e:
return None, f'Provided asset {e.asset_name} is not known to Rotkehlchen'
# Not catching DeserializationError here since we have asserts for the data
# being strings right above
try:
trade_type = deserialize_trade_type(str(data['otc_type']))
amount = deserialize_asset_amount(data['otc_amount'])
rate = deserialize_price(data['otc_rate'])
fee = deserialize_fee(data['otc_fee'])
except DeserializationError as e:
return None, f'Deserialization Error: {str(e)}'
try:
assert isinstance(data['otc_timestamp'], str)
timestamp = create_timestamp(data['otc_timestamp'],
formatstr='%d/%m/%Y %H:%M')
except ValueError as e:
return None, f'Could not process the given datetime: {e}'
log.debug(
'Creating OTC trade data',
sensitive_log=True,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
)
if data['otc_fee_currency'] not in (first, second):
return None, 'Trade fee currency should be one of the two in the currency pair'
if data['otc_type'] not in ('buy', 'sell'):
return None, 'Trade type can only be buy or sell'
trade = Trade(
timestamp=timestamp,
location=Location.EXTERNAL,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(data['otc_link']),
notes=str(data['otc_notes']),
)
return trade, ''
def test_coinbase_query_trade_history(function_scope_coinbase):
"""Test that coinbase trade history query works fine for the happy path"""
coinbase = function_scope_coinbase
with patch.object(coinbase.session,
'get',
side_effect=mock_normal_coinbase_query):
trades = coinbase.query_trade_history(
start_ts=0,
end_ts=TEST_END_TS,
only_cache=False,
)
warnings = coinbase.msg_aggregator.consume_warnings()
errors = coinbase.msg_aggregator.consume_errors()
assert len(warnings) == 0
assert len(errors) == 0
assert len(trades) == 2
expected_trades = [
Trade(
timestamp=1500705839,
location=Location.COINBASE,
base_asset=A_BTC,
quote_asset=A_USD,
trade_type=TradeType.BUY,
amount=FVal("486.34313725"),
rate=FVal("9.997920454875299055122012005"),
fee=FVal("1.01"),
fee_currency=A_USD,
link='9e14d574-30fa-5d85-b02c-6be0d851d61d',
),
Trade(
timestamp=1427402520,
location=Location.COINBASE,
base_asset=A_ETH,
quote_asset=A_USD,
trade_type=TradeType.SELL,
amount=FVal("100.45"),
rate=FVal("88.90014932802389248382279741"),
fee=FVal("10.1"),
fee_currency=A_USD,
link='1e14d574-30fa-5d85-b02c-6be0d851d61d',
)
]
assert trades == expected_trades
# and now try only a smaller time range
with patch.object(coinbase.session,
'get',
side_effect=mock_normal_coinbase_query):
trades = coinbase.query_trade_history(
start_ts=0,
end_ts=1451606400,
only_cache=False,
)
warnings = coinbase.msg_aggregator.consume_warnings()
errors = coinbase.msg_aggregator.consume_errors()
assert len(warnings) == 0
assert len(errors) == 0
assert len(trades) == 1
assert trades[0].trade_type == TradeType.SELL
assert trades[0].timestamp == 1427402520
def test_asset_conversion_choosing_fee_asset():
"""Test that the fee asset is correctly chosen when the received asset transaction
is created before the giving transaction.
"""
trade_a = {
"id": "REDACTED",
"type": "trade",
"status": "completed",
"amount": {
"amount": "-37734.034561",
"currency": "ETH",
},
"native_amount": {
"amount": "-79362.22",
"currency": "USD",
},
"description": None,
"created_at": "2021-10-12T13:23:56Z",
"updated_at": "2021-10-12T13:23:56Z",
"resource": "transaction",
"resource_path": "/v2/accounts/REDACTED/transactions/REDACTED",
"instant_exchange": False,
"trade": {
"id": "id_of_trade",
"resource": "trade",
"resource_path": "/v2/accounts/REDACTED/trades/id_of_trade",
},
"details": {
"title": "Converted from ETH",
"subtitle": "Using ETH Wallet",
"header": "Converted 37,734.034561 ETH ($79,362.22)",
"health": "positive",
"payment_method_name": "ETH Wallet",
},
"hide_native_amount": False,
}
trade_b = {
"id": "REDACTED",
"type": "trade",
"status": "completed",
"amount": {
"amount": "552.315885836",
"currency": "BTC",
},
"native_amount": {
"amount": "77827.94",
"currency": "USD",
},
"description": None,
"created_at": "2021-10-12T13:23:55Z",
"updated_at": "2021-10-12T13:23:57Z",
"resource": "transaction",
"resource_path": "/v2/accounts/REDACTED/transactions/REDACTED",
"instant_exchange": False,
"trade": {
"id": "id_of_trade",
"resource": "trade",
"resource_path": "/v2/accounts/REDACTED/trades/id_of_trade",
},
"details": {
"title": "Converted to BTC",
"subtitle": "Using ETH Wallet",
"header": "Converted 552.31588584 BTC ($77,827.94)",
"health": "positive",
"payment_method_name": "ETH Wallet",
},
"hide_native_amount": False,
}
trade = trade_from_conversion(trade_a, trade_b)
expected_trade = Trade(
timestamp=1634045036,
location=Location.COINBASE,
base_asset=A_ETH,
quote_asset=A_BTC,
trade_type=TradeType.SELL,
amount=FVal('37734.034561'),
rate=FVal('0.01463707478571204566189616471'),
fee=FVal('10.88821337581925051594581586'),
fee_currency=A_BTC,
link='id_of_trade',
)
assert trade == expected_trade
def query_online_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> Tuple[List[Trade], Tuple[Timestamp, Timestamp]]:
"""Queries coinbase pro for trades"""
log.debug('Query coinbasepro trade history',
start_ts=start_ts,
end_ts=end_ts)
self.first_connection()
trades = []
# first get all orders, to see which product ids we need to query fills for
orders = []
for batch in self._paginated_query(
endpoint='orders',
query_options={'status': 'done'},
):
orders.extend(batch)
queried_product_ids = set()
for order_entry in orders:
product_id = order_entry.get('product_id', None)
if product_id is None:
msg = (
'Skipping coinbasepro trade since it lacks a product_id. '
'Check logs for details')
self.msg_aggregator.add_error(msg)
log.error(
'Error processing a coinbasepro order.',
raw_trade=order_entry,
error=msg,
)
continue
if product_id in queried_product_ids or product_id not in self.available_products:
continue # already queried this product id or delisted product id
# Now let's get all fills for this product id
queried_product_ids.add(product_id)
fills = []
for batch in self._paginated_query(
endpoint='fills',
query_options={'product_id': product_id},
):
fills.extend(batch)
try:
base_asset, quote_asset = coinbasepro_to_worldpair(product_id)
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring the trade.', )
continue
for fill_entry in fills:
try:
timestamp = coinbasepro_deserialize_timestamp(
fill_entry, 'created_at')
if timestamp < start_ts or timestamp > end_ts:
continue
# Fee currency seems to always be quote asset
# https://github.com/ccxt/ccxt/blob/ddf3a15cbff01541f0b37c35891aa143bb7f9d7b/python/ccxt/coinbasepro.py#L724 # noqa: E501
trades.append(
Trade(
timestamp=timestamp,
location=Location.COINBASEPRO,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=TradeType.deserialize(
fill_entry['side']),
amount=deserialize_asset_amount(
fill_entry['size']),
rate=deserialize_price(fill_entry['price']),
fee=deserialize_fee(fill_entry['fee']),
fee_currency=quote_asset,
link=str(fill_entry['trade_id']) + '_' +
fill_entry['order_id'],
notes='',
))
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring the trade.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a coinbasepro trade. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a coinbasepro fill.',
raw_trade=fill_entry,
error=msg,
)
continue
return trades, (start_ts, end_ts)
else: # old v1 trades
amount = deserialize_asset_amount(raw_result['amount'])
fee_currency = quote_asset if trade_type == TradeType.SELL else base_asset
rate = deserialize_price(raw_result['dealPrice'])
trade_id = raw_result['id']
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
trade = Trade(
timestamp=timestamp,
location=Location.KUCOIN,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(trade_id),
notes='',
)
return trade
@overload
def _process_unsuccessful_response( # pylint: disable=no-self-use
self,
response: Response,
case: Literal[KucoinCase.API_KEY],
) -> Tuple[bool, str]:
...
def create_action(self, index: int, ts: Timestamp):
"""Create a random trade action on a random exchange depending
on the funds that are available in that exchange"""
# choose an exchange at random
exchange_name = random.choice(ALLOWED_EXCHANGES)
exchange = getattr(self, exchange_name)
# choose a random pair at that exchange
pair = exchange.choose_pair(
timestamp=ts,
price_query=self.query_historical_price,
)
print(
f'Creating trade {index + 1} / {self.trades_number} in {exchange_name}'
f' for the pair: {pair} at timestamp {ts}', )
# depending on our funds decide on what to do. Buy/sell
base, quote = pair_get_assets(pair)
if exchange.get_balance(base) is None:
action_type = TradeType.BUY
elif exchange.get_balance(quote) is None:
action_type = TradeType.SELL
else:
# TODO: trade the one we have most of
action_type = random.choice(list(TradeType))
# if we are buying we are going to spend from the quote asset
if action_type == TradeType.BUY:
spending_asset = quote
else: # selling spends from the base asset
spending_asset = base
# get a spending asset amount within our per-trade equivalent range and
# our available funds
spending_usd_rate = self.query_historical_price(
spending_asset, A_USD, ts)
max_usd_in_spending_asset = spending_usd_rate * exchange.get_balance(
spending_asset)
max_usd_equivalent_to_spend = min(max_usd_in_spending_asset,
MAX_TRADE_USD_VALUE)
rate = self.query_historical_price(base, quote, ts)
usd_to_spend = FVal(
random.uniform(0.01, float(max_usd_equivalent_to_spend)))
amount_in_spending_asset = usd_to_spend / spending_usd_rate
# if we are buying then the amount is the amount of asset we bought
if action_type == TradeType.BUY:
amount = amount_in_spending_asset / rate
# if we are selling the amount is the spending asset amount
else:
amount = amount_in_spending_asset
quote_asset_usd_rate = self.query_historical_price(quote, A_USD, ts)
fee_in_quote_currency = FVal(random.uniform(
0, MAX_FEE_USD_VALUE)) / quote_asset_usd_rate
# create the trade
trade = Trade(
timestamp=ts,
location=deserialize_location(exchange_name),
pair=pair,
trade_type=action_type,
amount=amount,
rate=rate,
fee=fee_in_quote_currency,
fee_currency=quote,
link='',
notes='',
)
logger.info(f'Created trade: {trade}')
# Adjust our global and per exchange accounting
if action_type == TradeType.BUY:
# we buy so we increase our base asset by amount
self.increase_asset(base, amount, exchange_name)
# and decrease quote by amount * rate
self.decrease_asset(quote, amount * rate, exchange_name)
else:
# we sell so we increase our quote asset
self.increase_asset(quote, amount * rate, exchange_name)
# and decrease our base asset
self.decrease_asset(base, amount, exchange_name)
# finally add it to the exchange
exchange.append_trade(trade)
def query_online_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> Tuple[List[Trade], Tuple[Timestamp, Timestamp]]:
"""Queries gemini for trades
"""
log.debug('Query gemini trade history', start_ts=start_ts, end_ts=end_ts)
trades = []
gemini_trades = []
for symbol in self.symbols:
gemini_trades = self._get_trades_for_symbol(
symbol=symbol,
start_ts=start_ts,
end_ts=end_ts,
)
for entry in gemini_trades:
try:
timestamp = deserialize_timestamp(entry['timestamp'])
if timestamp > end_ts:
break
base, quote = gemini_symbol_to_base_quote(symbol)
trades.append(Trade(
timestamp=timestamp,
location=Location.GEMINI,
base_asset=base,
quote_asset=quote,
trade_type=TradeType.deserialize(entry['type']),
amount=deserialize_asset_amount(entry['amount']),
rate=deserialize_price(entry['price']),
fee=deserialize_fee(entry['fee_amount']),
fee_currency=asset_from_gemini(entry['fee_currency']),
link=str(entry['tid']),
notes='',
))
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Gemini pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Gemini asset {e.asset_name}. '
f'Ignoring the trade.',
)
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a gemini trade. '
'Check logs for details. Ignoring it.',
)
log.error(
'Error processing a gemini trade.',
raw_trade=entry,
error=msg,
)
continue
return trades, (start_ts, end_ts)
def assert_poloniex_trades_result(
trades: List[Dict[str, Any]],
trades_to_check: Optional[Tuple[int, ...]] = None,
) -> None:
"""Convenience function to assert on the trades returned by poloniex's mock
'trades_to_check' is a tuple of indexes indicating which trades should be checked.
For example (1, 2) would mean that we have given two trades for checking and that
they corresponse to the second and third of the normally expected trades.
So the first trade is skipped.
The mock trade data are set in tests/utils/history.py
"""
if trades_to_check is None:
assert len(trades) == 3
trades_to_check = (0, 1, 2)
else:
assert len(trades) == len(trades_to_check)
for given_idx, idx in enumerate(trades_to_check):
trade = trades[given_idx]
expected_id = Trade(
**{k: v
for k, v in trades[given_idx].items()
if k != 'trade_id'}, ).identifier
assert trade['trade_id'] == expected_id
if idx == 0:
assert trade['timestamp'] == 1539713238
assert trade['location'] == 'poloniex'
assert trade['pair'] == 'XMR_ETH'
assert trade['trade_type'] == 'buy'
assert FVal(trade['amount']) == FVal('1.40308443')
assert FVal(trade['rate']) == FVal('0.06935244')
assert FVal(trade['fee']) == FVal('0.00140308443')
assert trade['fee_currency'] == 'XMR'
assert trade['link'] == '394131415'
assert trade['notes'] == ''
elif idx == 1:
assert trade['timestamp'] == 1539713237
assert trade['location'] == 'poloniex'
assert trade['pair'] == 'ETH_BTC'
assert trade['trade_type'] == 'buy'
assert FVal(trade['amount']) == FVal('1.40308443')
assert FVal(trade['rate']) == FVal('0.06935244')
assert FVal(trade['fee']) == FVal('0.00140308443')
assert trade['fee_currency'] == 'ETH'
assert trade['link'] == '394131413'
assert trade['notes'] == ''
elif idx == 2:
assert trade['timestamp'] == 1539713117
assert trade['location'] == 'poloniex'
assert trade['pair'] == 'ETH_BTC'
assert trade['trade_type'] == 'sell'
assert trade['amount'] == '1.40308443'
assert FVal(trade['rate']) == FVal('0.06935244')
assert FVal(trade['fee']) == FVal('0.0000973073287465092')
assert trade['fee_currency'] == 'BTC'
assert trade['link'] == '394131412'
assert trade['notes'] == ''
else:
raise AssertionError('index out of range')
def test_cointracking_data_import(rotkehlchen_server):
dir_path = os.path.dirname(os.path.realpath(__file__))
filepath = os.path.join(dir_path, 'data', 'cointracking_trades_list.csv')
# Check that for unknown source we get an error
response = rotkehlchen_server.import_data_from(source='other_source',
filepath=filepath)
assert response['result'] is False
assert 'unknown location' in response['message']
# Check that the test cointracking data are imported succesfully
rotkehlchen_server.import_data_from(source='cointracking_info',
filepath=filepath)
trades = rotkehlchen_server.rotkehlchen.data.db.get_trades()
asset_movements = rotkehlchen_server.rotkehlchen.data.db.get_asset_movements(
)
warnings = rotkehlchen_server.rotkehlchen.msg_aggregator.consume_warnings()
errors = rotkehlchen_server.rotkehlchen.msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 3
expected_trades = [
Trade(
timestamp=1566687660,
location=Location.COINBASE,
pair='ETH_EUR',
trade_type=TradeType.BUY,
amount=FVal('0.05772716'),
rate=FVal('190.3783245183029963712055123'),
fee=ZERO,
fee_currency=A_ETH,
link='',
notes='',
),
Trade(
timestamp=1567418400,
location=Location.EXTERNAL,
pair='BTC_USD',
trade_type=TradeType.BUY,
amount=FVal('0.00100000'),
rate=ZERO,
fee=ZERO,
fee_currency=A_BTC,
link='',
notes='Just a small gift from someone',
)
]
assert expected_trades == trades
expected_movements = [
AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.DEPOSIT,
timestamp=1565848620,
asset=A_XMR,
amount=FVal('5'),
fee_asset=A_XMR,
fee=ZERO,
link='',
),
AssetMovement(
location=Location.COINBASE,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=1566726120,
asset=A_ETH,
amount=FVal('0.05770427'),
fee_asset=A_ETH,
fee=ZERO,
link='',
)
]
assert expected_movements == asset_movements
def trade_from_poloniex(poloniex_trade: Dict[str, Any],
pair: TradePair) -> Trade:
"""Turn a poloniex trade returned from poloniex trade history to our common trade
history format
Throws:
- UnsupportedAsset due to asset_from_poloniex()
- DeserializationError due to the data being in unexpected format
- UnprocessableTradePair due to the pair data being in an unexpected format
"""
try:
trade_type = deserialize_trade_type(poloniex_trade['type'])
amount = deserialize_asset_amount(poloniex_trade['amount'])
rate = deserialize_price(poloniex_trade['rate'])
perc_fee = deserialize_fee(poloniex_trade['fee'])
base_currency = asset_from_poloniex(
get_pair_position_str(pair, 'first'))
quote_currency = asset_from_poloniex(
get_pair_position_str(pair, 'second'))
timestamp = deserialize_timestamp_from_poloniex_date(
poloniex_trade['date'])
except KeyError as e:
raise DeserializationError(
f'Poloniex trade deserialization error. Missing key entry for {str(e)} in trade dict',
)
cost = rate * amount
if trade_type == TradeType.BUY:
fee = amount * perc_fee
fee_currency = quote_currency
elif trade_type == TradeType.SELL:
fee = cost * perc_fee
fee_currency = base_currency
else:
raise DeserializationError(
f'Got unexpected trade type "{trade_type}" for poloniex trade')
if poloniex_trade['category'] == 'settlement':
if trade_type == TradeType.BUY:
trade_type = TradeType.SETTLEMENT_BUY
else:
trade_type = TradeType.SETTLEMENT_SELL
log.debug(
'Processing poloniex Trade',
sensitive_log=True,
timestamp=timestamp,
order_type=trade_type,
pair=pair,
base_currency=base_currency,
quote_currency=quote_currency,
amount=amount,
fee=fee,
rate=rate,
)
# Use the converted assets in our pair
pair = trade_pair_from_assets(base_currency, quote_currency)
# Since in Poloniex the base currency is the cost currency, iow in poloniex
# for BTC_ETH we buy ETH with BTC and sell ETH for BTC, we need to turn it
# into the Rotkehlchen way which is following the base/quote approach.
pair = invert_pair(pair)
return Trade(
timestamp=timestamp,
location='poloniex',
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
)
def test_query_trades_sandbox(sandbox_kuckoin, inquirer): # pylint: disable=unused-argument
"""The sandbox account has 6 trades. Below a list of the trades and their
timestamps in ascending mode.
- trade 1: 1612556651 -> skipped
- trade 2: 1612556693
- trade 3: 1612556765
- trade 4: 1612556765
- trade 5: 1612556765
- trade 6: 1612556794 -> skipped
By requesting trades from 1612556693 to 1612556765, the first and last trade
should be skipped.
"""
expected_trades = [
Trade(
timestamp=Timestamp(1612556765),
location=Location.KUCOIN,
pair=TradePair('ETH_BTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.02934995')),
rate=Price(FVal('0.046058')),
fee=Fee(FVal('9.4625999797E-7')),
fee_currency=Asset('BTC'),
link='601da9ddf73c300006194ec6',
notes='',
),
Trade(
timestamp=Timestamp(1612556765),
location=Location.KUCOIN,
pair=TradePair('ETH_BTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.02')),
rate=Price(FVal('0.04561')),
fee=Fee(FVal('6.3854E-7')),
fee_currency=Asset('BTC'),
link='601da9ddf73c300006194ec5',
notes='',
),
Trade(
timestamp=Timestamp(1612556765),
location=Location.KUCOIN,
pair=TradePair('ETH_BTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.06')),
rate=Price(FVal('0.0456')),
fee=Fee(FVal('0.0000019152')),
fee_currency=Asset('BTC'),
link='601da9ddf73c300006194ec4',
notes='',
),
Trade(
timestamp=Timestamp(1612556693),
location=Location.KUCOIN,
pair=TradePair('BTC_USDT'),
trade_type=TradeType.SELL,
amount=AssetAmount(FVal('0.0013')),
rate=Price(FVal('37624.4')),
fee=Fee(FVal('0.034238204')),
fee_currency=Asset('USDT'),
link='601da995e0ee8b00063a075c',
notes='',
),
]
trades = sandbox_kuckoin.query_online_trade_history(
start_ts=Timestamp(1612556693),
end_ts=Timestamp(1612556765),
)
assert trades == expected_trades
def test_add_trades(rotkehlchen_api_server):
"""Test that adding trades to the trades endpoint works as expected"""
new_trades = [{ # own chain to fiat
'timestamp': 1575640208,
'location': 'external',
'base_asset': 'BTC',
'quote_asset': 'EUR',
'trade_type': 'buy',
'amount': '0.5541',
'rate': '8422.1',
'fee': '0.55',
'fee_currency': 'USD',
'link': 'optional trader identifier',
'notes': 'optional notes',
}, { # own chain to eth token, with some optional fields (link,notes) missing
'timestamp': 1585640208,
'location': 'external',
'base_asset': 'ETH',
'quote_asset': A_AAVE.identifier,
'trade_type': 'buy',
'amount': '0.5541',
'rate': '8422.1',
'fee': '0.55',
'fee_currency': 'USD',
}, { # token to token, with all optional fields (fee,fee_currency,link,notes) missing
'timestamp': 1595640208,
'location': 'external',
'base_asset': A_DAI.identifier,
'quote_asset': A_AAVE.identifier,
'trade_type': 'buy',
'amount': '1.5541',
'rate': '22.1',
}]
# add multple trades
all_expected_trades = []
for new_trade in new_trades:
response = requests.put(
api_url_for(
rotkehlchen_api_server,
'tradesresource',
),
json=new_trade,
)
result = assert_proper_response_with_result(response)
# And check that the identifier is correctly generated when returning the trade
new_trade['trade_id'] = Trade(
**TradeSchema().load(new_trade)).identifier
expected_trade = new_trade.copy()
for x in ('fee', 'fee_currency', 'link', 'notes'):
expected_trade[x] = new_trade.get(x, None)
assert result == expected_trade
all_expected_trades.insert(0, expected_trade)
# and now make sure the trade is saved by querying for it
response = requests.get(
api_url_for(
rotkehlchen_api_server,
"tradesresource",
), )
result = assert_proper_response_with_result(response)
data = response.json()
assert data['message'] == ''
assert result['entries'] == [{
'entry': x,
'ignored_in_accounting': False
} for x in all_expected_trades] # noqa: E501
# Test trade with rate 0. Should fail
zero_rate_trade = {
'timestamp': 1575640208,
'location': 'external',
'base_asset': 'ETH',
'quote_asset': A_WETH.identifier,
'trade_type': 'buy',
'amount': '0.5541',
'rate': '0',
'fee': '0.01',
'fee_currency': 'USD',
'link': 'optional trader identifier',
'notes': 'optional notes',
}
response = requests.put(
api_url_for(
rotkehlchen_api_server,
"tradesresource",
),
json=zero_rate_trade,
)
assert_error_response(
response=response,
contained_in_msg='A zero rate is not allowed',
status_code=HTTPStatus.BAD_REQUEST,
)
def _consume_cryptocom_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cryptocom entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCryptocomEntry if importing of this entry is not supported.
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Transaction Kind']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='crypto.com',
)
description = csv_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees info until (Nov 2020) on crypto.com
# fees are not displayed in the export data
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if row_type in (
'crypto_purchase',
'crypto_exchange',
'referral_gift',
'referral_bonus',
'crypto_earn_interest_paid',
'referral_card_cashback',
'card_cashback_reverted',
'reimbursement',
):
# variable mapping to raw data
currency = csv_row['Currency']
to_currency = csv_row['To Currency']
native_currency = csv_row['Native Currency']
amount = csv_row['Amount']
to_amount = csv_row['To Amount']
native_amount = csv_row['Native Amount']
trade_type = TradeType.BUY if to_currency != native_currency else TradeType.SELL
if row_type == 'crypto_exchange':
# trades crypto to crypto
base_asset = Asset(to_currency)
quote_asset = Asset(currency)
if quote_asset is None:
raise DeserializationError('Got a trade entry with an empty quote asset')
base_amount_bought = deserialize_asset_amount(to_amount)
quote_amount_sold = deserialize_asset_amount(amount)
else:
base_asset = Asset(currency)
quote_asset = Asset(native_currency)
base_amount_bought = deserialize_asset_amount(amount)
quote_amount_sold = deserialize_asset_amount(native_amount)
rate = Price(abs(quote_amount_sold / base_amount_bought))
pair = TradePair(f'{base_asset.identifier}_{quote_asset.identifier}')
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
pair=pair,
trade_type=trade_type,
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type in ('crypto_withdrawal', 'crypto_deposit'):
if row_type == 'crypto_withdrawal':
category = AssetMovementCategory.WITHDRAWAL
amount = deserialize_asset_amount_force_positive(csv_row['Amount'])
else:
category = AssetMovementCategory.DEPOSIT
amount = deserialize_asset_amount(csv_row['Amount'])
asset = Asset(csv_row['Currency'])
asset_movement = AssetMovement(
location=Location.CRYPTOCOM,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif row_type in (
'crypto_earn_program_created',
'crypto_earn_program_withdrawn',
'lockup_lock',
'lockup_unlock',
'dynamic_coin_swap_bonus_exchange_deposit',
'crypto_wallet_swap_debited',
'crypto_wallet_swap_credited',
'lockup_swap_debited',
'lockup_swap_credited',
'lockup_swap_rebate',
'dynamic_coin_swap_bonus_exchange_deposit',
# we don't handle cryto.com exchange yet
'crypto_to_exchange_transfer',
'exchange_to_crypto_transfer',
# supercharger actions
'supercharger_deposit',
'supercharger_withdrawal',
# already handled using _import_cryptocom_double_entries
'dynamic_coin_swap_debited',
'dynamic_coin_swap_credited',
'dust_conversion_debited',
'dust_conversion_credited',
):
# those types are ignored because it doesn't affect the wallet balance
# or are not handled here
return
else:
raise UnsupportedCryptocomEntry(
f'Unknown entrype type "{row_type}" encountered during '
f'cryptocom data import. Ignoring entry',
)
def test_binance_query_trade_history(function_scope_binance):
"""Test that turning a binance trade as returned by the server to our format works"""
binance = function_scope_binance
def mock_my_trades(url, **kwargs): # pylint: disable=unused-argument
if 'myTrades' in url:
if 'symbol=BNBBTC' in url:
text = BINANCE_MYTRADES_RESPONSE
else:
text = '[]'
elif 'fiat/payments' in url:
match = TIMESTAMPS_RE.search(url)
assert match
groups = match.groups()
assert len(groups) == 2
from_ts, to_ts = [int(x) for x in groups]
if 'transactionType=0' in url:
if from_ts < 1624529919000 < to_ts:
text = BINANCE_FIATBUY_RESPONSE
else:
text = '[]'
elif 'transactionType=1' in url:
if from_ts < 1628529919000 < to_ts:
text = BINANCE_FIATSELL_RESPONSE
else:
text = '[]'
else:
raise AssertionError('Unexpected binance request in test')
else:
raise AssertionError('Unexpected binance request in test')
return MockResponse(200, text)
with patch.object(binance.session, 'get', side_effect=mock_my_trades):
trades = binance.query_trade_history(start_ts=0, end_ts=1638529919, only_cache=False)
expected_trades = [Trade(
timestamp=1499865549,
location=Location.BINANCE,
base_asset=A_BNB,
quote_asset=A_BTC,
trade_type=TradeType.BUY,
amount=FVal('12'),
rate=FVal('4.00000100'),
fee=FVal('10.10000000'),
fee_currency=A_BNB,
link='28457',
), Trade(
timestamp=1624529919,
location=Location.BINANCE,
base_asset=A_LUNA,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=FVal('4.462'),
rate=FVal('4.437472'),
fee=FVal('0.2'),
fee_currency=A_EUR,
link='353fca443f06466db0c4dc89f94f027a',
), Trade(
timestamp=1628529919,
location=Location.BINANCE,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.SELL,
amount=FVal('4.462'),
rate=FVal('4.437472'),
fee=FVal('0.2'),
fee_currency=A_EUR,
link='463fca443f06466db0c4dc89f94f027a',
)]
assert trades == expected_trades
def _import_cryptocom_double_entries(self, data: Any, double_type: str) -> None:
"""Look for events that have double entries and handle them as trades.
This method looks for `*_debited` and `*_credited` entries using the
same timestamp to handle them as one trade.
Known double_type: 'dynamic_coin_swap' or 'dust_conversion'
"""
double_rows: Dict[Any, Dict[str, Any]] = {}
debited_row = None
credited_row = None
for row in data:
if row['Transaction Kind'] == f'{double_type}_debited':
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='crypto.com',
)
if timestamp not in double_rows:
double_rows[timestamp] = {}
double_rows[timestamp]['debited'] = row
elif row['Transaction Kind'] == f'{double_type}_credited':
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='crypto.com',
)
if timestamp not in double_rows:
double_rows[timestamp] = {}
double_rows[timestamp]['credited'] = row
for timestamp in double_rows:
credited_row = double_rows[timestamp]['credited']
debited_row = double_rows[timestamp]['debited']
if credited_row is not None and debited_row is not None:
description = credited_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees here
fee = Fee(ZERO)
fee_currency = A_USD
base_asset = Asset(credited_row['Currency'])
quote_asset = Asset(debited_row['Currency'])
pair = TradePair(f'{base_asset.identifier}_{quote_asset.identifier}')
base_amount_bought = deserialize_asset_amount(credited_row['Amount'])
quote_amount_sold = deserialize_asset_amount(debited_row['Amount'])
rate = Price(abs(base_amount_bought / quote_amount_sold))
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
pair=pair,
trade_type=TradeType.BUY,
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
def test_query_owned_assets(data_dir, username):
"""Test the get_owned_assets with also an unknown asset in the DB"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
balances = deepcopy(asset_balances)
balances.extend([
DBAssetBalance(
category=BalanceType.ASSET,
time=Timestamp(1488326400),
asset=A_BTC,
amount='1',
usd_value='1222.66',
),
DBAssetBalance(
category=BalanceType.ASSET,
time=Timestamp(1489326500),
asset=A_XMR,
amount='2',
usd_value='33.8',
),
])
data.db.add_multiple_balances(balances)
cursor = data.db.conn.cursor()
cursor.execute(
'INSERT INTO timed_balances('
' time, currency, amount, usd_value, category) '
' VALUES(?, ?, ?, ?, ?)',
(1469326500, 'ADSADX', '10.1', '100.5', 'A'),
)
data.db.conn.commit()
# also make sure that assets from trades are included
data.db.add_trades([
Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
pair=TradePair('ETH_BTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
), Trade(
timestamp=Timestamp(99),
location=Location.EXTERNAL,
pair=TradePair('ETH_BTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(2)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
), Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
pair=TradePair('SDC_SDT-2'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
), Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
pair=TradePair('SUSHI_1INCH'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
), Trade(
timestamp=Timestamp(3),
location=Location.EXTERNAL,
pair=TradePair('SUSHI_1INCH'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(2)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
), Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
pair=TradePair('UNKNOWNTOKEN_BTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
),
])
assets_list = data.db.query_owned_assets()
assert set(assets_list) == {A_USD, A_ETH, A_DAI, A_BTC, A_XMR, Asset('SDC'), Asset('SDT-2'), Asset('SUSHI'), Asset('1INCH')} # noqa: E501
assert all(isinstance(x, Asset) for x in assets_list)
warnings = data.db.msg_aggregator.consume_warnings()
assert len(warnings) == 1
assert 'Unknown/unsupported asset ADSADX' in warnings[0]
def _consume_cointracking_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cointracking entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCointrackingEntry if importing of this entry is not supported.
- IndexError if the CSV file is corrupt
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Type']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date'],
formatstr='%d.%m.%Y %H:%M:%S',
location='cointracking.info',
)
notes = csv_row['Comment']
location = exchange_row_to_location(csv_row['Exchange'])
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if csv_row['Fee'] != '':
fee = deserialize_fee(csv_row['Fee'])
fee_currency = Asset(csv_row['Cur.Fee'])
if row_type in ('Gift/Tip', 'Trade', 'Income'):
base_asset = Asset(csv_row['Cur.Buy'])
quote_asset = None if csv_row['Cur.Sell'] == '' else Asset(csv_row['Cur.Sell'])
if quote_asset is None and row_type not in ('Gift/Tip', 'Income'):
raise DeserializationError('Got a trade entry with an empty quote asset')
if quote_asset is None:
# Really makes no difference as this is just a gift and the amount is zero
quote_asset = A_USD
pair = TradePair(f'{base_asset.identifier}_{quote_asset.identifier}')
base_amount_bought = deserialize_asset_amount(csv_row['Buy'])
if csv_row['Sell'] != '-':
quote_amount_sold = deserialize_asset_amount(csv_row['Sell'])
else:
quote_amount_sold = AssetAmount(ZERO)
rate = Price(quote_amount_sold / base_amount_bought)
trade = Trade(
timestamp=timestamp,
location=location,
pair=pair,
trade_type=TradeType.BUY, # It's always a buy during cointracking import
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type in ('Deposit', 'Withdrawal'):
category = deserialize_asset_movement_category(row_type.lower())
if category == AssetMovementCategory.DEPOSIT:
amount = deserialize_asset_amount(csv_row['Buy'])
asset = Asset(csv_row['Cur.Buy'])
else:
amount = deserialize_asset_amount_force_positive(csv_row['Sell'])
asset = Asset(csv_row['Cur.Sell'])
asset_movement = AssetMovement(
location=location,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_currency,
link='',
)
self.db.add_asset_movements([asset_movement])
else:
raise UnsupportedCointrackingEntry(
f'Unknown entrype type "{row_type}" encountered during cointracking '
f'data import. Ignoring entry',
)
def test_global_db_restore(globaldb, database):
"""
Check that the user can recreate assets information from the packaged
database with rotki (hard reset). The test adds a new asset, restores
the database and checks that the added token is not in there and that
the amount of assets is the expected
"""
# Add a custom eth token
address_to_delete = make_ethereum_address()
token_to_delete = EthereumToken.initialize(
address=address_to_delete,
decimals=18,
name='willdell',
symbol='DELME',
)
globaldb.add_asset(
asset_id='DELMEID1',
asset_type=AssetType.ETHEREUM_TOKEN,
data=token_to_delete,
)
# Add a token with underlying token
with_underlying_address = make_ethereum_address()
with_underlying = EthereumToken.initialize(
address=with_underlying_address,
decimals=18,
name="Not a scam",
symbol="NSCM",
started=0,
underlying_tokens=[
UnderlyingToken(
address=address_to_delete,
weight=1,
)
],
)
globaldb.add_asset(
asset_id='xDELMEID1',
asset_type=AssetType.ETHEREUM_TOKEN,
data=with_underlying,
)
# Add asset that is not a token
globaldb.add_asset(
asset_id='1',
asset_type=AssetType.OWN_CHAIN,
data={
'name': 'Lolcoin',
'symbol': 'LOLZ',
'started': 0,
},
)
# Add asset that is not a token
globaldb.add_asset(
asset_id='2',
asset_type=AssetType.OWN_CHAIN,
data={
'name': 'Lolcoin2',
'symbol': 'LOLZ2',
'started': 0,
},
)
database.add_asset_identifiers('1')
database.add_asset_identifiers('2')
# Try to reset DB it if we have a trade that uses a custom asset
buy_asset = symbol_to_asset_or_token('LOLZ2')
buy_amount = deserialize_asset_amount(1)
sold_asset = symbol_to_asset_or_token('LOLZ')
sold_amount = deserialize_asset_amount(2)
rate = Price(buy_amount / sold_amount)
trade = Trade(
timestamp=Timestamp(12312312),
location=Location.BLOCKFI,
base_asset=buy_asset,
quote_asset=sold_asset,
trade_type=TradeType.BUY,
amount=buy_amount,
rate=rate,
fee=None,
fee_currency=None,
link='',
notes="",
)
database.add_trades([trade])
status, _ = GlobalDBHandler().hard_reset_assets_list(database)
assert status is False
# Now do it without the trade
database.delete_trade(trade.identifier)
status, msg = GlobalDBHandler().hard_reset_assets_list(database, True)
assert status, msg
cursor = globaldb._conn.cursor()
query = f'SELECT COUNT(*) FROM ethereum_tokens where address == "{address_to_delete}";'
r = cursor.execute(query)
assert r.fetchone() == (0, ), 'Ethereum token should have been deleted'
query = f'SELECT COUNT(*) FROM assets where details_reference == "{address_to_delete}";'
r = cursor.execute(query)
assert r.fetchone() == (
0, ), 'Ethereum token should have been deleted from assets'
query = f'SELECT COUNT(*) FROM ethereum_tokens where address == "{with_underlying_address}";'
r = cursor.execute(query)
assert r.fetchone() == (
0,
), 'Token with underlying token should have been deleted from assets'
query = f'SELECT COUNT(*) FROM assets where details_reference == "{with_underlying_address}";'
r = cursor.execute(query)
assert r.fetchone() == (0, )
query = f'SELECT COUNT(*) FROM underlying_tokens_list where address == "{address_to_delete}";'
r = cursor.execute(query)
assert r.fetchone() == (0, )
query = 'SELECT COUNT(*) FROM assets where identifier == "1";'
r = cursor.execute(query)
assert r.fetchone() == (0, ), 'Non ethereum token should be deleted'
# Check that the user database is correctly updated
query = 'SELECT identifier from assets'
r = cursor.execute(query)
user_db_cursor = database.conn.cursor()
user_db_cursor.execute(query)
assert r.fetchall() == user_db_cursor.fetchall()
# Check that the number of assets is the expected
root_dir = Path(__file__).resolve().parent.parent.parent
builtin_database = root_dir / 'data' / 'global.db'
conn = sqlite3.connect(builtin_database)
cursor_clean_db = conn.cursor()
tokens_expected = cursor_clean_db.execute('SELECT COUNT(*) FROM assets;')
tokens_local = cursor.execute('SELECT COUNT(*) FROM assets;')
assert tokens_expected.fetchone() == tokens_local.fetchone()
cursor.execute('SELECT asset_id FROM user_owned_assets')
msg = 'asset id in trade should not be in the owned table'
assert "'2'" not in [entry[0] for entry in cursor.fetchall()], msg
conn.close()
def _consume_cointracking_entry(self, csv_row: List[str]) -> None:
"""Consumes a cointracking entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCointrackingEntry if importing of this entry is not supported.
- IndexError if the CSV file is corrupt
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row[1] # Type
timestamp = deserialize_timestamp_from_date(
date=csv_row[9],
formatstr='%d.%m.%Y %H:%M',
location='cointracking.info',
)
notes = csv_row[8]
location = exchange_row_to_location(csv_row[6])
if row_type == 'Gift/Tip' or row_type == 'Trade':
base_asset = Asset(csv_row[3])
quote_asset = None if csv_row[5] == '' else Asset(csv_row[5])
if not quote_asset and row_type != 'Gift/Tip':
raise DeserializationError(
'Got a trade entry with an empty quote asset')
if quote_asset is None:
# Really makes no difference as this is just a gift and the amount is zero
quote_asset = A_USD
pair = TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}')
base_amount_bought = deserialize_asset_amount(csv_row[2])
if csv_row[4] != '-':
quote_amount_sold = deserialize_asset_amount(csv_row[4])
else:
quote_amount_sold = AssetAmount(ZERO)
rate = Price(quote_amount_sold / base_amount_bought)
trade = Trade(
timestamp=timestamp,
location=location,
pair=pair,
trade_type=TradeType.
BUY, # It's always a buy during cointracking import
amount=base_amount_bought,
rate=rate,
fee=Fee(
ZERO), # There are no fees when import from cointracking
fee_currency=base_asset,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type == 'Deposit' or row_type == 'Withdrawal':
category = deserialize_asset_movement_category(row_type.lower())
if category == AssetMovementCategory.DEPOSIT:
amount = deserialize_asset_amount(csv_row[2])
asset = Asset(csv_row[3])
else:
amount = deserialize_asset_amount(csv_row[4])
asset = Asset(csv_row[5])
asset_movement = AssetMovement(
location=location,
category=category,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=Fee(ZERO),
link='',
)
self.db.add_asset_movements([asset_movement])
else:
raise UnsupportedCointrackingEntry(
f'Unknown entrype type "{row_type}" encountered during cointracking '
f'data import. Ignoring entry', )