def _check_column(attribute: str, index: int, sheet_id: str,
expected: Dict[str, Any],
got_columns: List[List[str]]): # noqa: E501
expected_value = FVal(expected[attribute])
got_value = FVal(got_columns[index][0])
msg = f'Sheet: {sheet_id}, row: {index + CSV_INDEX_OFFSET} {attribute} mismatch. {got_value} != {expected_value}' # noqa: E501
assert expected_value.is_close(got_value), msg
def assert_serialized_dicts_equal(
a: Dict,
b: Dict,
ignore_keys: Optional[List] = None,
length_list_keymap: Optional[Dict] = None,
max_diff: str = "1e-6",
) -> None:
"""Compares serialized dicts so that serialized numbers can be compared for equality"""
assert len(a) == len(
b), f"Dicts don't have the same key length {len(a)} != {len(b)}"
for a_key, a_val in a.items():
if ignore_keys and a_key in ignore_keys:
continue
if isinstance(a_val, FVal):
try:
compare_val = FVal(b[a_key])
except ValueError:
raise AssertionError(
f'Could not turn {a_key} value {b[a_key]} into an FVal')
msg = f"{a_key} doesn't match. {compare_val} != {a_val}"
assert compare_val.is_close(a_val, max_diff=max_diff), msg
elif isinstance(b[a_key], FVal):
try:
compare_val = FVal(a_val)
except ValueError:
raise AssertionError(
f'Could not turn {a_key} value {a[a_key]} into an FVal')
msg = f"{a_key} doesn't match. {compare_val} != {b[a_key]}"
assert compare_val.is_close(b[a_key], max_diff=max_diff), msg
elif isinstance(a_val, list):
max_length_to_check = None
if length_list_keymap and a_key in length_list_keymap:
max_length_to_check = length_list_keymap[a_key]
assert_serialized_lists_equal(
a=a_val,
b=b[a_key],
max_length_to_check=max_length_to_check,
ignore_keys=ignore_keys,
length_list_keymap=length_list_keymap,
)
else:
assert a_val == b[
a_key], f"{a_key} doesn't match. {a_val} != {b[a_key]}"
def test_end_to_end_tax_report_in_period(accountant):
result = accounting_history_process(
accountant=accountant,
start_ts=1483228800, # 01/01/2017
end_ts=1514764799, # 31/12/2017
history_list=trades_history,
loans_list=loans_list,
asset_movements_list=asset_movements_list,
eth_transaction_list=eth_tx_list,
margin_list=margin_history,
)
# Make sure that the "started_processing_timestamp" is the ts of the first
# action in history
assert accountant.started_processing_timestamp == 1446979735
# Make sure that the "currently_processing_timestamp" is the ts of the last
# action seen in history before end_ts
assert accountant.currently_processing_timestamp == 1511626623
result = result['overview']
general_trade_pl = FVal(result['general_trade_profit_loss'])
assert general_trade_pl.is_close('1506.96912912')
taxable_trade_pl = FVal(result['taxable_trade_profit_loss'])
assert taxable_trade_pl.is_close('642.652537097')
loan_profit = FVal(result['loan_profit'])
assert loan_profit.is_close('0.111881296004')
settlement_losses = FVal(result['settlement_losses'])
assert settlement_losses.is_close('10.7553789375')
asset_movement_fees = FVal(result['asset_movement_fees'])
assert asset_movement_fees.is_close('2.38526415')
ethereum_transaction_gas_costs = FVal(result['ethereum_transaction_gas_costs'])
assert ethereum_transaction_gas_costs.is_close('2.2617525')
margin_pl = FVal(result['margin_positions_profit_loss'])
assert margin_pl.is_close('234.61035')
expected_total_taxable_pl = (
taxable_trade_pl +
margin_pl +
loan_profit -
settlement_losses -
asset_movement_fees -
ethereum_transaction_gas_costs
)
total_taxable_pl = FVal(result['total_taxable_profit_loss'])
assert expected_total_taxable_pl.is_close(total_taxable_pl)
expected_total_pl = (
general_trade_pl +
margin_pl +
loan_profit -
settlement_losses -
asset_movement_fees -
ethereum_transaction_gas_costs
)
total_pl = FVal(result['total_profit_loss'])
assert expected_total_pl.is_close(total_pl)
def test_end_to_end_tax_report(accountant):
result = accounting_history_process(
accountant=accountant,
start_ts=0,
end_ts=1514764799, # 31/12/2017
history_list=trades_history,
loans_list=loans_list,
asset_movements_list=asset_movements_list,
eth_transaction_list=eth_tx_list,
margin_list=margin_history,
)
result = result['overview']
# Make sure that the "started_processing_timestamp" is the ts of the first
# action in history
assert accountant.started_processing_timestamp == 1446979735
# Make sure that the "currently_processing_timestamp" is the ts of the last
# action seen in history before end_ts
assert accountant.currently_processing_timestamp == 1511626623
general_trade_pl = FVal(result['general_trade_profit_loss'])
assert general_trade_pl.is_close('5032.30394444')
taxable_trade_pl = FVal(result['taxable_trade_profit_loss'])
assert taxable_trade_pl.is_close('3954.94067484')
loan_profit = FVal(result['loan_profit'])
assert loan_profit.is_close('0.114027511004')
settlement_losses = FVal(result['settlement_losses'])
assert settlement_losses.is_close('11.8554392326')
asset_movement_fees = FVal(result['asset_movement_fees'])
assert asset_movement_fees.is_close('2.39417915')
ethereum_transaction_gas_costs = FVal(result['ethereum_transaction_gas_costs'])
assert ethereum_transaction_gas_costs.is_close('2.7210025')
margin_pl = FVal(result['margin_positions_profit_loss'])
assert margin_pl.is_close('232.95481')
expected_total_taxable_pl = (
taxable_trade_pl +
margin_pl +
loan_profit -
settlement_losses -
asset_movement_fees -
ethereum_transaction_gas_costs
)
total_taxable_pl = FVal(result['total_taxable_profit_loss'])
assert expected_total_taxable_pl.is_close(total_taxable_pl)
expected_total_pl = (
general_trade_pl +
margin_pl +
loan_profit -
settlement_losses -
asset_movement_fees -
ethereum_transaction_gas_costs
)
total_pl = FVal(result['total_profit_loss'])
assert expected_total_pl.is_close(total_pl)
def test_end_to_end_tax_report_in_period(accountant):
result = accounting_history_process(
accountant=accountant,
start_ts=1483228800, # 01/01/2017
end_ts=1514764799, # 31/12/2017
history_list=trades_history,
loans_list=loans_list,
asset_movements_list=asset_movements_list,
eth_transaction_list=eth_tx_list,
margin_list=margin_history,
)
# Make sure that the "currently_processing_timestamp" is the ts of the last
# action seen in history before end_ts
assert accountant.currently_processing_timestamp == 1511626623
result = result['overview']
general_trade_pl = FVal(result['general_trade_profit_loss'])
assert general_trade_pl.is_close('1506.937290335')
taxable_trade_pl = FVal(result['taxable_trade_profit_loss'])
assert taxable_trade_pl.is_close('642.7084519791')
loan_profit = FVal(result['loan_profit'])
assert loan_profit.is_close('0.1140477')
settlement_losses = FVal(result['settlement_losses'])
assert settlement_losses.is_close('10.81727783')
asset_movement_fees = FVal(result['asset_movement_fees'])
assert asset_movement_fees.is_close('2.38205415')
ethereum_transaction_gas_costs = FVal(
result['ethereum_transaction_gas_costs'])
assert ethereum_transaction_gas_costs.is_close('2.276810')
margin_pl = FVal(result['margin_positions_profit_loss'])
assert margin_pl.is_close('234.5323965')
defi_pl = FVal(result['defi_profit_loss'])
assert defi_pl == ZERO
expected_total_taxable_pl = (taxable_trade_pl + margin_pl + loan_profit -
settlement_losses - asset_movement_fees -
ethereum_transaction_gas_costs)
total_taxable_pl = FVal(result['total_taxable_profit_loss'])
assert expected_total_taxable_pl.is_close(total_taxable_pl)
expected_total_pl = (general_trade_pl + margin_pl + loan_profit -
settlement_losses - asset_movement_fees -
ethereum_transaction_gas_costs)
total_pl = FVal(result['total_profit_loss'])
assert expected_total_pl.is_close(total_pl)
def test_end_to_end_tax_report(accountant):
result = accounting_history_process(
accountant=accountant,
start_ts=0,
end_ts=1514764799, # 31/12/2017
history_list=trades_history,
loans_list=loans_list,
asset_movements_list=asset_movements_list,
eth_transaction_list=eth_tx_list,
margin_list=margin_history,
)
result = result['overview']
# Make sure that the "currently_processing_timestamp" is the ts of the last
# action seen in history before end_ts
assert accountant.currently_processing_timestamp == 1511626623
general_trade_pl = FVal(result['general_trade_profit_loss'])
assert general_trade_pl.is_close('5032.272455644')
taxable_trade_pl = FVal(result['taxable_trade_profit_loss'])
assert taxable_trade_pl.is_close('3954.996939709')
loan_profit = FVal(result['loan_profit'])
assert loan_profit.is_close('0.116193915')
settlement_losses = FVal(result['settlement_losses'])
assert settlement_losses.is_close('11.91758472725')
asset_movement_fees = FVal(result['asset_movement_fees'])
assert asset_movement_fees.is_close('2.39096865')
ethereum_transaction_gas_costs = FVal(
result['ethereum_transaction_gas_costs'])
assert ethereum_transaction_gas_costs.is_close('2.736160')
margin_pl = FVal(result['margin_positions_profit_loss'])
assert margin_pl.is_close('232.8225695')
defi_pl = FVal(result['defi_profit_loss'])
assert defi_pl == ZERO
expected_total_taxable_pl = (taxable_trade_pl + margin_pl + loan_profit -
settlement_losses - asset_movement_fees -
ethereum_transaction_gas_costs)
total_taxable_pl = FVal(result['total_taxable_profit_loss'])
assert expected_total_taxable_pl.is_close(total_taxable_pl)
expected_total_pl = (general_trade_pl + margin_pl + loan_profit -
settlement_losses - asset_movement_fees -
ethereum_transaction_gas_costs)
total_pl = FVal(result['total_profit_loss'])
assert expected_total_pl.is_close(total_pl)
def assert_serialized_dicts_equal(
a: Dict,
b: Dict,
ignore_keys: Optional[List] = None,
length_list_keymap: Optional[Dict] = None,
max_diff: str = "1e-6",
same_key_length=True,
) -> None:
"""Compares serialized dicts so that serialized numbers can be compared for equality"""
if same_key_length:
assert len(a) == len(
b), f"Dicts don't have the same key length {len(a)} != {len(b)}"
for a_key, a_val in a.items():
if ignore_keys and a_key in ignore_keys:
continue
if isinstance(a_val, FVal):
try:
compare_val = FVal(b[a_key])
except ValueError:
raise AssertionError(
f'Could not turn {a_key} amount {b[a_key]} into an FVal')
msg = f"{a_key} amount doesn't match. {compare_val} != {a_val}"
assert compare_val.is_close(a_val, max_diff=max_diff), msg
elif isinstance(b[a_key], FVal):
try:
compare_val = FVal(a_val)
except ValueError:
raise AssertionError(
f'Could not turn {a_key} value {a[a_key]} into an FVal')
msg = f"{a_key} doesn't match. {compare_val} != {b[a_key]}"
assert compare_val.is_close(b[a_key], max_diff=max_diff), msg
elif isinstance(a_val, str) and isinstance(b[a_key], str):
if a_val == b[a_key]:
continue
if '%' in a_val:
raise AssertionError(f'{a_val} != {b[a_key]}')
# if strings are not equal, try to turn them to Fvals
try:
afval = FVal(a_val)
except ValueError:
raise AssertionError(
f'After string comparison failure could not turn {a_val} to a number '
f'to compare with {b[a_key]}', )
try:
bfval = FVal(b[a_key])
except ValueError:
raise AssertionError(
f'After string comparison failure could not turn {b[a_key]} to a number '
f'to compare with {b[a_key]}', )
msg = f"{a_key} doesn't match. {afval} != {bfval}"
assert afval.is_close(bfval, max_diff=max_diff), msg
elif isinstance(a_val,
dict) and 'amount' in a_val and 'usd_value' in a_val:
assert 'amount' in b[a_key]
assert 'usd_value' in b[a_key]
try:
compare_val = FVal(b[a_key]['amount'])
except ValueError:
raise AssertionError(
f'Could not turn {a_key} amount {b[a_key]} into an FVal')
msg = f"{a_key} amount doesn't match. {compare_val} != {a_val['amount']}"
assert compare_val.is_close(a_val['amount'],
max_diff=max_diff), msg
try:
compare_val = FVal(b[a_key]['usd_value'])
except ValueError:
raise AssertionError(
f'Could not turn {a_key} usd_value {b[a_key]} into an FVal'
)
msg = f"{a_key} usd_value doesn't match. {compare_val} != {a_val['usd_value']}"
assert compare_val.is_close(a_val['usd_value'],
max_diff=max_diff), msg
elif isinstance(a_val, list):
max_length_to_check = None
if length_list_keymap and a_key in length_list_keymap:
max_length_to_check = length_list_keymap[a_key]
assert_serialized_lists_equal(
a=a_val,
b=b[a_key],
max_length_to_check=max_length_to_check,
ignore_keys=ignore_keys,
length_list_keymap=length_list_keymap,
)
else:
assert a_val == b[
a_key], f"{a_key} doesn't match. {a_val} != {b[a_key]}"
def test_query_eth2_deposits_details_and_stats(rotkehlchen_api_server, ethereum_accounts):
"""This test uses real data and queries the eth2 details, deposits and daily stats"""
async_query = random.choice([False, True])
rotki = rotkehlchen_api_server.rest_api.rotkehlchen
setup = setup_balances(
rotki,
ethereum_accounts=ethereum_accounts,
btc_accounts=[],
original_queries=['logs', 'transactions', 'blocknobytime', 'beaconchain'],
)
with ExitStack() as stack:
setup.enter_blockchain_patches(stack)
response = requests.get(
api_url_for(
rotkehlchen_api_server,
'eth2stakedetailsresource',
), json={'async_query': async_query},
)
if async_query:
task_id = assert_ok_async_response(response)
outcome = wait_for_async_task(
rotkehlchen_api_server,
task_id,
timeout=ASYNC_TASK_WAIT_TIMEOUT * 5,
)
assert outcome['message'] == ''
details = outcome['result']
else:
details = assert_proper_response_with_result(response)
with ExitStack() as stack:
setup.enter_blockchain_patches(stack)
response = requests.get(
api_url_for(
rotkehlchen_api_server,
'eth2stakedepositsresource',
), json={'async_query': async_query},
)
if async_query:
task_id = assert_ok_async_response(response)
outcome = wait_for_async_task(
rotkehlchen_api_server,
task_id,
timeout=ASYNC_TASK_WAIT_TIMEOUT * 10,
)
assert outcome['message'] == ''
deposits = outcome['result']
else:
deposits = assert_proper_response_with_result(response)
expected_pubkey = '0xb016e31f633a21fbe42a015152399361184f1e2c0803d89823c224994af74a561c4ad8cfc94b18781d589d03e952cd5b' # noqa: E501
assert deposits[0] == {
'from_address': '0xfeF0E7635281eF8E3B705e9C5B86e1d3B0eAb397',
'tx_index': 15,
'pubkey': expected_pubkey,
'timestamp': 1604506685,
'tx_hash': '0xd9eca1c2a0c5ff2f25071713432b21cc4d0ff2e8963edc63a48478e395e08db1',
'value': {'amount': '32', 'usd_value': '32'},
'withdrawal_credentials': '0x004c7691c2085648f394ffaef851f3b1d51b95f7263114bc923fc5338f5fc499', # noqa: E501
}
assert FVal(details[0]['balance']['amount']) >= ZERO
assert FVal(details[0]['balance']['usd_value']) >= ZERO
assert details[0]['eth1_depositor'] == '0xfeF0E7635281eF8E3B705e9C5B86e1d3B0eAb397' # noqa: E501
assert details[0]['index'] == 9
assert details[0]['public_key'] == expected_pubkey
for duration in ('1d', '1w', '1m', '1y'):
performance = details[0][f'performance_{duration}']
# Can't assert for positive since they may go offline for a day and the test will fail
# https://twitter.com/LefterisJP/status/1361091757274972160
assert FVal(performance['amount']) is not None
assert FVal(performance['usd_value']) is not None
# for daily stats let's have 3 validators
new_index_1 = 43948
new_index_2 = 23948
response = requests.put(
api_url_for(
rotkehlchen_api_server,
'eth2validatorsresource',
), json={'validator_index': new_index_1},
)
assert_simple_ok_response(response)
response = requests.put(
api_url_for(
rotkehlchen_api_server,
'eth2validatorsresource',
), json={'validator_index': new_index_2},
)
assert_simple_ok_response(response)
# Query deposits again after including manually input validator
with ExitStack() as stack:
setup.enter_blockchain_patches(stack)
response = requests.get(
api_url_for(
rotkehlchen_api_server,
'eth2stakedepositsresource',
), json={'async_query': async_query},
)
if async_query:
task_id = assert_ok_async_response(response)
outcome = wait_for_async_task(
rotkehlchen_api_server,
task_id,
timeout=ASYNC_TASK_WAIT_TIMEOUT * 10,
)
assert outcome['message'] == ''
deposits = outcome['result']
else:
deposits = assert_proper_response_with_result(response)
assert len(deposits) == 3
warnings = rotki.msg_aggregator.consume_warnings()
errors = rotki.msg_aggregator.consume_errors()
assert len(warnings) == 0
assert len(errors) == 0
# Now query eth2 details also including manually input validators to see they work
with ExitStack() as stack:
setup.enter_blockchain_patches(stack)
response = requests.get(
api_url_for(
rotkehlchen_api_server,
'eth2stakedetailsresource',
), json={'async_query': async_query},
)
if async_query:
task_id = assert_ok_async_response(response)
outcome = wait_for_async_task(
rotkehlchen_api_server,
task_id,
timeout=ASYNC_TASK_WAIT_TIMEOUT * 5,
)
assert outcome['message'] == ''
details = outcome['result']
else:
details = assert_proper_response_with_result(response)
# The 2 new validators along with their depositor details should be there
assert len(details) == 3
assert details[0]['index'] == 9 # already checked above
assert details[1]['index'] == new_index_2
assert details[1]['eth1_depositor'] == '0x234EE9e35f8e9749A002fc42970D570DB716453B'
assert details[2]['index'] == new_index_1
assert details[2]['eth1_depositor'] == '0xc2288B408Dc872A1546F13E6eBFA9c94998316a2'
warnings = rotki.msg_aggregator.consume_warnings()
errors = rotki.msg_aggregator.consume_errors()
assert len(warnings) == 0
assert len(errors) == 0
# query daily stats, first without cache -- requesting all
json = {'only_cache': False}
response = requests.post(
api_url_for(
rotkehlchen_api_server,
'eth2dailystatsresource',
), json=json,
)
result = assert_proper_response_with_result(response)
total_stats = len(result['entries'])
assert total_stats == result['entries_total']
assert total_stats == result['entries_found']
full_sum_pnl = FVal(result['sum_pnl'])
full_sum_usd_value = FVal(result['sum_usd_value'])
calculated_sum_pnl = ZERO
calculated_sum_usd_value = ZERO
for entry in result['entries']:
calculated_sum_pnl += FVal(entry['pnl']['amount'])
calculated_sum_usd_value += FVal(entry['pnl']['usd_value'])
assert full_sum_pnl.is_close(calculated_sum_pnl)
assert full_sum_usd_value.is_close(calculated_sum_usd_value)
# filter by validator_index
queried_validators = [new_index_1, 9]
json = {'only_cache': True, 'validators': queried_validators}
response = requests.post(
api_url_for(
rotkehlchen_api_server,
'eth2dailystatsresource',
), json=json,
)
result = assert_proper_response_with_result(response)
assert result['entries_total'] == total_stats
assert result['entries_found'] <= total_stats
assert all(x['validator_index'] in queried_validators for x in result['entries'])
# filter by validator_index and timestamp
queried_validators = [new_index_1, 9]
from_ts = 1613779200
to_ts = 1632182400
json = {'only_cache': True, 'validators': queried_validators, 'from_timestamp': from_ts, 'to_timestamp': to_ts} # noqa: E501
response = requests.post(
api_url_for(
rotkehlchen_api_server,
'eth2dailystatsresource',
), json=json,
)
result = assert_proper_response_with_result(response)
assert result['entries_total'] == total_stats
assert result['entries_found'] <= total_stats
assert len(result['entries']) == result['entries_found']
full_sum_pnl = FVal(result['sum_pnl'])
full_sum_usd_value = FVal(result['sum_usd_value'])
calculated_sum_pnl = ZERO
calculated_sum_usd_value = ZERO
next_page_times = []
for idx, entry in enumerate(result['entries']):
calculated_sum_pnl += FVal(entry['pnl']['amount'])
calculated_sum_usd_value += FVal(entry['pnl']['usd_value'])
assert entry['validator_index'] in queried_validators
time = entry['timestamp']
assert time >= from_ts
assert time <= to_ts
if 5 <= idx <= 9:
next_page_times.append(time)
if idx >= result['entries_found'] - 1:
continue
assert entry['timestamp'] >= result['entries'][idx + 1]['timestamp']
assert full_sum_pnl.is_close(calculated_sum_pnl)
assert full_sum_usd_value.is_close(calculated_sum_usd_value)
# filter by validator_index and timestamp and add pagination
json = {'only_cache': True, 'validators': queried_validators, 'from_timestamp': from_ts, 'to_timestamp': to_ts, 'limit': 5, 'offset': 5} # noqa: E501
response = requests.post(
api_url_for(
rotkehlchen_api_server,
'eth2dailystatsresource',
), json=json,
)
result = assert_proper_response_with_result(response)
assert result['entries_total'] == total_stats
assert result['entries_found'] <= total_stats
assert FVal(result['sum_pnl']) == full_sum_pnl, 'pagination should show same sum'
assert FVal(result['sum_usd_value']) == full_sum_usd_value, 'pagination should show same sum'
assert len(result['entries']) == 5
for idx, entry in enumerate(result['entries']):
assert entry['validator_index'] in queried_validators
time = entry['timestamp']
assert time >= from_ts
assert time <= to_ts
if idx <= 4:
assert time == next_page_times[idx]
