def test_query_async_tasks(rotkehlchen_api_server_with_exchanges):
"""Test that querying the outcomes of async tasks works as expected
We don't mock price queries in this test only because that cause the tasks
list test below to fail since due to the mocking the tasks returns immediately and
does not wait on a gevent context switching. So if we mock we don't get to
test the task is still pending functionality.
"""
# async query balances of one specific exchange
server = rotkehlchen_api_server_with_exchanges
binance = try_get_first_exchange(
server.rest_api.rotkehlchen.exchange_manager,
Location.BINANCE) # noqa: E501
binance_patch = patch.object(binance.session,
'get',
side_effect=mock_binance_balance_response)
# Check querying the async taks resource when no async task is scheduled
response = requests.get(api_url_for(server, "asynctasksresource"))
result = assert_proper_response_with_result(response)
assert result == {'completed': [], 'pending': []}
# Create an async task
with binance_patch:
response = requests.get(api_url_for(
server,
'named_exchanges_balances_resource',
location='binance',
),
json={'async_query': True})
task_id = assert_ok_async_response(response)
# now check that there is a task
response = requests.get(api_url_for(server, 'asynctasksresource'))
result = assert_proper_response_with_result(response)
assert result == {'completed': [], 'pending': [task_id]}
# assert json_data['result'] == {'completed': [], 'pending': []}
# now query for the task result and see it's still pending (test for task lists)
response = requests.get(
api_url_for(server,
'specific_async_tasks_resource',
task_id=task_id), )
assert_proper_response(response)
json_data = response.json()
assert json_data['message'] == 'The task with id 0 is still pending'
assert json_data['result'] == {'status': 'pending', 'outcome': None}
while True:
# and now query for the task result and assert on it
response = requests.get(
api_url_for(server,
"specific_async_tasks_resource",
task_id=task_id), )
assert_proper_response(response)
json_data = response.json()
if json_data['result']['status'] == 'pending':
# context switch so that the greenlet to query balances can operate
gevent.sleep(1)
elif json_data['result']['status'] == 'completed':
break
else:
raise AssertionError(
f"Unexpected status: {json_data['result']['status']}")
assert json_data['message'] == ''
assert json_data['result']['status'] == 'completed'
# assert that there is an outcome
assert json_data['result']['outcome'] is not None
assert json_data['result']['outcome']['result'] is not None
assert json_data['result']['outcome']['message'] == ''
# Finally try to query an unknown task id and check proper error is returned
response = requests.get(
api_url_for(server, "specific_async_tasks_resource", task_id=568), )
assert_error_response(
response=response,
contained_in_msg='No task with id 568 found',
status_code=HTTPStatus.NOT_FOUND,
result_exists=True,
)
json_data = response.json()
assert json_data['result'] == {'status': 'not-found', 'outcome': None}
def test_edit_exchange_account(rotkehlchen_api_server_with_exchanges):
server = rotkehlchen_api_server_with_exchanges
rotki = rotkehlchen_api_server_with_exchanges.rest_api.rotkehlchen
kraken = try_get_first_exchange(rotki.exchange_manager, Location.KRAKEN)
poloniex = try_get_first_exchange(rotki.exchange_manager,
Location.POLONIEX)
assert kraken.name == 'mockkraken'
assert kraken.account_type == DEFAULT_KRAKEN_ACCOUNT_TYPE
assert poloniex.name == 'poloniex'
data = {
'name': 'mockkraken',
'location': 'kraken',
'new_name': 'my_kraken'
}
response = requests.patch(api_url_for(server, 'exchangesresource'),
json=data)
result = assert_proper_response_with_result(response)
assert result is True
kraken = try_get_first_exchange(rotki.exchange_manager, Location.KRAKEN)
assert kraken.name == 'my_kraken'
assert kraken.account_type == DEFAULT_KRAKEN_ACCOUNT_TYPE
data = {
'name': 'poloniex',
'location': 'poloniex',
'new_name': 'my_poloniex'
}
response = requests.patch(api_url_for(server, 'exchangesresource'),
json=data)
result = assert_proper_response_with_result(response)
assert result is True
poloniex = try_get_first_exchange(rotki.exchange_manager,
Location.POLONIEX)
assert poloniex.name == 'my_poloniex'
# Make sure that existing location exchange but wrong name returns error
data = {
'name': 'some_poloniex',
'location': 'poloniex',
'new_name': 'other_poloniex'
}
response = requests.patch(api_url_for(server, 'exchangesresource'),
json=data)
assert_error_response(
response=response,
status_code=HTTPStatus.CONFLICT,
contained_in_msg=
'Could not find poloniex exchange some_poloniex for editing',
)
# Make sure that real location but not registered returns error
data = {'name': 'kucoin', 'location': 'kucoin', 'new_name': 'other_kucoin'}
response = requests.patch(api_url_for(server, 'exchangesresource'),
json=data)
assert_error_response(
response=response,
status_code=HTTPStatus.CONFLICT,
contained_in_msg='Could not find kucoin exchange kucoin for editing',
)
# Make sure that not existing location returns error
data = {
'name': 'kucoin',
'location': 'fakeexchange',
'new_name': 'other_kucoin'
}
response = requests.patch(api_url_for(server, 'exchangesresource'),
json=data)
assert_error_response(
response=response,
status_code=HTTPStatus.BAD_REQUEST,
contained_in_msg='Failed to deserialize Location value fakeexchange',
)
def test_query_all_balances_ignore_cache(
rotkehlchen_api_server_with_exchanges,
ethereum_accounts,
btc_accounts,
):
"""Test that using the query all balances endpoint can ignore the cache"""
rotki = rotkehlchen_api_server_with_exchanges.rest_api.rotkehlchen
setup = setup_balances(rotki, ethereum_accounts, btc_accounts)
binance = try_get_first_exchange(rotki.exchange_manager, Location.BINANCE)
poloniex = try_get_first_exchange(rotki.exchange_manager,
Location.POLONIEX)
eth_query_patch = patch.object(
rotki.chain_manager,
'query_ethereum_balances',
wraps=rotki.chain_manager.query_ethereum_balances,
)
btc_query_patch = patch.object(
rotki.chain_manager,
'query_btc_balances',
wraps=rotki.chain_manager.query_btc_balances,
)
tokens_query_patch = patch.object(
rotki.chain_manager,
'query_ethereum_tokens',
wraps=rotki.chain_manager.query_ethereum_tokens,
)
original_binance_query_dict = binance.api_query_dict
binance_query_patch = patch.object(binance,
'api_query_dict',
wraps=binance.api_query_dict)
poloniex_query_patch = patch.object(poloniex,
'api_query_dict',
wraps=poloniex.api_query_dict)
with ExitStack() as stack:
stack.enter_context(setup.poloniex_patch)
stack.enter_context(setup.binance_patch)
etherscan_mock = stack.enter_context(setup.etherscan_patch)
stack.enter_context(setup.bitcoin_patch)
stack.enter_context(setup.ethtokens_max_chunks_patch)
stack.enter_context(setup.beaconchain_patch)
function_call_counters = []
function_call_counters.append(stack.enter_context(eth_query_patch))
function_call_counters.append(stack.enter_context(btc_query_patch))
function_call_counters.append(stack.enter_context(tokens_query_patch))
function_call_counters.append(stack.enter_context(binance_query_patch))
function_call_counters.append(
stack.enter_context(poloniex_query_patch))
# Query all balances for the first time and test it works
response = requests.get(
api_url_for(
rotkehlchen_api_server_with_exchanges,
'allbalancesresource',
), )
result = assert_proper_response_with_result(response)
assert_all_balances(
result=result,
db=rotki.data.db,
expected_data_in_db=True,
setup=setup,
)
for fn in function_call_counters:
if fn._mock_wraps == original_binance_query_dict:
assert fn.call_count == 3
else:
assert fn.call_count == 1
full_query_etherscan_count = etherscan_mock.call_count
# Query all balances second time and assert cache was used
response = requests.get(
api_url_for(
rotkehlchen_api_server_with_exchanges,
'allbalancesresource',
), )
result = assert_proper_response_with_result(response)
assert_all_balances(
result=result,
db=rotki.data.db,
expected_data_in_db=True,
setup=setup,
)
msg = 'call count should stay the same since cache should have been used'
for fn in function_call_counters:
if fn._mock_wraps == original_binance_query_dict:
assert fn.call_count == 3, msg
else:
assert fn.call_count == 1, msg
msg = 'etherscan call_count should have remained the same due to no token detection '
assert etherscan_mock.call_count == full_query_etherscan_count, msg
# Now query all balances but request cache ignoring
response = requests.get(
api_url_for(
rotkehlchen_api_server_with_exchanges,
'allbalancesresource',
),
json={'ignore_cache': True},
)
result = assert_proper_response_with_result(response)
assert_all_balances(
result=result,
db=rotki.data.db,
expected_data_in_db=True,
setup=setup,
)
msg = 'call count should increase since cache should have been ignored'
for fn in function_call_counters:
if fn._mock_wraps == original_binance_query_dict:
assert fn.call_count == 6, msg
else:
assert fn.call_count == 2, msg
msg = 'etherscan call count should have doubled after forced token detection'
# TODO: Figure out a correct formula for this
expected_count = full_query_etherscan_count * 2 - 1
assert etherscan_mock.call_count == expected_count, msg
def test_multiple_balance_queries_not_concurrent(
rotkehlchen_api_server_with_exchanges,
ethereum_accounts,
btc_accounts,
separate_blockchain_calls,
):
"""Test multiple different balance query requests happening concurrently
This tests that if multiple balance query requests happen concurrently we
do not end up doing them multiple times, but reuse the results thanks to cache.
Try running both all blockchain balances in one call and each blockchain call separately.
"""
rotki = rotkehlchen_api_server_with_exchanges.rest_api.rotkehlchen
setup = setup_balances(rotki, ethereum_accounts, btc_accounts)
multieth_balance_patch = patch.object(
rotki.chain_manager.ethereum,
'get_multieth_balance',
wraps=rotki.chain_manager.ethereum.get_multieth_balance,
)
btc_balances_patch = patch(
'rotkehlchen.chain.manager.get_bitcoin_addresses_balances',
wraps=get_bitcoin_addresses_balances,
)
binance = try_get_first_exchange(rotki.exchange_manager, Location.BINANCE)
binance_querydict_patch = patch.object(binance,
'api_query_dict',
wraps=binance.api_query_dict)
# Test all balances request by requesting to not save the data
with ExitStack() as stack:
setup.enter_all_patches(stack)
eth = stack.enter_context(multieth_balance_patch)
btc = stack.enter_context(btc_balances_patch)
bn = stack.enter_context(binance_querydict_patch)
response = requests.get(
api_url_for(
rotkehlchen_api_server_with_exchanges,
'allbalancesresource',
),
json={'async_query': True},
)
task_id_all = assert_ok_async_response(response)
response = requests.get(api_url_for(
rotkehlchen_api_server_with_exchanges,
'named_exchanges_balances_resource',
location='binance',
),
json={'async_query': True})
task_id_one_exchange = assert_ok_async_response(response)
if separate_blockchain_calls:
response = requests.get(api_url_for(
rotkehlchen_api_server_with_exchanges,
'blockchainbalancesresource',
),
json={
'async_query': True,
'blockchain': 'ETH'
})
task_id_blockchain_eth = assert_ok_async_response(response)
response = requests.get(api_url_for(
rotkehlchen_api_server_with_exchanges,
'blockchainbalancesresource',
),
json={
'async_query': True,
'blockchain': 'BTC'
})
task_id_blockchain_btc = assert_ok_async_response(response)
else:
response = requests.get(api_url_for(
rotkehlchen_api_server_with_exchanges,
'blockchainbalancesresource',
),
json={'async_query': True})
task_id_blockchain = assert_ok_async_response(response)
outcome_all = wait_for_async_task_with_result(
rotkehlchen_api_server_with_exchanges,
task_id_all,
timeout=ASYNC_TASK_WAIT_TIMEOUT * 2,
)
outcome_one_exchange = wait_for_async_task(
rotkehlchen_api_server_with_exchanges,
task_id_one_exchange,
timeout=ASYNC_TASK_WAIT_TIMEOUT * 2,
)
if separate_blockchain_calls:
outcome_eth = wait_for_async_task_with_result(
rotkehlchen_api_server_with_exchanges,
task_id_blockchain_eth,
timeout=ASYNC_TASK_WAIT_TIMEOUT * 2,
)
outcome_btc = wait_for_async_task_with_result(
rotkehlchen_api_server_with_exchanges,
task_id_blockchain_btc,
timeout=ASYNC_TASK_WAIT_TIMEOUT * 2,
)
else:
outcome_blockchain = wait_for_async_task_with_result(
rotkehlchen_api_server_with_exchanges,
task_id_blockchain,
timeout=ASYNC_TASK_WAIT_TIMEOUT * 2,
)
assert eth.call_count == 1, 'eth balance query should only fire once'
assert btc.call_count == 1, 'btc balance query should only happen once'
assert bn.call_count == 3, 'binance balance query should do 2 calls'
assert_all_balances(
result=outcome_all,
db=rotki.data.db,
expected_data_in_db=True,
setup=setup,
)
assert_binance_balances_result(outcome_one_exchange['result'])
if not separate_blockchain_calls:
outcome_eth = outcome_blockchain
outcome_btc = outcome_blockchain
assert_eth_balances_result(
rotki=rotki,
result=outcome_eth,
eth_accounts=ethereum_accounts,
eth_balances=setup.eth_balances,
token_balances=setup.token_balances,
also_btc=not separate_blockchain_calls,
)
assert_btc_balances_result(
result=outcome_btc,
btc_accounts=btc_accounts,
btc_balances=setup.btc_balances,
also_eth=not separate_blockchain_calls,
)
def setup_balances(
rotki,
ethereum_accounts: Optional[List[ChecksumEthAddress]],
btc_accounts: Optional[List[BTCAddress]],
eth_balances: Optional[List[str]] = None,
token_balances: Optional[Dict[EthereumToken, List[str]]] = None,
liabilities: Optional[Dict[EthereumToken, List[str]]] = None,
btc_balances: Optional[List[str]] = None,
manually_tracked_balances: Optional[List[ManuallyTrackedBalance]] = None,
original_queries: Optional[List[str]] = None,
extra_flags: Optional[List[str]] = None,
) -> BalancesTestSetup:
"""Setup the blockchain, exchange and fiat balances for some tests
When eth_balances, token_balances and btc_balances are not provided some
default values are provided.
"""
if ethereum_accounts is None:
ethereum_accounts = []
if btc_accounts is None:
btc_accounts = []
# Sanity checks for setup input
if eth_balances is not None:
msg = ('The eth balances should be a list with each '
'element representing balance of an account')
assert len(eth_balances) == len(ethereum_accounts)
else:
# Default test values
if len(ethereum_accounts) != 0:
eth_balances = ['1000000', '2000000']
else:
eth_balances = []
if token_balances is not None:
msg = 'token balances length does not match number of owned eth tokens'
for _, balances in token_balances.items():
msg = ('The token balances should be a list with each '
'element representing balance of an account')
assert len(balances) == len(ethereum_accounts), msg
else:
# Default test values
if len(ethereum_accounts) != 0:
token_balances = {A_RDN: ['0', '4000000']}
else:
token_balances = {}
if btc_balances is not None:
msg = ('The btc balances should be a list with each '
'element representing balance of an account')
assert len(btc_balances) == len(btc_accounts)
else:
# Default test values
if len(btc_accounts) != 0:
btc_balances = ['3000000', '5000000']
else:
btc_balances = []
eth_map: Dict[ChecksumEthAddress, Dict[Union[str, EthereumToken],
Any]] = {}
for idx, acc in enumerate(ethereum_accounts):
eth_map[acc] = {}
eth_map[acc]['ETH'] = eth_balances[idx]
for token in token_balances:
eth_map[acc][token] = token_balances[token][idx]
defi_balances_patch = None
if liabilities is not None:
def mock_add_defi_balances_to_token_and_totals():
# super hacky way of mocking this but well f**k it
if len(rotki.chain_manager.balances.eth) == 4:
d_liabilities = liabilities.copy()
else: # we know the only test this is used removes index 0 and 2
msg = 'Should be at removal of accounts and only have 2 left'
assert len(rotki.chain_manager.balances.eth) == 2, msg
d_liabilities = {
k: [x for idx, x in enumerate(v) if idx not in (0, 2)]
for k, v in liabilities.items()
}
for token, balances in d_liabilities.items():
for idx, balance in enumerate(balances):
balance = FVal(balance)
if balance == ZERO:
continue
account = ethereum_accounts[idx]
rotki.chain_manager.balances.eth[account].liabilities[
token] = Balance(balance)
rotki.chain_manager.totals.liabilities[token] += Balance(
balance)
defi_balances_patch = patch.object(
rotki.chain_manager,
'add_defi_balances_to_token_and_totals',
side_effect=mock_add_defi_balances_to_token_and_totals,
)
btc_map: Dict[BTCAddress, str] = {}
for idx, btc_acc in enumerate(btc_accounts):
btc_map[btc_acc] = btc_balances[idx]
binance = try_get_first_exchange(rotki.exchange_manager, Location.BINANCE)
binance_patch = patch_binance_balances_query(
binance) if binance else None # type: ignore
poloniex = try_get_first_exchange(rotki.exchange_manager,
Location.POLONIEX)
poloniex_patch = patch_poloniex_balances_query(
poloniex) if poloniex else None # type: ignore
etherscan_patch = mock_etherscan_query(
eth_map=eth_map,
etherscan=rotki.etherscan,
original_queries=original_queries,
original_requests_get=requests.get,
extra_flags=extra_flags,
)
beaconchain_patch = mock_beaconchain(
beaconchain=rotki.chain_manager.beaconchain,
original_queries=original_queries,
original_requests_get=requests.get,
)
# For ethtoken detection we can have bigger chunk length during tests since it's mocked anyway
ethtokens_max_chunks_patch = patch(
'rotkehlchen.chain.ethereum.tokens.ETHERSCAN_MAX_TOKEN_CHUNK_LENGTH',
new=800,
)
bitcoin_patch = mock_bitcoin_balances_query(
btc_map=btc_map,
original_requests_get=requests.get,
)
# Taken from BINANCE_BALANCES_RESPONSE from tests.utils.exchanges
binance_balances = {
A_ETH: FVal('4763368.68006011'),
A_BTC: FVal('4723846.89208129')
}
# Taken from POLONIEX_BALANCES_RESPONSE from tests.utils.exchanges
poloniex_balances = {A_ETH: FVal('11.0'), A_BTC: FVal('5.5')}
if manually_tracked_balances is None:
manually_tracked_balances = []
rotki.data.db.add_manually_tracked_balances(manually_tracked_balances)
return BalancesTestSetup(
eth_balances=eth_balances,
btc_balances=btc_balances,
token_balances=token_balances,
binance_balances=binance_balances,
poloniex_balances=poloniex_balances,
manually_tracked_balances=manually_tracked_balances,
poloniex_patch=poloniex_patch,
binance_patch=binance_patch,
etherscan_patch=etherscan_patch,
ethtokens_max_chunks_patch=ethtokens_max_chunks_patch,
bitcoin_patch=bitcoin_patch,
beaconchain_patch=beaconchain_patch,
defi_balances_patch=defi_balances_patch,
)