def test_add_remove_account_assure_all_balances_not_always_queried(blockchain):
"""Due to a programming mistake at addition and removal of blockchain accounts
after the first time all balances were queried every time. That slowed
everything down (https://github.com/rotki/rotki/issues/678).
This is a regression test for that behaviour
TODO: Is this still needed? Shouldn't it just be removed?
Had to add lots of mocks to make it not be a slow test
"""
addr1 = '0xe188c6BEBB81b96A65aa20dDB9e2aef62627fa4c'
addr2 = '0x78a087fCf440315b843632cFd6FDE6E5adcCc2C2'
etherscan_patch = mock_etherscan_query(
eth_map={
addr1: {
'ETH': 1
},
addr2: {
'ETH': 2
}
},
etherscan=blockchain.ethereum.etherscan,
original_requests_get=requests.get,
original_queries=[],
)
ethtokens_max_chunks_patch = patch(
'rotkehlchen.chain.ethereum.tokens.ETHERSCAN_MAX_TOKEN_CHUNK_LENGTH',
new=800,
)
with etherscan_patch, ethtokens_max_chunks_patch:
blockchain.add_blockchain_accounts(
blockchain=SupportedBlockchain.ETHEREUM,
accounts=[addr1],
)
assert addr1 in blockchain.accounts.eth
with etherscan_patch, ethtokens_max_chunks_patch, patch.object(
blockchain, 'query_balances') as mock: # noqa: E501
blockchain.remove_blockchain_accounts(
blockchain=SupportedBlockchain.ETHEREUM,
accounts=[addr1],
)
assert addr1 not in blockchain.accounts.eth
assert mock.call_count == 0, 'blockchain.query_balances() should not have been called'
addr2 = '0x78a087fCf440315b843632cFd6FDE6E5adcCc2C2'
with etherscan_patch, ethtokens_max_chunks_patch, patch.object(
blockchain, 'query_balances') as mock: # noqa: E501
blockchain.add_blockchain_accounts(
blockchain=SupportedBlockchain.ETHEREUM,
accounts=[addr2],
)
def test_detected_tokens_cache(ethtokens, inquirer): # pylint: disable=unused-argument
"""Test that a cache of the detected tokens is created and used at subsequent queries.
Also test that the cache can be ignored and recreated with a forced redetection
"""
addr1 = make_ethereum_address()
addr2 = make_ethereum_address()
eth_map = {addr1: {A_GNO: 5000, A_MKR: 4000}, addr2: {A_MKR: 6000}}
etherscan_patch = mock_etherscan_query(
eth_map=eth_map,
etherscan=ethtokens.ethereum.etherscan,
original_queries=None,
original_requests_get=requests.get,
extra_flags=None,
)
ethtokens_max_chunks_patch = patch(
'rotkehlchen.chain.ethereum.tokens.ETHERSCAN_MAX_TOKEN_CHUNK_LENGTH',
new=800,
)
with ethtokens_max_chunks_patch, etherscan_patch as etherscan_mock:
# Initially autodetect the tokens at the first call
result1, _ = ethtokens.query_tokens_for_addresses([addr1, addr2],
False)
initial_call_count = etherscan_mock.call_count
# Then in second call autodetect queries should not have been made, and DB cache used
result2, _ = ethtokens.query_tokens_for_addresses([addr1, addr2],
False)
call_count = etherscan_mock.call_count
assert call_count == initial_call_count + 2
# In the third call force re-detection
result3, _ = ethtokens.query_tokens_for_addresses([addr1, addr2], True)
call_count = etherscan_mock.call_count
assert call_count == initial_call_count + 2 + initial_call_count
assert result1 == result2 == result3
assert len(result1) == len(eth_map)
for key, entry in result1.items():
eth_map_entry = eth_map[key]
assert len(entry) == len(eth_map_entry)
for token, val in entry.items():
assert token_normalized_value(eth_map_entry[token],
token) == val
def test_ignored_tokens_in_query(ethtokens, inquirer): # pylint: disable=unused-argument
"""Test that if a token is ignored it's not included in the query"""
addr1 = make_ethereum_address()
addr2 = make_ethereum_address()
eth_map = {addr1: {A_GNO: 5000, A_MKR: 4000}, addr2: {A_MKR: 6000}}
etherscan_patch = mock_etherscan_query(
eth_map=eth_map,
etherscan=ethtokens.ethereum.etherscan,
original_queries=None,
original_requests_get=requests.get,
extra_flags=None,
)
ethtokens_max_chunks_patch = patch(
'rotkehlchen.chain.ethereum.tokens.ETHERSCAN_MAX_TOKEN_CHUNK_LENGTH',
new=800,
)
with ethtokens_max_chunks_patch, etherscan_patch:
result, _ = ethtokens.query_tokens_for_addresses([addr1, addr2], False)
assert len(result[addr1]) == 1
assert result[addr1][A_MKR] == FVal('4E-15')
assert len(result[addr2]) == 1
def test_multiple_concurrent_ethereum_blockchain_queries(blockchain):
"""Test that if there is multiple concurrent ETH blockchain queries
we don't end up double counting:
(1) the DeFi balances (2) the protocol balances such as DSR / makerdao vaults etc.
"""
addr1 = '0xe188c6BEBB81b96A65aa20dDB9e2aef62627fa4c'
addr2 = '0x78a087fCf440315b843632cFd6FDE6E5adcCc2C2'
etherscan_patch = mock_etherscan_query(
eth_map={
addr1: {
A_ETH: 1,
A_DAI: 1 * 10**18
},
addr2: {
A_ETH: 2
}
},
etherscan=blockchain.ethereum.etherscan,
original_requests_get=requests.get,
original_queries=None,
extra_flags=None,
)
ethtokens_max_chunks_patch = patch(
'rotkehlchen.chain.ethereum.tokens.ETHERSCAN_MAX_TOKEN_CHUNK_LENGTH',
new=800,
)
beaconchain_patch = mock_beaconchain(
blockchain.beaconchain,
original_queries=None,
original_requests_get=requests.get,
)
def mock_query_defi_balances():
blockchain.defi_balances = {
addr1: [
DefiProtocolBalances(
protocol=DefiProtocol('a', 'b', 'c', 1),
balance_type='Asset',
base_balance=DefiBalance(
token_address=A_DAI.ethereum_address,
token_name='DAI',
token_symbol='DAI',
balance=Balance(amount=FVal(1), usd_value=(1)),
),
underlying_balances=[
DefiBalance(
token_address=A_DAI.ethereum_address,
token_name='DAI',
token_symbol='DAI',
balance=Balance(amount=FVal(1), usd_value=(1)),
)
],
)
],
}
return blockchain.defi_balances
defi_balances_mock = patch.object(
blockchain,
'query_defi_balances',
wraps=mock_query_defi_balances,
)
def mock_add_defi_balances_to_token_and_totals():
"""This function will make sure all greenlets end up hitting the balance addition
at the same time thus double +++ counting balance ... in the way the code
was written before"""
gevent.sleep(2) # make sure all greenlets stop here
# and then let them all go in the same time in the adding
for account, defi_balances in blockchain.defi_balances.items():
blockchain._add_account_defi_balances_to_token_and_totals(
account=account,
balances=defi_balances,
)
add_defi_mock = patch.object(
blockchain,
'add_defi_balances_to_token_and_totals',
wraps=mock_add_defi_balances_to_token_and_totals,
)
with etherscan_patch, ethtokens_max_chunks_patch:
blockchain.add_blockchain_accounts(
blockchain=SupportedBlockchain.ETHEREUM,
accounts=[addr1, addr2],
)
assert addr1 in blockchain.accounts.eth
with etherscan_patch, ethtokens_max_chunks_patch, defi_balances_mock, add_defi_mock, beaconchain_patch: # noqa: E501
greenlets = [
gevent.spawn_later(0.01 * x, blockchain.query_ethereum_balances,
False) for x in range(5)
]
gevent.joinall(greenlets)
assert blockchain.totals.assets[A_DAI].amount == 2
assert blockchain.balances.eth[addr1].assets[A_DAI].amount == 2
def setup_balances(
rotki,
ethereum_accounts: Optional[List[ChecksumEthAddress]],
btc_accounts: Optional[List[BTCAddress]],
eth_balances: Optional[List[str]] = None,
token_balances: Optional[Dict[EthereumToken, List[str]]] = None,
liabilities: Optional[Dict[EthereumToken, List[str]]] = None,
btc_balances: Optional[List[str]] = None,
manually_tracked_balances: Optional[List[ManuallyTrackedBalance]] = None,
original_queries: Optional[List[str]] = None,
extra_flags: Optional[List[str]] = None,
) -> BalancesTestSetup:
"""Setup the blockchain, exchange and fiat balances for some tests
When eth_balances, token_balances and btc_balances are not provided some
default values are provided.
"""
if ethereum_accounts is None:
ethereum_accounts = []
if btc_accounts is None:
btc_accounts = []
# Sanity checks for setup input
if eth_balances is not None:
msg = ('The eth balances should be a list with each '
'element representing balance of an account')
assert len(eth_balances) == len(ethereum_accounts)
else:
# Default test values
if len(ethereum_accounts) != 0:
eth_balances = ['1000000', '2000000']
else:
eth_balances = []
if token_balances is not None:
msg = 'token balances length does not match number of owned eth tokens'
for _, balances in token_balances.items():
msg = ('The token balances should be a list with each '
'element representing balance of an account')
assert len(balances) == len(ethereum_accounts), msg
else:
# Default test values
if len(ethereum_accounts) != 0:
token_balances = {A_RDN: ['0', '4000000']}
else:
token_balances = {}
if btc_balances is not None:
msg = ('The btc balances should be a list with each '
'element representing balance of an account')
assert len(btc_balances) == len(btc_accounts)
else:
# Default test values
if len(btc_accounts) != 0:
btc_balances = ['3000000', '5000000']
else:
btc_balances = []
eth_map: Dict[ChecksumEthAddress, Dict[Union[str, EthereumToken],
Any]] = {}
for idx, acc in enumerate(ethereum_accounts):
eth_map[acc] = {}
eth_map[acc]['ETH'] = eth_balances[idx]
for token in token_balances:
eth_map[acc][token] = token_balances[token][idx]
defi_balances_patch = None
if liabilities is not None:
def mock_add_defi_balances_to_token_and_totals():
# super hacky way of mocking this but well f**k it
if len(rotki.chain_manager.balances.eth) == 4:
d_liabilities = liabilities.copy()
else: # we know the only test this is used removes index 0 and 2
msg = 'Should be at removal of accounts and only have 2 left'
assert len(rotki.chain_manager.balances.eth) == 2, msg
d_liabilities = {
k: [x for idx, x in enumerate(v) if idx not in (0, 2)]
for k, v in liabilities.items()
}
for token, balances in d_liabilities.items():
for idx, balance in enumerate(balances):
balance = FVal(balance)
if balance == ZERO:
continue
account = ethereum_accounts[idx]
rotki.chain_manager.balances.eth[account].liabilities[
token] = Balance(balance)
rotki.chain_manager.totals.liabilities[token] += Balance(
balance)
defi_balances_patch = patch.object(
rotki.chain_manager,
'add_defi_balances_to_token_and_totals',
side_effect=mock_add_defi_balances_to_token_and_totals,
)
btc_map: Dict[BTCAddress, str] = {}
for idx, btc_acc in enumerate(btc_accounts):
btc_map[btc_acc] = btc_balances[idx]
binance = try_get_first_exchange(rotki.exchange_manager, Location.BINANCE)
binance_patch = patch_binance_balances_query(
binance) if binance else None # type: ignore
poloniex = try_get_first_exchange(rotki.exchange_manager,
Location.POLONIEX)
poloniex_patch = patch_poloniex_balances_query(
poloniex) if poloniex else None # type: ignore
etherscan_patch = mock_etherscan_query(
eth_map=eth_map,
etherscan=rotki.etherscan,
original_queries=original_queries,
original_requests_get=requests.get,
extra_flags=extra_flags,
)
beaconchain_patch = mock_beaconchain(
beaconchain=rotki.chain_manager.beaconchain,
original_queries=original_queries,
original_requests_get=requests.get,
)
# For ethtoken detection we can have bigger chunk length during tests since it's mocked anyway
ethtokens_max_chunks_patch = patch(
'rotkehlchen.chain.ethereum.tokens.ETHERSCAN_MAX_TOKEN_CHUNK_LENGTH',
new=800,
)
bitcoin_patch = mock_bitcoin_balances_query(
btc_map=btc_map,
original_requests_get=requests.get,
)
# Taken from BINANCE_BALANCES_RESPONSE from tests.utils.exchanges
binance_balances = {
A_ETH: FVal('4763368.68006011'),
A_BTC: FVal('4723846.89208129')
}
# Taken from POLONIEX_BALANCES_RESPONSE from tests.utils.exchanges
poloniex_balances = {A_ETH: FVal('11.0'), A_BTC: FVal('5.5')}
if manually_tracked_balances is None:
manually_tracked_balances = []
rotki.data.db.add_manually_tracked_balances(manually_tracked_balances)
return BalancesTestSetup(
eth_balances=eth_balances,
btc_balances=btc_balances,
token_balances=token_balances,
binance_balances=binance_balances,
poloniex_balances=poloniex_balances,
manually_tracked_balances=manually_tracked_balances,
poloniex_patch=poloniex_patch,
binance_patch=binance_patch,
etherscan_patch=etherscan_patch,
ethtokens_max_chunks_patch=ethtokens_max_chunks_patch,
bitcoin_patch=bitcoin_patch,
beaconchain_patch=beaconchain_patch,
defi_balances_patch=defi_balances_patch,
)
def setup_balances(
rotki,
ethereum_accounts: Optional[List[ChecksumEthAddress]],
btc_accounts: Optional[List[BTCAddress]],
eth_balances: Optional[List[str]] = None,
token_balances: Optional[Dict[EthereumToken, List[str]]] = None,
btc_balances: Optional[List[str]] = None,
manually_tracked_balances: Optional[List[ManuallyTrackedBalance]] = None,
original_queries: Optional[List[str]] = None,
) -> BalancesTestSetup:
"""Setup the blockchain, exchange and fiat balances for some tests
When eth_balances, token_balances and btc_balances are not provided some
default values are provided.
"""
if ethereum_accounts is None:
ethereum_accounts = []
if btc_accounts is None:
btc_accounts = []
# Sanity checks for setup input
if eth_balances is not None:
msg = ('The eth balances should be a list with each '
'element representing balance of an account')
assert len(eth_balances) == len(ethereum_accounts)
else:
# Default test values
if len(ethereum_accounts) != 0:
eth_balances = ['1000000', '2000000']
else:
eth_balances = []
if token_balances is not None:
msg = 'token balances length does not match number of owned eth tokens'
for _, balances in token_balances.items():
msg = ('The token balances should be a list with each '
'element representing balance of an account')
assert len(balances) == len(ethereum_accounts), msg
else:
# Default test values
if len(ethereum_accounts) != 0:
token_balances = {A_RDN: ['0', '4000000']}
else:
token_balances = {}
if btc_balances is not None:
msg = ('The btc balances should be a list with each '
'element representing balance of an account')
assert len(btc_balances) == len(btc_accounts)
else:
# Default test values
if len(btc_accounts) != 0:
btc_balances = ['3000000', '5000000']
else:
btc_balances = []
eth_map: Dict[ChecksumEthAddress, Dict[Union[str, EthereumToken],
Any]] = {}
for idx, acc in enumerate(ethereum_accounts):
eth_map[acc] = {}
eth_map[acc]['ETH'] = eth_balances[idx]
for token in token_balances:
eth_map[acc][token] = token_balances[token][idx]
btc_map: Dict[BTCAddress, str] = {}
for idx, btc_acc in enumerate(btc_accounts):
btc_map[btc_acc] = btc_balances[idx]
binance = rotki.exchange_manager.connected_exchanges.get('binance', None)
binance_patch = patch_binance_balances_query(binance) if binance else None
poloniex = rotki.exchange_manager.connected_exchanges.get('poloniex', None)
poloniex_patch = patch_poloniex_balances_query(
poloniex) if poloniex else None
etherscan_patch = mock_etherscan_query(
eth_map=eth_map,
etherscan=rotki.etherscan,
original_queries=original_queries,
original_requests_get=requests.get,
)
beaconchain_patch = mock_beaconchain(
beaconchain=rotki.chain_manager.beaconchain,
original_queries=original_queries,
original_requests_get=requests.get,
)
# For ethtoken detection we can have bigger chunk length during tests since it's mocked anyway
ethtokens_max_chunks_patch = patch(
'rotkehlchen.chain.ethereum.tokens.ETHERSCAN_MAX_TOKEN_CHUNK_LENGTH',
new=800,
)
bitcoin_patch = mock_bitcoin_balances_query(
btc_map=btc_map,
original_requests_get=requests.get,
)
# Taken from BINANCE_BALANCES_RESPONSE from tests.utils.exchanges
binance_balances = {
'ETH': FVal('4763368.68006011'),
'BTC': FVal('4723846.89208129')
}
# Taken from POLONIEX_BALANCES_RESPONSE from tests.utils.exchanges
poloniex_balances = {'ETH': FVal('11.0'), 'BTC': FVal('5.5')}
if manually_tracked_balances is None:
manually_tracked_balances = []
rotki.data.db.add_manually_tracked_balances(manually_tracked_balances)
return BalancesTestSetup(
eth_balances=eth_balances,
btc_balances=btc_balances,
token_balances=token_balances,
binance_balances=binance_balances,
poloniex_balances=poloniex_balances,
manually_tracked_balances=manually_tracked_balances,
poloniex_patch=poloniex_patch,
binance_patch=binance_patch,
etherscan_patch=etherscan_patch,
ethtokens_max_chunks_patch=ethtokens_max_chunks_patch,
bitcoin_patch=bitcoin_patch,
beaconchain_patch=beaconchain_patch,
)
