def add_blockchain_accounts_to_db(db: DBHandler, blockchain_accounts: BlockchainAccounts) -> None:
db.add_blockchain_accounts(
SupportedBlockchain.ETHEREUM,
[BlockchainAccountData(address=x) for x in blockchain_accounts.eth],
)
db.add_blockchain_accounts(
SupportedBlockchain.BITCOIN,
[BlockchainAccountData(address=x) for x in blockchain_accounts.btc],
)
def test_remove_queried_address_on_account_remove(data_dir, username):
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
data.db.add_blockchain_accounts(
SupportedBlockchain.ETHEREUM,
[
BlockchainAccountData(
address='0xd36029d76af6fE4A356528e4Dc66B2C18123597D'),
],
)
queried_addresses = QueriedAddresses(data.db)
queried_addresses.add_queried_address_for_module(
'makerdao_vaults',
'0xd36029d76af6fE4A356528e4Dc66B2C18123597D',
)
addresses = queried_addresses.get_queried_addresses_for_module(
'makerdao_vaults')
assert '0xd36029d76af6fE4A356528e4Dc66B2C18123597D' in addresses
data.db.remove_blockchain_accounts(
SupportedBlockchain.ETHEREUM,
['0xd36029d76af6fE4A356528e4Dc66B2C18123597D'],
)
addresses = queried_addresses.get_queried_addresses_for_module(
'makerdao_vaults')
assert not addresses
def _init_database(
data_dir: FilePath,
password: str,
msg_aggregator: MessagesAggregator,
db_settings: Optional[Dict[str, Any]],
ignored_assets: Optional[List[Asset]],
blockchain_accounts: BlockchainAccounts,
include_etherscan_key: bool,
) -> DBHandler:
db = DBHandler(data_dir, password, msg_aggregator)
settings = {
# DO not submit usage analytics during tests
'submit_usage_analytics': False,
'main_currency': DEFAULT_TESTS_MAIN_CURRENCY,
}
# Set the given db_settings. The pre-set values have priority unless overriden here
if db_settings is not None:
for key, value in db_settings.items():
settings[key] = value
db.set_settings(ModifiableDBSettings(**settings))
if ignored_assets:
for asset in ignored_assets:
db.add_to_ignored_assets(asset)
# Make sure that the fixture provided accounts are in the blockchain
db.add_blockchain_accounts(
SupportedBlockchain.ETHEREUM,
[BlockchainAccountData(address=x) for x in blockchain_accounts.eth],
)
db.add_blockchain_accounts(
SupportedBlockchain.BITCOIN,
[BlockchainAccountData(address=x) for x in blockchain_accounts.btc],
)
if include_etherscan_key:
# Add the tests only etherscan API key
db.add_external_service_credentials([
ExternalServiceApiCredentials(
service=ExternalService.ETHERSCAN,
api_key=ApiKey('8JT7WQBB2VQP5C3416Y8X3S8GBA3CVZKP4'),
)
])
return db
def patch(
self,
blockchain: SupportedBlockchain,
accounts: List[Dict[str, Any]],
) -> Response:
account_data = [
BlockchainAccountData(
address=entry['address'],
label=entry['label'],
tags=entry['tags'],
) for entry in accounts
]
return self.rest_api.edit_blockchain_accounts(
blockchain=blockchain,
account_data=account_data,
)
def put(
self,
blockchain: SupportedBlockchain,
accounts: List[Dict[str, Any]],
async_query: bool,
) -> Response:
account_data = [
BlockchainAccountData(
address=entry['address'],
label=entry['label'],
tags=entry['tags'],
) for entry in accounts
]
return self.rest_api.add_blockchain_accounts(
blockchain=blockchain,
account_data=account_data,
async_query=async_query,
)
def _derive_xpub_addresses(self, xpub_data: XpubData,
new_xpub: bool) -> None:
"""Derives new xpub addresses, and adds all those until the addresses that
have not had any transactions to the tracked bitcoin addresses
Should be called with the lock acquired
May raise:
- RemoteError: if blockstream/blockchain.info and others can't be reached
"""
last_receiving_idx, last_change_idx = self.db.get_last_xpub_derived_indices(
xpub_data)
derived_addresses_data = derive_addresses_from_xpub_data(
xpub_data=xpub_data,
start_receiving_index=last_receiving_idx,
start_change_index=last_change_idx,
)
known_btc_addresses = self.db.get_blockchain_accounts().btc
new_addresses = []
new_balances = []
existing_address_data = []
for entry in derived_addresses_data:
if entry.address not in known_btc_addresses:
new_addresses.append(entry.address)
new_balances.append(entry.balance)
elif new_xpub:
existing_address_data.append(
BlockchainAccountData(
address=entry.address,
label=None,
tags=xpub_data.tags,
))
if new_xpub and xpub_data.tags:
insert_tag_mappings( # if we got tags add them to the xpub
cursor=self.db.conn.cursor(),
data=[xpub_data],
object_reference_keys=['xpub.xpub', 'derivation_path'],
)
if new_xpub and len(existing_address_data) != 0:
insert_tag_mappings( # if we got tags add them to the existing addresses too
cursor=self.db.conn.cursor(),
data=existing_address_data,
object_reference_keys=['address'],
)
if len(new_addresses) != 0:
self.chain_manager.add_blockchain_accounts(
blockchain=SupportedBlockchain.BITCOIN,
accounts=new_addresses,
already_queried_balances=new_balances,
)
self.db.add_blockchain_accounts(
blockchain=SupportedBlockchain.BITCOIN,
account_data=[
BlockchainAccountData(
address=x,
label=None,
tags=xpub_data.tags,
) for x in new_addresses
],
)
self.db.ensure_xpub_mappings_exist(
xpub=xpub_data.xpub.xpub, # type: ignore
derivation_path=xpub_data.derivation_path,
derived_addresses_data=derived_addresses_data,
)
def test_writing_fetching_data(data_dir, username):
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
data.db.add_blockchain_accounts(
SupportedBlockchain.BITCOIN,
[BlockchainAccountData(address='1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS')],
)
data.db.add_blockchain_accounts(
SupportedBlockchain.ETHEREUM,
[
BlockchainAccountData(
address='0xd36029d76af6fE4A356528e4Dc66B2C18123597D'),
BlockchainAccountData(
address='0x80B369799104a47e98A553f3329812a44A7FaCDc'),
],
)
accounts = data.db.get_blockchain_accounts()
assert isinstance(accounts, BlockchainAccounts)
assert accounts.btc == ['1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS']
# See that after addition the address has been checksummed
assert set(accounts.eth) == {
'0xd36029d76af6fE4A356528e4Dc66B2C18123597D',
'0x80B369799104a47e98A553f3329812a44A7FaCDc',
}
# Add existing account should fail
with pytest.raises(InputError): # pylint: disable=no-member
data.db.add_blockchain_accounts(
SupportedBlockchain.ETHEREUM,
[
BlockchainAccountData(
address='0xd36029d76af6fE4A356528e4Dc66B2C18123597D')
],
)
# Remove non-existing account
with pytest.raises(InputError):
data.db.remove_blockchain_accounts(
SupportedBlockchain.ETHEREUM,
['0x136029d76af6fE4A356528e4Dc66B2C18123597D'],
)
# Remove existing account
data.db.remove_blockchain_accounts(
SupportedBlockchain.ETHEREUM,
['0xd36029d76af6fE4A356528e4Dc66B2C18123597D'],
)
accounts = data.db.get_blockchain_accounts()
assert accounts.eth == ['0x80B369799104a47e98A553f3329812a44A7FaCDc']
result, _ = data.add_ignored_assets([A_DAO])
assert result
result, _ = data.add_ignored_assets([A_DOGE])
assert result
result, _ = data.add_ignored_assets([A_DOGE])
assert not result
ignored_assets = data.db.get_ignored_assets()
assert all(isinstance(asset, Asset) for asset in ignored_assets)
assert set(ignored_assets) == {A_DAO, A_DOGE}
# Test removing asset that is not in the list
result, msg = data.remove_ignored_assets([A_RDN])
assert 'not in ignored assets' in msg
assert not result
result, _ = data.remove_ignored_assets([A_DOGE])
assert result
assert data.db.get_ignored_assets() == [A_DAO]
# With nothing inserted in settings make sure default values are returned
result = data.db.get_settings()
last_write_diff = ts_now() - result.last_write_ts
# make sure last_write was within 3 secs
assert 0 <= last_write_diff < 3
expected_dict = {
'have_premium': False,
'eth_rpc_endpoint': 'http://localhost:8545',
'ksm_rpc_endpoint': 'http://localhost:9933',
'dot_rpc_endpoint': '',
'ui_floating_precision': DEFAULT_UI_FLOATING_PRECISION,
'version': ROTKEHLCHEN_DB_VERSION,
'include_crypto2crypto': DEFAULT_INCLUDE_CRYPTO2CRYPTO,
'include_gas_costs': DEFAULT_INCLUDE_GAS_COSTS,
'taxfree_after_period': YEAR_IN_SECONDS,
'balance_save_frequency': DEFAULT_BALANCE_SAVE_FREQUENCY,
'last_balance_save': 0,
'main_currency': DEFAULT_MAIN_CURRENCY.identifier,
'date_display_format': DEFAULT_DATE_DISPLAY_FORMAT,
'last_data_upload_ts': 0,
'premium_should_sync': False,
'submit_usage_analytics': True,
'last_write_ts': 0,
'active_modules': DEFAULT_ACTIVE_MODULES,
'frontend_settings': '',
'account_for_assets_movements': DEFAULT_ACCOUNT_FOR_ASSETS_MOVEMENTS,
'btc_derivation_gap_limit': DEFAULT_BTC_DERIVATION_GAP_LIMIT,
'calculate_past_cost_basis': DEFAULT_CALCULATE_PAST_COST_BASIS,
'display_date_in_localtime': DEFAULT_DISPLAY_DATE_IN_LOCALTIME,
'current_price_oracles': DEFAULT_CURRENT_PRICE_ORACLES,
'historical_price_oracles': DEFAULT_HISTORICAL_PRICE_ORACLES,
'taxable_ledger_actions': DEFAULT_TAXABLE_LEDGER_ACTIONS,
'pnl_csv_with_formulas': DEFAULT_PNL_CSV_WITH_FORMULAS,
'pnl_csv_have_summary': DEFAULT_PNL_CSV_HAVE_SUMMARY,
'ssf_0graph_multiplier': DEFAULT_SSF_0GRAPH_MULTIPLIER,
'last_data_migration': DEFAULT_LAST_DATA_MIGRATION,
}
assert len(expected_dict) == len(
DBSettings()), 'One or more settings are missing'
# Make sure that results are the same. Comparing like this since we ignore last
# write ts check
result_dict = result._asdict()
for key, value in expected_dict.items():
assert key in result_dict
if key != 'last_write_ts':
assert value == result_dict[key]
def test_writting_fetching_data(data_dir, username):
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
tokens = [A_GNO, A_RDN]
data.write_owned_eth_tokens(tokens)
result = data.db.get_owned_tokens()
assert set(tokens) == set(result)
data.db.add_blockchain_accounts(
SupportedBlockchain.BITCOIN,
[BlockchainAccountData(address='1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS')],
)
data.db.add_blockchain_accounts(
SupportedBlockchain.ETHEREUM,
[
BlockchainAccountData(
address='0xd36029d76af6fE4A356528e4Dc66B2C18123597D'),
BlockchainAccountData(
address='0x80B369799104a47e98A553f3329812a44A7FaCDc'),
],
)
accounts = data.db.get_blockchain_accounts()
assert isinstance(accounts, BlockchainAccounts)
assert accounts.btc == ['1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS']
# See that after addition the address has been checksummed
assert set(accounts.eth) == {
'0xd36029d76af6fE4A356528e4Dc66B2C18123597D',
'0x80B369799104a47e98A553f3329812a44A7FaCDc',
}
# Add existing account should fail
with pytest.raises(sqlcipher.IntegrityError): # pylint: disable=no-member
data.db.add_blockchain_accounts(
SupportedBlockchain.ETHEREUM,
[
BlockchainAccountData(
address='0xd36029d76af6fE4A356528e4Dc66B2C18123597D')
],
)
# Remove non-existing account
with pytest.raises(InputError):
data.db.remove_blockchain_accounts(
SupportedBlockchain.ETHEREUM,
['0x136029d76af6fE4A356528e4Dc66B2C18123597D'],
)
# Remove existing account
data.db.remove_blockchain_accounts(
SupportedBlockchain.ETHEREUM,
['0xd36029d76af6fE4A356528e4Dc66B2C18123597D'],
)
accounts = data.db.get_blockchain_accounts()
assert accounts.eth == ['0x80B369799104a47e98A553f3329812a44A7FaCDc']
result, _ = data.add_ignored_assets([A_DAO])
assert result
result, _ = data.add_ignored_assets([A_DOGE])
assert result
result, _ = data.add_ignored_assets([A_DOGE])
assert not result
ignored_assets = data.db.get_ignored_assets()
assert all(isinstance(asset, Asset) for asset in ignored_assets)
assert set(ignored_assets) == {A_DAO, A_DOGE}
# Test removing asset that is not in the list
result, msg = data.remove_ignored_assets([A_RDN])
assert 'not in ignored assets' in msg
assert not result
result, _ = data.remove_ignored_assets([A_DOGE])
assert result
assert data.db.get_ignored_assets() == [A_DAO]
# With nothing inserted in settings make sure default values are returned
result = data.db.get_settings()
last_write_diff = ts_now() - result.last_write_ts
# make sure last_write was within 3 secs
assert last_write_diff >= 0 and last_write_diff < 3
expected_dict = {
'have_premium': False,
'historical_data_start': DEFAULT_START_DATE,
'eth_rpc_endpoint': 'http://localhost:8545',
'ui_floating_precision': DEFAULT_UI_FLOATING_PRECISION,
'version': ROTKEHLCHEN_DB_VERSION,
'include_crypto2crypto': DEFAULT_INCLUDE_CRYPTO2CRYPTO,
'include_gas_costs': DEFAULT_INCLUDE_GAS_COSTS,
'taxfree_after_period': YEAR_IN_SECONDS,
'balance_save_frequency': DEFAULT_BALANCE_SAVE_FREQUENCY,
'last_balance_save': 0,
'main_currency': DEFAULT_MAIN_CURRENCY.identifier,
'anonymized_logs': DEFAULT_ANONYMIZED_LOGS,
'date_display_format': DEFAULT_DATE_DISPLAY_FORMAT,
'last_data_upload_ts': 0,
'premium_should_sync': False,
'submit_usage_analytics': True,
'last_write_ts': 0,
}
assert len(expected_dict) == len(
DBSettings()), 'One or more settings are missing'
# Make sure that results are the same. Comparing like this since we ignore last
# write ts check
result_dict = result._asdict()
for key, value in expected_dict.items():
assert key in result_dict
if key != 'last_write_ts':
assert value == result_dict[key]
def setup_db_for_xpub_tests(data_dir, username):
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
data.db.add_tag('public', 'foooo', 'ffffff', '000000')
data.db.add_tag('desktop', 'boooo', 'ffffff', '000000')
xpub = 'xpub68V4ZQQ62mea7ZUKn2urQu47Bdn2Wr7SxrBxBDDwE3kjytj361YBGSKDT4WoBrE5htrSB8eAMe59NPnKrcAbiv2veN5GQUmfdjRddD1Hxrk' # noqa: E501
derivation_path = 'm/0/0/0'
xpub_data1 = XpubData(
xpub=HDKey.from_xpub(xpub=xpub, path='m'),
derivation_path=derivation_path,
label='xpub1',
tags=['public', 'desktop'],
)
data.db.ensure_tags_exist([xpub_data1], action='adding', data_type='bitcoin_xpub')
insert_tag_mappings( # if we got tags add them to the xpub
cursor=data.db.conn.cursor(),
data=[xpub_data1],
object_reference_keys=['xpub.xpub', 'derivation_path'],
)
data.db.add_bitcoin_xpub(xpub_data1)
addr1 = '1LZypJUwJJRdfdndwvDmtAjrVYaHko136r'
addr2 = '1MKSdDCtBSXiE49vik8xUG2pTgTGGh5pqe'
addr3 = '12wxFzpjdymPk3xnHmdDLCTXUT9keY3XRd'
addr4 = '16zNpyv8KxChtjXnE5nYcPqcXcrSQXX2JW'
all_addresses = [addr1, addr2, addr3, addr4]
account_data = [BlockchainAccountData(x) for x in [addr1, addr2, addr3, addr4]]
data.db.add_blockchain_accounts(
blockchain=SupportedBlockchain.BITCOIN,
account_data=account_data,
)
insert_tag_mappings( # if we got tags add them to the existing addresses too
cursor=data.db.conn.cursor(),
data=account_data,
object_reference_keys=['address'],
)
data.db.ensure_xpub_mappings_exist(
xpub=xpub,
derivation_path=derivation_path,
derived_addresses_data=[
XpubDerivedAddressData(0, 0, addr1, ZERO),
XpubDerivedAddressData(0, 1, addr2, ZERO),
],
)
xpub = 'zpub6quTRdxqWmerHdiWVKZdLMp9FY641F1F171gfT2RS4D1FyHnutwFSMiab58Nbsdu4fXBaFwpy5xyGnKZ8d6xn2j4r4yNmQ3Yp3yDDxQUo3q' # noqa: E501
derivation_path = 'm/0'
xpub_data2 = XpubData(
xpub=HDKey.from_xpub(xpub=xpub, path='m'),
derivation_path=derivation_path,
)
data.db.add_bitcoin_xpub(xpub_data2)
addr1 = 'bc1qc3qcxs025ka9l6qn0q5cyvmnpwrqw2z49qwrx5'
addr2 = 'bc1qnus7355ecckmeyrmvv56mlm42lxvwa4wuq5aev'
addr3 = 'bc1qup7f8g5k3h5uqzfjed03ztgn8hhe542w69wc0g'
addr4 = 'bc1qr4r8vryfzexvhjrx5fh5uj0s2ead8awpqspqra'
all_addresses.extend([addr1, addr2, addr3, addr4])
data.db.add_blockchain_accounts(
blockchain=SupportedBlockchain.BITCOIN,
account_data=[BlockchainAccountData(x) for x in [addr1, addr2, addr3, addr4]],
)
data.db.ensure_xpub_mappings_exist(
xpub=xpub,
derivation_path=derivation_path,
derived_addresses_data=[
XpubDerivedAddressData(1, 0, addr1, ZERO),
XpubDerivedAddressData(1, 1, addr2, ZERO),
XpubDerivedAddressData(1, 2, addr3, ZERO),
XpubDerivedAddressData(1, 3, addr4, ZERO),
],
)
# Finally also add the same xpub as xpub1 with no derivation path
xpub = 'xpub68V4ZQQ62mea7ZUKn2urQu47Bdn2Wr7SxrBxBDDwE3kjytj361YBGSKDT4WoBrE5htrSB8eAMe59NPnKrcAbiv2veN5GQUmfdjRddD1Hxrk' # noqa: E501
derivation_path = None
xpub_data3 = XpubData(
xpub=HDKey.from_xpub(xpub=xpub, path='m'),
derivation_path=derivation_path,
)
data.db.add_bitcoin_xpub(xpub_data3)
return data.db, xpub_data1, xpub_data2, xpub_data3, all_addresses
