def add_blockchain_accounts_to_db(db: DBHandler, blockchain_accounts: BlockchainAccounts) -> None: db.add_blockchain_accounts( SupportedBlockchain.ETHEREUM, [BlockchainAccountData(address=x) for x in blockchain_accounts.eth], ) db.add_blockchain_accounts( SupportedBlockchain.BITCOIN, [BlockchainAccountData(address=x) for x in blockchain_accounts.btc], )
def test_remove_queried_address_on_account_remove(data_dir, username): msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) data.db.add_blockchain_accounts( SupportedBlockchain.ETHEREUM, [ BlockchainAccountData( address='0xd36029d76af6fE4A356528e4Dc66B2C18123597D'), ], ) queried_addresses = QueriedAddresses(data.db) queried_addresses.add_queried_address_for_module( 'makerdao_vaults', '0xd36029d76af6fE4A356528e4Dc66B2C18123597D', ) addresses = queried_addresses.get_queried_addresses_for_module( 'makerdao_vaults') assert '0xd36029d76af6fE4A356528e4Dc66B2C18123597D' in addresses data.db.remove_blockchain_accounts( SupportedBlockchain.ETHEREUM, ['0xd36029d76af6fE4A356528e4Dc66B2C18123597D'], ) addresses = queried_addresses.get_queried_addresses_for_module( 'makerdao_vaults') assert not addresses
def _init_database( data_dir: FilePath, password: str, msg_aggregator: MessagesAggregator, db_settings: Optional[Dict[str, Any]], ignored_assets: Optional[List[Asset]], blockchain_accounts: BlockchainAccounts, include_etherscan_key: bool, ) -> DBHandler: db = DBHandler(data_dir, password, msg_aggregator) settings = { # DO not submit usage analytics during tests 'submit_usage_analytics': False, 'main_currency': DEFAULT_TESTS_MAIN_CURRENCY, } # Set the given db_settings. The pre-set values have priority unless overriden here if db_settings is not None: for key, value in db_settings.items(): settings[key] = value db.set_settings(ModifiableDBSettings(**settings)) if ignored_assets: for asset in ignored_assets: db.add_to_ignored_assets(asset) # Make sure that the fixture provided accounts are in the blockchain db.add_blockchain_accounts( SupportedBlockchain.ETHEREUM, [BlockchainAccountData(address=x) for x in blockchain_accounts.eth], ) db.add_blockchain_accounts( SupportedBlockchain.BITCOIN, [BlockchainAccountData(address=x) for x in blockchain_accounts.btc], ) if include_etherscan_key: # Add the tests only etherscan API key db.add_external_service_credentials([ ExternalServiceApiCredentials( service=ExternalService.ETHERSCAN, api_key=ApiKey('8JT7WQBB2VQP5C3416Y8X3S8GBA3CVZKP4'), ) ]) return db
def patch( self, blockchain: SupportedBlockchain, accounts: List[Dict[str, Any]], ) -> Response: account_data = [ BlockchainAccountData( address=entry['address'], label=entry['label'], tags=entry['tags'], ) for entry in accounts ] return self.rest_api.edit_blockchain_accounts( blockchain=blockchain, account_data=account_data, )
def put( self, blockchain: SupportedBlockchain, accounts: List[Dict[str, Any]], async_query: bool, ) -> Response: account_data = [ BlockchainAccountData( address=entry['address'], label=entry['label'], tags=entry['tags'], ) for entry in accounts ] return self.rest_api.add_blockchain_accounts( blockchain=blockchain, account_data=account_data, async_query=async_query, )
def _derive_xpub_addresses(self, xpub_data: XpubData, new_xpub: bool) -> None: """Derives new xpub addresses, and adds all those until the addresses that have not had any transactions to the tracked bitcoin addresses Should be called with the lock acquired May raise: - RemoteError: if blockstream/blockchain.info and others can't be reached """ last_receiving_idx, last_change_idx = self.db.get_last_xpub_derived_indices( xpub_data) derived_addresses_data = derive_addresses_from_xpub_data( xpub_data=xpub_data, start_receiving_index=last_receiving_idx, start_change_index=last_change_idx, ) known_btc_addresses = self.db.get_blockchain_accounts().btc new_addresses = [] new_balances = [] existing_address_data = [] for entry in derived_addresses_data: if entry.address not in known_btc_addresses: new_addresses.append(entry.address) new_balances.append(entry.balance) elif new_xpub: existing_address_data.append( BlockchainAccountData( address=entry.address, label=None, tags=xpub_data.tags, )) if new_xpub and xpub_data.tags: insert_tag_mappings( # if we got tags add them to the xpub cursor=self.db.conn.cursor(), data=[xpub_data], object_reference_keys=['xpub.xpub', 'derivation_path'], ) if new_xpub and len(existing_address_data) != 0: insert_tag_mappings( # if we got tags add them to the existing addresses too cursor=self.db.conn.cursor(), data=existing_address_data, object_reference_keys=['address'], ) if len(new_addresses) != 0: self.chain_manager.add_blockchain_accounts( blockchain=SupportedBlockchain.BITCOIN, accounts=new_addresses, already_queried_balances=new_balances, ) self.db.add_blockchain_accounts( blockchain=SupportedBlockchain.BITCOIN, account_data=[ BlockchainAccountData( address=x, label=None, tags=xpub_data.tags, ) for x in new_addresses ], ) self.db.ensure_xpub_mappings_exist( xpub=xpub_data.xpub.xpub, # type: ignore derivation_path=xpub_data.derivation_path, derived_addresses_data=derived_addresses_data, )
def test_writing_fetching_data(data_dir, username): msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) data.db.add_blockchain_accounts( SupportedBlockchain.BITCOIN, [BlockchainAccountData(address='1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS')], ) data.db.add_blockchain_accounts( SupportedBlockchain.ETHEREUM, [ BlockchainAccountData( address='0xd36029d76af6fE4A356528e4Dc66B2C18123597D'), BlockchainAccountData( address='0x80B369799104a47e98A553f3329812a44A7FaCDc'), ], ) accounts = data.db.get_blockchain_accounts() assert isinstance(accounts, BlockchainAccounts) assert accounts.btc == ['1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS'] # See that after addition the address has been checksummed assert set(accounts.eth) == { '0xd36029d76af6fE4A356528e4Dc66B2C18123597D', '0x80B369799104a47e98A553f3329812a44A7FaCDc', } # Add existing account should fail with pytest.raises(InputError): # pylint: disable=no-member data.db.add_blockchain_accounts( SupportedBlockchain.ETHEREUM, [ BlockchainAccountData( address='0xd36029d76af6fE4A356528e4Dc66B2C18123597D') ], ) # Remove non-existing account with pytest.raises(InputError): data.db.remove_blockchain_accounts( SupportedBlockchain.ETHEREUM, ['0x136029d76af6fE4A356528e4Dc66B2C18123597D'], ) # Remove existing account data.db.remove_blockchain_accounts( SupportedBlockchain.ETHEREUM, ['0xd36029d76af6fE4A356528e4Dc66B2C18123597D'], ) accounts = data.db.get_blockchain_accounts() assert accounts.eth == ['0x80B369799104a47e98A553f3329812a44A7FaCDc'] result, _ = data.add_ignored_assets([A_DAO]) assert result result, _ = data.add_ignored_assets([A_DOGE]) assert result result, _ = data.add_ignored_assets([A_DOGE]) assert not result ignored_assets = data.db.get_ignored_assets() assert all(isinstance(asset, Asset) for asset in ignored_assets) assert set(ignored_assets) == {A_DAO, A_DOGE} # Test removing asset that is not in the list result, msg = data.remove_ignored_assets([A_RDN]) assert 'not in ignored assets' in msg assert not result result, _ = data.remove_ignored_assets([A_DOGE]) assert result assert data.db.get_ignored_assets() == [A_DAO] # With nothing inserted in settings make sure default values are returned result = data.db.get_settings() last_write_diff = ts_now() - result.last_write_ts # make sure last_write was within 3 secs assert 0 <= last_write_diff < 3 expected_dict = { 'have_premium': False, 'eth_rpc_endpoint': 'http://localhost:8545', 'ksm_rpc_endpoint': 'http://localhost:9933', 'dot_rpc_endpoint': '', 'ui_floating_precision': DEFAULT_UI_FLOATING_PRECISION, 'version': ROTKEHLCHEN_DB_VERSION, 'include_crypto2crypto': DEFAULT_INCLUDE_CRYPTO2CRYPTO, 'include_gas_costs': DEFAULT_INCLUDE_GAS_COSTS, 'taxfree_after_period': YEAR_IN_SECONDS, 'balance_save_frequency': DEFAULT_BALANCE_SAVE_FREQUENCY, 'last_balance_save': 0, 'main_currency': DEFAULT_MAIN_CURRENCY.identifier, 'date_display_format': DEFAULT_DATE_DISPLAY_FORMAT, 'last_data_upload_ts': 0, 'premium_should_sync': False, 'submit_usage_analytics': True, 'last_write_ts': 0, 'active_modules': DEFAULT_ACTIVE_MODULES, 'frontend_settings': '', 'account_for_assets_movements': DEFAULT_ACCOUNT_FOR_ASSETS_MOVEMENTS, 'btc_derivation_gap_limit': DEFAULT_BTC_DERIVATION_GAP_LIMIT, 'calculate_past_cost_basis': DEFAULT_CALCULATE_PAST_COST_BASIS, 'display_date_in_localtime': DEFAULT_DISPLAY_DATE_IN_LOCALTIME, 'current_price_oracles': DEFAULT_CURRENT_PRICE_ORACLES, 'historical_price_oracles': DEFAULT_HISTORICAL_PRICE_ORACLES, 'taxable_ledger_actions': DEFAULT_TAXABLE_LEDGER_ACTIONS, 'pnl_csv_with_formulas': DEFAULT_PNL_CSV_WITH_FORMULAS, 'pnl_csv_have_summary': DEFAULT_PNL_CSV_HAVE_SUMMARY, 'ssf_0graph_multiplier': DEFAULT_SSF_0GRAPH_MULTIPLIER, 'last_data_migration': DEFAULT_LAST_DATA_MIGRATION, } assert len(expected_dict) == len( DBSettings()), 'One or more settings are missing' # Make sure that results are the same. Comparing like this since we ignore last # write ts check result_dict = result._asdict() for key, value in expected_dict.items(): assert key in result_dict if key != 'last_write_ts': assert value == result_dict[key]
def test_writting_fetching_data(data_dir, username): msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) tokens = [A_GNO, A_RDN] data.write_owned_eth_tokens(tokens) result = data.db.get_owned_tokens() assert set(tokens) == set(result) data.db.add_blockchain_accounts( SupportedBlockchain.BITCOIN, [BlockchainAccountData(address='1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS')], ) data.db.add_blockchain_accounts( SupportedBlockchain.ETHEREUM, [ BlockchainAccountData( address='0xd36029d76af6fE4A356528e4Dc66B2C18123597D'), BlockchainAccountData( address='0x80B369799104a47e98A553f3329812a44A7FaCDc'), ], ) accounts = data.db.get_blockchain_accounts() assert isinstance(accounts, BlockchainAccounts) assert accounts.btc == ['1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS'] # See that after addition the address has been checksummed assert set(accounts.eth) == { '0xd36029d76af6fE4A356528e4Dc66B2C18123597D', '0x80B369799104a47e98A553f3329812a44A7FaCDc', } # Add existing account should fail with pytest.raises(sqlcipher.IntegrityError): # pylint: disable=no-member data.db.add_blockchain_accounts( SupportedBlockchain.ETHEREUM, [ BlockchainAccountData( address='0xd36029d76af6fE4A356528e4Dc66B2C18123597D') ], ) # Remove non-existing account with pytest.raises(InputError): data.db.remove_blockchain_accounts( SupportedBlockchain.ETHEREUM, ['0x136029d76af6fE4A356528e4Dc66B2C18123597D'], ) # Remove existing account data.db.remove_blockchain_accounts( SupportedBlockchain.ETHEREUM, ['0xd36029d76af6fE4A356528e4Dc66B2C18123597D'], ) accounts = data.db.get_blockchain_accounts() assert accounts.eth == ['0x80B369799104a47e98A553f3329812a44A7FaCDc'] result, _ = data.add_ignored_assets([A_DAO]) assert result result, _ = data.add_ignored_assets([A_DOGE]) assert result result, _ = data.add_ignored_assets([A_DOGE]) assert not result ignored_assets = data.db.get_ignored_assets() assert all(isinstance(asset, Asset) for asset in ignored_assets) assert set(ignored_assets) == {A_DAO, A_DOGE} # Test removing asset that is not in the list result, msg = data.remove_ignored_assets([A_RDN]) assert 'not in ignored assets' in msg assert not result result, _ = data.remove_ignored_assets([A_DOGE]) assert result assert data.db.get_ignored_assets() == [A_DAO] # With nothing inserted in settings make sure default values are returned result = data.db.get_settings() last_write_diff = ts_now() - result.last_write_ts # make sure last_write was within 3 secs assert last_write_diff >= 0 and last_write_diff < 3 expected_dict = { 'have_premium': False, 'historical_data_start': DEFAULT_START_DATE, 'eth_rpc_endpoint': 'http://localhost:8545', 'ui_floating_precision': DEFAULT_UI_FLOATING_PRECISION, 'version': ROTKEHLCHEN_DB_VERSION, 'include_crypto2crypto': DEFAULT_INCLUDE_CRYPTO2CRYPTO, 'include_gas_costs': DEFAULT_INCLUDE_GAS_COSTS, 'taxfree_after_period': YEAR_IN_SECONDS, 'balance_save_frequency': DEFAULT_BALANCE_SAVE_FREQUENCY, 'last_balance_save': 0, 'main_currency': DEFAULT_MAIN_CURRENCY.identifier, 'anonymized_logs': DEFAULT_ANONYMIZED_LOGS, 'date_display_format': DEFAULT_DATE_DISPLAY_FORMAT, 'last_data_upload_ts': 0, 'premium_should_sync': False, 'submit_usage_analytics': True, 'last_write_ts': 0, } assert len(expected_dict) == len( DBSettings()), 'One or more settings are missing' # Make sure that results are the same. Comparing like this since we ignore last # write ts check result_dict = result._asdict() for key, value in expected_dict.items(): assert key in result_dict if key != 'last_write_ts': assert value == result_dict[key]
def setup_db_for_xpub_tests(data_dir, username): msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) data.db.add_tag('public', 'foooo', 'ffffff', '000000') data.db.add_tag('desktop', 'boooo', 'ffffff', '000000') xpub = 'xpub68V4ZQQ62mea7ZUKn2urQu47Bdn2Wr7SxrBxBDDwE3kjytj361YBGSKDT4WoBrE5htrSB8eAMe59NPnKrcAbiv2veN5GQUmfdjRddD1Hxrk' # noqa: E501 derivation_path = 'm/0/0/0' xpub_data1 = XpubData( xpub=HDKey.from_xpub(xpub=xpub, path='m'), derivation_path=derivation_path, label='xpub1', tags=['public', 'desktop'], ) data.db.ensure_tags_exist([xpub_data1], action='adding', data_type='bitcoin_xpub') insert_tag_mappings( # if we got tags add them to the xpub cursor=data.db.conn.cursor(), data=[xpub_data1], object_reference_keys=['xpub.xpub', 'derivation_path'], ) data.db.add_bitcoin_xpub(xpub_data1) addr1 = '1LZypJUwJJRdfdndwvDmtAjrVYaHko136r' addr2 = '1MKSdDCtBSXiE49vik8xUG2pTgTGGh5pqe' addr3 = '12wxFzpjdymPk3xnHmdDLCTXUT9keY3XRd' addr4 = '16zNpyv8KxChtjXnE5nYcPqcXcrSQXX2JW' all_addresses = [addr1, addr2, addr3, addr4] account_data = [BlockchainAccountData(x) for x in [addr1, addr2, addr3, addr4]] data.db.add_blockchain_accounts( blockchain=SupportedBlockchain.BITCOIN, account_data=account_data, ) insert_tag_mappings( # if we got tags add them to the existing addresses too cursor=data.db.conn.cursor(), data=account_data, object_reference_keys=['address'], ) data.db.ensure_xpub_mappings_exist( xpub=xpub, derivation_path=derivation_path, derived_addresses_data=[ XpubDerivedAddressData(0, 0, addr1, ZERO), XpubDerivedAddressData(0, 1, addr2, ZERO), ], ) xpub = 'zpub6quTRdxqWmerHdiWVKZdLMp9FY641F1F171gfT2RS4D1FyHnutwFSMiab58Nbsdu4fXBaFwpy5xyGnKZ8d6xn2j4r4yNmQ3Yp3yDDxQUo3q' # noqa: E501 derivation_path = 'm/0' xpub_data2 = XpubData( xpub=HDKey.from_xpub(xpub=xpub, path='m'), derivation_path=derivation_path, ) data.db.add_bitcoin_xpub(xpub_data2) addr1 = 'bc1qc3qcxs025ka9l6qn0q5cyvmnpwrqw2z49qwrx5' addr2 = 'bc1qnus7355ecckmeyrmvv56mlm42lxvwa4wuq5aev' addr3 = 'bc1qup7f8g5k3h5uqzfjed03ztgn8hhe542w69wc0g' addr4 = 'bc1qr4r8vryfzexvhjrx5fh5uj0s2ead8awpqspqra' all_addresses.extend([addr1, addr2, addr3, addr4]) data.db.add_blockchain_accounts( blockchain=SupportedBlockchain.BITCOIN, account_data=[BlockchainAccountData(x) for x in [addr1, addr2, addr3, addr4]], ) data.db.ensure_xpub_mappings_exist( xpub=xpub, derivation_path=derivation_path, derived_addresses_data=[ XpubDerivedAddressData(1, 0, addr1, ZERO), XpubDerivedAddressData(1, 1, addr2, ZERO), XpubDerivedAddressData(1, 2, addr3, ZERO), XpubDerivedAddressData(1, 3, addr4, ZERO), ], ) # Finally also add the same xpub as xpub1 with no derivation path xpub = 'xpub68V4ZQQ62mea7ZUKn2urQu47Bdn2Wr7SxrBxBDDwE3kjytj361YBGSKDT4WoBrE5htrSB8eAMe59NPnKrcAbiv2veN5GQUmfdjRddD1Hxrk' # noqa: E501 derivation_path = None xpub_data3 = XpubData( xpub=HDKey.from_xpub(xpub=xpub, path='m'), derivation_path=derivation_path, ) data.db.add_bitcoin_xpub(xpub_data3) return data.db, xpub_data1, xpub_data2, xpub_data3, all_addresses