def price_historian( accounting_data_dir, inquirer, # pylint: disable=unused-argument should_mock_price_queries, mocked_price_queries, cryptocompare, ): # Since this is a singleton and we want it initialized everytime the fixture # is called make sure its instance is always starting from scratch PriceHistorian._PriceHistorian__instance = None historian = PriceHistorian( data_directory=accounting_data_dir, history_date_start=TEST_HISTORY_DATA_START, cryptocompare=cryptocompare, ) if should_mock_price_queries: def mock_historical_price_query(from_asset, to_asset, timestamp): if from_asset == to_asset: return FVal(1) try: price = mocked_price_queries[from_asset.identifier][ to_asset.identifier][timestamp] except KeyError: raise AssertionError( f'No mocked price found from {from_asset.identifier} to ' f'{to_asset.identifier} at {timestamp}', ) return price historian.query_historical_price = mock_historical_price_query return historian
def price_historian( data_dir, inquirer, # pylint: disable=unused-argument should_mock_price_queries, mocked_price_queries, cryptocompare, session_coingecko, default_mock_price_value, historical_price_oracles_order, ): # Since this is a singleton and we want it initialized everytime the fixture # is called make sure its instance is always starting from scratch PriceHistorian._PriceHistorian__instance = None historian = PriceHistorian( data_directory=data_dir, cryptocompare=cryptocompare, coingecko=session_coingecko, ) historian.set_oracles_order(historical_price_oracles_order) maybe_mock_historical_price_queries( historian=historian, should_mock_price_queries=should_mock_price_queries, mocked_price_queries=mocked_price_queries, default_mock_value=default_mock_price_value, ) return historian
def _adjust_to_cryptocompare_price_incosistencies( price: Price, from_asset: Asset, to_asset: Asset, timestamp: Timestamp, ) -> Price: """Doublecheck against the USD rate, and if incosistencies are found then take the USD adjusted price. This is due to incosistencies in the provided historical data from cryptocompare. https://github.com/rotki/rotki/issues/221 Note: Since 12/01/2019 this seems to no longer be happening, but I will keep the code around just in case a regression is introduced on the side of cryptocompare. May raise: - PriceQueryUnsupportedAsset if the from asset is known to miss from cryptocompare - NoPriceForGivenTimestamp if we can't find a price for the asset in the given timestamp from cryptocompare - RemoteError if there is a problem reaching the cryptocompare server or with reading the response returned by the server """ from_asset_usd = PriceHistorian().query_historical_price( from_asset=from_asset, to_asset=A_USD, timestamp=timestamp, ) to_asset_usd = PriceHistorian().query_historical_price( from_asset=to_asset, to_asset=A_USD, timestamp=timestamp, ) usd_invert_conversion = Price(from_asset_usd / to_asset_usd) abs_diff = abs(usd_invert_conversion - price) relative_difference = abs_diff / max(price, usd_invert_conversion) if relative_difference >= FVal('0.1'): log.warning( 'Cryptocompare historical price data are incosistent.' 'Taking USD adjusted price. Check github issue #221', from_asset=from_asset, to_asset=to_asset, incosistent_price=price, usd_price=from_asset_usd, adjusted_price=usd_invert_conversion, ) return usd_invert_conversion return price
def set_settings(self, settings: ModifiableDBSettings) -> Tuple[bool, str]: """Tries to set new settings. Returns True in success or False with message if error""" with self.lock: if settings.eth_rpc_endpoint is not None: result, msg = self.chain_manager.set_eth_rpc_endpoint( settings.eth_rpc_endpoint) if not result: return False, msg if settings.ksm_rpc_endpoint is not None: result, msg = self.chain_manager.set_ksm_rpc_endpoint( settings.ksm_rpc_endpoint) if not result: return False, msg if settings.kraken_account_type is not None: kraken = self.exchange_manager.get('kraken') if kraken: kraken.set_account_type( settings.kraken_account_type) # type: ignore if settings.btc_derivation_gap_limit is not None: self.chain_manager.btc_derivation_gap_limit = settings.btc_derivation_gap_limit if settings.current_price_oracles is not None: Inquirer().set_oracles_order(settings.current_price_oracles) if settings.historical_price_oracles is not None: PriceHistorian().set_oracles_order( settings.historical_price_oracles) self.data.db.set_settings(settings) return True, ''
def unlock_user(self, user, password, create_new, sync_approval, api_key, api_secret): # unlock or create the DB self.password = password user_dir = self.data.unlock(user, password, create_new) self.try_premium_at_start(api_key, api_secret, create_new, sync_approval, user_dir) secret_data = self.data.db.get_exchange_secrets() self.cache_data_filename = os.path.join(self.data_dir, 'cache_data.json') historical_data_start = self.data.historical_start_date() self.trades_historian = TradesHistorian( self.data_dir, self.data.db, self.data.get_eth_accounts(), historical_data_start, ) self.price_historian = PriceHistorian( self.data_dir, historical_data_start, ) self.accountant = Accountant(price_historian=self.price_historian, profit_currency=self.data.main_currency(), create_csv=True) self.inquirer = Inquirer(kraken=self.kraken) self.initialize_exchanges(secret_data) self.blockchain = Blockchain(self.data.db.get_blockchain_accounts(), self.data.eth_tokens, self.data.db.get_owned_tokens(), self.inquirer, self.args.ethrpc_port)
def query_historical_price(from_asset: Asset, to_asset: Asset, timestamp: Timestamp): return PriceHistorian().query_historical_price( from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, )
def query_historical_price( from_asset: Asset, to_asset: Asset, timestamp: Timestamp, ) -> FVal: price = PriceHistorian().query_historical_price(from_asset, to_asset, timestamp) return price
def unlock_user(self, user, password, create_new, sync_approval, api_key, api_secret): log.info( 'Unlocking user', user=user, create_new=create_new, sync_approval=sync_approval, ) # unlock or create the DB self.password = password self.user_directory = self.data.unlock(user, password, create_new) self.try_premium_at_start( api_key=api_key, api_secret=api_secret, create_new=create_new, sync_approval=sync_approval, ) secret_data = self.data.db.get_exchange_secrets() settings = self.data.db.get_settings() historical_data_start = settings['historical_data_start'] eth_rpc_port = settings['eth_rpc_port'] self.trades_historian = TradesHistorian( user_directory=self.user_directory, db=self.data.db, eth_accounts=self.data.get_eth_accounts(), historical_data_start=historical_data_start, ) self.inquirer = Inquirer(data_dir=self.data_dir, kraken=self.kraken) price_historian = PriceHistorian( data_directory=self.data_dir, history_date_start=historical_data_start, inquirer=self.inquirer, ) db_settings = self.data.db.get_settings() self.accountant = Accountant( price_historian=price_historian, profit_currency=self.data.main_currency(), user_directory=self.user_directory, create_csv=True, ignored_assets=self.data.db.get_ignored_assets(), include_crypto2crypto=db_settings['include_crypto2crypto'], taxfree_after_period=db_settings['taxfree_after_period'], include_gas_costs=db_settings['include_gas_costs'], ) # Initialize the rotkehlchen logger LoggingSettings(anonymized_logs=db_settings['anonymized_logs']) self.initialize_exchanges(secret_data) ethchain = Ethchain(eth_rpc_port) self.blockchain = Blockchain( blockchain_accounts=self.data.db.get_blockchain_accounts(), all_eth_tokens=self.data.eth_tokens, owned_eth_tokens=self.data.db.get_owned_tokens(), inquirer=self.inquirer, ethchain=ethchain, )
def __init__(self, data_directory='.'): self.data_directory = Path(data_directory) self.config = yaml.load(open(self.data_directory / 'buchfink.yaml', 'r'), Loader=yaml.SafeLoader) self.reports_directory = self.data_directory / "reports" self.trades_directory = self.data_directory / "trades" self.cache_directory = self.data_directory / "cache" self.reports_directory.mkdir(exist_ok=True) self.trades_directory.mkdir(exist_ok=True) self.cache_directory.mkdir(exist_ok=True) (self.cache_directory / 'cryptocompare').mkdir(exist_ok=True) (self.cache_directory / 'history').mkdir(exist_ok=True) (self.cache_directory / 'inquirer').mkdir(exist_ok=True) self.cryptocompare = Cryptocompare( self.cache_directory / 'cryptocompare', self) self.historian = PriceHistorian(self.cache_directory / 'history', '01/01/2014', self.cryptocompare) self.inquirer = Inquirer(self.cache_directory / 'inquirer', self.cryptocompare) self.msg_aggregator = MessagesAggregator() self.greenlet_manager = GreenletManager( msg_aggregator=self.msg_aggregator) # Initialize blockchain querying modules self.etherscan = Etherscan(database=self, msg_aggregator=self.msg_aggregator) self.all_eth_tokens = AssetResolver().get_all_eth_tokens() self.alethio = Alethio( database=self, msg_aggregator=self.msg_aggregator, all_eth_tokens=self.all_eth_tokens, ) self.ethereum_manager = EthereumManager( ethrpc_endpoint=self.get_eth_rpc_endpoint(), etherscan=self.etherscan, msg_aggregator=self.msg_aggregator, ) #self.chain_manager = ChainManager( # blockchain_accounts=[], # owned_eth_tokens=[], # ethereum_manager=self.ethereum_manager, # msg_aggregator=self.msg_aggregator, # alethio=alethio, # greenlet_manager=self.greenlet_manager, # premium=False, # eth_modules=ethereum_modules, #) self.ethereum_analyzer = EthereumAnalyzer( ethereum_manager=self.ethereum_manager, database=self, )
def _adjust_to_cryptocompare_price_incosistencies( price: Price, from_asset: Asset, to_asset: Asset, timestamp: Timestamp, ) -> Price: """Doublecheck against the USD rate, and if incosistencies are found then take the USD adjusted price. This is due to incosistencies in the provided historical data from cryptocompare. https://github.com/rotkehlchenio/rotkehlchen/issues/221 Note: Since 12/01/2019 this seems to no longer be happening, but I will keep the code around just in case a regression is introduced on the side of cryptocompare. """ from_asset_usd = PriceHistorian().query_historical_price( from_asset=from_asset, to_asset=A_USD, timestamp=timestamp, ) to_asset_usd = PriceHistorian().query_historical_price( from_asset=to_asset, to_asset=A_USD, timestamp=timestamp, ) usd_invert_conversion = from_asset_usd / to_asset_usd abs_diff = abs(usd_invert_conversion - price) relative_difference = abs_diff / max(price, usd_invert_conversion) if relative_difference >= FVal('0.1'): log.warning( 'Cryptocompare historical price data are incosistent.' 'Taking USD adjusted price. Check github issue #221', from_asset=from_asset, to_asset=to_asset, incosistent_price=price, usd_price=from_asset_usd, adjusted_price=usd_invert_conversion, ) return usd_invert_conversion return price
def price_historian( accounting_data_dir, inquirer, should_mock_price_queries, mocked_price_queries, ): historian = PriceHistorian( data_directory=accounting_data_dir, history_date_start=TEST_HISTORY_DATA_START, inquirer=inquirer, ) if should_mock_price_queries: def mock_historical_price_query(from_asset, to_asset, timestamp): if from_asset == to_asset: return FVal(1) return mocked_price_queries[from_asset][to_asset][timestamp] historian.query_historical_price = mock_historical_price_query return historian
def get_rate_in_profit_currency(self, asset: Asset, timestamp: Timestamp) -> FVal: if asset == self.profit_currency: rate = FVal(1) else: rate = PriceHistorian().query_historical_price( from_asset=asset, to_asset=self.profit_currency, timestamp=timestamp, ) assert isinstance(rate, (FVal, int)) # TODO Remove. Is temporary assert return rate
def initialize_mock_rotkehlchen_instance( rotki, start_with_logged_in_user, start_with_valid_premium, db_password, rotki_premium_credentials, username, blockchain_accounts, owned_eth_tokens, include_etherscan_key, include_cryptocompare_key, should_mock_price_queries, mocked_price_queries, ethereum_modules, db_settings, ignored_assets, tags, manually_tracked_balances, default_mock_price_value, ): if start_with_logged_in_user: rotki.unlock_user( user=username, password=db_password, create_new=True, sync_approval='no', premium_credentials=None, given_ethereum_modules=ethereum_modules, ) if start_with_valid_premium: rotki.premium = Premium(rotki_premium_credentials) rotki.premium_sync_manager.premium = rotki.premium # After unlocking when all objects are created we need to also include # customized fixtures that may have been set by the tests rotki.chain_manager.owned_eth_tokens = owned_eth_tokens rotki.chain_manager.accounts = blockchain_accounts add_settings_to_test_db(rotki.data.db, db_settings, ignored_assets) maybe_include_etherscan_key(rotki.data.db, include_etherscan_key) maybe_include_cryptocompare_key(rotki.data.db, include_cryptocompare_key) add_blockchain_accounts_to_db(rotki.data.db, blockchain_accounts) add_tags_to_test_db(rotki.data.db, tags) add_manually_tracked_balances_to_test_db(rotki.data.db, manually_tracked_balances) maybe_mock_historical_price_queries( historian=PriceHistorian(), should_mock_price_queries=should_mock_price_queries, mocked_price_queries=mocked_price_queries, default_mock_value=default_mock_price_value, )
def price_historian( accounting_data_dir, inquirer, # pylint: disable=unused-argument should_mock_price_queries, mocked_price_queries, ): # Since this is a singleton and we want it initialized everytime the fixture # is called make sure its instance is always starting from scratch PriceHistorian._PriceHistorian__instance = None historian = PriceHistorian( data_directory=accounting_data_dir, history_date_start=TEST_HISTORY_DATA_START, cryptocompare=Cryptocompare(data_directory=accounting_data_dir), ) if should_mock_price_queries: def mock_historical_price_query(from_asset, to_asset, timestamp): if from_asset == to_asset: return FVal(1) return mocked_price_queries[from_asset][to_asset][timestamp] historian.query_historical_price = mock_historical_price_query return historian
def get_fee_in_profit_currency(self, trade: Trade) -> Fee: """Get the profit_currency rate of the fee of the given trade May raise: - PriceQueryUnknownFromAsset if the from asset is known to miss from cryptocompare - NoPriceForGivenTimestamp if we can't find a price for the asset in the given timestamp from the price oracle - RemoteError if there is a problem reaching the price oracle server or with reading the response returned by the server """ fee_rate = PriceHistorian().query_historical_price( from_asset=trade.fee_currency, to_asset=self.profit_currency, timestamp=trade.timestamp, ) return Fee(fee_rate * trade.fee)
def init_accounting_tests(history_list, margin_list, start_ts, end_ts): price_historian = PriceHistorian('/home/lefteris/.rotkehlchen') accountant = Accountant( price_historian=price_historian, profit_currency='EUR', create_csv=False ) accountant.process_history( start_ts=start_ts, end_ts=end_ts, trade_history=trades_from_dictlist(history_list, start_ts, end_ts), margin_history=trades_from_dictlist(margin_list, start_ts, end_ts), loan_history=list(), asset_movements=list(), eth_transactions=list() ) return accountant
def init_accounting_tests(history_list, margin_list, start_ts, end_ts): # TODO: This should become part of test fixtures. Also HAS to be deleted at teardown user_directory = mkdtemp() price_historian = PriceHistorian(user_directory) accountant = Accountant(price_historian=price_historian, profit_currency='EUR', user_directory=user_directory, create_csv=False, ignored_assets=[]) accountant.process_history( start_ts=start_ts, end_ts=end_ts, trade_history=trades_from_dictlist(history_list, start_ts, end_ts), margin_history=trades_from_dictlist(margin_list, start_ts, end_ts), loan_history=list(), asset_movements=list(), eth_transactions=list()) return accountant
def unlock_user(self, user, password, create_new, sync_approval, api_key, api_secret): # unlock or create the DB self.password = password user_dir = self.data.unlock(user, password, create_new) self.try_premium_at_start(api_key, api_secret, create_new, sync_approval, user_dir) secret_data = self.data.db.get_exchange_secrets() settings = self.data.db.get_settings() historical_data_start = settings['historical_data_start'] eth_rpc_port = settings['eth_rpc_port'] self.trades_historian = TradesHistorian( self.data_dir, self.data.db, self.data.get_eth_accounts(), historical_data_start, ) self.price_historian = PriceHistorian( self.data_dir, historical_data_start, ) db_settings = self.data.db.get_settings() self.accountant = Accountant( price_historian=self.price_historian, profit_currency=self.data.main_currency(), user_directory=user_dir, create_csv=True, ignored_assets=self.data.db.get_ignored_assets(), include_crypto2crypto=db_settings['include_crypto2crypto'], taxfree_after_period=db_settings['taxfree_after_period'], ) self.inquirer = Inquirer(kraken=self.kraken) self.initialize_exchanges(secret_data) ethchain = Ethchain(eth_rpc_port) self.blockchain = Blockchain( blockchain_accounts=self.data.db.get_blockchain_accounts(), all_eth_tokens=self.data.eth_tokens, owned_eth_tokens=self.data.db.get_owned_tokens(), inquirer=self.inquirer, ethchain=ethchain, )
def get_rate_in_profit_currency(self, asset: Asset, timestamp: Timestamp) -> FVal: """Get the profit_currency price of asset in the given timestamp May raise: - PriceQueryUnknownFromAsset if the from asset is known to miss from cryptocompare - NoPriceForGivenTimestamp if we can't find a price for the asset in the given timestamp from the price oracle - RemoteError if there is a problem reaching the price oracle server or with reading the response returned by the server """ if asset == self.profit_currency: rate = FVal(1) else: rate = PriceHistorian().query_historical_price( from_asset=asset, to_asset=self.profit_currency, timestamp=timestamp, ) return rate
def price_historian( data_dir, inquirer, # pylint: disable=unused-argument should_mock_price_queries, mocked_price_queries, cryptocompare, ): # Since this is a singleton and we want it initialized everytime the fixture # is called make sure its instance is always starting from scratch PriceHistorian._PriceHistorian__instance = None historian = PriceHistorian( data_directory=data_dir, history_date_start=TEST_HISTORY_DATA_START, cryptocompare=cryptocompare, ) maybe_mock_historical_price_queries( historian=historian, should_mock_price_queries=should_mock_price_queries, mocked_price_queries=mocked_price_queries, ) return historian
def _update_events_value( self, staking_events: ADXStakingEvents, ) -> None: # Update amounts for unbonds and unbond requests bond_id_bond_map: Dict[HexStr, Optional[Bond]] = { bond.bond_id: bond for bond in staking_events.bonds } for event in ( staking_events.unbonds + staking_events. unbond_requests # type: ignore # mypy bug concatenating lists ): has_bond = True bond = bond_id_bond_map.get(event.bond_id, None) if bond: event.value = Balance(amount=bond.value.amount) event.pool_id = bond.pool_id elif event.bond_id not in bond_id_bond_map: bond_id_bond_map[event.bond_id] = None db_bonds = cast( List[Bond], self.database.get_adex_events( bond_id=event.bond_id, event_type=AdexEventType.BOND, )) if db_bonds: db_bond = db_bonds[0] bond_id_bond_map[event.bond_id] = db_bond event.value = Balance(amount=db_bond.value.amount) event.pool_id = db_bond.pool_id else: has_bond = False else: has_bond = False if has_bond is False: msg = (f'Failed to update AdEx event data. ' f'Unable to find the related bond event: {event}') self.msg_aggregator.add_error(msg) # Update usd_value for all events token_usd_price_history: Dict[Tuple[EthereumToken, Timestamp], Price] = {} for event in staking_events.get_all( ): # type: ignore # event can have all types # NB: channel withdraw events can have nullable token if isinstance(event, ChannelWithdraw): if event.token is None: continue key = (event.token, event.timestamp) token = event.token else: key = (A_ADX, event.timestamp) token = A_ADX usd_price = token_usd_price_history.get(key, None) if usd_price is None: usd_price = PriceHistorian().query_historical_price( from_asset=token, to_asset=A_USD, timestamp=event.timestamp, ) event.value.usd_value = event.value.amount * usd_price token_usd_price_history[key] = usd_price
def unlock_user( self, user: str, password: str, create_new: bool, sync_approval: str, premium_credentials: Optional[PremiumCredentials], ) -> None: """Unlocks an existing user or creates a new one if `create_new` is True May raise: - PremiumAuthenticationError if the password can't unlock the database. - AuthenticationError if premium_credentials are given and are invalid or can't authenticate with the server - DBUpgradeError if the rotki DB version is newer than the software or there is a DB upgrade and there is an error. """ log.info( 'Unlocking user', user=user, create_new=create_new, sync_approval=sync_approval, ) # unlock or create the DB self.password = password self.user_directory = self.data.unlock(user, password, create_new) self.data_importer = DataImporter(db=self.data.db) self.last_data_upload_ts = self.data.db.get_last_data_upload_ts() self.premium_sync_manager = PremiumSyncManager(data=self.data, password=password) # set the DB in the external services instances that need it self.cryptocompare.set_database(self.data.db) # Anything that was set above here has to be cleaned in case of failure in the next step # by reset_after_failed_account_creation_or_login() try: self.premium = self.premium_sync_manager.try_premium_at_start( given_premium_credentials=premium_credentials, username=user, create_new=create_new, sync_approval=sync_approval, ) except PremiumAuthenticationError: # Reraise it only if this is during the creation of a new account where # the premium credentials were given by the user if create_new: raise # else let's just continue. User signed in succesfully, but he just # has unauthenticable/invalid premium credentials remaining in his DB settings = self.get_settings() maybe_submit_usage_analytics(settings.submit_usage_analytics) self.etherscan = Etherscan(database=self.data.db, msg_aggregator=self.msg_aggregator) alethio = Alethio( database=self.data.db, msg_aggregator=self.msg_aggregator, all_eth_tokens=self.all_eth_tokens, ) historical_data_start = settings.historical_data_start eth_rpc_endpoint = settings.eth_rpc_endpoint # Initialize the price historian singleton PriceHistorian( data_directory=self.data_dir, history_date_start=historical_data_start, cryptocompare=self.cryptocompare, ) self.accountant = Accountant( db=self.data.db, user_directory=self.user_directory, msg_aggregator=self.msg_aggregator, create_csv=True, ) # Initialize the rotkehlchen logger LoggingSettings(anonymized_logs=settings.anonymized_logs) exchange_credentials = self.data.db.get_exchange_credentials() self.exchange_manager.initialize_exchanges( exchange_credentials=exchange_credentials, database=self.data.db, ) # Initialize blockchain querying modules ethchain = Ethchain( ethrpc_endpoint=eth_rpc_endpoint, etherscan=self.etherscan, msg_aggregator=self.msg_aggregator, ) makerdao = MakerDAO( ethchain=ethchain, database=self.data.db, msg_aggregator=self.msg_aggregator, ) self.chain_manager = ChainManager( blockchain_accounts=self.data.db.get_blockchain_accounts(), owned_eth_tokens=self.data.db.get_owned_tokens(), ethchain=ethchain, msg_aggregator=self.msg_aggregator, alethio=alethio, greenlet_manager=self.greenlet_manager, eth_modules={'makerdao': makerdao}, ) self.ethereum_analyzer = EthereumAnalyzer( ethchain=ethchain, database=self.data.db, ) self.trades_historian = TradesHistorian( user_directory=self.user_directory, db=self.data.db, msg_aggregator=self.msg_aggregator, exchange_manager=self.exchange_manager, chain_manager=self.chain_manager, ) self.user_is_logged_in = True
def unlock_user( self, user: str, password: str, create_new: bool, sync_approval: Literal['yes', 'no', 'unknown'], premium_credentials: Optional[PremiumCredentials], initial_settings: Optional[ModifiableDBSettings] = None, ) -> None: """Unlocks an existing user or creates a new one if `create_new` is True May raise: - PremiumAuthenticationError if the password can't unlock the database. - AuthenticationError if premium_credentials are given and are invalid or can't authenticate with the server - DBUpgradeError if the rotki DB version is newer than the software or there is a DB upgrade and there is an error. - SystemPermissionError if the directory or DB file can not be accessed """ log.info( 'Unlocking user', user=user, create_new=create_new, sync_approval=sync_approval, initial_settings=initial_settings, ) # unlock or create the DB self.password = password self.user_directory = self.data.unlock(user, password, create_new, initial_settings) self.data_importer = DataImporter(db=self.data.db) self.last_data_upload_ts = self.data.db.get_last_data_upload_ts() self.premium_sync_manager = PremiumSyncManager(data=self.data, password=password) # set the DB in the external services instances that need it self.cryptocompare.set_database(self.data.db) # Anything that was set above here has to be cleaned in case of failure in the next step # by reset_after_failed_account_creation_or_login() try: self.premium = self.premium_sync_manager.try_premium_at_start( given_premium_credentials=premium_credentials, username=user, create_new=create_new, sync_approval=sync_approval, ) except PremiumAuthenticationError: # Reraise it only if this is during the creation of a new account where # the premium credentials were given by the user if create_new: raise # else let's just continue. User signed in succesfully, but he just # has unauthenticable/invalid premium credentials remaining in his DB settings = self.get_settings() self.greenlet_manager.spawn_and_track( task_name='submit_usage_analytics', method=maybe_submit_usage_analytics, should_submit=settings.submit_usage_analytics, ) self.etherscan = Etherscan(database=self.data.db, msg_aggregator=self.msg_aggregator) historical_data_start = settings.historical_data_start eth_rpc_endpoint = settings.eth_rpc_endpoint # Initialize the price historian singleton PriceHistorian( data_directory=self.data_dir, history_date_start=historical_data_start, cryptocompare=self.cryptocompare, ) self.accountant = Accountant( db=self.data.db, user_directory=self.user_directory, msg_aggregator=self.msg_aggregator, create_csv=True, ) # Initialize the rotkehlchen logger LoggingSettings(anonymized_logs=settings.anonymized_logs) exchange_credentials = self.data.db.get_exchange_credentials() self.exchange_manager.initialize_exchanges( exchange_credentials=exchange_credentials, database=self.data.db, ) # Initialize blockchain querying modules ethereum_manager = EthereumManager( ethrpc_endpoint=eth_rpc_endpoint, etherscan=self.etherscan, msg_aggregator=self.msg_aggregator, greenlet_manager=self.greenlet_manager, connect_at_start=ETHEREUM_NODES_TO_CONNECT_AT_START, ) self.chain_manager = ChainManager( blockchain_accounts=self.data.db.get_blockchain_accounts(), ethereum_manager=ethereum_manager, msg_aggregator=self.msg_aggregator, database=self.data.db, greenlet_manager=self.greenlet_manager, premium=self.premium, eth_modules=settings.active_modules, ) self.ethereum_analyzer = EthereumAnalyzer( ethereum_manager=ethereum_manager, database=self.data.db, ) self.trades_historian = TradesHistorian( user_directory=self.user_directory, db=self.data.db, msg_aggregator=self.msg_aggregator, exchange_manager=self.exchange_manager, chain_manager=self.chain_manager, ) self.user_is_logged_in = True log.debug('User unlocking complete')
def price_historian(accounting_data_dir): return PriceHistorian(accounting_data_dir, TEST_HISTORY_DATA_START)
def query_historical_price( self, from_asset: Asset, to_asset: Asset, timestamp: Timestamp, historical_data_start: Timestamp, ) -> Price: """ Query the historical price on `timestamp` for `from_asset` in `to_asset`. So how much `to_asset` does 1 unit of `from_asset` cost. May raise: - PriceQueryUnsupportedAsset if from/to asset is known to miss from cryptocompare - NoPriceForGivenTimestamp if we can't find a price for the asset in the given timestamp from cryptocompare - RemoteError if there is a problem reaching the cryptocompare server or with reading the response returned by the server """ try: data = self.get_historical_data( from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, historical_data_start=historical_data_start, ) except UnsupportedAsset as e: raise PriceQueryUnsupportedAsset(e.asset_name) # all data are sorted and timestamps are always increasing by 1 hour # find the closest entry to the provided timestamp if timestamp >= data[0].time: # convert_to_int can't raise here due to its input index = convert_to_int((timestamp - data[0].time) / 3600, accept_only_exact=False) # print("timestamp: {} index: {} data_length: {}".format(timestamp, index, len(data))) diff = abs(data[index].time - timestamp) if index + 1 <= len(data) - 1: diff_p1 = abs(data[index + 1].time - timestamp) if diff_p1 < diff: index = index + 1 if data[index].high is None or data[index].low is None: # If we get some None in the hourly set price to 0 so that we check alternatives price = Price(ZERO) else: price = Price((data[index].high + data[index].low) / 2) else: # no price found in the historical data from/to asset, try alternatives price = Price(ZERO) if price == 0: if from_asset != 'BTC' and to_asset != 'BTC': log.debug( f"Couldn't find historical price from {from_asset} to " f"{to_asset} at timestamp {timestamp}. Comparing with BTC...", ) # Just get the BTC price asset_btc_price = PriceHistorian().query_historical_price( from_asset=from_asset, to_asset=A_BTC, timestamp=timestamp, ) btc_to_asset_price = PriceHistorian().query_historical_price( from_asset=A_BTC, to_asset=to_asset, timestamp=timestamp, ) price = Price(asset_btc_price * btc_to_asset_price) else: log.debug( f"Couldn't find historical price from {from_asset} to " f"{to_asset} at timestamp {timestamp} through cryptocompare." f" Attempting to get daily price...", ) price = self.query_endpoint_pricehistorical( from_asset, to_asset, timestamp) comparison_to_nonusd_fiat = ((to_asset.is_fiat() and to_asset != A_USD) or (from_asset.is_fiat() and from_asset != A_USD)) if comparison_to_nonusd_fiat: price = self._adjust_to_cryptocompare_price_incosistencies( price=price, from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, ) if price == 0: raise NoPriceForGivenTimestamp( from_asset=from_asset, to_asset=to_asset, date=timestamp_to_date(timestamp, formatstr='%d/%m/%Y, %H:%M:%S'), ) log.debug( 'Got historical price', from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, price=price, ) return price
def query_historical_price( self, from_asset: Asset, to_asset: Asset, timestamp: Timestamp, historical_data_start: Timestamp, ) -> Price: """ Query the historical price on `timestamp` for `from_asset` in `to_asset`. So how much `to_asset` does 1 unit of `from_asset` cost. May raise: - PriceQueryUnsupportedAsset if from/to asset is known to miss from cryptocompare - NoPriceForGivenTimestamp if we can't find a price for the asset in the given timestamp from cryptocompare - RemoteError if there is a problem reaching the cryptocompare server or with reading the response returned by the server """ try: data = self.get_historical_data( from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, historical_data_start=historical_data_start, ) except UnsupportedAsset as e: raise PriceQueryUnsupportedAsset(e.asset_name) price = Price(ZERO) # all data are sorted and timestamps are always increasing by 1 hour # find the closest entry to the provided timestamp if timestamp >= data[0].time: index_in_bounds = True # convert_to_int can't raise here due to its input index = convert_to_int((timestamp - data[0].time) / 3600, accept_only_exact=False) if index > len(data) - 1: # index out of bounds # Try to see if index - 1 is there and if yes take it if index > len(data): index = index - 1 else: # give up. This happened: https://github.com/rotki/rotki/issues/1534 log.error( f'Expected data index in cryptocompare historical hour price ' f'not found. Queried price of: {from_asset.identifier} in ' f'{to_asset.identifier} at {timestamp}. Data ' f'index: {index}. Length of returned data: {len(data)}. ' f'https://github.com/rotki/rotki/issues/1534. Attempting other methods...', ) index_in_bounds = False if index_in_bounds: diff = abs(data[index].time - timestamp) if index + 1 <= len(data) - 1: diff_p1 = abs(data[index + 1].time - timestamp) if diff_p1 < diff: index = index + 1 if data[index].high is not None and data[index].low is not None: price = Price((data[index].high + data[index].low) / 2) else: # no price found in the historical data from/to asset, try alternatives price = Price(ZERO) if price == 0: if from_asset != 'BTC' and to_asset != 'BTC': log.debug( f"Couldn't find historical price from {from_asset} to " f"{to_asset} at timestamp {timestamp}. Comparing with BTC...", ) # Just get the BTC price asset_btc_price = PriceHistorian().query_historical_price( from_asset=from_asset, to_asset=A_BTC, timestamp=timestamp, ) btc_to_asset_price = PriceHistorian().query_historical_price( from_asset=A_BTC, to_asset=to_asset, timestamp=timestamp, ) price = Price(asset_btc_price * btc_to_asset_price) else: log.debug( f"Couldn't find historical price from {from_asset} to " f"{to_asset} at timestamp {timestamp} through cryptocompare." f" Attempting to get daily price...", ) price = self.query_endpoint_pricehistorical( from_asset, to_asset, timestamp) comparison_to_nonusd_fiat = ((to_asset.is_fiat() and to_asset != A_USD) or (from_asset.is_fiat() and from_asset != A_USD)) if comparison_to_nonusd_fiat: price = self._adjust_to_cryptocompare_price_incosistencies( price=price, from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, ) if price == 0: raise NoPriceForGivenTimestamp( from_asset=from_asset, to_asset=to_asset, date=timestamp_to_date(timestamp, formatstr='%d/%m/%Y, %H:%M:%S'), ) log.debug( 'Got historical price', from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, price=price, ) return price
def query_historical_price( self, from_asset: Asset, to_asset: Asset, timestamp: Timestamp, historical_data_start: Timestamp, ) -> Price: if from_asset in KNOWN_TO_MISS_FROM_CRYPTOCOMPARE: raise PriceQueryUnknownFromAsset(from_asset) data = self.get_historical_data( from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, historical_data_start=historical_data_start, ) # all data are sorted and timestamps are always increasing by 1 hour # find the closest entry to the provided timestamp if timestamp >= data[0].time: index = convert_to_int((timestamp - data[0].time) / 3600, accept_only_exact=False) # print("timestamp: {} index: {} data_length: {}".format(timestamp, index, len(data))) diff = abs(data[index].time - timestamp) if index + 1 <= len(data) - 1: diff_p1 = abs(data[index + 1].time - timestamp) if diff_p1 < diff: index = index + 1 if data[index].high is None or data[index].low is None: # If we get some None in the hourly set price to 0 so that we check alternatives price = Price(ZERO) else: price = (data[index].high + data[index].low) / 2 else: # no price found in the historical data from/to asset, try alternatives price = Price(ZERO) if price == 0: if from_asset != 'BTC' and to_asset != 'BTC': log.debug( f"Couldn't find historical price from {from_asset} to " f"{to_asset} at timestamp {timestamp}. Comparing with BTC...", ) # Just get the BTC price asset_btc_price = PriceHistorian().query_historical_price( from_asset=from_asset, to_asset=A_BTC, timestamp=timestamp, ) btc_to_asset_price = PriceHistorian().query_historical_price( from_asset=A_BTC, to_asset=to_asset, timestamp=timestamp, ) price = asset_btc_price * btc_to_asset_price else: log.debug( f"Couldn't find historical price from {from_asset} to " f"{to_asset} at timestamp {timestamp} through cryptocompare." f" Attempting to get daily price...", ) price = self.query_endpoint_pricehistorical( from_asset, to_asset, timestamp) comparison_to_nonusd_fiat = ((to_asset.is_fiat() and to_asset != A_USD) or (from_asset.is_fiat() and from_asset != A_USD)) if comparison_to_nonusd_fiat: price = self._adjust_to_cryptocompare_price_incosistencies( price=price, from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, ) if price == 0: raise NoPriceForGivenTimestamp( from_asset, to_asset, timestamp_to_date(timestamp, formatstr='%d/%m/%Y, %H:%M:%S'), ) log.debug( 'Got historical price', from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, price=price, ) return price
def unlock_user( self, user: str, password: str, create_new: bool, sync_approval: bool, api_key: ApiKey, api_secret: ApiSecret, ) -> None: """Unlocks an existing user or creates a new one if `create_new` is True""" log.info( 'Unlocking user', user=user, create_new=create_new, sync_approval=sync_approval, ) # unlock or create the DB self.password = password self.user_directory = self.data.unlock(user, password, create_new) self.last_data_upload_ts = self.data.db.get_last_data_upload_ts() self.premium_sync_manager = PremiumSyncManager(data=self.data, password=password) try: self.premium = self.premium_sync_manager.try_premium_at_start( api_key=api_key, api_secret=api_secret, username=user, create_new=create_new, sync_approval=sync_approval, ) except AuthenticationError: # It means that our credentials were not accepted by the server # or some other error happened pass exchange_credentials = self.data.db.get_exchange_credentials() settings = self.data.db.get_settings() historical_data_start = settings['historical_data_start'] eth_rpc_endpoint = settings['eth_rpc_endpoint'] self.trades_historian = TradesHistorian( user_directory=self.user_directory, db=self.data.db, eth_accounts=self.data.get_eth_accounts(), msg_aggregator=self.msg_aggregator, ) # Initialize the price historian singleton PriceHistorian( data_directory=self.data_dir, history_date_start=historical_data_start, cryptocompare=Cryptocompare(data_directory=self.data_dir), ) db_settings = self.data.db.get_settings() self.accountant = Accountant( profit_currency=self.data.main_currency(), user_directory=self.user_directory, msg_aggregator=self.msg_aggregator, create_csv=True, ignored_assets=self.data.db.get_ignored_assets(), include_crypto2crypto=db_settings['include_crypto2crypto'], taxfree_after_period=db_settings['taxfree_after_period'], include_gas_costs=db_settings['include_gas_costs'], ) # Initialize the rotkehlchen logger LoggingSettings(anonymized_logs=db_settings['anonymized_logs']) self.initialize_exchanges(exchange_credentials) ethchain = Ethchain(eth_rpc_endpoint) self.blockchain = Blockchain( blockchain_accounts=self.data.db.get_blockchain_accounts(), owned_eth_tokens=self.data.db.get_owned_tokens(), ethchain=ethchain, msg_aggregator=self.msg_aggregator, ) self.user_is_logged_in = True
def unlock_user( self, user: str, password: str, create_new: bool, sync_approval: Literal['yes', 'no', 'unknown'], premium_credentials: Optional[PremiumCredentials], initial_settings: Optional[ModifiableDBSettings] = None, ) -> None: """Unlocks an existing user or creates a new one if `create_new` is True May raise: - PremiumAuthenticationError if the password can't unlock the database. - AuthenticationError if premium_credentials are given and are invalid or can't authenticate with the server - DBUpgradeError if the rotki DB version is newer than the software or there is a DB upgrade and there is an error. - SystemPermissionError if the directory or DB file can not be accessed """ log.info( 'Unlocking user', user=user, create_new=create_new, sync_approval=sync_approval, initial_settings=initial_settings, ) # unlock or create the DB self.password = password self.user_directory = self.data.unlock(user, password, create_new, initial_settings) self.data_importer = DataImporter(db=self.data.db) self.last_data_upload_ts = self.data.db.get_last_data_upload_ts() self.premium_sync_manager = PremiumSyncManager(data=self.data, password=password) # set the DB in the external services instances that need it self.cryptocompare.set_database(self.data.db) # Anything that was set above here has to be cleaned in case of failure in the next step # by reset_after_failed_account_creation_or_login() try: self.premium = self.premium_sync_manager.try_premium_at_start( given_premium_credentials=premium_credentials, username=user, create_new=create_new, sync_approval=sync_approval, ) except PremiumAuthenticationError: # Reraise it only if this is during the creation of a new account where # the premium credentials were given by the user if create_new: raise self.msg_aggregator.add_warning( 'Could not authenticate the Rotki premium API keys found in the DB.' ' Has your subscription expired?', ) # else let's just continue. User signed in succesfully, but he just # has unauthenticable/invalid premium credentials remaining in his DB settings = self.get_settings() self.greenlet_manager.spawn_and_track( after_seconds=None, task_name='submit_usage_analytics', exception_is_error=False, method=maybe_submit_usage_analytics, should_submit=settings.submit_usage_analytics, ) self.etherscan = Etherscan(database=self.data.db, msg_aggregator=self.msg_aggregator) self.beaconchain = BeaconChain(database=self.data.db, msg_aggregator=self.msg_aggregator) eth_rpc_endpoint = settings.eth_rpc_endpoint # Initialize the price historian singleton PriceHistorian( data_directory=self.data_dir, cryptocompare=self.cryptocompare, coingecko=self.coingecko, ) PriceHistorian().set_oracles_order(settings.historical_price_oracles) self.accountant = Accountant( db=self.data.db, user_directory=self.user_directory, msg_aggregator=self.msg_aggregator, create_csv=True, ) # Initialize the rotkehlchen logger LoggingSettings(anonymized_logs=settings.anonymized_logs) exchange_credentials = self.data.db.get_exchange_credentials() self.exchange_manager.initialize_exchanges( exchange_credentials=exchange_credentials, database=self.data.db, ) # Initialize blockchain querying modules ethereum_manager = EthereumManager( ethrpc_endpoint=eth_rpc_endpoint, etherscan=self.etherscan, database=self.data.db, msg_aggregator=self.msg_aggregator, greenlet_manager=self.greenlet_manager, connect_at_start=ETHEREUM_NODES_TO_CONNECT_AT_START, ) kusama_manager = SubstrateManager( chain=SubstrateChain.KUSAMA, msg_aggregator=self.msg_aggregator, greenlet_manager=self.greenlet_manager, connect_at_start=KUSAMA_NODES_TO_CONNECT_AT_START, connect_on_startup=self._connect_ksm_manager_on_startup(), own_rpc_endpoint=settings.ksm_rpc_endpoint, ) Inquirer().inject_ethereum(ethereum_manager) Inquirer().set_oracles_order(settings.current_price_oracles) self.chain_manager = ChainManager( blockchain_accounts=self.data.db.get_blockchain_accounts(), ethereum_manager=ethereum_manager, kusama_manager=kusama_manager, msg_aggregator=self.msg_aggregator, database=self.data.db, greenlet_manager=self.greenlet_manager, premium=self.premium, eth_modules=settings.active_modules, data_directory=self.data_dir, beaconchain=self.beaconchain, btc_derivation_gap_limit=settings.btc_derivation_gap_limit, ) self.events_historian = EventsHistorian( user_directory=self.user_directory, db=self.data.db, msg_aggregator=self.msg_aggregator, exchange_manager=self.exchange_manager, chain_manager=self.chain_manager, ) self.task_manager = TaskManager( max_tasks_num=DEFAULT_MAX_TASKS_NUM, greenlet_manager=self.greenlet_manager, api_task_greenlets=self.api_task_greenlets, database=self.data.db, cryptocompare=self.cryptocompare, premium_sync_manager=self.premium_sync_manager, chain_manager=self.chain_manager, exchange_manager=self.exchange_manager, ) self.user_is_logged_in = True log.debug('User unlocking complete')
def initialize_mock_rotkehlchen_instance( rotki, start_with_logged_in_user, start_with_valid_premium, db_password, rotki_premium_credentials, username, blockchain_accounts, include_etherscan_key, include_cryptocompare_key, should_mock_price_queries, mocked_price_queries, ethereum_modules, db_settings, ignored_assets, tags, manually_tracked_balances, default_mock_price_value, ethereum_manager_connect_at_start, kusama_manager_connect_at_start, eth_rpc_endpoint, ksm_rpc_endpoint, aave_use_graph, max_tasks_num, ): if not start_with_logged_in_user: return # Mock the initial get settings to include the specified ethereum modules def mock_get_settings() -> DBSettings: settings = DBSettings( active_modules=ethereum_modules, eth_rpc_endpoint=eth_rpc_endpoint, ksm_rpc_endpoint=ksm_rpc_endpoint, ) return settings settings_patch = patch.object(rotki, 'get_settings', side_effect=mock_get_settings) # Do not connect to the usual nodes at start by default. Do not want to spam # them during our tests. It's configurable per test, with the default being nothing eth_rpcconnect_patch = patch( 'rotkehlchen.rotkehlchen.ETHEREUM_NODES_TO_CONNECT_AT_START', new=ethereum_manager_connect_at_start, ) ksm_rpcconnect_patch = patch( 'rotkehlchen.rotkehlchen.KUSAMA_NODES_TO_CONNECT_AT_START', new=kusama_manager_connect_at_start, ) ksm_connect_on_startup_patch = patch.object( rotki, '_connect_ksm_manager_on_startup', return_value=bool(blockchain_accounts.ksm), ) with settings_patch, eth_rpcconnect_patch, ksm_rpcconnect_patch, ksm_connect_on_startup_patch: rotki.unlock_user( user=username, password=db_password, create_new=True, sync_approval='no', premium_credentials=None, ) # configure when task manager should run for tests rotki.task_manager.max_tasks_num = max_tasks_num if start_with_valid_premium: rotki.premium = Premium(rotki_premium_credentials) rotki.premium_sync_manager.premium = rotki.premium # After unlocking when all objects are created we need to also include # customized fixtures that may have been set by the tests rotki.chain_manager.accounts = blockchain_accounts add_settings_to_test_db(rotki.data.db, db_settings, ignored_assets) maybe_include_etherscan_key(rotki.data.db, include_etherscan_key) maybe_include_cryptocompare_key(rotki.data.db, include_cryptocompare_key) add_blockchain_accounts_to_db(rotki.data.db, blockchain_accounts) add_tags_to_test_db(rotki.data.db, tags) add_manually_tracked_balances_to_test_db(rotki.data.db, manually_tracked_balances) maybe_mock_historical_price_queries( historian=PriceHistorian(), should_mock_price_queries=should_mock_price_queries, mocked_price_queries=mocked_price_queries, default_mock_value=default_mock_price_value, ) wait_until_all_nodes_connected( ethereum_manager_connect_at_start=ethereum_manager_connect_at_start, ethereum=rotki.chain_manager.ethereum, ) wait_until_all_substrate_nodes_connected( substrate_manager_connect_at_start=kusama_manager_connect_at_start, substrate_manager=rotki.chain_manager.kusama, ) aave = rotki.chain_manager.get_module('aave') if aave: aave.use_graph = aave_use_graph