def test_cryptocompare_historical_data_use_cached_price(data_dir, database): """Test that the cryptocompare cache is used and also properly deserialized""" # Create a cache file for SNGLS_BTC contents = """{"start_time": 0, "end_time": 1439390800, "data": [{"time": 1438387200, "close": 10, "high": 10, "low": 10, "open": 10, "volumefrom": 10, "volumeto": 10}, {"time": 1438390800, "close": 20, "high": 20, "low": 20, "open": 20, "volumefrom": 20, "volumeto": 20}]}""" with open(os.path.join(data_dir, 'price_history_SNGLS_BTC.json'), 'w') as f: f.write(contents) cc = Cryptocompare(data_directory=data_dir, database=database) with patch.object(cc, 'query_endpoint_histohour') as histohour_mock: result = cc.get_historical_data( from_asset=A_SNGLS, to_asset=A_BTC, timestamp=1438390801, historical_data_start=0, ) # make sure that histohour was not called, in essence that the cache was used assert histohour_mock.call_count == 0 assert len(result) == 2 assert isinstance(result[0].low, FVal) assert result[0].low == FVal(10) assert isinstance(result[0].high, FVal) assert result[0].high == FVal(10) assert isinstance(result[1].low, FVal) assert result[1].low == FVal(20) assert isinstance(result[1].high, FVal) assert result[1].high == FVal(20)
def test_cryptocompare_historical_data_price( data_dir, database, from_asset, to_asset, timestamp, price, ): """Test that the cryptocompare histohour data retrieval works and price is returned """ cc = Cryptocompare(data_directory=data_dir, database=database) # Get lots of historical prices from at least 1 query after the ts we need result = cc.get_historical_data( from_asset=from_asset, to_asset=to_asset, timestamp=timestamp + 2020 * 3600, only_check_cache=False, ) # Query the ts we need from the cached data result_price = cc._retrieve_price_from_data( data=result, from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, ) assert result_price == price
def test_cryptocompare_histohour_data_going_backward(data_dir, database, freezer): """Test that the cryptocompare histohour data retrieval works properly This test checks that doing an additional query in the past workd properly and that the cached data are properly appended to the cached result. In production this scenario should not happen often. Only way to happen if cryptocompare somehow adds older data than what was previously queried. """ # first timestamp cryptocompare has histohour BTC/USD when queried from this test is btc_start_ts = 1279936800 # first timestamp cryptocompare has histohour BTC/USD is: 1279940400 now_ts = btc_start_ts + 3600 * 2000 + 122 # create a cache file for BTC_USD contents = """{"start_time": 1301536800, "end_time": 1301540400, "data": [{"time": 1301536800, "close": 0.298, "high": 0.298, "low": 0.298, "open": 0.298, "volumefrom": 0.298, "volumeto": 0.298}, {"time": 1301540400, "close": 0.298, "high": 0.298, "low": 0.298, "open": 0.298, "volumefrom": 0.298, "volumeto": 0.298}]}""" price_history_dir = get_or_make_price_history_dir(data_dir) with open(price_history_dir / f'{PRICE_HISTORY_FILE_PREFIX}BTC_USD.json', 'w') as f: f.write(contents) freezer.move_to(datetime.fromtimestamp(now_ts)) cc = Cryptocompare(data_directory=data_dir, database=database) result = cc.get_historical_data( from_asset=A_BTC, to_asset=A_USD, timestamp=now_ts - 3600 * 2 - 55, only_check_cache=False, ) cache_key = PairCacheKey('BTC_USD') assert len(result) == CRYPTOCOMPARE_HOURQUERYLIMIT * 3 + 2 check_cc_result(result, forward=False) assert cache_key in cc.price_history assert cc.price_history[cache_key].start_time == btc_start_ts assert cc.price_history[cache_key].end_time == now_ts check_cc_result(cc.price_history[cache_key].data, forward=False)
def test_cryptocompare_historical_data_price( data_dir, database, from_asset, to_asset, timestamp, price, ): """Test that the cryptocompare histohour data retrieval works and price is returned """ cc = Cryptocompare(data_directory=data_dir, database=database) # Get lots of historical prices from at least 1 query after the ts we need cc.query_and_store_historical_data( from_asset=from_asset, to_asset=to_asset, timestamp=timestamp + 2020 * 3600, ) # Query the ts we need directly from the cached data price_cache_entry = GlobalDBHandler().get_historical_price( from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, max_seconds_distance=3600, source=HistoricalPriceOracle.CRYPTOCOMPARE, ) assert price_cache_entry.price == price
def __init__(self, args: argparse.Namespace) -> None: """Initialize the Rotkehlchen object May Raise: - SystemPermissionError if the given data directory's permissions are not correct. """ self.lock = Semaphore() self.lock.acquire() # Can also be None after unlock if premium credentials did not # authenticate or premium server temporarily offline self.premium: Optional[Premium] = None self.user_is_logged_in: bool = False configure_logging(args) self.sleep_secs = args.sleep_secs if args.data_dir is None: self.data_dir = default_data_directory() else: self.data_dir = Path(args.data_dir) if not os.access(self.data_dir, os.W_OK | os.R_OK): raise SystemPermissionError( f'The given data directory {self.data_dir} is not readable or writable', ) self.args = args self.msg_aggregator = MessagesAggregator() self.greenlet_manager = GreenletManager(msg_aggregator=self.msg_aggregator) self.exchange_manager = ExchangeManager(msg_aggregator=self.msg_aggregator) # Initialize the AssetResolver singleton AssetResolver(data_directory=self.data_dir) self.data = DataHandler(self.data_dir, self.msg_aggregator) self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None) self.coingecko = Coingecko() self.icon_manager = IconManager(data_dir=self.data_dir, coingecko=self.coingecko) self.greenlet_manager.spawn_and_track( after_seconds=None, task_name='periodically_query_icons_until_all_cached', method=self.icon_manager.periodically_query_icons_until_all_cached, batch_size=ICONS_BATCH_SIZE, sleep_time_secs=ICONS_QUERY_SLEEP, ) # Initialize the Inquirer singleton Inquirer( data_dir=self.data_dir, cryptocompare=self.cryptocompare, coingecko=self.coingecko, ) # Keeps how many trades we have found per location. Used for free user limiting self.actions_per_location: Dict[str, Dict[Location, int]] = { 'trade': defaultdict(int), 'asset_movement': defaultdict(int), } self.lock.release() self.shutdown_event = gevent.event.Event()
def __init__(self, args: argparse.Namespace) -> None: """Initialize the Rotkehlchen object This runs during backend initialization so it should be as light as possible. May Raise: - SystemPermissionError if the given data directory's permissions are not correct. """ # Can also be None after unlock if premium credentials did not # authenticate or premium server temporarily offline self.premium: Optional[Premium] = None self.user_is_logged_in: bool = False configure_logging(args) self.sleep_secs = args.sleep_secs if args.data_dir is None: self.data_dir = default_data_directory() else: self.data_dir = Path(args.data_dir) self.data_dir.mkdir(parents=True, exist_ok=True) if not os.access(self.data_dir, os.W_OK | os.R_OK): raise SystemPermissionError( f'The given data directory {self.data_dir} is not readable or writable', ) self.main_loop_spawned = False self.args = args self.api_task_greenlets: List[gevent.Greenlet] = [] self.msg_aggregator = MessagesAggregator() self.greenlet_manager = GreenletManager( msg_aggregator=self.msg_aggregator) self.exchange_manager = ExchangeManager( msg_aggregator=self.msg_aggregator) # Initialize the GlobalDBHandler singleton. Has to be initialized BEFORE asset resolver GlobalDBHandler(data_dir=self.data_dir) self.data = DataHandler(self.data_dir, self.msg_aggregator) self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None) self.coingecko = Coingecko() self.icon_manager = IconManager(data_dir=self.data_dir, coingecko=self.coingecko) self.assets_updater = AssetsUpdater(self.msg_aggregator) # Initialize the Inquirer singleton Inquirer( data_dir=self.data_dir, cryptocompare=self.cryptocompare, coingecko=self.coingecko, ) # Keeps how many trades we have found per location. Used for free user limiting self.actions_per_location: Dict[str, Dict[Location, int]] = { 'trade': defaultdict(int), 'asset_movement': defaultdict(int), } self.task_manager: Optional[TaskManager] = None self.shutdown_event = gevent.event.Event()
def __init__(self, args: argparse.Namespace) -> None: self.lock = Semaphore() self.lock.acquire() # Can also be None after unlock if premium credentials did not # authenticate or premium server temporarily offline self.premium: Optional[Premium] = None self.user_is_logged_in = False logfilename = None if args.logtarget == 'file': logfilename = args.logfile if args.loglevel == 'debug': loglevel = logging.DEBUG elif args.loglevel == 'info': loglevel = logging.INFO elif args.loglevel == 'warn': loglevel = logging.WARN elif args.loglevel == 'error': loglevel = logging.ERROR elif args.loglevel == 'critical': loglevel = logging.CRITICAL else: raise AssertionError('Should never get here. Illegal log value') logging.basicConfig( filename=logfilename, filemode='w', level=loglevel, format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s', datefmt='%d/%m/%Y %H:%M:%S %Z', ) if not args.logfromothermodules: logging.getLogger('urllib3').setLevel(logging.CRITICAL) logging.getLogger('urllib3.connectionpool').setLevel( logging.CRITICAL) self.sleep_secs = args.sleep_secs self.data_dir = args.data_dir self.args = args self.msg_aggregator = MessagesAggregator() self.greenlet_manager = GreenletManager( msg_aggregator=self.msg_aggregator) self.exchange_manager = ExchangeManager( msg_aggregator=self.msg_aggregator) self.all_eth_tokens = AssetResolver().get_all_eth_tokens() self.data = DataHandler(self.data_dir, self.msg_aggregator) self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None) # Initialize the Inquirer singleton Inquirer(data_dir=self.data_dir, cryptocompare=self.cryptocompare) self.lock.release() self.shutdown_event = gevent.event.Event()
def test_cryptocompare_query_pricehistorical(accounting_data_dir): """Test that cryptocompare price historical query works fine""" cc = Cryptocompare(data_directory=accounting_data_dir) price = cc.query_endpoint_pricehistorical( from_asset=A_SNGLS, to_asset=A_BTC, timestamp=1475413990, ) # Just test a price is returned assert price
def test_cryptocompare_historical_data_use_cached_price(data_dir, database, historical_price_test_data): # pylint: disable=unused-argument # noqa: E501 """Test that the cryptocompare cache is used""" cc = Cryptocompare(data_directory=data_dir, database=database) with patch.object(cc, 'query_endpoint_histohour') as histohour_mock: result = cc.query_historical_price( from_asset=A_ETH, to_asset=A_EUR, timestamp=1511627623, ) # make sure that histohour was not called, in essence that the cache was used assert histohour_mock.call_count == 0 assert result == FVal(396.56)
def test_cryptocompare_histohour_data_going_backward(data_dir, database, freezer): """Test that the cryptocompare histohour data retrieval works properly This test checks that doing an additional query in the past workd properly and that the cached data are properly appended to the cached result. In production this scenario should not happen often. Only way to happen if cryptocompare somehow adds older data than what was previously queried. """ globaldb = GlobalDBHandler() # first timestamp cryptocompare has histohour BTC/USD when queried from this test is btc_start_ts = 1279936800 # first timestamp cryptocompare has histohour BTC/USD is: 1279940400 now_ts = btc_start_ts + 3600 * 2000 + 122 # create a cache file for BTC_USD cache_data = [ HistoricalPrice( from_asset=A_BTC, to_asset=A_USD, source=HistoricalPriceOracle.CRYPTOCOMPARE, timestamp=Timestamp(1301536800), price=Price(FVal('0.298')), ), HistoricalPrice( from_asset=A_BTC, to_asset=A_USD, source=HistoricalPriceOracle.CRYPTOCOMPARE, timestamp=Timestamp(1301540400), price=Price(FVal('0.298')), ) ] globaldb.add_historical_prices(cache_data) freezer.move_to(datetime.fromtimestamp(now_ts)) cc = Cryptocompare(data_directory=data_dir, database=database) cc.query_and_store_historical_data( from_asset=A_BTC, to_asset=A_USD, timestamp=now_ts - 3600 * 2 - 55, ) result = get_globaldb_cache_entries(from_asset=A_BTC, to_asset=A_USD) assert len(result) == CRYPTOCOMPARE_HOURQUERYLIMIT * 3 + 2 check_cc_result(result, forward=False) data_range = globaldb.get_historical_price_range( A_BTC, A_USD, HistoricalPriceOracle.CRYPTOCOMPARE) # noqa: E501 assert data_range[0] == btc_start_ts assert data_range[ 1] == 1301540400 # that's the closest ts to now_ts cc returns
def create_inquirer(data_directory, should_mock_current_price_queries, mocked_prices) -> Inquirer: # Since this is a singleton and we want it initialized everytime the fixture # is called make sure its instance is always starting from scratch Inquirer._Inquirer__instance = None # type: ignore # Get a cryptocompare without a DB since invoking DB fixture here causes problems # of existing user for some tests cryptocompare = Cryptocompare(data_directory=data_directory, database=None) gecko = Coingecko(data_directory=data_directory) inquirer = Inquirer(data_dir=data_directory, cryptocompare=cryptocompare, coingecko=gecko) if not should_mock_current_price_queries: return inquirer def mock_find_usd_price(asset): # pylint: disable=unused-argument return mocked_prices.get(asset, FVal('1.5')) inquirer.find_usd_price = mock_find_usd_price # type: ignore def mock_query_fiat_pair(base, quote): # pylint: disable=unused-argument return FVal(1) inquirer.query_fiat_pair = mock_query_fiat_pair # type: ignore return inquirer
def test_price_queries(price_historian, data_dir, database): """Test some historical price queries. Make sure that we test some assets not in cryptocompare but in coigecko so the backup mechanism triggers and works""" # These should hit cryptocompare assert price_historian.query_historical_price(A_BTC, A_EUR, 1479200704) == FVal('663.66') assert price_historian.query_historical_price( A_XMR, A_BTC, 1579200704) == FVal('0.007526') # this should hit the cryptocompare cache we are creating here contents = """{"start_time": 0, "end_time": 1439390800, "data": [{"time": 1438387200, "close": 10, "high": 10, "low": 10, "open": 10, "volumefrom": 10, "volumeto": 10}, {"time": 1438390800, "close": 20, "high": 20, "low": 20, "open": 20, "volumefrom": 20, "volumeto": 20}]}""" price_history_dir = get_or_make_price_history_dir(data_dir) with open(price_history_dir / f'{PRICE_HISTORY_FILE_PREFIX}DASH_USD.json', 'w') as f: f.write(contents) price_historian._PriceHistorian__instance._cryptocompare = Cryptocompare( data_directory=data_dir, database=database, ) price_historian.set_oracles_order(price_historian._oracles) assert price_historian.query_historical_price(A_DASH, A_USD, 1438387700) == FVal('10') # this should hit coingecko, since cornichon is not in cryptocompare cornichon = Asset('CORN-2') expected_price = FVal('0.07830444726516915') assert price_historian.query_historical_price(cornichon, A_USD, 1608854400) == expected_price
def test_empty_histohour(data_dir, database, freezer): """Histohour can be empty and can have also floating point zeros like in CHI/EUR This test makes sure that an empty list is returned at the very first all zeros result that also has floating point and querying stops. If cryptocompare actually fixes their zero historical price problem this test can go away """ now_ts = 1610365553 freezer.move_to(datetime.fromtimestamp(now_ts)) cc = Cryptocompare(data_directory=data_dir, database=database) result = cc.get_historical_data( from_asset=Asset('CHI'), to_asset=Asset('EUR'), timestamp=now_ts, only_check_cache=False, ) assert len(result) == 0
def test_get_cached_data_metadata(data_dir, database): """Test that the get_cached_data_metadata function works correctly and returns just the metadata by reading part ofthe file """ contents = """{"start_time": 1301536800, "end_time": 1301540400, "data": [{"time": 1301536800, "close": 0.298, "high": 0.298, "low": 0.298, "open": 0.298, "volumefrom": 0.298, "volumeto": 0.298}, {"time": 1301540400, "close": 0.298, "high": 0.298, "low": 0.298, "open": 0.298, "volumefrom": 0.298, "volumeto": 0.298}]}""" price_history_dir = get_or_make_price_history_dir(data_dir) with open(price_history_dir / f'{PRICE_HISTORY_FILE_PREFIX}BTC_USD.json', 'w') as f: f.write(contents) cc = Cryptocompare(data_directory=data_dir, database=database) result = cc.get_cached_data_metadata( from_asset=A_BTC, to_asset=A_USD, ) assert result is not None assert result[0] == 1301536800 assert result[1] == 1301540400
def __init__(self, data_directory='.'): self.data_directory = Path(data_directory) self.config = yaml.load(open(self.data_directory / 'buchfink.yaml', 'r'), Loader=yaml.SafeLoader) self.reports_directory = self.data_directory / "reports" self.trades_directory = self.data_directory / "trades" self.cache_directory = self.data_directory / "cache" self.reports_directory.mkdir(exist_ok=True) self.trades_directory.mkdir(exist_ok=True) self.cache_directory.mkdir(exist_ok=True) (self.cache_directory / 'cryptocompare').mkdir(exist_ok=True) (self.cache_directory / 'history').mkdir(exist_ok=True) (self.cache_directory / 'inquirer').mkdir(exist_ok=True) self.cryptocompare = Cryptocompare( self.cache_directory / 'cryptocompare', self) self.historian = PriceHistorian(self.cache_directory / 'history', '01/01/2014', self.cryptocompare) self.inquirer = Inquirer(self.cache_directory / 'inquirer', self.cryptocompare) self.msg_aggregator = MessagesAggregator() self.greenlet_manager = GreenletManager( msg_aggregator=self.msg_aggregator) # Initialize blockchain querying modules self.etherscan = Etherscan(database=self, msg_aggregator=self.msg_aggregator) self.all_eth_tokens = AssetResolver().get_all_eth_tokens() self.alethio = Alethio( database=self, msg_aggregator=self.msg_aggregator, all_eth_tokens=self.all_eth_tokens, ) self.ethereum_manager = EthereumManager( ethrpc_endpoint=self.get_eth_rpc_endpoint(), etherscan=self.etherscan, msg_aggregator=self.msg_aggregator, ) #self.chain_manager = ChainManager( # blockchain_accounts=[], # owned_eth_tokens=[], # ethereum_manager=self.ethereum_manager, # msg_aggregator=self.msg_aggregator, # alethio=alethio, # greenlet_manager=self.greenlet_manager, # premium=False, # eth_modules=ethereum_modules, #) self.ethereum_analyzer = EthereumAnalyzer( ethereum_manager=self.ethereum_manager, database=self, )
def test_cryptocompare_histohour_data_going_forward(data_dir, database, freezer): """Test that the cryptocompare histohour data retrieval works properly This test checks that doing an additional query in the future works properly and appends the cached data with the newly returned data """ # first timestamp cryptocompare has histohour BTC/USD when queried from this test is btc_start_ts = 1279940400 now_ts = btc_start_ts + 3600 * 2000 + 122 freezer.move_to(datetime.fromtimestamp(now_ts)) cc = Cryptocompare(data_directory=data_dir, database=database) result = cc.get_historical_data( from_asset=A_BTC, to_asset=A_USD, timestamp=now_ts - 3600 * 2 - 55, only_check_cache=False, ) cache_key = PairCacheKey('BTC_USD') assert len(result) == CRYPTOCOMPARE_HOURQUERYLIMIT + 1 assert all(x.low == x.high == FVal('0.05454') for x in result) assert cache_key in cc.price_history assert cc.price_history[cache_key].start_time == btc_start_ts assert cc.price_history[cache_key].end_time == now_ts assert all(x.low == x.high == FVal('0.05454') for x in cc.price_history[cache_key].data) # now let's move a bit to the future and query again to see the cache is appended to now_ts = now_ts + 3600 * 2000 * 2 + 4700 freezer.move_to(datetime.fromtimestamp(now_ts)) result = cc.get_historical_data( from_asset=A_BTC, to_asset=A_USD, timestamp=now_ts - 3600 * 4 - 55, only_check_cache=False, ) assert len(result) == CRYPTOCOMPARE_HOURQUERYLIMIT * 3 + 2 check_cc_result(result, forward=True) assert cache_key in cc.price_history assert cc.price_history[cache_key].start_time == btc_start_ts assert cc.price_history[cache_key].end_time == now_ts check_cc_result(cc.price_history[cache_key].data, forward=True)
def __init__(self, args: argparse.Namespace) -> None: """Initialize the Rotkehlchen object May Raise: - SystemPermissionError if the given data directory's permissions are not correct. """ self.lock = Semaphore() self.lock.acquire() # Can also be None after unlock if premium credentials did not # authenticate or premium server temporarily offline self.premium: Optional[Premium] = None self.user_is_logged_in: bool = False configure_logging(args) self.sleep_secs = args.sleep_secs if args.data_dir is None: self.data_dir = default_data_directory() else: self.data_dir = Path(args.data_dir) if not os.access(self.data_dir, os.W_OK | os.R_OK): raise SystemPermissionError( f'The given data directory {self.data_dir} is not readable or writable', ) self.args = args self.msg_aggregator = MessagesAggregator() self.greenlet_manager = GreenletManager( msg_aggregator=self.msg_aggregator) self.exchange_manager = ExchangeManager( msg_aggregator=self.msg_aggregator) self.data = DataHandler(self.data_dir, self.msg_aggregator) self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None) # Initialize the Inquirer singleton Inquirer(data_dir=self.data_dir, cryptocompare=self.cryptocompare) self.lock.release() self.shutdown_event = gevent.event.Event()
def test_get_cached_data_metadata(data_dir, database): """Test that the get_cached_data_metadata function works correctly and returns just the metadata by reading part ofthe file The json cache data in production are saved as one line files. So here we also keep it in one line on purpose. The previous regex we used failed with 1 line json file """ contents = """{"start_time": 1301536800, "end_time": 1301540400, "data": [{"time": 1301536800, "close": 0.298, "high": 0.298, "low": 0.298, "open": 0.298, "volumefrom": 0.298, "volumeto": 0.298}, {"time": 1301540400, "close": 0.298, "high": 0.298, "low": 0.298, "open": 0.298, "volumefrom": 0.298, "volumeto": 0.298}]}""" # noqa: E501 price_history_dir = get_or_make_price_history_dir(data_dir) with open(price_history_dir / f'{PRICE_HISTORY_FILE_PREFIX}BTC_USD.json', 'w') as f: f.write(contents) cc = Cryptocompare(data_directory=data_dir, database=database) # make sure that _read_cachefile_metadata runs and they are read from file and not from memory cc.price_history = {} result = cc.get_cached_data_metadata( from_asset=A_BTC, to_asset=A_USD, ) assert result is not None assert result[0] == 1301536800 assert result[1] == 1301540400
def test_cryptocompare_histohour_query_old_ts_xcp( accounting_data_dir, price_historian, # pylint: disable=unused-argument ): """Test that as a result of this query a crash does not happen. Regression for: https://github.com/rotkehlchenio/rotkehlchen/issues/432 Unfortunately still no price is found so we have to expect a NoPriceForGivenTimestamp This test is now skipped since it's a subset of: test_end_to_end_tax_report::test_unknown_cryptocompare_asset_and_price_not_found_in_history When more price data sources are introduced then this should probably be unskipped to focus on the cryptocompare case. But at the moment both tests follow the same path and are probably slow due to the price querying. """ with pytest.raises(NoPriceForGivenTimestamp): cc = Cryptocompare(data_directory=accounting_data_dir) cc.query_historical_price( from_asset=Asset('XCP'), to_asset=A_USD, timestamp=1392685761, historical_data_start=1438387200, )
def test_price_queries(price_historian, data_dir, database): """Test some historical price queries. Make sure that we test some assets not in cryptocompare but in coigecko so the backup mechanism triggers and works""" # These should hit cryptocompare assert price_historian.query_historical_price(A_BTC, A_EUR, 1479200704) == FVal('663.66') assert price_historian.query_historical_price( A_XMR, A_BTC, 1579200704) == FVal('0.007526') # this should hit the cryptocompare cache we are creating here cache_data = [ HistoricalPrice( from_asset=A_DASH, to_asset=A_USD, source=HistoricalPriceOracle.CRYPTOCOMPARE, timestamp=Timestamp(1438387200), price=Price(FVal('10')), ), HistoricalPrice( from_asset=A_DASH, to_asset=A_USD, source=HistoricalPriceOracle.CRYPTOCOMPARE, timestamp=Timestamp(1438390800), price=Price(FVal('20')), ) ] GlobalDBHandler().add_historical_prices(cache_data) price_historian._PriceHistorian__instance._cryptocompare = Cryptocompare( data_directory=data_dir, database=database, ) price_historian.set_oracles_order(price_historian._oracles) assert price_historian.query_historical_price(A_DASH, A_USD, 1438387700) == FVal('10') # this should hit coingecko, since cornichon is not in cryptocompare expected_price = FVal('0.07830444726516915') assert price_historian.query_historical_price(A_CORN, A_USD, 1608854400) == expected_price
def test_cryptocompare_histohour_data_going_forward(data_dir, database, freezer): """Test that the cryptocompare histohour data retrieval works properly This test checks that doing an additional query in the future works properly and appends the cached data with the newly returned data """ # first timestamp cryptocompare has histohour BTC/USD when queried from this test is btc_start_ts = 1279940400 now_ts = btc_start_ts + 3600 * 2000 + 122 freezer.move_to(datetime.fromtimestamp(now_ts)) cc = Cryptocompare(data_directory=data_dir, database=database) cc.query_and_store_historical_data( from_asset=A_BTC, to_asset=A_USD, timestamp=now_ts - 3600 * 2 - 55, ) globaldb = GlobalDBHandler() result = get_globaldb_cache_entries(from_asset=A_BTC, to_asset=A_USD) assert len(result) == CRYPTOCOMPARE_HOURQUERYLIMIT + 1 assert all(x.price == Price(FVal(0.05454)) for x in result) data_range = globaldb.get_historical_price_range( A_BTC, A_USD, HistoricalPriceOracle.CRYPTOCOMPARE) # noqa: E501 assert data_range[0] == btc_start_ts assert data_range[ 1] == 1287140400 # that's the closest ts to now_ts cc returns # now let's move a bit to the future and query again to see the cache is appended to now_ts = now_ts + 3600 * 2000 * 2 + 4700 freezer.move_to(datetime.fromtimestamp(now_ts)) cc.query_and_store_historical_data( from_asset=A_BTC, to_asset=A_USD, timestamp=now_ts - 3600 * 4 - 55, ) result = get_globaldb_cache_entries(from_asset=A_BTC, to_asset=A_USD) assert len(result) == CRYPTOCOMPARE_HOURQUERYLIMIT * 3 + 2 check_cc_result(result, forward=True) data_range = globaldb.get_historical_price_range( A_BTC, A_USD, HistoricalPriceOracle.CRYPTOCOMPARE) # noqa: E501 assert data_range[0] == btc_start_ts assert data_range[ 1] == 1301544000 # that's the closest ts to now_ts cc returns
def price_historian( accounting_data_dir, inquirer, # pylint: disable=unused-argument should_mock_price_queries, mocked_price_queries, ): # Since this is a singleton and we want it initialized everytime the fixture # is called make sure its instance is always starting from scratch PriceHistorian._PriceHistorian__instance = None historian = PriceHistorian( data_directory=accounting_data_dir, history_date_start=TEST_HISTORY_DATA_START, cryptocompare=Cryptocompare(data_directory=accounting_data_dir), ) if should_mock_price_queries: def mock_historical_price_query(from_asset, to_asset, timestamp): if from_asset == to_asset: return FVal(1) return mocked_price_queries[from_asset][to_asset][timestamp] historian.query_historical_price = mock_historical_price_query return historian
class Rotkehlchen(): def __init__(self, args: argparse.Namespace) -> None: """Initialize the Rotkehlchen object May Raise: - SystemPermissionError if the given data directory's permissions are not correct. """ self.lock = Semaphore() self.lock.acquire() # Can also be None after unlock if premium credentials did not # authenticate or premium server temporarily offline self.premium: Optional[Premium] = None self.user_is_logged_in: bool = False configure_logging(args) self.sleep_secs = args.sleep_secs if args.data_dir is None: self.data_dir = default_data_directory() else: self.data_dir = Path(args.data_dir) if not os.access(self.data_dir, os.W_OK | os.R_OK): raise SystemPermissionError( f'The given data directory {self.data_dir} is not readable or writable', ) self.main_loop_spawned = False self.args = args self.api_task_greenlets: List[gevent.Greenlet] = [] self.msg_aggregator = MessagesAggregator() self.greenlet_manager = GreenletManager( msg_aggregator=self.msg_aggregator) self.exchange_manager = ExchangeManager( msg_aggregator=self.msg_aggregator) # Initialize the AssetResolver singleton AssetResolver(data_directory=self.data_dir) self.data = DataHandler(self.data_dir, self.msg_aggregator) self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None) self.coingecko = Coingecko(data_directory=self.data_dir) self.icon_manager = IconManager(data_dir=self.data_dir, coingecko=self.coingecko) self.greenlet_manager.spawn_and_track( after_seconds=None, task_name='periodically_query_icons_until_all_cached', exception_is_error=False, method=self.icon_manager.periodically_query_icons_until_all_cached, batch_size=ICONS_BATCH_SIZE, sleep_time_secs=ICONS_QUERY_SLEEP, ) # Initialize the Inquirer singleton Inquirer( data_dir=self.data_dir, cryptocompare=self.cryptocompare, coingecko=self.coingecko, ) # Keeps how many trades we have found per location. Used for free user limiting self.actions_per_location: Dict[str, Dict[Location, int]] = { 'trade': defaultdict(int), 'asset_movement': defaultdict(int), } self.lock.release() self.task_manager: Optional[TaskManager] = None self.shutdown_event = gevent.event.Event() def reset_after_failed_account_creation_or_login(self) -> None: """If the account creation or login failed make sure that the Rotki instance is clear Tricky instances are when after either failed premium credentials or user refusal to sync premium databases we relogged in. """ self.cryptocompare.db = None def unlock_user( self, user: str, password: str, create_new: bool, sync_approval: Literal['yes', 'no', 'unknown'], premium_credentials: Optional[PremiumCredentials], initial_settings: Optional[ModifiableDBSettings] = None, ) -> None: """Unlocks an existing user or creates a new one if `create_new` is True May raise: - PremiumAuthenticationError if the password can't unlock the database. - AuthenticationError if premium_credentials are given and are invalid or can't authenticate with the server - DBUpgradeError if the rotki DB version is newer than the software or there is a DB upgrade and there is an error. - SystemPermissionError if the directory or DB file can not be accessed """ log.info( 'Unlocking user', user=user, create_new=create_new, sync_approval=sync_approval, initial_settings=initial_settings, ) # unlock or create the DB self.password = password self.user_directory = self.data.unlock(user, password, create_new, initial_settings) self.data_importer = DataImporter(db=self.data.db) self.last_data_upload_ts = self.data.db.get_last_data_upload_ts() self.premium_sync_manager = PremiumSyncManager(data=self.data, password=password) # set the DB in the external services instances that need it self.cryptocompare.set_database(self.data.db) # Anything that was set above here has to be cleaned in case of failure in the next step # by reset_after_failed_account_creation_or_login() try: self.premium = self.premium_sync_manager.try_premium_at_start( given_premium_credentials=premium_credentials, username=user, create_new=create_new, sync_approval=sync_approval, ) except PremiumAuthenticationError: # Reraise it only if this is during the creation of a new account where # the premium credentials were given by the user if create_new: raise self.msg_aggregator.add_warning( 'Could not authenticate the Rotki premium API keys found in the DB.' ' Has your subscription expired?', ) # else let's just continue. User signed in succesfully, but he just # has unauthenticable/invalid premium credentials remaining in his DB settings = self.get_settings() self.greenlet_manager.spawn_and_track( after_seconds=None, task_name='submit_usage_analytics', exception_is_error=False, method=maybe_submit_usage_analytics, should_submit=settings.submit_usage_analytics, ) self.etherscan = Etherscan(database=self.data.db, msg_aggregator=self.msg_aggregator) self.beaconchain = BeaconChain(database=self.data.db, msg_aggregator=self.msg_aggregator) eth_rpc_endpoint = settings.eth_rpc_endpoint # Initialize the price historian singleton PriceHistorian( data_directory=self.data_dir, cryptocompare=self.cryptocompare, coingecko=self.coingecko, ) PriceHistorian().set_oracles_order(settings.historical_price_oracles) self.accountant = Accountant( db=self.data.db, user_directory=self.user_directory, msg_aggregator=self.msg_aggregator, create_csv=True, ) # Initialize the rotkehlchen logger LoggingSettings(anonymized_logs=settings.anonymized_logs) exchange_credentials = self.data.db.get_exchange_credentials() self.exchange_manager.initialize_exchanges( exchange_credentials=exchange_credentials, database=self.data.db, ) # Initialize blockchain querying modules ethereum_manager = EthereumManager( ethrpc_endpoint=eth_rpc_endpoint, etherscan=self.etherscan, database=self.data.db, msg_aggregator=self.msg_aggregator, greenlet_manager=self.greenlet_manager, connect_at_start=ETHEREUM_NODES_TO_CONNECT_AT_START, ) kusama_manager = SubstrateManager( chain=SubstrateChain.KUSAMA, msg_aggregator=self.msg_aggregator, greenlet_manager=self.greenlet_manager, connect_at_start=KUSAMA_NODES_TO_CONNECT_AT_START, connect_on_startup=self._connect_ksm_manager_on_startup(), own_rpc_endpoint=settings.ksm_rpc_endpoint, ) Inquirer().inject_ethereum(ethereum_manager) Inquirer().set_oracles_order(settings.current_price_oracles) self.chain_manager = ChainManager( blockchain_accounts=self.data.db.get_blockchain_accounts(), ethereum_manager=ethereum_manager, kusama_manager=kusama_manager, msg_aggregator=self.msg_aggregator, database=self.data.db, greenlet_manager=self.greenlet_manager, premium=self.premium, eth_modules=settings.active_modules, data_directory=self.data_dir, beaconchain=self.beaconchain, btc_derivation_gap_limit=settings.btc_derivation_gap_limit, ) self.events_historian = EventsHistorian( user_directory=self.user_directory, db=self.data.db, msg_aggregator=self.msg_aggregator, exchange_manager=self.exchange_manager, chain_manager=self.chain_manager, ) self.task_manager = TaskManager( max_tasks_num=DEFAULT_MAX_TASKS_NUM, greenlet_manager=self.greenlet_manager, api_task_greenlets=self.api_task_greenlets, database=self.data.db, cryptocompare=self.cryptocompare, premium_sync_manager=self.premium_sync_manager, chain_manager=self.chain_manager, exchange_manager=self.exchange_manager, ) self.user_is_logged_in = True log.debug('User unlocking complete') def logout(self) -> None: if not self.user_is_logged_in: return user = self.data.username log.info( 'Logging out user', user=user, ) self.greenlet_manager.clear() del self.chain_manager self.exchange_manager.delete_all_exchanges() # Reset rotkehlchen logger to default LoggingSettings(anonymized_logs=DEFAULT_ANONYMIZED_LOGS) del self.accountant del self.events_historian del self.data_importer if self.premium is not None: del self.premium self.data.logout() self.password = '' self.cryptocompare.unset_database() # Make sure no messages leak to other user sessions self.msg_aggregator.consume_errors() self.msg_aggregator.consume_warnings() self.task_manager = None self.user_is_logged_in = False log.info( 'User successfully logged out', user=user, ) def set_premium_credentials(self, credentials: PremiumCredentials) -> None: """ Sets the premium credentials for Rotki Raises PremiumAuthenticationError if the given key is rejected by the Rotkehlchen server """ log.info('Setting new premium credentials') if self.premium is not None: self.premium.set_credentials(credentials) else: self.premium = premium_create_and_verify(credentials) self.data.db.set_rotkehlchen_premium(credentials) def delete_premium_credentials(self) -> Tuple[bool, str]: """Deletes the premium credentials for Rotki""" msg = '' success = self.data.db.del_rotkehlchen_premium() if success is False: msg = 'The database was unable to delete the Premium keys for the logged-in user' self.deactivate_premium_status() return success, msg def deactivate_premium_status(self) -> None: """Deactivate premium in the current session""" self.premium = None self.premium_sync_manager.premium = None self.chain_manager.deactivate_premium_status() def start(self) -> gevent.Greenlet: assert not self.main_loop_spawned, 'Tried to spawn the main loop twice' greenlet = gevent.spawn(self.main_loop) self.main_loop_spawned = True return greenlet def main_loop(self) -> None: """Rotki main loop that fires often and runs the task manager's scheduler""" while self.shutdown_event.wait( timeout=MAIN_LOOP_SECS_DELAY) is not True: if self.task_manager is not None: self.task_manager.schedule() def get_blockchain_account_data( self, blockchain: SupportedBlockchain, ) -> Union[List[BlockchainAccountData], Dict[str, Any]]: account_data = self.data.db.get_blockchain_account_data(blockchain) if blockchain != SupportedBlockchain.BITCOIN: return account_data xpub_data = self.data.db.get_bitcoin_xpub_data() addresses_to_account_data = {x.address: x for x in account_data} address_to_xpub_mappings = self.data.db.get_addresses_to_xpub_mapping( list(addresses_to_account_data.keys()), # type: ignore ) xpub_mappings: Dict['XpubData', List[BlockchainAccountData]] = {} for address, xpub_entry in address_to_xpub_mappings.items(): if xpub_entry not in xpub_mappings: xpub_mappings[xpub_entry] = [] xpub_mappings[xpub_entry].append( addresses_to_account_data[address]) data: Dict[str, Any] = {'standalone': [], 'xpubs': []} # Add xpub data for xpub_entry in xpub_data: data_entry = xpub_entry.serialize() addresses = xpub_mappings.get(xpub_entry, None) data_entry['addresses'] = addresses if addresses and len( addresses) != 0 else None data['xpubs'].append(data_entry) # Add standalone addresses for account in account_data: if account.address not in address_to_xpub_mappings: data['standalone'].append(account) return data def add_blockchain_accounts( self, blockchain: SupportedBlockchain, account_data: List[BlockchainAccountData], ) -> BlockchainBalancesUpdate: """Adds new blockchain accounts Adds the accounts to the blockchain instance and queries them to get the updated balances. Also adds them in the DB May raise: - EthSyncError from modify_blockchain_account - InputError if the given accounts list is empty. - TagConstraintError if any of the given account data contain unknown tags. - RemoteError if an external service such as Etherscan is queried and there is a problem with its query. """ self.data.db.ensure_tags_exist( given_data=account_data, action='adding', data_type='blockchain accounts', ) address_type = blockchain.get_address_type() updated_balances = self.chain_manager.add_blockchain_accounts( blockchain=blockchain, accounts=[address_type(entry.address) for entry in account_data], ) self.data.db.add_blockchain_accounts( blockchain=blockchain, account_data=account_data, ) return updated_balances def edit_blockchain_accounts( self, blockchain: SupportedBlockchain, account_data: List[BlockchainAccountData], ) -> None: """Edits blockchain accounts Edits blockchain account data for the given accounts May raise: - InputError if the given accounts list is empty or if any of the accounts to edit do not exist. - TagConstraintError if any of the given account data contain unknown tags. """ # First check for validity of account data addresses if len(account_data) == 0: raise InputError( 'Empty list of blockchain account data to edit was given') accounts = [x.address for x in account_data] unknown_accounts = set(accounts).difference( self.chain_manager.accounts.get(blockchain)) if len(unknown_accounts) != 0: raise InputError( f'Tried to edit unknown {blockchain.value} ' f'accounts {",".join(unknown_accounts)}', ) self.data.db.ensure_tags_exist( given_data=account_data, action='editing', data_type='blockchain accounts', ) # Finally edit the accounts self.data.db.edit_blockchain_accounts( blockchain=blockchain, account_data=account_data, ) def remove_blockchain_accounts( self, blockchain: SupportedBlockchain, accounts: ListOfBlockchainAddresses, ) -> BlockchainBalancesUpdate: """Removes blockchain accounts Removes the accounts from the blockchain instance and queries them to get the updated balances. Also removes them from the DB May raise: - RemoteError if an external service such as Etherscan is queried and there is a problem with its query. - InputError if a non-existing account was given to remove """ balances_update = self.chain_manager.remove_blockchain_accounts( blockchain=blockchain, accounts=accounts, ) self.data.db.remove_blockchain_accounts(blockchain, accounts) return balances_update def get_history_query_status(self) -> Dict[str, str]: if self.events_historian.progress < FVal('100'): processing_state = self.events_historian.processing_state_name progress = self.events_historian.progress / 2 elif self.accountant.first_processed_timestamp == -1: processing_state = 'Processing all retrieved historical events' progress = FVal(50) else: processing_state = 'Processing all retrieved historical events' # start_ts is min of the query start or the first action timestamp since action # processing can start well before query start to calculate cost basis start_ts = min( self.accountant.events.query_start_ts, self.accountant.first_processed_timestamp, ) diff = self.accountant.events.query_end_ts - start_ts progress = 50 + 100 * ( FVal(self.accountant.currently_processing_timestamp - start_ts) / FVal(diff) / 2) return { 'processing_state': str(processing_state), 'total_progress': str(progress) } def process_history( self, start_ts: Timestamp, end_ts: Timestamp, ) -> Tuple[Dict[str, Any], str]: ( error_or_empty, history, loan_history, asset_movements, eth_transactions, defi_events, ledger_actions, ) = self.events_historian.get_history( start_ts=start_ts, end_ts=end_ts, has_premium=self.premium is not None, ) result = self.accountant.process_history( start_ts=start_ts, end_ts=end_ts, trade_history=history, loan_history=loan_history, asset_movements=asset_movements, eth_transactions=eth_transactions, defi_events=defi_events, ledger_actions=ledger_actions, ) return result, error_or_empty @overload def _apply_actions_limit( self, location: Location, action_type: Literal['trade'], location_actions: TRADES_LIST, all_actions: TRADES_LIST, ) -> TRADES_LIST: ... @overload def _apply_actions_limit( self, location: Location, action_type: Literal['asset_movement'], location_actions: List[AssetMovement], all_actions: List[AssetMovement], ) -> List[AssetMovement]: ... def _apply_actions_limit( self, location: Location, action_type: Literal['trade', 'asset_movement'], location_actions: Union[TRADES_LIST, List[AssetMovement]], all_actions: Union[TRADES_LIST, List[AssetMovement]], ) -> Union[TRADES_LIST, List[AssetMovement]]: """Take as many actions from location actions and add them to all actions as the limit permits Returns the modified (or not) all_actions """ # If we are already at or above the limit return current actions disregarding this location actions_mapping = self.actions_per_location[action_type] current_num_actions = sum(x for _, x in actions_mapping.items()) limit = LIMITS_MAPPING[action_type] if current_num_actions >= limit: return all_actions # Find out how many more actions can we return, and depending on that get # the number of actions from the location actions and add them to the total remaining_num_actions = limit - current_num_actions if remaining_num_actions < 0: remaining_num_actions = 0 num_actions_to_take = min(len(location_actions), remaining_num_actions) actions_mapping[location] = num_actions_to_take all_actions.extend( location_actions[0:num_actions_to_take]) # type: ignore return all_actions def query_trades( self, from_ts: Timestamp, to_ts: Timestamp, location: Optional[Location], ) -> TRADES_LIST: """Queries trades for the given location and time range. If no location is given then all external, all exchange and DEX trades are queried. DEX Trades are queried only if the user has premium If the user does not have premium then a trade limit is applied. May raise: - RemoteError: If there are problems connecting to any of the remote exchanges """ trades: TRADES_LIST if location is not None: trades = self.query_location_trades(from_ts, to_ts, location) else: trades = self.query_location_trades(from_ts, to_ts, Location.EXTERNAL) # crypto.com is not an API key supported exchange but user can import from CSV trades.extend( self.query_location_trades(from_ts, to_ts, Location.CRYPTOCOM)) for name, exchange in self.exchange_manager.connected_exchanges.items( ): exchange_trades = exchange.query_trade_history( start_ts=from_ts, end_ts=to_ts) if self.premium is None: trades = self._apply_actions_limit( location=deserialize_location(name), action_type='trade', location_actions=exchange_trades, all_actions=trades, ) else: trades.extend(exchange_trades) # for all trades we also need uniswap trades if self.premium is not None: uniswap = self.chain_manager.uniswap if uniswap is not None: trades.extend( uniswap.get_trades( addresses=self.chain_manager. queried_addresses_for_module('uniswap'), from_timestamp=from_ts, to_timestamp=to_ts, ), ) # return trades with most recent first trades.sort(key=lambda x: x.timestamp, reverse=True) return trades def query_location_trades( self, from_ts: Timestamp, to_ts: Timestamp, location: Location, ) -> TRADES_LIST: # clear the trades queried for this location self.actions_per_location['trade'][location] = 0 location_trades: TRADES_LIST if location in (Location.EXTERNAL, Location.CRYPTOCOM): location_trades = self.data.db.get_trades( # type: ignore # list invariance from_ts=from_ts, to_ts=to_ts, location=location, ) elif location == Location.UNISWAP: if self.premium is not None: uniswap = self.chain_manager.uniswap if uniswap is not None: location_trades = uniswap.get_trades( # type: ignore # list invariance addresses=self.chain_manager. queried_addresses_for_module('uniswap'), from_timestamp=from_ts, to_timestamp=to_ts, ) else: # should only be an exchange exchange = self.exchange_manager.get(str(location)) if not exchange: logger.warning( f'Tried to query trades from {location} which is either not an ' f'exchange or not an exchange the user has connected to', ) return [] location_trades = exchange.query_trade_history(start_ts=from_ts, end_ts=to_ts) trades: TRADES_LIST = [] if self.premium is None: trades = self._apply_actions_limit( location=location, action_type='trade', location_actions=location_trades, all_actions=trades, ) else: trades = location_trades return trades def query_balances( self, requested_save_data: bool = False, timestamp: Timestamp = None, ignore_cache: bool = False, ) -> Dict[str, Any]: """Query all balances rotkehlchen can see. If requested_save_data is True then the data are always saved in the DB, if it is False then data are saved if self.data.should_save_balances() is True. If timestamp is None then the current timestamp is used. If a timestamp is given then that is the time that the balances are going to be saved in the DB If ignore_cache is True then all underlying calls that have a cache ignore it Returns a dictionary with the queried balances. """ log.info('query_balances called', requested_save_data=requested_save_data) balances: Dict[str, Dict[Asset, Balance]] = {} problem_free = True for _, exchange in self.exchange_manager.connected_exchanges.items(): exchange_balances, _ = exchange.query_balances( ignore_cache=ignore_cache) # If we got an error, disregard that exchange but make sure we don't save data if not isinstance(exchange_balances, dict): problem_free = False else: balances[exchange.name] = exchange_balances liabilities: Dict[Asset, Balance] try: blockchain_result = self.chain_manager.query_balances( blockchain=None, force_token_detection=ignore_cache, ignore_cache=ignore_cache, ) balances[str( Location.BLOCKCHAIN)] = blockchain_result.totals.assets liabilities = blockchain_result.totals.liabilities except (RemoteError, EthSyncError) as e: problem_free = False liabilities = {} log.error(f'Querying blockchain balances failed due to: {str(e)}') balances = account_for_manually_tracked_balances(db=self.data.db, balances=balances) # Calculate usd totals assets_total_balance: DefaultDict[Asset, Balance] = defaultdict(Balance) total_usd_per_location: Dict[str, FVal] = {} for location, asset_balance in balances.items(): total_usd_per_location[location] = ZERO for asset, balance in asset_balance.items(): assets_total_balance[asset] += balance total_usd_per_location[location] += balance.usd_value net_usd = sum((balance.usd_value for _, balance in assets_total_balance.items()), ZERO) liabilities_total_usd = sum( (liability.usd_value for _, liability in liabilities.items()), ZERO) # noqa: E501 net_usd -= liabilities_total_usd # Calculate location stats location_stats: Dict[str, Any] = {} for location, total_usd in total_usd_per_location.items(): if location == str(Location.BLOCKCHAIN): total_usd -= liabilities_total_usd percentage = (total_usd / net_usd).to_percentage() if net_usd != ZERO else '0%' location_stats[location] = { 'usd_value': total_usd, 'percentage_of_net_value': percentage, } # Calculate 'percentage_of_net_value' per asset assets_total_balance_as_dict: Dict[Asset, Dict[str, Any]] = { asset: balance.to_dict() for asset, balance in assets_total_balance.items() } liabilities_as_dict: Dict[Asset, Dict[str, Any]] = { asset: balance.to_dict() for asset, balance in liabilities.items() } for asset, balance_dict in assets_total_balance_as_dict.items(): percentage = (balance_dict['usd_value'] / net_usd).to_percentage( ) if net_usd != ZERO else '0%' # noqa: E501 assets_total_balance_as_dict[asset][ 'percentage_of_net_value'] = percentage for asset, balance_dict in liabilities_as_dict.items(): percentage = (balance_dict['usd_value'] / net_usd).to_percentage( ) if net_usd != ZERO else '0%' # noqa: E501 liabilities_as_dict[asset]['percentage_of_net_value'] = percentage # Compose balances response result_dict = { 'assets': assets_total_balance_as_dict, 'liabilities': liabilities_as_dict, 'location': location_stats, 'net_usd': net_usd, } allowed_to_save = requested_save_data or self.data.should_save_balances( ) if problem_free and allowed_to_save: if not timestamp: timestamp = Timestamp(int(time.time())) self.data.db.save_balances_data(data=result_dict, timestamp=timestamp) log.debug('query_balances data saved') else: log.debug( 'query_balances data not saved', allowed_to_save=allowed_to_save, problem_free=problem_free, ) return result_dict def _query_exchange_asset_movements( self, from_ts: Timestamp, to_ts: Timestamp, all_movements: List[AssetMovement], exchange: Union[ExchangeInterface, Location], ) -> List[AssetMovement]: if isinstance(exchange, ExchangeInterface): location = deserialize_location(exchange.name) # clear the asset movements queried for this exchange self.actions_per_location['asset_movement'][location] = 0 location_movements = exchange.query_deposits_withdrawals( start_ts=from_ts, end_ts=to_ts, ) else: assert isinstance(exchange, Location), 'only a location should make it here' assert exchange == Location.CRYPTOCOM, 'only cryptocom should make it here' location = exchange # cryptocom has no exchange integration but we may have DB entries self.actions_per_location['asset_movement'][location] = 0 location_movements = self.data.db.get_asset_movements( from_ts=from_ts, to_ts=to_ts, location=location, ) movements: List[AssetMovement] = [] if self.premium is None: movements = self._apply_actions_limit( location=location, action_type='asset_movement', location_actions=location_movements, all_actions=all_movements, ) else: all_movements.extend(location_movements) movements = all_movements return movements def query_asset_movements( self, from_ts: Timestamp, to_ts: Timestamp, location: Optional[Location], ) -> List[AssetMovement]: """Queries AssetMovements for the given location and time range. If no location is given then all exchange asset movements are queried. If the user does not have premium then a limit is applied. May raise: - RemoteError: If there are problems connecting to any of the remote exchanges """ movements: List[AssetMovement] = [] if location is not None: if location == Location.CRYPTOCOM: movements = self._query_exchange_asset_movements( from_ts=from_ts, to_ts=to_ts, all_movements=movements, exchange=Location.CRYPTOCOM, ) else: exchange = self.exchange_manager.get(str(location)) if not exchange: logger.warning( f'Tried to query deposits/withdrawals from {location} which is either ' f'not at exchange or not an exchange the user has connected to', ) return [] movements = self._query_exchange_asset_movements( from_ts=from_ts, to_ts=to_ts, all_movements=movements, exchange=exchange, ) else: # cryptocom has no exchange integration but we may have DB entries due to csv import movements = self._query_exchange_asset_movements( from_ts=from_ts, to_ts=to_ts, all_movements=movements, exchange=Location.CRYPTOCOM, ) for _, exchange in self.exchange_manager.connected_exchanges.items( ): self._query_exchange_asset_movements( from_ts=from_ts, to_ts=to_ts, all_movements=movements, exchange=exchange, ) # return movements with most recent first movements.sort(key=lambda x: x.timestamp, reverse=True) return movements def set_settings(self, settings: ModifiableDBSettings) -> Tuple[bool, str]: """Tries to set new settings. Returns True in success or False with message if error""" with self.lock: if settings.eth_rpc_endpoint is not None: result, msg = self.chain_manager.set_eth_rpc_endpoint( settings.eth_rpc_endpoint) if not result: return False, msg if settings.ksm_rpc_endpoint is not None: result, msg = self.chain_manager.set_ksm_rpc_endpoint( settings.ksm_rpc_endpoint) if not result: return False, msg if settings.kraken_account_type is not None: kraken = self.exchange_manager.get('kraken') if kraken: kraken.set_account_type( settings.kraken_account_type) # type: ignore if settings.btc_derivation_gap_limit is not None: self.chain_manager.btc_derivation_gap_limit = settings.btc_derivation_gap_limit if settings.current_price_oracles is not None: Inquirer().set_oracles_order(settings.current_price_oracles) if settings.historical_price_oracles is not None: PriceHistorian().set_oracles_order( settings.historical_price_oracles) self.data.db.set_settings(settings) return True, '' def get_settings(self) -> DBSettings: """Returns the db settings with a check whether premium is active or not""" db_settings = self.data.db.get_settings( have_premium=self.premium is not None) return db_settings def setup_exchange( self, name: str, api_key: ApiKey, api_secret: ApiSecret, passphrase: Optional[str] = None, ) -> Tuple[bool, str]: """ Setup a new exchange with an api key and an api secret and optionally a passphrase By default the api keys are always validated unless validate is False. """ is_success, msg = self.exchange_manager.setup_exchange( name=name, api_key=api_key, api_secret=api_secret, database=self.data.db, passphrase=passphrase, ) if is_success: # Success, save the result in the DB self.data.db.add_exchange(name, api_key, api_secret, passphrase=passphrase) return is_success, msg def remove_exchange(self, name: str) -> Tuple[bool, str]: if not self.exchange_manager.has_exchange(name): return False, 'Exchange {} is not registered'.format(name) self.exchange_manager.delete_exchange(name) # Success, remove it also from the DB self.data.db.remove_exchange(name) self.data.db.delete_used_query_range_for_exchange(name) return True, '' def query_periodic_data(self) -> Dict[str, Union[bool, Timestamp]]: """Query for frequently changing data""" result: Dict[str, Union[bool, Timestamp]] = {} if self.user_is_logged_in: result[ 'last_balance_save'] = self.data.db.get_last_balance_save_time( ) result[ 'eth_node_connection'] = self.chain_manager.ethereum.web3_mapping.get( NodeName.OWN, None) is not None # noqa : E501 result['last_data_upload_ts'] = Timestamp( self.premium_sync_manager.last_data_upload_ts) # noqa : E501 return result def shutdown(self) -> None: self.logout() self.shutdown_event.set() def _connect_ksm_manager_on_startup(self) -> bool: return bool(self.data.db.get_blockchain_accounts().ksm) def create_oracle_cache( self, oracle: HistoricalPriceOracle, from_asset: Asset, to_asset: Asset, purge_old: bool, ) -> None: """Creates the cache of the given asset pair from the start of time until now for the given oracle. if purge_old is true then any old cache in memory and in a file is purged May raise: - RemoteError if there is a problem reaching the oracle - UnsupportedAsset if any of the two assets is not supported by the oracle """ if oracle != HistoricalPriceOracle.CRYPTOCOMPARE: return # only for cryptocompare for now self.cryptocompare.create_cache(from_asset, to_asset, purge_old) def delete_oracle_cache( self, oracle: HistoricalPriceOracle, from_asset: Asset, to_asset: Asset, ) -> None: if oracle != HistoricalPriceOracle.CRYPTOCOMPARE: return # only for cryptocompare for now self.cryptocompare.delete_cache(from_asset, to_asset) def get_oracle_cache( self, oracle: HistoricalPriceOracle) -> List[Dict[str, Any]]: if oracle != HistoricalPriceOracle.CRYPTOCOMPARE: return [] # only for cryptocompare for now return self.cryptocompare.get_all_cache_data()
def main(): arg_parser = aggregator_args() args = arg_parser.parse_args() msg_aggregator = MessagesAggregator() user_data_dir = Path(default_data_directory()) / args.db_user database = DBHandler( user_data_dir=user_data_dir, password=args.db_password, msg_aggregator=msg_aggregator, ) our_data = AssetResolver().assets paprika = CoinPaprika() cmc = None cmc_list = None root_path = os.path.dirname( os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) data_directory = f'{Path.home()}/.rotkehlchen' if args.cmc_api_key: cmc = Coinmarketcap( data_directory=data_directory, api_key=args.cmc_api_key, ) cmc_list = cmc.get_cryptocyrrency_map() cryptocompare = Cryptocompare(data_directory=data_directory, database=database) paprika_coins_list = paprika.get_coins_list() cryptocompare_coins_map = cryptocompare.all_coins() if args.input_file: if not os.path.isfile(args.input_file): print(f'Given input file {args.input_file} is not a file') sys.exit(1) with open(args.input_file, 'r') as f: input_data = rlk_jsonloads(f.read()) given_symbols = set(input_data.keys()) current_symbols = set(our_data.keys()) if not given_symbols.isdisjoint(current_symbols): print( f'The following given input symbols already exist in the ' f'all_assets.json file {given_symbols.intersection(current_symbols)}', ) sys.exit(1) # If an input file is given, iterate only its assets and perform checks for asset_symbol in input_data.keys(): input_data = process_asset( our_data=input_data, asset_symbol=asset_symbol, paprika_coins_list=paprika_coins_list, paprika=paprika, cmc_list=cmc_list, cryptocompare_coins_map=cryptocompare_coins_map, always_keep_our_time=args.always_keep_our_time, ) # and now combine the two dictionaries to get the final one. Note that no # checks are perfomed for what was in all_assets.json before the script # ran in this case our_data = {**our_data, **input_data} else: # Iterate all of the assets of the all_assets.json file and perform checks for asset_symbol in our_data.keys(): our_data = process_asset( our_data=our_data, asset_symbol=asset_symbol, paprika_coins_list=paprika_coins_list, paprika=paprika, cmc_list=cmc_list, cryptocompare_coins_map=cryptocompare_coins_map, always_keep_our_time=args.always_keep_our_time, ) # Finally overwrite the all_assets.json with the modified assets with open( os.path.join(root_path, 'rotkehlchen', 'data', 'all_assets.json'), 'w') as f: f.write(json.dumps( our_data, sort_keys=True, indent=4, ), )
def create_inquirer( data_directory, should_mock_current_price_queries, mocked_prices, current_price_oracles_order, ethereum_manager, ignore_mocked_prices_for=None, ) -> Inquirer: # Since this is a singleton and we want it initialized everytime the fixture # is called make sure its instance is always starting from scratch Inquirer._Inquirer__instance = None # type: ignore # Get a cryptocompare without a DB since invoking DB fixture here causes problems # of existing user for some tests cryptocompare = Cryptocompare(data_directory=data_directory, database=None) gecko = Coingecko() inquirer = Inquirer( data_dir=data_directory, cryptocompare=cryptocompare, coingecko=gecko, ) if ethereum_manager is not None: inquirer.inject_ethereum(ethereum_manager) uniswap_v2_oracle = UniswapV2Oracle(ethereum_manager) uniswap_v3_oracle = UniswapV3Oracle(ethereum_manager) saddle_oracle = SaddleOracle(ethereum_manager) Inquirer().add_defi_oracles( uniswap_v2=uniswap_v2_oracle, uniswap_v3=uniswap_v3_oracle, saddle=saddle_oracle, ) inquirer.set_oracles_order(current_price_oracles_order) if not should_mock_current_price_queries: return inquirer def mock_find_price( from_asset, to_asset, ignore_cache: bool = False, # pylint: disable=unused-argument ): return mocked_prices.get((from_asset, to_asset), FVal('1.5')) def mock_find_usd_price(asset, ignore_cache: bool = False): # pylint: disable=unused-argument return mocked_prices.get(asset, FVal('1.5')) if ignore_mocked_prices_for is None: inquirer.find_price = mock_find_price # type: ignore inquirer.find_usd_price = mock_find_usd_price # type: ignore else: def mock_some_prices(from_asset, to_asset, ignore_cache=False): if from_asset.symbol in ignore_mocked_prices_for: return inquirer.find_price_old(from_asset, to_asset, ignore_cache) return mock_find_price(from_asset, to_asset, ignore_cache) def mock_some_usd_prices(asset, ignore_cache=False): if asset.symbol in ignore_mocked_prices_for: return inquirer.find_usd_price_old(asset, ignore_cache) return mock_find_usd_price(asset, ignore_cache) inquirer.find_price_old = inquirer.find_price # type: ignore inquirer.find_usd_price_old = inquirer.find_usd_price # type: ignore inquirer.find_price = mock_some_prices # type: ignore inquirer.find_usd_price = mock_some_usd_prices # type: ignore def mock_query_fiat_pair(base, quote): # pylint: disable=unused-argument return FVal(1) inquirer._query_fiat_pair = mock_query_fiat_pair # type: ignore return inquirer
class Rotkehlchen(): def __init__(self, args: argparse.Namespace) -> None: """Initialize the Rotkehlchen object May Raise: - SystemPermissionError if the given data directory's permissions are not correct. """ self.lock = Semaphore() self.lock.acquire() # Can also be None after unlock if premium credentials did not # authenticate or premium server temporarily offline self.premium: Optional[Premium] = None self.user_is_logged_in: bool = False configure_logging(args) self.sleep_secs = args.sleep_secs if args.data_dir is None: self.data_dir = default_data_directory() else: self.data_dir = Path(args.data_dir) if not os.access(self.data_dir, os.W_OK | os.R_OK): raise SystemPermissionError( f'The given data directory {self.data_dir} is not readable or writable', ) self.args = args self.msg_aggregator = MessagesAggregator() self.greenlet_manager = GreenletManager( msg_aggregator=self.msg_aggregator) self.exchange_manager = ExchangeManager( msg_aggregator=self.msg_aggregator) self.data = DataHandler(self.data_dir, self.msg_aggregator) self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None) # Initialize the Inquirer singleton Inquirer(data_dir=self.data_dir, cryptocompare=self.cryptocompare) self.lock.release() self.shutdown_event = gevent.event.Event() def reset_after_failed_account_creation_or_login(self) -> None: """If the account creation or login failed make sure that the Rotki instance is clear Tricky instances are when after either failed premium credentials or user refusal to sync premium databases we relogged in. """ self.cryptocompare.db = None def unlock_user( self, user: str, password: str, create_new: bool, sync_approval: Literal['yes', 'no', 'unknown'], premium_credentials: Optional[PremiumCredentials], initial_settings: Optional[ModifiableDBSettings] = None, ) -> None: """Unlocks an existing user or creates a new one if `create_new` is True May raise: - PremiumAuthenticationError if the password can't unlock the database. - AuthenticationError if premium_credentials are given and are invalid or can't authenticate with the server - DBUpgradeError if the rotki DB version is newer than the software or there is a DB upgrade and there is an error. - SystemPermissionError if the directory or DB file can not be accessed """ log.info( 'Unlocking user', user=user, create_new=create_new, sync_approval=sync_approval, initial_settings=initial_settings, ) # unlock or create the DB self.password = password self.user_directory = self.data.unlock(user, password, create_new, initial_settings) self.data_importer = DataImporter(db=self.data.db) self.last_data_upload_ts = self.data.db.get_last_data_upload_ts() self.premium_sync_manager = PremiumSyncManager(data=self.data, password=password) # set the DB in the external services instances that need it self.cryptocompare.set_database(self.data.db) # Anything that was set above here has to be cleaned in case of failure in the next step # by reset_after_failed_account_creation_or_login() try: self.premium = self.premium_sync_manager.try_premium_at_start( given_premium_credentials=premium_credentials, username=user, create_new=create_new, sync_approval=sync_approval, ) except PremiumAuthenticationError: # Reraise it only if this is during the creation of a new account where # the premium credentials were given by the user if create_new: raise # else let's just continue. User signed in succesfully, but he just # has unauthenticable/invalid premium credentials remaining in his DB settings = self.get_settings() self.greenlet_manager.spawn_and_track( task_name='submit_usage_analytics', method=maybe_submit_usage_analytics, should_submit=settings.submit_usage_analytics, ) self.etherscan = Etherscan(database=self.data.db, msg_aggregator=self.msg_aggregator) historical_data_start = settings.historical_data_start eth_rpc_endpoint = settings.eth_rpc_endpoint # Initialize the price historian singleton PriceHistorian( data_directory=self.data_dir, history_date_start=historical_data_start, cryptocompare=self.cryptocompare, ) self.accountant = Accountant( db=self.data.db, user_directory=self.user_directory, msg_aggregator=self.msg_aggregator, create_csv=True, ) # Initialize the rotkehlchen logger LoggingSettings(anonymized_logs=settings.anonymized_logs) exchange_credentials = self.data.db.get_exchange_credentials() self.exchange_manager.initialize_exchanges( exchange_credentials=exchange_credentials, database=self.data.db, ) # Initialize blockchain querying modules ethereum_manager = EthereumManager( ethrpc_endpoint=eth_rpc_endpoint, etherscan=self.etherscan, msg_aggregator=self.msg_aggregator, greenlet_manager=self.greenlet_manager, connect_at_start=ETHEREUM_NODES_TO_CONNECT_AT_START, ) self.chain_manager = ChainManager( blockchain_accounts=self.data.db.get_blockchain_accounts(), ethereum_manager=ethereum_manager, msg_aggregator=self.msg_aggregator, database=self.data.db, greenlet_manager=self.greenlet_manager, premium=self.premium, eth_modules=settings.active_modules, ) self.ethereum_analyzer = EthereumAnalyzer( ethereum_manager=ethereum_manager, database=self.data.db, ) self.trades_historian = TradesHistorian( user_directory=self.user_directory, db=self.data.db, msg_aggregator=self.msg_aggregator, exchange_manager=self.exchange_manager, chain_manager=self.chain_manager, ) self.user_is_logged_in = True log.debug('User unlocking complete') def logout(self) -> None: if not self.user_is_logged_in: return user = self.data.username log.info( 'Logging out user', user=user, ) del self.chain_manager self.exchange_manager.delete_all_exchanges() # Reset rotkehlchen logger to default LoggingSettings(anonymized_logs=DEFAULT_ANONYMIZED_LOGS) del self.accountant del self.trades_historian del self.data_importer if self.premium is not None: del self.premium self.data.logout() self.password = '' self.cryptocompare.unset_database() # Make sure no messages leak to other user sessions self.msg_aggregator.consume_errors() self.msg_aggregator.consume_warnings() self.user_is_logged_in = False log.info( 'User successfully logged out', user=user, ) def set_premium_credentials(self, credentials: PremiumCredentials) -> None: """ Sets the premium credentials for Rotki Raises PremiumAuthenticationError if the given key is rejected by the Rotkehlchen server """ log.info('Setting new premium credentials') if self.premium is not None: self.premium.set_credentials(credentials) else: self.premium = premium_create_and_verify(credentials) self.data.db.set_rotkehlchen_premium(credentials) def delete_premium_credentials(self, name: str) -> Tuple[bool, str]: """Deletes the premium credentials for Rotki""" success: bool msg = '' if name != self.data.username: msg = f'Provided user "{name}" is not the logged in user' success = False success = self.data.db.del_rotkehlchen_premium() if success is False: msg = 'The database was unable to delete the Premium keys for the logged-in user' self.deactivate_premium_status() return success, msg def deactivate_premium_status(self) -> None: """Deactivate premium in the current session""" self.premium = None self.premium_sync_manager.premium = None self.chain_manager.deactivate_premium_status() def start(self) -> gevent.Greenlet: return gevent.spawn(self.main_loop) def main_loop(self) -> None: """Rotki main loop that fires often and manages many different tasks Each task remembers the last time it run sucesfully and know how often it should run. So each task manages itself. """ while self.shutdown_event.wait(MAIN_LOOP_SECS_DELAY) is not True: if self.user_is_logged_in: log.debug('Main loop start') self.premium_sync_manager.maybe_upload_data_to_server() self.ethereum_analyzer.analyze_ethereum_transactions() log.debug('Main loop end') def add_blockchain_accounts( self, blockchain: SupportedBlockchain, account_data: List[BlockchainAccountData], ) -> BlockchainBalancesUpdate: """Adds new blockchain accounts Adds the accounts to the blockchain instance and queries them to get the updated balances. Also adds them in the DB May raise: - EthSyncError from modify_blockchain_account - InputError if the given accounts list is empty. - TagConstraintError if any of the given account data contain unknown tags. - RemoteError if an external service such as Etherscan is queried and there is a problem with its query. """ self.data.db.ensure_tags_exist( given_data=account_data, action='adding', data_type='blockchain accounts', ) address_type = blockchain.get_address_type() updated_balances = self.chain_manager.add_blockchain_accounts( blockchain=blockchain, accounts=[address_type(entry.address) for entry in account_data], ) self.data.db.add_blockchain_accounts( blockchain=blockchain, account_data=account_data, ) return updated_balances def edit_blockchain_accounts( self, blockchain: SupportedBlockchain, account_data: List[BlockchainAccountData], ) -> None: """Edits blockchain accounts Edits blockchain account data for the given accounts May raise: - InputError if the given accounts list is empty or if any of the accounts to edit do not exist. - TagConstraintError if any of the given account data contain unknown tags. """ # First check for validity of account data addresses if len(account_data) == 0: raise InputError( 'Empty list of blockchain account data to edit was given') accounts = [x.address for x in account_data] unknown_accounts = set(accounts).difference( self.chain_manager.accounts.get(blockchain)) if len(unknown_accounts) != 0: raise InputError( f'Tried to edit unknown {blockchain.value} ' f'accounts {",".join(unknown_accounts)}', ) self.data.db.ensure_tags_exist( given_data=account_data, action='editing', data_type='blockchain accounts', ) # Finally edit the accounts self.data.db.edit_blockchain_accounts( blockchain=blockchain, account_data=account_data, ) return None def remove_blockchain_accounts( self, blockchain: SupportedBlockchain, accounts: ListOfBlockchainAddresses, ) -> BlockchainBalancesUpdate: """Removes blockchain accounts Removes the accounts from the blockchain instance and queries them to get the updated balances. Also removes them from the DB May raise: - RemoteError if an external service such as Etherscan is queried and there is a problem with its query. - InputError if a non-existing account was given to remove """ balances_update = self.chain_manager.remove_blockchain_accounts( blockchain=blockchain, accounts=accounts, ) self.data.db.remove_blockchain_accounts(blockchain, accounts) return balances_update def process_history( self, start_ts: Timestamp, end_ts: Timestamp, ) -> Tuple[Dict[str, Any], str]: ( error_or_empty, history, loan_history, asset_movements, eth_transactions, defi_events, ) = self.trades_historian.get_history( start_ts=start_ts, end_ts=end_ts, has_premium=self.premium is not None, ) result = self.accountant.process_history( start_ts=start_ts, end_ts=end_ts, trade_history=history, loan_history=loan_history, asset_movements=asset_movements, eth_transactions=eth_transactions, defi_events=defi_events, ) return result, error_or_empty def query_balances( self, requested_save_data: bool = False, timestamp: Timestamp = None, ignore_cache: bool = False, ) -> Dict[str, Any]: """Query all balances rotkehlchen can see. If requested_save_data is True then the data are always saved in the DB, if it is False then data are saved if self.data.should_save_balances() is True. If timestamp is None then the current timestamp is used. If a timestamp is given then that is the time that the balances are going to be saved in the DB If ignore_cache is True then all underlying calls that have a cache ignore it Returns a dictionary with the queried balances. """ log.info('query_balances called', requested_save_data=requested_save_data) balances = {} problem_free = True for _, exchange in self.exchange_manager.connected_exchanges.items(): exchange_balances, _ = exchange.query_balances( ignore_cache=ignore_cache) # If we got an error, disregard that exchange but make sure we don't save data if not isinstance(exchange_balances, dict): problem_free = False else: balances[exchange.name] = exchange_balances try: blockchain_result = self.chain_manager.query_balances( blockchain=None, force_token_detection=ignore_cache, ignore_cache=ignore_cache, ) balances['blockchain'] = { asset: balance.to_dict() for asset, balance in blockchain_result.totals.items() } except (RemoteError, EthSyncError) as e: problem_free = False log.error(f'Querying blockchain balances failed due to: {str(e)}') balances = account_for_manually_tracked_balances(db=self.data.db, balances=balances) combined = combine_stat_dicts([v for k, v in balances.items()]) total_usd_per_location = [(k, dict_get_sumof(v, 'usd_value')) for k, v in balances.items()] # calculate net usd value net_usd = FVal(0) for _, v in combined.items(): net_usd += FVal(v['usd_value']) stats: Dict[str, Any] = { 'location': {}, 'net_usd': net_usd, } for entry in total_usd_per_location: name = entry[0] total = entry[1] if net_usd != FVal(0): percentage = (total / net_usd).to_percentage() else: percentage = '0%' stats['location'][name] = { 'usd_value': total, 'percentage_of_net_value': percentage, } for k, v in combined.items(): if net_usd != FVal(0): percentage = (v['usd_value'] / net_usd).to_percentage() else: percentage = '0%' combined[k]['percentage_of_net_value'] = percentage result_dict = merge_dicts(combined, stats) allowed_to_save = requested_save_data or self.data.should_save_balances( ) if problem_free and allowed_to_save: if not timestamp: timestamp = Timestamp(int(time.time())) self.data.save_balances_data(data=result_dict, timestamp=timestamp) log.debug('query_balances data saved') else: log.debug( 'query_balances data not saved', allowed_to_save=allowed_to_save, problem_free=problem_free, ) # After adding it to the saved file we can overlay additional data that # is not required to be saved in the history file try: details = self.accountant.events.details for asset, (tax_free_amount, average_buy_value) in details.items(): if asset not in result_dict: continue result_dict[asset]['tax_free_amount'] = tax_free_amount result_dict[asset]['average_buy_value'] = average_buy_value current_price = result_dict[asset]['usd_value'] / result_dict[ asset]['amount'] if average_buy_value != FVal(0): result_dict[asset]['percent_change'] = ( ((current_price - average_buy_value) / average_buy_value) * 100) else: result_dict[asset]['percent_change'] = 'INF' except AttributeError: pass return result_dict def set_settings(self, settings: ModifiableDBSettings) -> Tuple[bool, str]: """Tries to set new settings. Returns True in success or False with message if error""" with self.lock: if settings.eth_rpc_endpoint is not None: result, msg = self.chain_manager.set_eth_rpc_endpoint( settings.eth_rpc_endpoint) if not result: return False, msg if settings.kraken_account_type is not None: kraken = self.exchange_manager.get('kraken') if kraken: kraken.set_account_type( settings.kraken_account_type) # type: ignore self.data.db.set_settings(settings) return True, '' def get_settings(self) -> DBSettings: """Returns the db settings with a check whether premium is active or not""" db_settings = self.data.db.get_settings( have_premium=self.premium is not None) return db_settings def setup_exchange( self, name: str, api_key: ApiKey, api_secret: ApiSecret, passphrase: Optional[str] = None, ) -> Tuple[bool, str]: """ Setup a new exchange with an api key and an api secret and optionally a passphrase By default the api keys are always validated unless validate is False. """ is_success, msg = self.exchange_manager.setup_exchange( name=name, api_key=api_key, api_secret=api_secret, database=self.data.db, passphrase=passphrase, ) if is_success: # Success, save the result in the DB self.data.db.add_exchange(name, api_key, api_secret, passphrase=passphrase) return is_success, msg def remove_exchange(self, name: str) -> Tuple[bool, str]: if not self.exchange_manager.has_exchange(name): return False, 'Exchange {} is not registered'.format(name) self.exchange_manager.delete_exchange(name) # Success, remove it also from the DB self.data.db.remove_exchange(name) self.data.db.delete_used_query_range_for_exchange(name) return True, '' def query_periodic_data(self) -> Dict[str, Union[bool, Timestamp]]: """Query for frequently changing data""" result: Dict[str, Union[bool, Timestamp]] = {} if self.user_is_logged_in: result[ 'last_balance_save'] = self.data.db.get_last_balance_save_time( ) result[ 'eth_node_connection'] = self.chain_manager.ethereum.web3_mapping.get( NodeName.OWN, None) is not None # noqa : E501 result[ 'history_process_start_ts'] = self.accountant.started_processing_timestamp result[ 'history_process_current_ts'] = self.accountant.currently_processing_timestamp return result def shutdown(self) -> None: self.logout() self.shutdown_event.set()
def fixture_cryptocompare(data_dir, database): return Cryptocompare(data_directory=data_dir, database=database)
class Rotkehlchen(): def __init__(self, args: argparse.Namespace) -> None: self.lock = Semaphore() self.lock.acquire() # Can also be None after unlock if premium credentials did not # authenticate or premium server temporarily offline self.premium: Optional[Premium] = None self.user_is_logged_in = False logfilename = None if args.logtarget == 'file': logfilename = args.logfile if args.loglevel == 'debug': loglevel = logging.DEBUG elif args.loglevel == 'info': loglevel = logging.INFO elif args.loglevel == 'warn': loglevel = logging.WARN elif args.loglevel == 'error': loglevel = logging.ERROR elif args.loglevel == 'critical': loglevel = logging.CRITICAL else: raise AssertionError('Should never get here. Illegal log value') logging.basicConfig( filename=logfilename, filemode='w', level=loglevel, format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s', datefmt='%d/%m/%Y %H:%M:%S %Z', ) if not args.logfromothermodules: logging.getLogger('urllib3').setLevel(logging.CRITICAL) logging.getLogger('urllib3.connectionpool').setLevel( logging.CRITICAL) self.sleep_secs = args.sleep_secs self.data_dir = args.data_dir self.args = args self.msg_aggregator = MessagesAggregator() self.greenlet_manager = GreenletManager( msg_aggregator=self.msg_aggregator) self.exchange_manager = ExchangeManager( msg_aggregator=self.msg_aggregator) self.all_eth_tokens = AssetResolver().get_all_eth_tokens() self.data = DataHandler(self.data_dir, self.msg_aggregator) self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None) # Initialize the Inquirer singleton Inquirer(data_dir=self.data_dir, cryptocompare=self.cryptocompare) self.lock.release() self.shutdown_event = gevent.event.Event() def reset_after_failed_account_creation_or_login(self) -> None: """If the account creation or login failed make sure that the Rotki instance is clear Tricky instances are when after either failed premium credentials or user refusal to sync premium databases we relogged in. """ self.cryptocompare.db = None def unlock_user( self, user: str, password: str, create_new: bool, sync_approval: str, premium_credentials: Optional[PremiumCredentials], ) -> None: """Unlocks an existing user or creates a new one if `create_new` is True May raise: - PremiumAuthenticationError if the password can't unlock the database. - AuthenticationError if premium_credentials are given and are invalid or can't authenticate with the server - DBUpgradeError if the rotki DB version is newer than the software or there is a DB upgrade and there is an error. """ log.info( 'Unlocking user', user=user, create_new=create_new, sync_approval=sync_approval, ) # unlock or create the DB self.password = password self.user_directory = self.data.unlock(user, password, create_new) self.data_importer = DataImporter(db=self.data.db) self.last_data_upload_ts = self.data.db.get_last_data_upload_ts() self.premium_sync_manager = PremiumSyncManager(data=self.data, password=password) # set the DB in the external services instances that need it self.cryptocompare.set_database(self.data.db) # Anything that was set above here has to be cleaned in case of failure in the next step # by reset_after_failed_account_creation_or_login() try: self.premium = self.premium_sync_manager.try_premium_at_start( given_premium_credentials=premium_credentials, username=user, create_new=create_new, sync_approval=sync_approval, ) except PremiumAuthenticationError: # Reraise it only if this is during the creation of a new account where # the premium credentials were given by the user if create_new: raise # else let's just continue. User signed in succesfully, but he just # has unauthenticable/invalid premium credentials remaining in his DB settings = self.get_settings() maybe_submit_usage_analytics(settings.submit_usage_analytics) self.etherscan = Etherscan(database=self.data.db, msg_aggregator=self.msg_aggregator) alethio = Alethio( database=self.data.db, msg_aggregator=self.msg_aggregator, all_eth_tokens=self.all_eth_tokens, ) historical_data_start = settings.historical_data_start eth_rpc_endpoint = settings.eth_rpc_endpoint # Initialize the price historian singleton PriceHistorian( data_directory=self.data_dir, history_date_start=historical_data_start, cryptocompare=self.cryptocompare, ) self.accountant = Accountant( db=self.data.db, user_directory=self.user_directory, msg_aggregator=self.msg_aggregator, create_csv=True, ) # Initialize the rotkehlchen logger LoggingSettings(anonymized_logs=settings.anonymized_logs) exchange_credentials = self.data.db.get_exchange_credentials() self.exchange_manager.initialize_exchanges( exchange_credentials=exchange_credentials, database=self.data.db, ) # Initialize blockchain querying modules ethchain = Ethchain( ethrpc_endpoint=eth_rpc_endpoint, etherscan=self.etherscan, msg_aggregator=self.msg_aggregator, ) makerdao = MakerDAO( ethchain=ethchain, database=self.data.db, msg_aggregator=self.msg_aggregator, ) self.chain_manager = ChainManager( blockchain_accounts=self.data.db.get_blockchain_accounts(), owned_eth_tokens=self.data.db.get_owned_tokens(), ethchain=ethchain, msg_aggregator=self.msg_aggregator, alethio=alethio, greenlet_manager=self.greenlet_manager, eth_modules={'makerdao': makerdao}, ) self.ethereum_analyzer = EthereumAnalyzer( ethchain=ethchain, database=self.data.db, ) self.trades_historian = TradesHistorian( user_directory=self.user_directory, db=self.data.db, msg_aggregator=self.msg_aggregator, exchange_manager=self.exchange_manager, chain_manager=self.chain_manager, ) self.user_is_logged_in = True def logout(self) -> None: if not self.user_is_logged_in: return user = self.data.username log.info( 'Logging out user', user=user, ) del self.chain_manager self.exchange_manager.delete_all_exchanges() # Reset rotkehlchen logger to default LoggingSettings(anonymized_logs=DEFAULT_ANONYMIZED_LOGS) del self.accountant del self.trades_historian del self.data_importer if self.premium is not None: del self.premium self.data.logout() self.password = '' self.cryptocompare.unset_database() # Make sure no messages leak to other user sessions self.msg_aggregator.consume_errors() self.msg_aggregator.consume_warnings() self.user_is_logged_in = False log.info( 'User successfully logged out', user=user, ) def set_premium_credentials(self, credentials: PremiumCredentials) -> None: """ Sets the premium credentials for Rotki Raises PremiumAuthenticationError if the given key is rejected by the Rotkehlchen server """ log.info('Setting new premium credentials') if self.premium is not None: self.premium.set_credentials(credentials) else: self.premium = premium_create_and_verify(credentials) self.data.db.set_rotkehlchen_premium(credentials) def start(self) -> gevent.Greenlet: return gevent.spawn(self.main_loop) def main_loop(self) -> None: """Rotki main loop that fires often and manages many different tasks Each task remembers the last time it run sucesfully and know how often it should run. So each task manages itself. """ while self.shutdown_event.wait(MAIN_LOOP_SECS_DELAY) is not True: if self.user_is_logged_in: log.debug('Main loop start') self.premium_sync_manager.maybe_upload_data_to_server() self.ethereum_analyzer.analyze_ethereum_transactions() log.debug('Main loop end') def add_blockchain_accounts( self, blockchain: SupportedBlockchain, account_data: List[BlockchainAccountData], ) -> BlockchainBalancesUpdate: """Adds new blockchain accounts Adds the accounts to the blockchain instance and queries them to get the updated balances. Also adds them in the DB May raise: - EthSyncError from modify_blockchain_account - InputError if the given accounts list is empty. - TagConstraintError if any of the given account data contain unknown tags. - RemoteError if an external service such as Etherscan is queried and there is a problem with its query. """ # First see if any of the given tags for the accounts do not exist in the DB existing_tags = self.data.db.get_tags() existing_tag_keys = existing_tags.keys() unknown_tags: Set[str] = set() for entry in account_data: if entry.tags is not None: unknown_tags.update( set(entry.tags).difference(existing_tag_keys)) if len(unknown_tags) != 0: raise TagConstraintError( f'When adding blockchain accounts, unknown tags ' f'{", ".join(unknown_tags)} were found', ) updated_balances = self.chain_manager.add_blockchain_accounts( blockchain=blockchain, accounts=[entry.address for entry in account_data], ) self.data.db.add_blockchain_accounts( blockchain=blockchain, account_data=account_data, ) return updated_balances def edit_blockchain_accounts( self, blockchain: SupportedBlockchain, account_data: List[BlockchainAccountData], ) -> None: """Edits blockchain accounts Edits blockchain account data for the given accounts May raise: - InputError if the given accounts list is empty or if any of the accounts to edit do not exist. - TagConstraintError if any of the given account data contain unknown tags. """ # First check for validity of account data addresses if len(account_data) == 0: raise InputError( 'Empty list of blockchain account data to edit was given') accounts = [x.address for x in account_data] unknown_accounts = set(accounts).difference( self.chain_manager.accounts.get(blockchain)) if len(unknown_accounts) != 0: raise InputError( f'Tried to edit unknown {blockchain.value} ' f'accounts {",".join(unknown_accounts)}', ) # Then See if any of the given tags for the accounts do not exist in the DB existing_tags = self.data.db.get_tags() existing_tag_keys = existing_tags.keys() unknown_tags: Set[str] = set() for entry in account_data: if entry.tags is not None: unknown_tags.update( set(entry.tags).difference(existing_tag_keys)) if len(unknown_tags) != 0: raise TagConstraintError( f'When editing blockchain accounts, unknown tags ' f'{", ".join(unknown_tags)} were found', ) # Finally edit the accounts self.data.db.edit_blockchain_accounts( blockchain=blockchain, account_data=account_data, ) return None def remove_blockchain_accounts( self, blockchain: SupportedBlockchain, accounts: ListOfBlockchainAddresses, ) -> BlockchainBalancesUpdate: """Removes blockchain accounts Removes the accounts from the blockchain instance and queries them to get the updated balances. Also removes them from the DB May raise: - RemoteError if an external service such as Etherscan is queried and there is a problem with its query. - InputError if a non-existing account was given to remove """ balances_update = self.chain_manager.remove_blockchain_accounts( blockchain=blockchain, accounts=accounts, ) self.data.db.remove_blockchain_accounts(blockchain, accounts) return balances_update def add_owned_eth_tokens( self, tokens: List[EthereumToken], ) -> BlockchainBalancesUpdate: """Adds tokens to the blockchain state and updates balance of all accounts May raise: - InputError if some of the tokens already exist - RemoteError if an external service such as Etherscan is queried and there is a problem with its query. - EthSyncError if querying the token balances through a provided ethereum client and the chain is not synced """ new_data = self.chain_manager.track_new_tokens(tokens) self.data.write_owned_eth_tokens(self.chain_manager.owned_eth_tokens) return new_data def remove_owned_eth_tokens( self, tokens: List[EthereumToken], ) -> BlockchainBalancesUpdate: """ Removes tokens from the state and stops their balance from being tracked for each account May raise: - RemoteError if an external service such as Etherscan is queried and there is a problem with its query. - EthSyncError if querying the token balances through a provided ethereum client and the chain is not synced """ new_data = self.chain_manager.remove_eth_tokens(tokens) self.data.write_owned_eth_tokens(self.chain_manager.owned_eth_tokens) return new_data def process_history( self, start_ts: Timestamp, end_ts: Timestamp, ) -> Tuple[Dict[str, Any], str]: ( error_or_empty, history, loan_history, asset_movements, eth_transactions, ) = self.trades_historian.get_history( start_ts=start_ts, end_ts=end_ts, has_premium=self.premium is not None, ) result = self.accountant.process_history( start_ts=start_ts, end_ts=end_ts, trade_history=history, loan_history=loan_history, asset_movements=asset_movements, eth_transactions=eth_transactions, ) return result, error_or_empty def query_fiat_balances(self) -> Dict[Asset, Dict[str, FVal]]: result = {} balances = self.data.get_fiat_balances() for currency, str_amount in balances.items(): amount = FVal(str_amount) usd_rate = Inquirer().query_fiat_pair(currency, A_USD) result[currency] = { 'amount': amount, 'usd_value': amount * usd_rate, } return result def query_balances( self, requested_save_data: bool = True, timestamp: Timestamp = None, ignore_cache: bool = False, ) -> Dict[str, Any]: """Query all balances rotkehlchen can see. If requested_save_data is True then the data are saved in the DB. If timestamp is None then the current timestamp is used. If a timestamp is given then that is the time that the balances are going to be saved in the DB If ignore_cache is True then all underlying calls that have a cache ignore it Returns a dictionary with the queried balances. """ log.info('query_balances called', requested_save_data=requested_save_data) balances = {} problem_free = True for _, exchange in self.exchange_manager.connected_exchanges.items(): exchange_balances, _ = exchange.query_balances( ignore_cache=ignore_cache) # If we got an error, disregard that exchange but make sure we don't save data if not isinstance(exchange_balances, dict): problem_free = False else: balances[exchange.name] = exchange_balances try: blockchain_result = self.chain_manager.query_balances( blockchain=None, ignore_cache=ignore_cache, ) balances['blockchain'] = { asset: balance.to_dict() for asset, balance in blockchain_result.totals.items() } except (RemoteError, EthSyncError) as e: problem_free = False log.error(f'Querying blockchain balances failed due to: {str(e)}') result = self.query_fiat_balances() if result != {}: balances['banks'] = result combined = combine_stat_dicts([v for k, v in balances.items()]) total_usd_per_location = [(k, dict_get_sumof(v, 'usd_value')) for k, v in balances.items()] # calculate net usd value net_usd = FVal(0) for _, v in combined.items(): net_usd += FVal(v['usd_value']) stats: Dict[str, Any] = { 'location': {}, 'net_usd': net_usd, } for entry in total_usd_per_location: name = entry[0] total = entry[1] if net_usd != FVal(0): percentage = (total / net_usd).to_percentage() else: percentage = '0%' stats['location'][name] = { 'usd_value': total, 'percentage_of_net_value': percentage, } for k, v in combined.items(): if net_usd != FVal(0): percentage = (v['usd_value'] / net_usd).to_percentage() else: percentage = '0%' combined[k]['percentage_of_net_value'] = percentage result_dict = merge_dicts(combined, stats) allowed_to_save = requested_save_data and self.data.should_save_balances( ) if problem_free and allowed_to_save: if not timestamp: timestamp = Timestamp(int(time.time())) self.data.save_balances_data(data=result_dict, timestamp=timestamp) log.debug('query_balances data saved') else: log.debug( 'query_balances data not saved', allowed_to_save=allowed_to_save, problem_free=problem_free, ) # After adding it to the saved file we can overlay additional data that # is not required to be saved in the history file try: details = self.accountant.events.details for asset, (tax_free_amount, average_buy_value) in details.items(): if asset not in result_dict: continue result_dict[asset]['tax_free_amount'] = tax_free_amount result_dict[asset]['average_buy_value'] = average_buy_value current_price = result_dict[asset]['usd_value'] / result_dict[ asset]['amount'] if average_buy_value != FVal(0): result_dict[asset]['percent_change'] = ( ((current_price - average_buy_value) / average_buy_value) * 100) else: result_dict[asset]['percent_change'] = 'INF' except AttributeError: pass return result_dict def set_settings(self, settings: ModifiableDBSettings) -> Tuple[bool, str]: """Tries to set new settings. Returns True in success or False with message if error""" with self.lock: if settings.eth_rpc_endpoint is not None: result, msg = self.chain_manager.set_eth_rpc_endpoint( settings.eth_rpc_endpoint) if not result: return False, msg self.data.db.set_settings(settings) return True, '' def get_settings(self) -> DBSettings: """Returns the db settings with a check whether premium is active or not""" db_settings = self.data.db.get_settings( have_premium=self.premium is not None) return db_settings def setup_exchange( self, name: str, api_key: ApiKey, api_secret: ApiSecret, passphrase: Optional[str] = None, ) -> Tuple[bool, str]: """ Setup a new exchange with an api key and an api secret By default the api keys are always validated unless validate is False. """ is_success, msg = self.exchange_manager.setup_exchange( name=name, api_key=api_key, api_secret=api_secret, database=self.data.db, passphrase=passphrase, ) if is_success: # Success, save the result in the DB self.data.db.add_exchange(name, api_key, api_secret, passphrase=passphrase) return is_success, msg def remove_exchange(self, name: str) -> Tuple[bool, str]: if not self.exchange_manager.has_exchange(name): return False, 'Exchange {} is not registered'.format(name) self.exchange_manager.delete_exchange(name) # Success, remove it also from the DB self.data.db.remove_exchange(name) self.data.db.delete_used_query_range_for_exchange(name) return True, '' def query_periodic_data(self) -> Dict[str, Union[bool, Timestamp]]: """Query for frequently changing data""" result: Dict[str, Union[bool, Timestamp]] = {} if self.user_is_logged_in: result[ 'last_balance_save'] = self.data.db.get_last_balance_save_time( ) result[ 'eth_node_connection'] = self.chain_manager.ethchain.connected result[ 'history_process_start_ts'] = self.accountant.started_processing_timestamp result[ 'history_process_current_ts'] = self.accountant.currently_processing_timestamp return result def shutdown(self) -> None: self.logout() self.shutdown_event.set()
def fixture_session_cryptocompare(session_data_dir, session_database): return Cryptocompare(data_directory=session_data_dir, database=session_database)
class Rotkehlchen(): def __init__(self, args: argparse.Namespace) -> None: """Initialize the Rotkehlchen object This runs during backend initialization so it should be as light as possible. May Raise: - SystemPermissionError if the given data directory's permissions are not correct. """ # Can also be None after unlock if premium credentials did not # authenticate or premium server temporarily offline self.premium: Optional[Premium] = None self.user_is_logged_in: bool = False configure_logging(args) self.sleep_secs = args.sleep_secs if args.data_dir is None: self.data_dir = default_data_directory() else: self.data_dir = Path(args.data_dir) self.data_dir.mkdir(parents=True, exist_ok=True) if not os.access(self.data_dir, os.W_OK | os.R_OK): raise SystemPermissionError( f'The given data directory {self.data_dir} is not readable or writable', ) self.main_loop_spawned = False self.args = args self.api_task_greenlets: List[gevent.Greenlet] = [] self.msg_aggregator = MessagesAggregator() self.greenlet_manager = GreenletManager(msg_aggregator=self.msg_aggregator) self.rotki_notifier = RotkiNotifier(greenlet_manager=self.greenlet_manager) self.msg_aggregator.rotki_notifier = self.rotki_notifier self.exchange_manager = ExchangeManager(msg_aggregator=self.msg_aggregator) # Initialize the GlobalDBHandler singleton. Has to be initialized BEFORE asset resolver GlobalDBHandler(data_dir=self.data_dir) self.data = DataHandler(self.data_dir, self.msg_aggregator) self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None) self.coingecko = Coingecko() self.icon_manager = IconManager(data_dir=self.data_dir, coingecko=self.coingecko) self.assets_updater = AssetsUpdater(self.msg_aggregator) # Initialize the Inquirer singleton Inquirer( data_dir=self.data_dir, cryptocompare=self.cryptocompare, coingecko=self.coingecko, ) self.task_manager: Optional[TaskManager] = None self.shutdown_event = gevent.event.Event() def reset_after_failed_account_creation_or_login(self) -> None: """If the account creation or login failed make sure that the rotki instance is clear Tricky instances are when after either failed premium credentials or user refusal to sync premium databases we relogged in """ self.cryptocompare.db = None def unlock_user( self, user: str, password: str, create_new: bool, sync_approval: Literal['yes', 'no', 'unknown'], premium_credentials: Optional[PremiumCredentials], initial_settings: Optional[ModifiableDBSettings] = None, sync_database: bool = True, ) -> None: """Unlocks an existing user or creates a new one if `create_new` is True May raise: - PremiumAuthenticationError if the password can't unlock the database. - AuthenticationError if premium_credentials are given and are invalid or can't authenticate with the server - DBUpgradeError if the rotki DB version is newer than the software or there is a DB upgrade and there is an error. - SystemPermissionError if the directory or DB file can not be accessed """ log.info( 'Unlocking user', user=user, create_new=create_new, sync_approval=sync_approval, sync_database=sync_database, initial_settings=initial_settings, ) # unlock or create the DB self.password = password self.user_directory = self.data.unlock(user, password, create_new, initial_settings) self.data_importer = DataImporter(db=self.data.db) self.last_data_upload_ts = self.data.db.get_last_data_upload_ts() self.premium_sync_manager = PremiumSyncManager(data=self.data, password=password) # set the DB in the external services instances that need it self.cryptocompare.set_database(self.data.db) # Anything that was set above here has to be cleaned in case of failure in the next step # by reset_after_failed_account_creation_or_login() try: self.premium = self.premium_sync_manager.try_premium_at_start( given_premium_credentials=premium_credentials, username=user, create_new=create_new, sync_approval=sync_approval, sync_database=sync_database, ) except PremiumAuthenticationError: # Reraise it only if this is during the creation of a new account where # the premium credentials were given by the user if create_new: raise self.msg_aggregator.add_warning( 'Could not authenticate the rotki premium API keys found in the DB.' ' Has your subscription expired?', ) # else let's just continue. User signed in succesfully, but he just # has unauthenticable/invalid premium credentials remaining in his DB settings = self.get_settings() self.greenlet_manager.spawn_and_track( after_seconds=None, task_name='submit_usage_analytics', exception_is_error=False, method=maybe_submit_usage_analytics, data_dir=self.data_dir, should_submit=settings.submit_usage_analytics, ) self.etherscan = Etherscan(database=self.data.db, msg_aggregator=self.msg_aggregator) self.beaconchain = BeaconChain(database=self.data.db, msg_aggregator=self.msg_aggregator) eth_rpc_endpoint = settings.eth_rpc_endpoint # Initialize the price historian singleton PriceHistorian( data_directory=self.data_dir, cryptocompare=self.cryptocompare, coingecko=self.coingecko, ) PriceHistorian().set_oracles_order(settings.historical_price_oracles) self.accountant = Accountant( db=self.data.db, user_directory=self.user_directory, msg_aggregator=self.msg_aggregator, create_csv=True, premium=self.premium, ) exchange_credentials = self.data.db.get_exchange_credentials() self.exchange_manager.initialize_exchanges( exchange_credentials=exchange_credentials, database=self.data.db, ) # Initialize blockchain querying modules ethereum_manager = EthereumManager( ethrpc_endpoint=eth_rpc_endpoint, etherscan=self.etherscan, msg_aggregator=self.msg_aggregator, greenlet_manager=self.greenlet_manager, connect_at_start=ETHEREUM_NODES_TO_CONNECT_AT_START, ) kusama_manager = SubstrateManager( chain=SubstrateChain.KUSAMA, msg_aggregator=self.msg_aggregator, greenlet_manager=self.greenlet_manager, connect_at_start=KUSAMA_NODES_TO_CONNECT_AT_START, connect_on_startup=self._connect_ksm_manager_on_startup(), own_rpc_endpoint=settings.ksm_rpc_endpoint, ) polkadot_manager = SubstrateManager( chain=SubstrateChain.POLKADOT, msg_aggregator=self.msg_aggregator, greenlet_manager=self.greenlet_manager, connect_at_start=POLKADOT_NODES_TO_CONNECT_AT_START, connect_on_startup=self._connect_dot_manager_on_startup(), own_rpc_endpoint=settings.dot_rpc_endpoint, ) self.covalent_avalanche = Covalent( database=self.data.db, msg_aggregator=self.msg_aggregator, chain_id=chains_id['avalanche'], ) avalanche_manager = AvalancheManager( avaxrpc_endpoint="https://api.avax.network/ext/bc/C/rpc", covalent=self.covalent_avalanche, msg_aggregator=self.msg_aggregator, ) Inquirer().inject_ethereum(ethereum_manager) Inquirer().set_oracles_order(settings.current_price_oracles) self.chain_manager = ChainManager( blockchain_accounts=self.data.db.get_blockchain_accounts(), ethereum_manager=ethereum_manager, kusama_manager=kusama_manager, polkadot_manager=polkadot_manager, avalanche_manager=avalanche_manager, msg_aggregator=self.msg_aggregator, database=self.data.db, greenlet_manager=self.greenlet_manager, premium=self.premium, eth_modules=settings.active_modules, data_directory=self.data_dir, beaconchain=self.beaconchain, btc_derivation_gap_limit=settings.btc_derivation_gap_limit, ) self.events_historian = EventsHistorian( user_directory=self.user_directory, db=self.data.db, msg_aggregator=self.msg_aggregator, exchange_manager=self.exchange_manager, chain_manager=self.chain_manager, ) self.task_manager = TaskManager( max_tasks_num=DEFAULT_MAX_TASKS_NUM, greenlet_manager=self.greenlet_manager, api_task_greenlets=self.api_task_greenlets, database=self.data.db, cryptocompare=self.cryptocompare, premium_sync_manager=self.premium_sync_manager, chain_manager=self.chain_manager, exchange_manager=self.exchange_manager, ) DataMigrationManager(self).maybe_migrate_data() self.greenlet_manager.spawn_and_track( after_seconds=5, task_name='periodically_query_icons_until_all_cached', exception_is_error=False, method=self.icon_manager.periodically_query_icons_until_all_cached, batch_size=ICONS_BATCH_SIZE, sleep_time_secs=ICONS_QUERY_SLEEP, ) self.user_is_logged_in = True log.debug('User unlocking complete') def _logout(self) -> None: if not self.user_is_logged_in: return user = self.data.username log.info( 'Logging out user', user=user, ) self.deactivate_premium_status() self.greenlet_manager.clear() del self.chain_manager self.exchange_manager.delete_all_exchanges() del self.accountant del self.events_historian del self.data_importer self.data.logout() self.password = '' self.cryptocompare.unset_database() # Make sure no messages leak to other user sessions self.msg_aggregator.consume_errors() self.msg_aggregator.consume_warnings() self.task_manager = None self.user_is_logged_in = False log.info( 'User successfully logged out', user=user, ) def logout(self) -> None: if self.task_manager is None: # no user logged in? return with self.task_manager.schedule_lock: self._logout() def set_premium_credentials(self, credentials: PremiumCredentials) -> None: """ Sets the premium credentials for rotki Raises PremiumAuthenticationError if the given key is rejected by the Rotkehlchen server """ log.info('Setting new premium credentials') if self.premium is not None: self.premium.set_credentials(credentials) else: self.premium = premium_create_and_verify(credentials) self.premium_sync_manager.premium = self.premium self.accountant.activate_premium_status(self.premium) self.chain_manager.activate_premium_status(self.premium) self.data.db.set_rotkehlchen_premium(credentials) def deactivate_premium_status(self) -> None: """Deactivate premium in the current session""" self.premium = None self.premium_sync_manager.premium = None self.accountant.deactivate_premium_status() self.chain_manager.deactivate_premium_status() def delete_premium_credentials(self) -> Tuple[bool, str]: """Deletes the premium credentials for rotki""" msg = '' success = self.data.db.del_rotkehlchen_premium() if success is False: msg = 'The database was unable to delete the Premium keys for the logged-in user' self.deactivate_premium_status() return success, msg def start(self) -> gevent.Greenlet: assert not self.main_loop_spawned, 'Tried to spawn the main loop twice' greenlet = gevent.spawn(self.main_loop) self.main_loop_spawned = True return greenlet def main_loop(self) -> None: """rotki main loop that fires often and runs the task manager's scheduler""" while self.shutdown_event.wait(timeout=MAIN_LOOP_SECS_DELAY) is not True: if self.task_manager is not None: self.task_manager.schedule() def get_blockchain_account_data( self, blockchain: SupportedBlockchain, ) -> Union[List[BlockchainAccountData], Dict[str, Any]]: account_data = self.data.db.get_blockchain_account_data(blockchain) if blockchain != SupportedBlockchain.BITCOIN: return account_data xpub_data = self.data.db.get_bitcoin_xpub_data() addresses_to_account_data = {x.address: x for x in account_data} address_to_xpub_mappings = self.data.db.get_addresses_to_xpub_mapping( list(addresses_to_account_data.keys()), # type: ignore ) xpub_mappings: Dict['XpubData', List[BlockchainAccountData]] = {} for address, xpub_entry in address_to_xpub_mappings.items(): if xpub_entry not in xpub_mappings: xpub_mappings[xpub_entry] = [] xpub_mappings[xpub_entry].append(addresses_to_account_data[address]) data: Dict[str, Any] = {'standalone': [], 'xpubs': []} # Add xpub data for xpub_entry in xpub_data: data_entry = xpub_entry.serialize() addresses = xpub_mappings.get(xpub_entry, None) data_entry['addresses'] = addresses if addresses and len(addresses) != 0 else None data['xpubs'].append(data_entry) # Add standalone addresses for account in account_data: if account.address not in address_to_xpub_mappings: data['standalone'].append(account) return data def add_blockchain_accounts( self, blockchain: SupportedBlockchain, account_data: List[BlockchainAccountData], ) -> BlockchainBalancesUpdate: """Adds new blockchain accounts Adds the accounts to the blockchain instance and queries them to get the updated balances. Also adds them in the DB May raise: - EthSyncError from modify_blockchain_account - InputError if the given accounts list is empty. - TagConstraintError if any of the given account data contain unknown tags. - RemoteError if an external service such as Etherscan is queried and there is a problem with its query. """ self.data.db.ensure_tags_exist( given_data=account_data, action='adding', data_type='blockchain accounts', ) address_type = blockchain.get_address_type() updated_balances = self.chain_manager.add_blockchain_accounts( blockchain=blockchain, accounts=[address_type(entry.address) for entry in account_data], ) self.data.db.add_blockchain_accounts( blockchain=blockchain, account_data=account_data, ) return updated_balances def edit_blockchain_accounts( self, blockchain: SupportedBlockchain, account_data: List[BlockchainAccountData], ) -> None: """Edits blockchain accounts Edits blockchain account data for the given accounts May raise: - InputError if the given accounts list is empty or if any of the accounts to edit do not exist. - TagConstraintError if any of the given account data contain unknown tags. """ # First check for validity of account data addresses if len(account_data) == 0: raise InputError('Empty list of blockchain account data to edit was given') accounts = [x.address for x in account_data] unknown_accounts = set(accounts).difference(self.chain_manager.accounts.get(blockchain)) if len(unknown_accounts) != 0: raise InputError( f'Tried to edit unknown {blockchain.value} ' f'accounts {",".join(unknown_accounts)}', ) self.data.db.ensure_tags_exist( given_data=account_data, action='editing', data_type='blockchain accounts', ) # Finally edit the accounts self.data.db.edit_blockchain_accounts( blockchain=blockchain, account_data=account_data, ) def remove_blockchain_accounts( self, blockchain: SupportedBlockchain, accounts: ListOfBlockchainAddresses, ) -> BlockchainBalancesUpdate: """Removes blockchain accounts Removes the accounts from the blockchain instance and queries them to get the updated balances. Also removes them from the DB May raise: - RemoteError if an external service such as Etherscan is queried and there is a problem with its query. - InputError if a non-existing account was given to remove """ balances_update = self.chain_manager.remove_blockchain_accounts( blockchain=blockchain, accounts=accounts, ) self.data.db.remove_blockchain_accounts(blockchain, accounts) return balances_update def get_history_query_status(self) -> Dict[str, str]: if self.events_historian.progress < FVal('100'): processing_state = self.events_historian.processing_state_name progress = self.events_historian.progress / 2 elif self.accountant.first_processed_timestamp == -1: processing_state = 'Processing all retrieved historical events' progress = FVal(50) else: processing_state = 'Processing all retrieved historical events' # start_ts is min of the query start or the first action timestamp since action # processing can start well before query start to calculate cost basis start_ts = min( self.accountant.events.query_start_ts, self.accountant.first_processed_timestamp, ) diff = self.accountant.events.query_end_ts - start_ts progress = 50 + 100 * ( FVal(self.accountant.currently_processing_timestamp - start_ts) / FVal(diff) / 2) return {'processing_state': str(processing_state), 'total_progress': str(progress)} def process_history( self, start_ts: Timestamp, end_ts: Timestamp, ) -> Tuple[int, str]: ( error_or_empty, history, loan_history, asset_movements, eth_transactions, defi_events, ledger_actions, ) = self.events_historian.get_history( start_ts=start_ts, end_ts=end_ts, has_premium=self.premium is not None, ) report_id = self.accountant.process_history( start_ts=start_ts, end_ts=end_ts, trade_history=history, loan_history=loan_history, asset_movements=asset_movements, eth_transactions=eth_transactions, defi_events=defi_events, ledger_actions=ledger_actions, ) return report_id, error_or_empty def query_balances( self, requested_save_data: bool = False, save_despite_errors: bool = False, timestamp: Timestamp = None, ignore_cache: bool = False, ) -> Dict[str, Any]: """Query all balances rotkehlchen can see. If requested_save_data is True then the data are always saved in the DB, if it is False then data are saved if self.data.should_save_balances() is True. If save_despite_errors is True then even if there is any error the snapshot will be saved. If timestamp is None then the current timestamp is used. If a timestamp is given then that is the time that the balances are going to be saved in the DB If ignore_cache is True then all underlying calls that have a cache ignore it Returns a dictionary with the queried balances. """ log.info( 'query_balances called', requested_save_data=requested_save_data, save_despite_errors=save_despite_errors, ) balances: Dict[str, Dict[Asset, Balance]] = {} problem_free = True for exchange in self.exchange_manager.iterate_exchanges(): exchange_balances, error_msg = exchange.query_balances(ignore_cache=ignore_cache) # If we got an error, disregard that exchange but make sure we don't save data if not isinstance(exchange_balances, dict): problem_free = False self.msg_aggregator.add_message( message_type=WSMessageType.BALANCE_SNAPSHOT_ERROR, data={'location': exchange.name, 'error': error_msg}, ) else: location_str = str(exchange.location) if location_str not in balances: balances[location_str] = exchange_balances else: # multiple exchange of same type. Combine balances balances[location_str] = combine_dicts( balances[location_str], exchange_balances, ) liabilities: Dict[Asset, Balance] try: blockchain_result = self.chain_manager.query_balances( blockchain=None, force_token_detection=ignore_cache, ignore_cache=ignore_cache, ) if len(blockchain_result.totals.assets) != 0: balances[str(Location.BLOCKCHAIN)] = blockchain_result.totals.assets liabilities = blockchain_result.totals.liabilities except (RemoteError, EthSyncError) as e: problem_free = False liabilities = {} log.error(f'Querying blockchain balances failed due to: {str(e)}') self.msg_aggregator.add_message( message_type=WSMessageType.BALANCE_SNAPSHOT_ERROR, data={'location': 'blockchain balances query', 'error': str(e)}, ) manually_tracked_liabilities = get_manually_tracked_balances( db=self.data.db, balance_type=BalanceType.LIABILITY, ) manual_liabilities_as_dict: DefaultDict[Asset, Balance] = defaultdict(Balance) for manual_liability in manually_tracked_liabilities: manual_liabilities_as_dict[manual_liability.asset] += manual_liability.value liabilities = combine_dicts(liabilities, manual_liabilities_as_dict) # retrieve loopring balances if module is activated if self.chain_manager.get_module('loopring'): try: loopring_balances = self.chain_manager.get_loopring_balances() except RemoteError as e: problem_free = False self.msg_aggregator.add_message( message_type=WSMessageType.BALANCE_SNAPSHOT_ERROR, data={'location': 'loopring', 'error': str(e)}, ) else: if len(loopring_balances) != 0: balances[str(Location.LOOPRING)] = loopring_balances # retrieve nft balances if module is activated nfts = self.chain_manager.get_module('nfts') if nfts is not None: try: nft_mapping = nfts.get_balances( addresses=self.chain_manager.queried_addresses_for_module('nfts'), return_zero_values=False, ignore_cache=False, ) except RemoteError as e: problem_free = False self.msg_aggregator.add_message( message_type=WSMessageType.BALANCE_SNAPSHOT_ERROR, data={'location': 'nfts', 'error': str(e)}, ) else: if len(nft_mapping) != 0: if str(Location.BLOCKCHAIN) not in balances: balances[str(Location.BLOCKCHAIN)] = {} for _, nft_balances in nft_mapping.items(): for balance_entry in nft_balances: balances[str(Location.BLOCKCHAIN)][Asset( balance_entry['id'])] = Balance( amount=FVal(1), usd_value=balance_entry['usd_price'], ) balances = account_for_manually_tracked_asset_balances(db=self.data.db, balances=balances) # Calculate usd totals assets_total_balance: DefaultDict[Asset, Balance] = defaultdict(Balance) total_usd_per_location: Dict[str, FVal] = {} for location, asset_balance in balances.items(): total_usd_per_location[location] = ZERO for asset, balance in asset_balance.items(): assets_total_balance[asset] += balance total_usd_per_location[location] += balance.usd_value net_usd = sum((balance.usd_value for _, balance in assets_total_balance.items()), ZERO) liabilities_total_usd = sum((liability.usd_value for _, liability in liabilities.items()), ZERO) # noqa: E501 net_usd -= liabilities_total_usd # Calculate location stats location_stats: Dict[str, Any] = {} for location, total_usd in total_usd_per_location.items(): if location == str(Location.BLOCKCHAIN): total_usd -= liabilities_total_usd percentage = (total_usd / net_usd).to_percentage() if net_usd != ZERO else '0%' location_stats[location] = { 'usd_value': total_usd, 'percentage_of_net_value': percentage, } # Calculate 'percentage_of_net_value' per asset assets_total_balance_as_dict: Dict[Asset, Dict[str, Any]] = { asset: balance.to_dict() for asset, balance in assets_total_balance.items() } liabilities_as_dict: Dict[Asset, Dict[str, Any]] = { asset: balance.to_dict() for asset, balance in liabilities.items() } for asset, balance_dict in assets_total_balance_as_dict.items(): percentage = (balance_dict['usd_value'] / net_usd).to_percentage() if net_usd != ZERO else '0%' # noqa: E501 assets_total_balance_as_dict[asset]['percentage_of_net_value'] = percentage for asset, balance_dict in liabilities_as_dict.items(): percentage = (balance_dict['usd_value'] / net_usd).to_percentage() if net_usd != ZERO else '0%' # noqa: E501 liabilities_as_dict[asset]['percentage_of_net_value'] = percentage # Compose balances response result_dict = { 'assets': assets_total_balance_as_dict, 'liabilities': liabilities_as_dict, 'location': location_stats, 'net_usd': net_usd, } allowed_to_save = requested_save_data or self.data.should_save_balances() if (problem_free or save_despite_errors) and allowed_to_save: if not timestamp: timestamp = Timestamp(int(time.time())) self.data.db.save_balances_data(data=result_dict, timestamp=timestamp) log.debug('query_balances data saved') else: log.debug( 'query_balances data not saved', allowed_to_save=allowed_to_save, problem_free=problem_free, save_despite_errors=save_despite_errors, ) return result_dict def set_settings(self, settings: ModifiableDBSettings) -> Tuple[bool, str]: """Tries to set new settings. Returns True in success or False with message if error""" if settings.eth_rpc_endpoint is not None: result, msg = self.chain_manager.set_eth_rpc_endpoint(settings.eth_rpc_endpoint) if not result: return False, msg if settings.ksm_rpc_endpoint is not None: result, msg = self.chain_manager.set_ksm_rpc_endpoint(settings.ksm_rpc_endpoint) if not result: return False, msg if settings.dot_rpc_endpoint is not None: result, msg = self.chain_manager.set_dot_rpc_endpoint(settings.dot_rpc_endpoint) if not result: return False, msg if settings.btc_derivation_gap_limit is not None: self.chain_manager.btc_derivation_gap_limit = settings.btc_derivation_gap_limit if settings.current_price_oracles is not None: Inquirer().set_oracles_order(settings.current_price_oracles) if settings.historical_price_oracles is not None: PriceHistorian().set_oracles_order(settings.historical_price_oracles) if settings.active_modules is not None: self.chain_manager.process_new_modules_list(settings.active_modules) self.data.db.set_settings(settings) return True, '' def get_settings(self) -> DBSettings: """Returns the db settings with a check whether premium is active or not""" db_settings = self.data.db.get_settings(have_premium=self.premium is not None) return db_settings def setup_exchange( self, name: str, location: Location, api_key: ApiKey, api_secret: ApiSecret, passphrase: Optional[str] = None, kraken_account_type: Optional['KrakenAccountType'] = None, PAIRS: Optional[List[str]] = None, # noqa: N803 ftx_subaccount: Optional[str] = None, ) -> Tuple[bool, str]: """ Setup a new exchange with an api key and an api secret and optionally a passphrase """ is_success, msg = self.exchange_manager.setup_exchange( name=name, location=location, api_key=api_key, api_secret=api_secret, database=self.data.db, passphrase=passphrase, ftx_subaccount=ftx_subaccount, PAIRS=PAIRS, ) if is_success: # Success, save the result in the DB self.data.db.add_exchange( name=name, location=location, api_key=api_key, api_secret=api_secret, passphrase=passphrase, kraken_account_type=kraken_account_type, PAIRS=PAIRS, ftx_subaccount=ftx_subaccount, ) return is_success, msg def remove_exchange(self, name: str, location: Location) -> Tuple[bool, str]: if self.exchange_manager.get_exchange(name=name, location=location) is None: return False, f'{str(location)} exchange {name} is not registered' self.exchange_manager.delete_exchange(name=name, location=location) # Success, remove it also from the DB self.data.db.remove_exchange(name=name, location=location) if self.exchange_manager.connected_exchanges.get(location) is None: # was last exchange of the location type. Delete used query ranges self.data.db.delete_used_query_range_for_exchange(location) return True, '' def query_periodic_data(self) -> Dict[str, Union[bool, Timestamp]]: """Query for frequently changing data""" result: Dict[str, Union[bool, Timestamp]] = {} if self.user_is_logged_in: result['last_balance_save'] = self.data.db.get_last_balance_save_time() result['eth_node_connection'] = self.chain_manager.ethereum.web3_mapping.get(NodeName.OWN, None) is not None # noqa : E501 result['last_data_upload_ts'] = Timestamp(self.premium_sync_manager.last_data_upload_ts) # noqa : E501 return result def shutdown(self) -> None: self.logout() self.shutdown_event.set() def _connect_ksm_manager_on_startup(self) -> bool: return bool(self.data.db.get_blockchain_accounts().ksm) def _connect_dot_manager_on_startup(self) -> bool: return bool(self.data.db.get_blockchain_accounts().dot) def create_oracle_cache( self, oracle: HistoricalPriceOracle, from_asset: Asset, to_asset: Asset, purge_old: bool, ) -> None: """Creates the cache of the given asset pair from the start of time until now for the given oracle. if purge_old is true then any old cache in memory and in a file is purged May raise: - RemoteError if there is a problem reaching the oracle - UnsupportedAsset if any of the two assets is not supported by the oracle """ if oracle != HistoricalPriceOracle.CRYPTOCOMPARE: return # only for cryptocompare for now self.cryptocompare.create_cache(from_asset, to_asset, purge_old)