class Rotkehlchen(object): def __init__(self, args): self.lock = Semaphore() self.lock.acquire() self.results_cache: typing.ResultCache = dict() self.premium = None self.connected_exchanges = [] logfilename = None if args.logtarget == 'file': logfilename = args.logfile if args.loglevel == 'debug': loglevel = logging.DEBUG elif args.loglevel == 'info': loglevel = logging.INFO elif args.loglevel == 'warn': loglevel = logging.WARN elif args.loglevel == 'error': loglevel = logging.ERROR elif args.loglevel == 'critical': loglevel = logging.CRITICAL else: raise ValueError('Should never get here. Illegal log value') logging.basicConfig( filename=logfilename, filemode='w', level=loglevel, format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s', datefmt='%d/%m/%Y %H:%M:%S %Z', ) if not args.logfromothermodules: logging.getLogger('zerorpc').setLevel(logging.CRITICAL) logging.getLogger('zerorpc.channel').setLevel(logging.CRITICAL) logging.getLogger('urllib3').setLevel(logging.CRITICAL) logging.getLogger('urllib3.connectionpool').setLevel( logging.CRITICAL) self.sleep_secs = args.sleep_secs self.data_dir = args.data_dir self.args = args self.last_data_upload_ts = 0 self.poloniex = None self.kraken = None self.bittrex = None self.bitmex = None self.binance = None self.data = DataHandler(self.data_dir) self.lock.release() self.shutdown_event = gevent.event.Event() def initialize_exchanges(self, secret_data): # initialize exchanges for which we have keys and are not already initialized if self.kraken is None and 'kraken' in secret_data: self.kraken = Kraken( str.encode(secret_data['kraken']['api_key']), str.encode(secret_data['kraken']['api_secret']), self.data_dir) self.connected_exchanges.append('kraken') self.trades_historian.set_exchange('kraken', self.kraken) if self.poloniex is None and 'poloniex' in secret_data: self.poloniex = Poloniex( str.encode(secret_data['poloniex']['api_key']), str.encode(secret_data['poloniex']['api_secret']), self.inquirer, self.data_dir) self.connected_exchanges.append('poloniex') self.trades_historian.set_exchange('poloniex', self.poloniex) if self.bittrex is None and 'bittrex' in secret_data: self.bittrex = Bittrex( str.encode(secret_data['bittrex']['api_key']), str.encode(secret_data['bittrex']['api_secret']), self.inquirer, self.data_dir) self.connected_exchanges.append('bittrex') self.trades_historian.set_exchange('bittrex', self.bittrex) if self.binance is None and 'binance' in secret_data: self.binance = Binance( str.encode(secret_data['binance']['api_key']), str.encode(secret_data['binance']['api_secret']), self.inquirer, self.data_dir) self.connected_exchanges.append('binance') self.trades_historian.set_exchange('binance', self.binance) if self.bitmex is None and 'bitmex' in secret_data: self.bitmex = Bitmex( str.encode(secret_data['bitmex']['api_key']), str.encode(secret_data['bitmex']['api_secret']), self.inquirer, self.data_dir) self.connected_exchanges.append('bitmex') self.trades_historian.set_exchange('bitmex', self.bitmex) def remove_all_exchanges(self): if self.kraken is not None: self.delete_exchange_data('kraken') if self.poloniex is not None: self.delete_exchange_data('poloniex') if self.bittrex is not None: self.delete_exchange_data('bittrex') if self.binance is not None: self.delete_exchange_data('binance') if self.bitmex is not None: self.delete_exchange_data('bitmex') def try_premium_at_start(self, api_key, api_secret, create_new, sync_approval, user_dir): """Check if new user provided api pair or we already got one in the DB""" if api_key != '': self.premium, valid, empty_or_error = premium_create_and_verify( api_key, api_secret) if not valid: log.error('Given API key is invalid') # At this point we are at a new user trying to create an account with # premium API keys and we failed. But a directory was created. Remove it. shutil.rmtree(user_dir) raise AuthenticationError( 'Could not verify keys for the new account. ' '{}'.format(empty_or_error)) else: # If we got premium initialize it and try to sync with the server premium_credentials = self.data.db.get_rotkehlchen_premium() if premium_credentials: api_key = premium_credentials[0] api_secret = premium_credentials[1] self.premium, valid, empty_or_error = premium_create_and_verify( api_key, api_secret) if not valid: log.error( 'The API keys found in the Database are not valid. Perhaps ' 'they expired?') del self.premium self.premium = None return else: # no premium credentials in the DB return if self.can_sync_data_from_server(): if sync_approval == 'unknown' and not create_new: log.info('DB data at server newer than local') raise PermissionError( 'Rotkehlchen Server has newer version of your DB data. ' 'Should we replace local data with the server\'s?') elif sync_approval == 'yes' or sync_approval == 'unknown' and create_new: log.info('User approved data sync from server') if self.sync_data_from_server(): if create_new: # if we successfully synced data from the server and this is # a new account, make sure the api keys are properly stored # in the DB self.data.db.set_rotkehlchen_premium( api_key, api_secret) else: log.debug('Could sync data from server but user refused') def unlock_user(self, user, password, create_new, sync_approval, api_key, api_secret): log.info( 'Unlocking user', user=user, create_new=create_new, sync_approval=sync_approval, ) # unlock or create the DB self.password = password user_dir = self.data.unlock(user, password, create_new) self.try_premium_at_start(api_key, api_secret, create_new, sync_approval, user_dir) secret_data = self.data.db.get_exchange_secrets() settings = self.data.db.get_settings() historical_data_start = settings['historical_data_start'] eth_rpc_port = settings['eth_rpc_port'] self.trades_historian = TradesHistorian( self.data_dir, self.data.db, self.data.get_eth_accounts(), historical_data_start, ) price_historian = PriceHistorian( self.data_dir, historical_data_start, ) db_settings = self.data.db.get_settings() self.accountant = Accountant( price_historian=price_historian, profit_currency=self.data.main_currency(), user_directory=user_dir, create_csv=True, ignored_assets=self.data.db.get_ignored_assets(), include_crypto2crypto=db_settings['include_crypto2crypto'], taxfree_after_period=db_settings['taxfree_after_period'], include_gas_costs=db_settings['include_gas_costs']) # Initialize the rotkehlchen logger LoggingSettings(anonymized_logs=db_settings['anonymized_logs']) self.inquirer = Inquirer(kraken=self.kraken) self.initialize_exchanges(secret_data) ethchain = Ethchain(eth_rpc_port) self.blockchain = Blockchain( blockchain_accounts=self.data.db.get_blockchain_accounts(), all_eth_tokens=self.data.eth_tokens, owned_eth_tokens=self.data.db.get_owned_tokens(), inquirer=self.inquirer, ethchain=ethchain, ) def logout(self): user = self.data.username, log.info( 'Logging out user', user=user, ) del self.blockchain self.blockchain = None self.remove_all_exchanges() # Reset rotkehlchen logger to default LoggingSettings(anonymized_logs=DEFAULT_ANONYMIZED_LOGS) del self.inquirer self.inquirer = None del self.accountant self.accountant = None del self.trades_historian self.trades_historian = None if self.premium is not None: del self.premium self.premium = None self.data.logout() self.password = None log.info( 'User successfully logged out', user=user, ) def set_premium_credentials(self, api_key, api_secret): log.info('Setting new premium credentials') if self.premium is not None: valid, empty_or_error = self.premium.set_credentials( api_key, api_secret) else: self.premium, valid, empty_or_error = premium_create_and_verify( api_key, api_secret) if valid: self.data.set_premium_credentials(api_key, api_secret) return True, '' log.error('Setting new premium credentials failed', error=empty_or_error) return False, empty_or_error def maybe_upload_data_to_server(self): # upload only if unlocked user has premium if self.premium is None: return # upload only once per hour diff = ts_now() - self.last_data_upload_ts if diff > 3600: self.upload_data_to_server() def upload_data_to_server(self): log.debug('upload to server -- start') data, our_hash = self.data.compress_and_encrypt_db(self.password) success, result_or_error = self.premium.query_last_data_metadata() if not success: log.debug( 'upload to server -- query last metadata failed', error=result_or_error, ) return log.debug( 'CAN_PUSH', ours=our_hash, theirs=result_or_error['data_hash'], ) if our_hash == result_or_error['data_hash']: log.debug('upload to server -- same hash') # same hash -- no need to upload anything return our_last_write_ts = self.data.db.get_last_write_ts() if our_last_write_ts <= result_or_error['last_modify_ts']: # Server's DB was modified after our local DB log.debug("CAN_PUSH -> 3") log.debug('upload to server -- remote db more recent than local') return success, result_or_error = self.premium.upload_data( data, our_hash, our_last_write_ts, 'zlib') if not success: log.debug('upload to server -- upload error', error=result_or_error) return self.last_data_upload_ts = ts_now() log.debug('upload to server -- success') def can_sync_data_from_server(self): log.debug('sync data from server -- start') data, our_hash = self.data.compress_and_encrypt_db(self.password) success, result_or_error = self.premium.query_last_data_metadata() if not success: log.debug('sync data from server failed', error=result_or_error) return False log.debug( 'CAN_PULL', ours=our_hash, theirs=result_or_error['data_hash'], ) if our_hash == result_or_error['data_hash']: log.debug('sync from server -- same hash') # same hash -- no need to get anything return False our_last_write_ts = self.data.db.get_last_write_ts() if our_last_write_ts >= result_or_error['last_modify_ts']: # Local DB is newer than Server DB log.debug('sync from server -- local DB more recent than remote') return False return True def sync_data_from_server(self): success, error_or_result = self.premium.pull_data() if not success: log.debug('sync from server -- pulling failed.', error=error_or_result) return False self.data.decompress_and_decrypt_db(self.password, error_or_result['data']) return True def start(self): return gevent.spawn(self.main_loop) def main_loop(self): while True and not self.shutdown_event.is_set(): log.debug('Main loop start') if self.poloniex is not None: self.poloniex.main_logic() if self.kraken is not None: self.kraken.main_logic() self.maybe_upload_data_to_server() log.debug('Main loop end') gevent.sleep(MAIN_LOOP_SECS_DELAY) def add_blockchain_account(self, blockchain, account): try: new_data = self.blockchain.add_blockchain_account( blockchain, account) except (InputError, EthSyncError) as e: return simple_result(False, str(e)) self.data.add_blockchain_account(blockchain, account) return accounts_result(new_data['per_account'], new_data['totals']) def remove_blockchain_account(self, blockchain, account): try: new_data = self.blockchain.remove_blockchain_account( blockchain, account) except (InputError, EthSyncError) as e: return simple_result(False, str(e)) self.data.remove_blockchain_account(blockchain, account) return accounts_result(new_data['per_account'], new_data['totals']) def add_owned_eth_tokens(self, tokens): try: new_data = self.blockchain.track_new_tokens(tokens) except (InputError, EthSyncError) as e: return simple_result(False, str(e)) self.data.write_owned_eth_tokens(self.blockchain.owned_eth_tokens) return accounts_result(new_data['per_account'], new_data['totals']) def remove_owned_eth_tokens(self, tokens): try: new_data = self.blockchain.remove_eth_tokens(tokens) except InputError as e: return simple_result(False, str(e)) self.data.write_owned_eth_tokens(self.blockchain.owned_eth_tokens) return accounts_result(new_data['per_account'], new_data['totals']) def process_history(self, start_ts, end_ts): ( error_or_empty, history, margin_history, loan_history, asset_movements, eth_transactions ) = self.trades_historian.get_history( start_ts= 0, # For entire history processing we need to have full history available end_ts=ts_now(), end_at_least_ts=end_ts) result = self.accountant.process_history(start_ts, end_ts, history, margin_history, loan_history, asset_movements, eth_transactions) return result, error_or_empty def query_fiat_balances(self): log.info('query_fiat_balances called') result = {} balances = self.data.get_fiat_balances() for currency, amount in balances.items(): amount = FVal(amount) usd_rate = query_fiat_pair(currency, 'USD') result[currency] = { 'amount': amount, 'usd_value': amount * usd_rate } return result def query_balances(self, requested_save_data=False): log.info('query_balances called', requested_save_data=requested_save_data) balances = {} problem_free = True for exchange in self.connected_exchanges: exchange_balances, msg = getattr(self, exchange).query_balances() # If we got an error, disregard that exchange but make sure we don't save data if not exchange_balances: problem_free = False else: balances[exchange] = exchange_balances result, error_or_empty = self.blockchain.query_balances() if error_or_empty == '': balances['blockchain'] = result['totals'] else: problem_free = False result = self.query_fiat_balances() if result != {}: balances['banks'] = result combined = combine_stat_dicts([v for k, v in balances.items()]) total_usd_per_location = [(k, dict_get_sumof(v, 'usd_value')) for k, v in balances.items()] # calculate net usd value net_usd = FVal(0) for k, v in combined.items(): net_usd += FVal(v['usd_value']) stats = {'location': {}, 'net_usd': net_usd} for entry in total_usd_per_location: name = entry[0] total = entry[1] if net_usd != FVal(0): percentage = (total / net_usd).to_percentage() else: percentage = '0%' stats['location'][name] = { 'usd_value': total, 'percentage_of_net_value': percentage, } for k, v in combined.items(): if net_usd != FVal(0): percentage = (v['usd_value'] / net_usd).to_percentage() else: percentage = '0%' combined[k]['percentage_of_net_value'] = percentage result_dict = merge_dicts(combined, stats) allowed_to_save = requested_save_data or self.data.should_save_balances( ) if problem_free and allowed_to_save: self.data.save_balances_data(result_dict) # After adding it to the saved file we can overlay additional data that # is not required to be saved in the history file try: details = self.data.accountant.details for asset, (tax_free_amount, average_buy_value) in details.items(): if asset not in result_dict: continue result_dict[asset]['tax_free_amount'] = tax_free_amount result_dict[asset]['average_buy_value'] = average_buy_value current_price = result_dict[asset]['usd_value'] / result_dict[ asset]['amount'] if average_buy_value != FVal(0): result_dict[asset]['percent_change'] = ( ((current_price - average_buy_value) / average_buy_value) * 100) else: result_dict[asset]['percent_change'] = 'INF' except AttributeError: pass return result_dict def set_main_currency(self, currency): with self.lock: self.data.set_main_currency(currency, self.accountant) if currency != 'USD': self.usd_to_main_currency_rate = query_fiat_pair( 'USD', currency) def set_settings(self, settings): log.info('Add new settings') message = '' with self.lock: if 'eth_rpc_port' in settings: result, msg = self.blockchain.set_eth_rpc_port( settings['eth_rpc_port']) if not result: # Don't save it in the DB del settings['eth_rpc_port'] message += "\nEthereum RPC port not set: " + msg if 'main_currency' in settings: main_currency = settings['main_currency'] if main_currency != 'USD': self.usd_to_main_currency_rate = query_fiat_pair( 'USD', main_currency) res, msg = self.accountant.customize(settings) if not res: message += '\n' + msg return False, message _, msg, = self.data.set_settings(settings, self.accountant) if msg != '': message += '\n' + msg # Always return success here but with a message return True, message def usd_to_main_currency(self, amount): main_currency = self.data.main_currency() if main_currency != 'USD' and not hasattr(self, 'usd_to_main_currency_rate'): self.usd_to_main_currency_rate = query_fiat_pair( 'USD', main_currency) return self.usd_to_main_currency_rate * amount def setup_exchange(self, name, api_key, api_secret): log.info('setup_exchange', name=name) if name not in SUPPORTED_EXCHANGES: return False, 'Attempted to register unsupported exchange {}'.format( name) if getattr(self, name) is not None: return False, 'Exchange {} is already registered'.format(name) secret_data = {} secret_data[name] = { 'api_key': api_key, 'api_secret': api_secret, } self.initialize_exchanges(secret_data) exchange = getattr(self, name) result, message = exchange.validate_api_key() if not result: log.error( 'Failed to validate API key for exchange', name=name, error=message, ) self.delete_exchange_data(name) return False, message # Success, save the result in the DB self.data.db.add_exchange(name, api_key, api_secret) return True, '' def delete_exchange_data(self, name): self.connected_exchanges.remove(name) self.trades_historian.set_exchange(name, None) delattr(self, name) setattr(self, name, None) def remove_exchange(self, name): if getattr(self, name) is None: return False, 'Exchange {} is not registered'.format(name) self.delete_exchange_data(name) # Success, remove it also from the DB self.data.db.remove_exchange(name) return True, '' def shutdown(self): log.info("Shutting Down") self.shutdown_event.set()
def test_data_init_and_password(data_dir, username): """DB Creation logic and tables at start testing""" msg_aggregator = MessagesAggregator() # Creating a new data dir should work data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) assert os.path.exists(os.path.join(data_dir, username)) # Trying to re-create it should throw with pytest.raises(AuthenticationError): data.unlock(username, '123', create_new=True) # Trying to unlock a non-existing user without create_new should throw with pytest.raises(AuthenticationError): data.unlock('otheruser', '123', create_new=False) # now relogin and check all tables are there del data data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=False) cursor = data.db.conn.cursor() cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") results = cursor.fetchall() results = [result[0] for result in results] assert set(results) == set(TABLES_AT_INIT) # finally logging in with wrong password should also fail del data data = DataHandler(data_dir, msg_aggregator) with pytest.raises(AuthenticationError): data.unlock(username, '1234', create_new=False)
class Rotkehlchen(): def __init__(self, args: argparse.Namespace) -> None: """Initialize the Rotkehlchen object May Raise: - SystemPermissionError if the given data directory's permissions are not correct. """ self.lock = Semaphore() self.lock.acquire() # Can also be None after unlock if premium credentials did not # authenticate or premium server temporarily offline self.premium: Optional[Premium] = None self.user_is_logged_in = False logfilename = None if args.logtarget == 'file': logfilename = args.logfile if args.loglevel == 'debug': loglevel = logging.DEBUG elif args.loglevel == 'info': loglevel = logging.INFO elif args.loglevel == 'warn': loglevel = logging.WARN elif args.loglevel == 'error': loglevel = logging.ERROR elif args.loglevel == 'critical': loglevel = logging.CRITICAL else: raise AssertionError('Should never get here. Illegal log value') logging.basicConfig( filename=logfilename, filemode='w', level=loglevel, format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s', datefmt='%d/%m/%Y %H:%M:%S %Z', ) if not args.logfromothermodules: logging.getLogger('urllib3').setLevel(logging.CRITICAL) logging.getLogger('urllib3.connectionpool').setLevel( logging.CRITICAL) self.sleep_secs = args.sleep_secs self.data_dir = args.data_dir if not os.access(self.data_dir, os.W_OK | os.R_OK): raise SystemPermissionError( f'The given data directory {self.data_dir} is not readable or writable', ) self.args = args self.msg_aggregator = MessagesAggregator() self.greenlet_manager = GreenletManager( msg_aggregator=self.msg_aggregator) self.exchange_manager = ExchangeManager( msg_aggregator=self.msg_aggregator) self.all_eth_tokens = AssetResolver().get_all_eth_tokens() self.data = DataHandler(self.data_dir, self.msg_aggregator) self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None) # Initialize the Inquirer singleton Inquirer(data_dir=self.data_dir, cryptocompare=self.cryptocompare) self.lock.release() self.shutdown_event = gevent.event.Event() def reset_after_failed_account_creation_or_login(self) -> None: """If the account creation or login failed make sure that the Rotki instance is clear Tricky instances are when after either failed premium credentials or user refusal to sync premium databases we relogged in. """ self.cryptocompare.db = None def unlock_user( self, user: str, password: str, create_new: bool, sync_approval: Literal['yes', 'no', 'unknown'], premium_credentials: Optional[PremiumCredentials], given_ethereum_modules: Optional[List[str]] = None, ) -> None: """Unlocks an existing user or creates a new one if `create_new` is True May raise: - PremiumAuthenticationError if the password can't unlock the database. - AuthenticationError if premium_credentials are given and are invalid or can't authenticate with the server - DBUpgradeError if the rotki DB version is newer than the software or there is a DB upgrade and there is an error. - SystemPermissionError if the directory or DB file can not be accessed """ log.info( 'Unlocking user', user=user, create_new=create_new, sync_approval=sync_approval, ) if given_ethereum_modules is None: ethereum_modules = ['makerdao'] # Default: ALL else: ethereum_modules = given_ethereum_modules # unlock or create the DB self.password = password self.user_directory = self.data.unlock(user, password, create_new) self.data_importer = DataImporter(db=self.data.db) self.last_data_upload_ts = self.data.db.get_last_data_upload_ts() self.premium_sync_manager = PremiumSyncManager(data=self.data, password=password) # set the DB in the external services instances that need it self.cryptocompare.set_database(self.data.db) # Anything that was set above here has to be cleaned in case of failure in the next step # by reset_after_failed_account_creation_or_login() try: self.premium = self.premium_sync_manager.try_premium_at_start( given_premium_credentials=premium_credentials, username=user, create_new=create_new, sync_approval=sync_approval, ) except PremiumAuthenticationError: # Reraise it only if this is during the creation of a new account where # the premium credentials were given by the user if create_new: raise # else let's just continue. User signed in succesfully, but he just # has unauthenticable/invalid premium credentials remaining in his DB settings = self.get_settings() maybe_submit_usage_analytics(settings.submit_usage_analytics) self.etherscan = Etherscan(database=self.data.db, msg_aggregator=self.msg_aggregator) alethio = Alethio( database=self.data.db, msg_aggregator=self.msg_aggregator, all_eth_tokens=self.all_eth_tokens, ) historical_data_start = settings.historical_data_start eth_rpc_endpoint = settings.eth_rpc_endpoint # Initialize the price historian singleton PriceHistorian( data_directory=self.data_dir, history_date_start=historical_data_start, cryptocompare=self.cryptocompare, ) self.accountant = Accountant( db=self.data.db, user_directory=self.user_directory, msg_aggregator=self.msg_aggregator, create_csv=True, ) # Initialize the rotkehlchen logger LoggingSettings(anonymized_logs=settings.anonymized_logs) exchange_credentials = self.data.db.get_exchange_credentials() self.exchange_manager.initialize_exchanges( exchange_credentials=exchange_credentials, database=self.data.db, ) # Initialize blockchain querying modules ethereum_manager = EthereumManager( ethrpc_endpoint=eth_rpc_endpoint, etherscan=self.etherscan, msg_aggregator=self.msg_aggregator, ) self.chain_manager = ChainManager( blockchain_accounts=self.data.db.get_blockchain_accounts(), owned_eth_tokens=self.data.db.get_owned_tokens(), ethereum_manager=ethereum_manager, msg_aggregator=self.msg_aggregator, alethio=alethio, greenlet_manager=self.greenlet_manager, eth_modules=ethereum_modules, ) self.ethereum_analyzer = EthereumAnalyzer( ethereum_manager=ethereum_manager, database=self.data.db, ) self.trades_historian = TradesHistorian( user_directory=self.user_directory, db=self.data.db, msg_aggregator=self.msg_aggregator, exchange_manager=self.exchange_manager, chain_manager=self.chain_manager, ) self.user_is_logged_in = True def logout(self) -> None: if not self.user_is_logged_in: return user = self.data.username log.info( 'Logging out user', user=user, ) del self.chain_manager self.exchange_manager.delete_all_exchanges() # Reset rotkehlchen logger to default LoggingSettings(anonymized_logs=DEFAULT_ANONYMIZED_LOGS) del self.accountant del self.trades_historian del self.data_importer if self.premium is not None: del self.premium self.data.logout() self.password = '' self.cryptocompare.unset_database() # Make sure no messages leak to other user sessions self.msg_aggregator.consume_errors() self.msg_aggregator.consume_warnings() self.user_is_logged_in = False log.info( 'User successfully logged out', user=user, ) def set_premium_credentials(self, credentials: PremiumCredentials) -> None: """ Sets the premium credentials for Rotki Raises PremiumAuthenticationError if the given key is rejected by the Rotkehlchen server """ log.info('Setting new premium credentials') if self.premium is not None: self.premium.set_credentials(credentials) else: self.premium = premium_create_and_verify(credentials) self.data.db.set_rotkehlchen_premium(credentials) def start(self) -> gevent.Greenlet: return gevent.spawn(self.main_loop) def main_loop(self) -> None: """Rotki main loop that fires often and manages many different tasks Each task remembers the last time it run sucesfully and know how often it should run. So each task manages itself. """ while self.shutdown_event.wait(MAIN_LOOP_SECS_DELAY) is not True: if self.user_is_logged_in: log.debug('Main loop start') self.premium_sync_manager.maybe_upload_data_to_server() self.ethereum_analyzer.analyze_ethereum_transactions() log.debug('Main loop end') def add_blockchain_accounts( self, blockchain: SupportedBlockchain, account_data: List[BlockchainAccountData], ) -> BlockchainBalancesUpdate: """Adds new blockchain accounts Adds the accounts to the blockchain instance and queries them to get the updated balances. Also adds them in the DB May raise: - EthSyncError from modify_blockchain_account - InputError if the given accounts list is empty. - TagConstraintError if any of the given account data contain unknown tags. - RemoteError if an external service such as Etherscan is queried and there is a problem with its query. """ self.data.db.ensure_tags_exist( given_data=account_data, action='adding', data_type='blockchain accounts', ) address_type = blockchain.get_address_type() updated_balances = self.chain_manager.add_blockchain_accounts( blockchain=blockchain, accounts=[address_type(entry.address) for entry in account_data], ) self.data.db.add_blockchain_accounts( blockchain=blockchain, account_data=account_data, ) return updated_balances def edit_blockchain_accounts( self, blockchain: SupportedBlockchain, account_data: List[BlockchainAccountData], ) -> None: """Edits blockchain accounts Edits blockchain account data for the given accounts May raise: - InputError if the given accounts list is empty or if any of the accounts to edit do not exist. - TagConstraintError if any of the given account data contain unknown tags. """ # First check for validity of account data addresses if len(account_data) == 0: raise InputError( 'Empty list of blockchain account data to edit was given') accounts = [x.address for x in account_data] unknown_accounts = set(accounts).difference( self.chain_manager.accounts.get(blockchain)) if len(unknown_accounts) != 0: raise InputError( f'Tried to edit unknown {blockchain.value} ' f'accounts {",".join(unknown_accounts)}', ) self.data.db.ensure_tags_exist( given_data=account_data, action='editing', data_type='blockchain accounts', ) # Finally edit the accounts self.data.db.edit_blockchain_accounts( blockchain=blockchain, account_data=account_data, ) return None def remove_blockchain_accounts( self, blockchain: SupportedBlockchain, accounts: ListOfBlockchainAddresses, ) -> BlockchainBalancesUpdate: """Removes blockchain accounts Removes the accounts from the blockchain instance and queries them to get the updated balances. Also removes them from the DB May raise: - RemoteError if an external service such as Etherscan is queried and there is a problem with its query. - InputError if a non-existing account was given to remove """ balances_update = self.chain_manager.remove_blockchain_accounts( blockchain=blockchain, accounts=accounts, ) self.data.db.remove_blockchain_accounts(blockchain, accounts) return balances_update def add_owned_eth_tokens( self, tokens: List[EthereumToken], ) -> BlockchainBalancesUpdate: """Adds tokens to the blockchain state and updates balance of all accounts May raise: - InputError if some of the tokens already exist - RemoteError if an external service such as Etherscan is queried and there is a problem with its query. - EthSyncError if querying the token balances through a provided ethereum client and the chain is not synced """ new_data = self.chain_manager.track_new_tokens(tokens) self.data.write_owned_eth_tokens(self.chain_manager.owned_eth_tokens) return new_data def remove_owned_eth_tokens( self, tokens: List[EthereumToken], ) -> BlockchainBalancesUpdate: """ Removes tokens from the state and stops their balance from being tracked for each account May raise: - RemoteError if an external service such as Etherscan is queried and there is a problem with its query. - EthSyncError if querying the token balances through a provided ethereum client and the chain is not synced """ new_data = self.chain_manager.remove_eth_tokens(tokens) self.data.write_owned_eth_tokens(self.chain_manager.owned_eth_tokens) return new_data def process_history( self, start_ts: Timestamp, end_ts: Timestamp, ) -> Tuple[Dict[str, Any], str]: ( error_or_empty, history, loan_history, asset_movements, eth_transactions, ) = self.trades_historian.get_history( start_ts=start_ts, end_ts=end_ts, has_premium=self.premium is not None, ) result = self.accountant.process_history( start_ts=start_ts, end_ts=end_ts, trade_history=history, loan_history=loan_history, asset_movements=asset_movements, eth_transactions=eth_transactions, ) return result, error_or_empty def query_fiat_balances(self) -> Dict[Asset, Dict[str, FVal]]: result = {} balances = self.data.get_fiat_balances() for currency, str_amount in balances.items(): amount = FVal(str_amount) usd_rate = Inquirer().query_fiat_pair(currency, A_USD) result[currency] = { 'amount': amount, 'usd_value': amount * usd_rate, } return result def query_balances( self, requested_save_data: bool = True, timestamp: Timestamp = None, ignore_cache: bool = False, ) -> Dict[str, Any]: """Query all balances rotkehlchen can see. If requested_save_data is True then the data are saved in the DB. If timestamp is None then the current timestamp is used. If a timestamp is given then that is the time that the balances are going to be saved in the DB If ignore_cache is True then all underlying calls that have a cache ignore it Returns a dictionary with the queried balances. """ log.info('query_balances called', requested_save_data=requested_save_data) balances = {} problem_free = True for _, exchange in self.exchange_manager.connected_exchanges.items(): exchange_balances, _ = exchange.query_balances( ignore_cache=ignore_cache) # If we got an error, disregard that exchange but make sure we don't save data if not isinstance(exchange_balances, dict): problem_free = False else: balances[exchange.name] = exchange_balances try: blockchain_result = self.chain_manager.query_balances( blockchain=None, ignore_cache=ignore_cache, ) balances['blockchain'] = { asset: balance.to_dict() for asset, balance in blockchain_result.totals.items() } except (RemoteError, EthSyncError) as e: problem_free = False log.error(f'Querying blockchain balances failed due to: {str(e)}') result = self.query_fiat_balances() if result != {}: balances['banks'] = result balances = account_for_manually_tracked_balances(db=self.data.db, balances=balances) combined = combine_stat_dicts([v for k, v in balances.items()]) total_usd_per_location = [(k, dict_get_sumof(v, 'usd_value')) for k, v in balances.items()] # calculate net usd value net_usd = FVal(0) for _, v in combined.items(): net_usd += FVal(v['usd_value']) stats: Dict[str, Any] = { 'location': {}, 'net_usd': net_usd, } for entry in total_usd_per_location: name = entry[0] total = entry[1] if net_usd != FVal(0): percentage = (total / net_usd).to_percentage() else: percentage = '0%' stats['location'][name] = { 'usd_value': total, 'percentage_of_net_value': percentage, } for k, v in combined.items(): if net_usd != FVal(0): percentage = (v['usd_value'] / net_usd).to_percentage() else: percentage = '0%' combined[k]['percentage_of_net_value'] = percentage result_dict = merge_dicts(combined, stats) allowed_to_save = requested_save_data and self.data.should_save_balances( ) if problem_free and allowed_to_save: if not timestamp: timestamp = Timestamp(int(time.time())) self.data.save_balances_data(data=result_dict, timestamp=timestamp) log.debug('query_balances data saved') else: log.debug( 'query_balances data not saved', allowed_to_save=allowed_to_save, problem_free=problem_free, ) # After adding it to the saved file we can overlay additional data that # is not required to be saved in the history file try: details = self.accountant.events.details for asset, (tax_free_amount, average_buy_value) in details.items(): if asset not in result_dict: continue result_dict[asset]['tax_free_amount'] = tax_free_amount result_dict[asset]['average_buy_value'] = average_buy_value current_price = result_dict[asset]['usd_value'] / result_dict[ asset]['amount'] if average_buy_value != FVal(0): result_dict[asset]['percent_change'] = ( ((current_price - average_buy_value) / average_buy_value) * 100) else: result_dict[asset]['percent_change'] = 'INF' except AttributeError: pass return result_dict def set_settings(self, settings: ModifiableDBSettings) -> Tuple[bool, str]: """Tries to set new settings. Returns True in success or False with message if error""" with self.lock: if settings.eth_rpc_endpoint is not None: result, msg = self.chain_manager.set_eth_rpc_endpoint( settings.eth_rpc_endpoint) if not result: return False, msg if settings.kraken_account_type is not None: kraken = self.exchange_manager.get('kraken') if kraken: kraken.set_account_type( settings.kraken_account_type) # type: ignore self.data.db.set_settings(settings) return True, '' def get_settings(self) -> DBSettings: """Returns the db settings with a check whether premium is active or not""" db_settings = self.data.db.get_settings( have_premium=self.premium is not None) return db_settings def setup_exchange( self, name: str, api_key: ApiKey, api_secret: ApiSecret, passphrase: Optional[str] = None, ) -> Tuple[bool, str]: """ Setup a new exchange with an api key and an api secret and optionally a passphrase By default the api keys are always validated unless validate is False. """ is_success, msg = self.exchange_manager.setup_exchange( name=name, api_key=api_key, api_secret=api_secret, database=self.data.db, passphrase=passphrase, ) if is_success: # Success, save the result in the DB self.data.db.add_exchange(name, api_key, api_secret, passphrase=passphrase) return is_success, msg def remove_exchange(self, name: str) -> Tuple[bool, str]: if not self.exchange_manager.has_exchange(name): return False, 'Exchange {} is not registered'.format(name) self.exchange_manager.delete_exchange(name) # Success, remove it also from the DB self.data.db.remove_exchange(name) self.data.db.delete_used_query_range_for_exchange(name) return True, '' def query_periodic_data(self) -> Dict[str, Union[bool, Timestamp]]: """Query for frequently changing data""" result: Dict[str, Union[bool, Timestamp]] = {} if self.user_is_logged_in: result[ 'last_balance_save'] = self.data.db.get_last_balance_save_time( ) result[ 'eth_node_connection'] = self.chain_manager.ethereum.connected result[ 'history_process_start_ts'] = self.accountant.started_processing_timestamp result[ 'history_process_current_ts'] = self.accountant.currently_processing_timestamp return result def shutdown(self) -> None: self.logout() self.shutdown_event.set()
def test_writing_fetching_data(data_dir, username): msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) data.db.add_blockchain_accounts( SupportedBlockchain.BITCOIN, [BlockchainAccountData(address='1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS')], ) data.db.add_blockchain_accounts( SupportedBlockchain.ETHEREUM, [ BlockchainAccountData( address='0xd36029d76af6fE4A356528e4Dc66B2C18123597D'), BlockchainAccountData( address='0x80B369799104a47e98A553f3329812a44A7FaCDc'), ], ) accounts = data.db.get_blockchain_accounts() assert isinstance(accounts, BlockchainAccounts) assert accounts.btc == ['1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS'] # See that after addition the address has been checksummed assert set(accounts.eth) == { '0xd36029d76af6fE4A356528e4Dc66B2C18123597D', '0x80B369799104a47e98A553f3329812a44A7FaCDc', } # Add existing account should fail with pytest.raises(sqlcipher.IntegrityError): # pylint: disable=no-member data.db.add_blockchain_accounts( SupportedBlockchain.ETHEREUM, [ BlockchainAccountData( address='0xd36029d76af6fE4A356528e4Dc66B2C18123597D') ], ) # Remove non-existing account with pytest.raises(InputError): data.db.remove_blockchain_accounts( SupportedBlockchain.ETHEREUM, ['0x136029d76af6fE4A356528e4Dc66B2C18123597D'], ) # Remove existing account data.db.remove_blockchain_accounts( SupportedBlockchain.ETHEREUM, ['0xd36029d76af6fE4A356528e4Dc66B2C18123597D'], ) accounts = data.db.get_blockchain_accounts() assert accounts.eth == ['0x80B369799104a47e98A553f3329812a44A7FaCDc'] result, _ = data.add_ignored_assets([A_DAO]) assert result result, _ = data.add_ignored_assets([A_DOGE]) assert result result, _ = data.add_ignored_assets([A_DOGE]) assert not result ignored_assets = data.db.get_ignored_assets() assert all(isinstance(asset, Asset) for asset in ignored_assets) assert set(ignored_assets) == {A_DAO, A_DOGE} # Test removing asset that is not in the list result, msg = data.remove_ignored_assets([A_RDN]) assert 'not in ignored assets' in msg assert not result result, _ = data.remove_ignored_assets([A_DOGE]) assert result assert data.db.get_ignored_assets() == [A_DAO] # With nothing inserted in settings make sure default values are returned result = data.db.get_settings() last_write_diff = ts_now() - result.last_write_ts # make sure last_write was within 3 secs assert last_write_diff >= 0 and last_write_diff < 3 expected_dict = { 'have_premium': False, 'historical_data_start': DEFAULT_START_DATE, 'eth_rpc_endpoint': 'http://localhost:8545', 'ui_floating_precision': DEFAULT_UI_FLOATING_PRECISION, 'version': ROTKEHLCHEN_DB_VERSION, 'include_crypto2crypto': DEFAULT_INCLUDE_CRYPTO2CRYPTO, 'include_gas_costs': DEFAULT_INCLUDE_GAS_COSTS, 'taxfree_after_period': YEAR_IN_SECONDS, 'balance_save_frequency': DEFAULT_BALANCE_SAVE_FREQUENCY, 'last_balance_save': 0, 'main_currency': DEFAULT_MAIN_CURRENCY.identifier, 'anonymized_logs': DEFAULT_ANONYMIZED_LOGS, 'date_display_format': DEFAULT_DATE_DISPLAY_FORMAT, 'last_data_upload_ts': 0, 'premium_should_sync': False, 'submit_usage_analytics': True, 'last_write_ts': 0, 'kraken_account_type': DEFAULT_KRAKEN_ACCOUNT_TYPE, 'active_modules': DEFAULT_ACTIVE_MODULES, 'frontend_settings': '', } assert len(expected_dict) == len( DBSettings()), 'One or more settings are missing' # Make sure that results are the same. Comparing like this since we ignore last # write ts check result_dict = result._asdict() for key, value in expected_dict.items(): assert key in result_dict if key != 'last_write_ts': assert value == result_dict[key]
def test_add_margin_positions(data_dir, username): """Test that adding and retrieving margin positions from the DB works fine. Also duplicates should be ignored and an error returned """ msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) margin1 = MarginPosition( location=Location.BITMEX, open_time=1451606400, close_time=1451616500, profit_loss=FVal('1.0'), pl_currency=A_BTC, fee=Fee(FVal('0.01')), fee_currency=A_EUR, link='', notes='', ) margin2 = MarginPosition( location=Location.BITMEX, open_time=1451626500, close_time=1451636500, profit_loss=FVal('0.5'), pl_currency=A_BTC, fee=Fee(FVal('0.01')), fee_currency=A_EUR, link='', notes='', ) margin3 = MarginPosition( location=Location.POLONIEX, open_time=1452636501, close_time=1459836501, profit_loss=FVal('2.5'), pl_currency=A_BTC, fee=Fee(FVal('0.01')), fee_currency=A_EUR, link='', notes='', ) # Add and retrieve the first 2 margins. All should be fine. data.db.add_margin_positions([margin1, margin2]) errors = msg_aggregator.consume_errors() warnings = msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 0 returned_margins = data.db.get_margin_positions() assert returned_margins == [margin1, margin2] # Add the last 2 margins. Since margin2 already exists in the DB it should be # ignored and a warning should be shown data.db.add_margin_positions([margin2, margin3]) errors = msg_aggregator.consume_errors() warnings = msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 1 returned_margins = data.db.get_margin_positions() assert returned_margins == [margin1, margin2, margin3]
def test_query_owned_assets(data_dir, username): """Test the get_owned_assets with also an unknown asset in the DB""" msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) balances = deepcopy(asset_balances) balances.extend([ DBAssetBalance( category=BalanceType.ASSET, time=Timestamp(1488326400), asset=A_BTC, amount='1', usd_value='1222.66', ), DBAssetBalance( category=BalanceType.ASSET, time=Timestamp(1489326500), asset=A_XMR, amount='2', usd_value='33.8', ), ]) data.db.add_multiple_balances(balances) data.db.conn.commit() # also make sure that assets from trades are included data.db.add_trades([ Trade( timestamp=Timestamp(1), location=Location.EXTERNAL, base_asset=A_ETH, quote_asset=A_BTC, trade_type=TradeType.BUY, amount=AssetAmount(FVal(1)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), Trade( timestamp=Timestamp(99), location=Location.EXTERNAL, base_asset=A_ETH, quote_asset=A_BTC, trade_type=TradeType.BUY, amount=AssetAmount(FVal(2)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), Trade( timestamp=Timestamp(1), location=Location.EXTERNAL, base_asset=A_SDC, quote_asset=A_SDT2, trade_type=TradeType.BUY, amount=AssetAmount(FVal(1)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), Trade( timestamp=Timestamp(1), location=Location.EXTERNAL, base_asset=A_SUSHI, quote_asset=A_1INCH, trade_type=TradeType.BUY, amount=AssetAmount(FVal(1)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), Trade( timestamp=Timestamp(3), location=Location.EXTERNAL, base_asset=A_SUSHI, quote_asset=A_1INCH, trade_type=TradeType.BUY, amount=AssetAmount(FVal(2)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), ]) assets_list = data.db.query_owned_assets() assert set(assets_list) == {A_USD, A_ETH, A_DAI, A_BTC, A_XMR, A_SDC, A_SDT2, A_SUSHI, A_1INCH} # noqa: E501 assert all(isinstance(x, Asset) for x in assets_list) warnings = data.db.msg_aggregator.consume_warnings() assert len(warnings) == 0
def test_add_asset_movements(data_dir, username, caplog): """Test that adding and retrieving asset movements from the DB works fine. Also duplicates should be ignored and an error returned """ msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) movement1 = AssetMovement( location=Location.BITMEX, category=AssetMovementCategory.DEPOSIT, address=None, transaction_id=None, timestamp=1451606400, asset=A_BTC, amount=FVal('1.0'), fee_asset=A_EUR, fee=Fee(FVal('0')), link='', ) movement2 = AssetMovement( location=Location.POLONIEX, category=AssetMovementCategory.WITHDRAWAL, address='0xfoo', transaction_id='0xboo', timestamp=1451608501, asset=A_ETH, amount=FVal('1.0'), fee_asset=A_EUR, fee=Fee(FVal('0.01')), link='', ) movement3 = AssetMovement( location=Location.BITTREX, category=AssetMovementCategory.WITHDRAWAL, address='0xcoo', transaction_id='0xdoo', timestamp=1461708501, asset=A_ETH, amount=FVal('1.0'), fee_asset=A_EUR, fee=Fee(FVal('0.01')), link='', ) # Add and retrieve the first 2 margins. All should be fine. data.db.add_asset_movements([movement1, movement2]) errors = msg_aggregator.consume_errors() warnings = msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 0 returned_movements = data.db.get_asset_movements() assert returned_movements == [movement1, movement2] # Add the last 2 movements. Since movement2 already exists in the DB it should be # ignored and a warning should be logged data.db.add_asset_movements([movement2, movement3]) assert ( 'Did not add "withdrawal of ETH with id 94405f38c7b86dd2e7943164d' '67ff44a32d56cef25840b3f5568e23c037fae0a' ) in caplog.text returned_movements = data.db.get_asset_movements() assert returned_movements == [movement1, movement2, movement3]
def setup_db_for_xpub_tests(data_dir, username): msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) data.db.add_tag('public', 'foooo', 'ffffff', '000000') data.db.add_tag('desktop', 'boooo', 'ffffff', '000000') xpub = 'xpub68V4ZQQ62mea7ZUKn2urQu47Bdn2Wr7SxrBxBDDwE3kjytj361YBGSKDT4WoBrE5htrSB8eAMe59NPnKrcAbiv2veN5GQUmfdjRddD1Hxrk' # noqa: E501 derivation_path = 'm/0/0/0' xpub_data1 = XpubData( xpub=HDKey.from_xpub(xpub=xpub, path='m'), derivation_path=derivation_path, label='xpub1', tags=['public', 'desktop'], ) data.db.ensure_tags_exist([xpub_data1], action='adding', data_type='bitcoin_xpub') insert_tag_mappings( # if we got tags add them to the xpub cursor=data.db.conn.cursor(), data=[xpub_data1], object_reference_keys=['xpub.xpub', 'derivation_path'], ) data.db.add_bitcoin_xpub(xpub_data1) addr1 = '1LZypJUwJJRdfdndwvDmtAjrVYaHko136r' addr2 = '1MKSdDCtBSXiE49vik8xUG2pTgTGGh5pqe' addr3 = '12wxFzpjdymPk3xnHmdDLCTXUT9keY3XRd' addr4 = '16zNpyv8KxChtjXnE5nYcPqcXcrSQXX2JW' all_addresses = [addr1, addr2, addr3, addr4] account_data = [ BlockchainAccountData(x) for x in [addr1, addr2, addr3, addr4] ] data.db.add_blockchain_accounts( blockchain=SupportedBlockchain.BITCOIN, account_data=account_data, ) insert_tag_mappings( # if we got tags add them to the existing addresses too cursor=data.db.conn.cursor(), data=account_data, object_reference_keys=['address'], ) data.db.ensure_xpub_mappings_exist( xpub=xpub, derivation_path=derivation_path, derived_addresses_data=[ XpubDerivedAddressData(0, 0, addr1, ZERO), XpubDerivedAddressData(0, 1, addr2, ZERO), ], ) xpub = 'zpub6quTRdxqWmerHdiWVKZdLMp9FY641F1F171gfT2RS4D1FyHnutwFSMiab58Nbsdu4fXBaFwpy5xyGnKZ8d6xn2j4r4yNmQ3Yp3yDDxQUo3q' # noqa: E501 derivation_path = 'm/0' xpub_data2 = XpubData( xpub=HDKey.from_xpub(xpub=xpub, path='m'), derivation_path=derivation_path, ) data.db.add_bitcoin_xpub(xpub_data2) addr1 = 'bc1qc3qcxs025ka9l6qn0q5cyvmnpwrqw2z49qwrx5' addr2 = 'bc1qnus7355ecckmeyrmvv56mlm42lxvwa4wuq5aev' addr3 = 'bc1qup7f8g5k3h5uqzfjed03ztgn8hhe542w69wc0g' addr4 = 'bc1qr4r8vryfzexvhjrx5fh5uj0s2ead8awpqspqra' all_addresses.extend([addr1, addr2, addr3, addr4]) data.db.add_blockchain_accounts( blockchain=SupportedBlockchain.BITCOIN, account_data=[ BlockchainAccountData(x) for x in [addr1, addr2, addr3, addr4] ], ) data.db.ensure_xpub_mappings_exist( xpub=xpub, derivation_path=derivation_path, derived_addresses_data=[ XpubDerivedAddressData(1, 0, addr1, ZERO), XpubDerivedAddressData(1, 1, addr2, ZERO), XpubDerivedAddressData(1, 2, addr3, ZERO), XpubDerivedAddressData(1, 3, addr4, ZERO), ], ) # Finally also add the same xpub as xpub1 with no derivation path xpub = 'xpub68V4ZQQ62mea7ZUKn2urQu47Bdn2Wr7SxrBxBDDwE3kjytj361YBGSKDT4WoBrE5htrSB8eAMe59NPnKrcAbiv2veN5GQUmfdjRddD1Hxrk' # noqa: E501 derivation_path = None xpub_data3 = XpubData( xpub=HDKey.from_xpub(xpub=xpub, path='m'), derivation_path=derivation_path, ) data.db.add_bitcoin_xpub(xpub_data3) return data.db, xpub_data1, xpub_data2, xpub_data3, all_addresses
def test_query_trades_including_ammswaps(data_dir, username): """Test that querying trades succesfully queries from both the trades and ammswaps""" msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) trades = [ Trade( timestamp=1, location=Location.EXTERNAL, base_asset=A_ETH, quote_asset=A_USDC, trade_type=TradeType.BUY, amount=FVal(1), rate=Price(FVal(1.5)), fee=None, fee_currency=None, link='', notes=None, ), Trade( timestamp=2, location=Location.KRAKEN, base_asset=A_BTC, quote_asset=A_EUR, trade_type=TradeType.SELL, amount=FVal(1), rate=Price(FVal(1.5)), fee=None, fee_currency=None, link='', notes=None, ), Trade( timestamp=3, location=Location.POLONIEX, base_asset=A_ETH, quote_asset=A_EUR, trade_type=TradeType.BUY, amount=FVal(2), rate=Price(FVal(1.5)), fee=None, fee_currency=None, link='', notes=None, ), ] swaps = [ AMMSwap( tx_hash='0x1', log_index=1, address='0xfoo', from_address='0xfrom', to_address='0xto', timestamp=2, location=Location.UNISWAP, token0=A_DAI, token1=A_USDC, amount0_in=FVal(1), amount1_in=ZERO, amount0_out=ZERO, amount1_out=FVal(2), ), AMMSwap( tx_hash='0x2', log_index=2, address='0xfoo', from_address='0xfrom', to_address='0xto', timestamp=3, location=Location.BALANCER, token0=A_UNI, token1=A_USDC, amount0_in=FVal(1), amount1_in=ZERO, amount0_out=ZERO, amount1_out=FVal(1.5), ), AMMSwap( tx_hash='0x2', log_index=3, address='0xfoo', from_address='0xfrom', to_address='0xto', timestamp=3, location=Location.BALANCER, token0=A_USDC, token1=A_DAI, amount0_in=FVal(1.5), amount1_in=ZERO, amount0_out=ZERO, amount1_out=FVal(3.5), ), AMMSwap( tx_hash='0x3', log_index=1, address='0xfoo', from_address='0xfrom', to_address='0xto', timestamp=4, location=Location.SUSHISWAP, token0=A_UNI, token1=A_DAI, amount0_in=ZERO, amount1_in=FVal(2), amount0_out=FVal(5), amount1_out=ZERO, ), AMMSwap( tx_hash='0x3', log_index=2, address='0xfoo', from_address='0xfrom', to_address='0xto', timestamp=4, location=Location.SUSHISWAP, token0=A_DAI, token1=A_USDC, amount0_in=FVal(5), amount1_in=ZERO, amount0_out=ZERO, amount1_out=FVal(4.95), ), AMMSwap( tx_hash='0x3', log_index=3, address='0xfoo', from_address='0xfrom', to_address='0xto', timestamp=4, location=Location.SUSHISWAP, token0=A_GNO, token1=A_USDC, amount0_in=ZERO, amount1_in=FVal(4.95), amount0_out=FVal(8.2), amount1_out=ZERO, ), AMMSwap( tx_hash='0x4', log_index=5, address='0xfoo', from_address='0xfrom', to_address='0xto', timestamp=11, location=Location.UNISWAP, token0=A_GNO, token1=A_USDC, amount0_in=FVal(5), amount1_in=FVal(6), amount0_out=ZERO, amount1_out=FVal(4.95), ), AMMSwap( tx_hash='0x4', log_index=10, address='0xfoo', from_address='0xfrom', to_address='0xto', timestamp=11, location=Location.UNISWAP, token0=A_USDC, token1=A_UNI, amount0_in=FVal(4.95), amount1_in=ZERO, amount0_out=ZERO, amount1_out=FVal(5.4), ), AMMSwap( tx_hash='0x5', log_index=1, address='0xfoo', from_address='0xfrom', to_address='0xto', timestamp=14, location=Location.SUSHISWAP, token0=A_UNI, token1=A_USDC, amount0_in=FVal(4.5), amount1_in=FVal(6), amount0_out=ZERO, amount1_out=FVal(3.2), ), AMMSwap( tx_hash='0x5', log_index=3, address='0xfoo', from_address='0xfrom', to_address='0xto', timestamp=14, location=Location.SUSHISWAP, token0=A_USDC, token1=A_GNO, amount0_in=FVal(5.55), amount1_in=ZERO, amount0_out=FVal(2.15), amount1_out=FVal(5.4), ), ] data.db.add_trades(trades) data.db.add_amm_swaps(swaps) swap1_trade = Trade( timestamp=swaps[0].timestamp, location=swaps[0].location, base_asset=swaps[0].token1, quote_asset=swaps[0].token0, trade_type=TradeType.BUY, amount=swaps[0].amount1_out, rate=swaps[0].amount0_in / swaps[0].amount1_out, fee=None, fee_currency=None, link=swaps[0].tx_hash, notes=None, ) swap2_trade = Trade( timestamp=swaps[1].timestamp, location=swaps[1].location, base_asset=swaps[2].token1, quote_asset=swaps[1].token0, trade_type=TradeType.BUY, amount=swaps[2].amount1_out, rate=swaps[1].amount0_in / swaps[2].amount1_out, fee=None, fee_currency=None, link=swaps[1].tx_hash, notes=None, ) swap3_trade = Trade( timestamp=swaps[3].timestamp, location=swaps[3].location, base_asset=swaps[5].token0, quote_asset=swaps[3].token1, trade_type=TradeType.BUY, amount=swaps[5].amount0_out, rate=swaps[3].amount1_in / swaps[5].amount0_out, fee=None, fee_currency=None, link=swaps[3].tx_hash, notes=None, ) swap4_trade1 = Trade( timestamp=swaps[6].timestamp, location=swaps[6].location, base_asset=swaps[7].token1, quote_asset=swaps[6].token0, trade_type=TradeType.BUY, amount=swaps[7].amount1_out / 2, rate=swaps[6].amount0_in / (swaps[7].amount1_out / 2), fee=None, fee_currency=None, link=swaps[6].tx_hash, notes=None, ) swap4_trade2 = Trade( timestamp=swaps[6].timestamp, location=swaps[6].location, base_asset=swaps[7].token1, quote_asset=swaps[6].token1, trade_type=TradeType.BUY, amount=swaps[7].amount1_out / 2, rate=swaps[6].amount1_in / (swaps[7].amount1_out / 2), fee=None, fee_currency=None, link=swaps[6].tx_hash, notes=None, ) swap5_trade1 = Trade( timestamp=swaps[8].timestamp, location=swaps[8].location, base_asset=swaps[9].token1, quote_asset=swaps[8].token1, trade_type=TradeType.BUY, amount=swaps[9].amount1_out, rate=swaps[8].amount1_in / swaps[9].amount1_out, fee=None, fee_currency=None, link=swaps[8].tx_hash, notes=None, ) swap5_trade2 = Trade( timestamp=swaps[8].timestamp, location=swaps[8].location, base_asset=swaps[9].token0, quote_asset=swaps[8].token0, trade_type=TradeType.BUY, amount=swaps[9].amount0_out, rate=swaps[8].amount0_in / swaps[9].amount0_out, fee=None, fee_currency=None, link=swaps[8].tx_hash, notes=None, ) # Get all trades returned_trades = data.db.get_trades(filter_query=TradesFilterQuery.make(), has_premium=True) assert len(returned_trades) == 10 assert returned_trades[0] == trades[0] assert returned_trades[2] == trades[1] assert returned_trades[4] == trades[2] assert_trades_equal(returned_trades[1], swap1_trade) assert_trades_equal(returned_trades[3], swap2_trade) assert_trades_equal(returned_trades[5], swap3_trade) assert_trades_equal(returned_trades[6], swap4_trade2) assert_trades_equal(returned_trades[7], swap4_trade1) assert_trades_equal(returned_trades[8], swap5_trade1) assert_trades_equal(returned_trades[9], swap5_trade2) # Get last 5 trades returned_trades = data.db.get_trades( filter_query=TradesFilterQuery.make(limit=5, offset=5), has_premium=True, ) assert len(returned_trades) == 5 assert_trades_equal(returned_trades[0], swap3_trade) assert_trades_equal(returned_trades[1], swap4_trade2) assert_trades_equal(returned_trades[2], swap4_trade1) assert_trades_equal(returned_trades[3], swap5_trade1) assert_trades_equal(returned_trades[4], swap5_trade2) # Get first 5 trades that are in uniswap and that buy USDC returned_trades = data.db.get_trades( filter_query=TradesFilterQuery.make( limit=5, offset=0, location=Location.UNISWAP, base_asset=A_USDC, ), has_premium=True, ) assert len(returned_trades) == 1 assert_trades_equal(returned_trades[0], swap1_trade) # Get all trades with quote asset USDC returned_trades = data.db.get_trades( filter_query=TradesFilterQuery.make(quote_asset=A_USDC), has_premium=True, ) assert len(returned_trades) == 3 assert_trades_equal(returned_trades[0], trades[0]) assert_trades_equal(returned_trades[1], swap4_trade2) assert_trades_equal(returned_trades[2], swap5_trade1) # Get all trades as non premium user with 2 free trades as limit limit_patch = patch( target='rotkehlchen.db.dbhandler.FREE_TRADES_LIMIT', new=2, ) with limit_patch: returned_trades, total_found = data.db.get_trades_and_limit_info( filter_query=TradesFilterQuery.make(), has_premium=False, ) # trades should be the latest 2 assert total_found == 3 # the 3 normal trades -- free users don't see swaps assert len(returned_trades) == 2 assert_trades_equal(returned_trades[0], trades[1]) assert_trades_equal(returned_trades[1], trades[2]) # Get filtered trades as non premium user with 2 free trades as limit limit_patch = patch( target='rotkehlchen.db.dbhandler.FREE_TRADES_LIMIT', new=2, ) with limit_patch: returned_trades, total_found = data.db.get_trades_and_limit_info( filter_query=TradesFilterQuery.make(from_ts=1, to_ts=2), has_premium=False, ) # trades should be the second one since the free limit includes the last 2 only assert total_found == 2, 'total found for filter should be 2' assert len(returned_trades) == 1 assert_trades_equal(returned_trades[0], trades[1])
def test_writting_fetching_external_trades(data_dir, username): data = DataHandler(data_dir) data.unlock(username, '123', create_new=True) # add 2 trades and check they are in the DB trade1 = { 'otc_timestamp': '10/03/2018 23:30', 'otc_pair': 'ETH_EUR', 'otc_type': 'buy', 'otc_amount': '10', 'otc_rate': '100', 'otc_fee': '0.001', 'otc_fee_currency': 'ETH', 'otc_link': 'a link', 'otc_notes': 'a note', } trade2 = { 'otc_timestamp': '10/03/2018 23:35', 'otc_pair': 'ETH_EUR', 'otc_type': 'buy', 'otc_amount': '5', 'otc_rate': '100', 'otc_fee': '0.001', 'otc_fee_currency': 'ETH', 'otc_link': 'a link 2', 'otc_notes': 'a note 2', } result, _, = data.add_external_trade(trade1) assert result result, _ = data.add_external_trade(trade2) assert result result = data.get_external_trades() del result[0]['id'] assert result[0] == from_otc_trade(trade1) del result[1]['id'] assert result[1] == from_otc_trade(trade2) # edit a trade and check the edit made it in the DB trade1['otc_rate'] = '120' trade1['otc_id'] = 1 result, _ = data.edit_external_trade(trade1) assert result result = data.get_external_trades() assert result[0] == from_otc_trade(trade1) del result[1]['id'] assert result[1] == from_otc_trade(trade2) # try to edit a non-existing trade trade1['otc_rate'] = '160' trade1['otc_id'] = 5 result, _ = data.edit_external_trade(trade1) assert not result trade1['otc_rate'] = '120' trade1['otc_id'] = 1 result = data.get_external_trades() assert result[0] == from_otc_trade(trade1) del result[1]['id'] assert result[1] == from_otc_trade(trade2) # try to delete non-existing trade result, _ = data.delete_external_trade(6) assert not result # delete an external trade result, _ = data.delete_external_trade(1) result = data.get_external_trades() del result[0]['id'] assert result[0] == from_otc_trade(trade2)
def test_writting_fetching_data(data_dir, username): data = DataHandler(data_dir) data.unlock(username, '123', create_new=True) tokens = ['GNO', 'RDN'] data.write_owned_eth_tokens(tokens) result = data.db.get_owned_tokens() assert set(tokens) == set(result) data.add_blockchain_account('BTC', '1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS') data.add_blockchain_account('ETH', '0xd36029d76af6fE4A356528e4Dc66B2C18123597D') data.add_blockchain_account('ETH', '0x80b369799104a47e98a553f3329812a44a7facdc') accounts = data.db.get_blockchain_accounts() assert len(accounts) == 2 assert accounts['BTC'] == ['1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS'] assert set(accounts['ETH']) == set([ '0xd36029d76af6fE4A356528e4Dc66B2C18123597D', to_checksum_address('0x80b369799104a47e98a553f3329812a44a7facdc') ]) # Add existing account should fail with pytest.raises(sqlcipher.IntegrityError): data.add_blockchain_account( 'ETH', '0xd36029d76af6fE4A356528e4Dc66B2C18123597D') # Remove non-existing account with pytest.raises(InputError): data.remove_blockchain_account( 'ETH', '0x136029d76af6fE4A356528e4Dc66B2C18123597D') # Remove existing account data.remove_blockchain_account( 'ETH', '0xd36029d76af6fE4A356528e4Dc66B2C18123597D') accounts = data.db.get_blockchain_accounts() assert accounts['ETH'] == [ to_checksum_address('0x80b369799104a47e98a553f3329812a44a7facdc') ] result, _ = data.add_ignored_asset('DAO') assert result result, _ = data.add_ignored_asset('DOGE') assert result result, _ = data.add_ignored_asset('DOGE') assert not result assert set(data.db.get_ignored_assets()) == set(['DAO', 'DOGE']) result, _ = data.remove_ignored_asset('XXX') assert not result result, _ = data.remove_ignored_asset('DOGE') assert result assert data.db.get_ignored_assets() == ['DAO'] # With nothing inserted in settings make sure default values are returned result = data.db.get_settings() last_write_diff = ts_now() - result['last_write_ts'] # make sure last_write was within 3 secs assert last_write_diff >= 0 and last_write_diff < 3 del result['last_write_ts'] assert result == { 'historical_data_start': DEFAULT_START_DATE, 'eth_rpc_port': '8545', 'ui_floating_precision': DEFAULT_UI_FLOATING_PRECISION, 'db_version': ROTKEHLCHEN_DB_VERSION, 'include_crypto2crypto': True, 'taxfree_after_period': YEAR_IN_SECONDS, 'balance_save_frequency': DEFAULT_BALANCE_SAVE_FREQUENCY, 'last_balance_save': 0, } # Check setting non-existing settings. Should be ignored _, msg = data.set_settings({'nonexisting_setting': 1}, accountant=None) assert msg != '' and 'nonexisting_setting' in msg _, msg = data.set_settings( { 'nonexisting_setting': 1, 'eth_rpc_port': '8555', 'ui_floating_precision': 3, }, accountant=None) assert msg != '' and 'nonexisting_setting' in msg # Now check nothing funny made it in the db result = data.db.get_settings() assert result['eth_rpc_port'] == '8555' assert result['ui_floating_precision'] == 3 assert 'nonexisting_setting' not in result
def test_add_and_get_yearn_vault_events(data_dir, username): """Test that get yearn vault events works fine and returns only events for what we need""" msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) addr1 = make_ethereum_address() addr1_events = [YearnVaultEvent( event_type='deposit', from_asset=A_DAI, from_value=Balance(amount=FVal(1), usd_value=FVal(1)), to_asset=Asset('yDAI'), to_value=Balance(amount=FVal(1), usd_value=FVal(1)), realized_pnl=None, block_number=1, timestamp=Timestamp(1), tx_hash='0x01653e88600a6492ad6e9ae2af415c990e623479057e4e93b163e65cfb2d4436', log_index=1, ), YearnVaultEvent( event_type='withdraw', from_asset=Asset('yDAI'), from_value=Balance(amount=FVal(1), usd_value=FVal(1)), to_asset=A_DAI, to_value=Balance(amount=FVal(1), usd_value=FVal(1)), realized_pnl=Balance(amount=FVal('0.01'), usd_value=FVal('0.01')), block_number=2, timestamp=Timestamp(2), tx_hash='0x4147da3e5d3c0565a99192ce0b32182ab30b8e1067921d9b2a8ef3bd60b7e2ce', log_index=2, )] data.db.add_yearn_vaults_events(address=addr1, events=addr1_events) addr2 = make_ethereum_address() addr2_events = [YearnVaultEvent( event_type='deposit', from_asset=A_DAI, from_value=Balance(amount=FVal(1), usd_value=FVal(1)), to_asset=Asset('yDAI'), to_value=Balance(amount=FVal(1), usd_value=FVal(1)), realized_pnl=None, block_number=1, timestamp=Timestamp(1), tx_hash='0x8c094d58f33e8dedcd348cb33b58f3bd447602f1fecb99e51b1c2868029eab55', log_index=1, ), YearnVaultEvent( event_type='withdraw', from_asset=Asset('yDAI'), from_value=Balance(amount=FVal(1), usd_value=FVal(1)), to_asset=A_DAI, to_value=Balance(amount=FVal(1), usd_value=FVal(1)), realized_pnl=Balance(amount=FVal('0.01'), usd_value=FVal('0.01')), block_number=2, timestamp=Timestamp(2), tx_hash='0x58c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de', log_index=2, )] data.db.add_yearn_vaults_events(address=addr2, events=addr2_events) events = data.db.get_yearn_vaults_events(address=addr1, vault=YEARN_VAULTS['yDAI']) assert events == addr1_events events = data.db.get_yearn_vaults_events(address=addr2, vault=YEARN_VAULTS['yDAI']) assert events == addr2_events
def test_query_owned_assets(data_dir, username): """Test the get_owned_assets with also an unknown asset in the DB""" msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) balances = deepcopy(asset_balances) balances.extend([ DBAssetBalance( category=BalanceType.ASSET, time=Timestamp(1488326400), asset=A_BTC, amount='1', usd_value='1222.66', ), DBAssetBalance( category=BalanceType.ASSET, time=Timestamp(1489326500), asset=A_XMR, amount='2', usd_value='33.8', ), ]) data.db.add_multiple_balances(balances) cursor = data.db.conn.cursor() cursor.execute( 'INSERT INTO timed_balances(' ' time, currency, amount, usd_value, category) ' ' VALUES(?, ?, ?, ?, ?)', (1469326500, 'ADSADX', '10.1', '100.5', 'A'), ) data.db.conn.commit() # also make sure that assets from trades are included data.db.add_trades([ Trade( timestamp=Timestamp(1), location=Location.EXTERNAL, pair=TradePair('ETH_BTC'), trade_type=TradeType.BUY, amount=AssetAmount(FVal(1)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), Trade( timestamp=Timestamp(99), location=Location.EXTERNAL, pair=TradePair('ETH_BTC'), trade_type=TradeType.BUY, amount=AssetAmount(FVal(2)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), Trade( timestamp=Timestamp(1), location=Location.EXTERNAL, pair=TradePair('SDC_SDT-2'), trade_type=TradeType.BUY, amount=AssetAmount(FVal(1)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), Trade( timestamp=Timestamp(1), location=Location.EXTERNAL, pair=TradePair('SUSHI_1INCH'), trade_type=TradeType.BUY, amount=AssetAmount(FVal(1)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), Trade( timestamp=Timestamp(3), location=Location.EXTERNAL, pair=TradePair('SUSHI_1INCH'), trade_type=TradeType.BUY, amount=AssetAmount(FVal(2)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), Trade( timestamp=Timestamp(1), location=Location.EXTERNAL, pair=TradePair('UNKNOWNTOKEN_BTC'), trade_type=TradeType.BUY, amount=AssetAmount(FVal(1)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_BTC, link='', notes='', ), ]) assets_list = data.db.query_owned_assets() assert set(assets_list) == { A_USD, A_ETH, A_DAI, A_BTC, A_XMR, Asset('SDC'), Asset('SDT-2'), Asset('SUSHI'), Asset('1INCH') } # noqa: E501 assert all(isinstance(x, Asset) for x in assets_list) warnings = data.db.msg_aggregator.consume_warnings() assert len(warnings) == 1 assert 'Unknown/unsupported asset ADSADX' in warnings[0]
def test_add_and_get_aave_events(data_dir, username): """Test that get aave events works fine and returns only events for what we need""" msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) addr1 = make_ethereum_address() addr1_events = [AaveDepositWithdrawalEvent( event_type='deposit', asset=A_DAI, atoken=A_ADAI_V1, value=Balance(amount=FVal(1), usd_value=FVal(1)), block_number=1, timestamp=Timestamp(1), tx_hash='0x01653e88600a6492ad6e9ae2af415c990e623479057e4e93b163e65cfb2d4436', log_index=1, ), AaveDepositWithdrawalEvent( event_type='withdrawal', asset=A_DAI, atoken=A_ADAI_V1, value=Balance(amount=FVal(1), usd_value=FVal(1)), block_number=2, timestamp=Timestamp(2), tx_hash='0x4147da3e5d3c0565a99192ce0b32182ab30b8e1067921d9b2a8ef3bd60b7e2ce', log_index=2, )] data.db.add_aave_events(address=addr1, events=addr1_events) addr2 = make_ethereum_address() addr2_events = [AaveDepositWithdrawalEvent( event_type='deposit', asset=A_DAI, atoken=A_ADAI_V1, value=Balance(amount=FVal(1), usd_value=FVal(1)), block_number=1, timestamp=Timestamp(1), tx_hash='0x8c094d58f33e8dedcd348cb33b58f3bd447602f1fecb99e51b1c2868029eab55', log_index=1, ), AaveDepositWithdrawalEvent( event_type='withdrawal', asset=A_DAI, atoken=A_ADAI_V1, value=Balance(amount=FVal(1), usd_value=FVal(1)), block_number=2, timestamp=Timestamp(2), tx_hash='0x58c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de', log_index=2, )] data.db.add_aave_events(address=addr2, events=addr2_events) # addr3 has all types of aave events so we test serialization/deserialization addr3 = make_ethereum_address() addr3_events = [AaveDepositWithdrawalEvent( event_type='deposit', asset=A_DAI, atoken=A_ADAI_V1, value=Balance(amount=FVal(1), usd_value=FVal(1)), block_number=1, timestamp=Timestamp(1), tx_hash='0x9e394d58f33e8dedcd348cb33b58f3bd447602f1fecb99e51b1c2868029eab55', log_index=1, ), AaveDepositWithdrawalEvent( event_type='withdrawal', asset=A_DAI, atoken=A_ADAI_V1, value=Balance(amount=FVal(1), usd_value=FVal(1)), block_number=2, timestamp=Timestamp(2), tx_hash='0x4c167445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de', log_index=2, ), AaveInterestEvent( event_type='interest', asset=A_WBTC, value=Balance(amount=FVal(1), usd_value=FVal(1)), block_number=4, timestamp=Timestamp(4), tx_hash='0x49c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de', log_index=4, ), AaveBorrowEvent( event_type='borrow', asset=A_ETH, value=Balance(amount=FVal(1), usd_value=FVal(1)), block_number=5, timestamp=Timestamp(5), tx_hash='0x19c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de', log_index=5, borrow_rate_mode='stable', borrow_rate=FVal('0.05233232323423432'), accrued_borrow_interest=FVal('5.112234'), ), AaveRepayEvent( event_type='repay', asset=A_MANA, value=Balance(amount=FVal(1), usd_value=FVal(1)), block_number=6, timestamp=Timestamp(6), tx_hash='0x29c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de', log_index=6, fee=Balance(amount=FVal('0.1'), usd_value=FVal('0.1')), ), AaveLiquidationEvent( event_type='liquidation', collateral_asset=A_ETH, collateral_balance=Balance(amount=FVal(1), usd_value=FVal(1)), principal_asset=A_ETH, principal_balance=Balance(amount=FVal(1), usd_value=FVal(1)), block_number=7, log_index=7, timestamp=Timestamp(7), tx_hash='0x39c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de', )] data.db.add_aave_events(address=addr3, events=addr3_events) events = data.db.get_aave_events(address=addr1, atoken=A_ADAI_V1) assert events == addr1_events events = data.db.get_aave_events(address=addr2, atoken=A_ADAI_V1) assert events == addr2_events events = data.db.get_aave_events(address=addr3) assert events == addr3_events # check that all aave events are properly hashable (aka can go in a set) test_set = set() for event in addr3_events: test_set.add(event) assert len(test_set) == len(addr3_events)
def __init__(self, args: argparse.Namespace) -> None: """Initialize the Rotkehlchen object May Raise: - SystemPermissionError if the given data directory's permissions are not correct. """ self.lock = Semaphore() self.lock.acquire() # Can also be None after unlock if premium credentials did not # authenticate or premium server temporarily offline self.premium: Optional[Premium] = None self.user_is_logged_in: bool = False configure_logging(args) self.sleep_secs = args.sleep_secs if args.data_dir is None: self.data_dir = default_data_directory() else: self.data_dir = Path(args.data_dir) if not os.access(self.data_dir, os.W_OK | os.R_OK): raise SystemPermissionError( f'The given data directory {self.data_dir} is not readable or writable', ) self.args = args self.msg_aggregator = MessagesAggregator() self.greenlet_manager = GreenletManager( msg_aggregator=self.msg_aggregator) self.exchange_manager = ExchangeManager( msg_aggregator=self.msg_aggregator) # Initialize the AssetResolver singleton AssetResolver(data_directory=self.data_dir) self.data = DataHandler(self.data_dir, self.msg_aggregator) self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None) self.coingecko = Coingecko() self.icon_manager = IconManager(data_dir=self.data_dir, coingecko=self.coingecko) self.greenlet_manager.spawn_and_track( after_seconds=None, task_name='periodically_query_icons_until_all_cached', method=self.icon_manager.periodically_query_icons_until_all_cached, batch_size=ICONS_BATCH_SIZE, sleep_time_secs=ICONS_QUERY_SLEEP, ) # Initialize the Inquirer singleton Inquirer( data_dir=self.data_dir, cryptocompare=self.cryptocompare, coingecko=self.coingecko, ) # Keeps how many trades we have found per location. Used for free user limiting self.actions_per_location: Dict[str, Dict[Location, int]] = { 'trade': defaultdict(int), 'asset_movement': defaultdict(int), } self.lock.release() self.shutdown_event = gevent.event.Event()
class Rotkehlchen(): def __init__(self, args: argparse.Namespace) -> None: """Initialize the Rotkehlchen object This runs during backend initialization so it should be as light as possible. May Raise: - SystemPermissionError if the given data directory's permissions are not correct. """ # Can also be None after unlock if premium credentials did not # authenticate or premium server temporarily offline self.premium: Optional[Premium] = None self.user_is_logged_in: bool = False configure_logging(args) self.sleep_secs = args.sleep_secs if args.data_dir is None: self.data_dir = default_data_directory() else: self.data_dir = Path(args.data_dir) self.data_dir.mkdir(parents=True, exist_ok=True) if not os.access(self.data_dir, os.W_OK | os.R_OK): raise SystemPermissionError( f'The given data directory {self.data_dir} is not readable or writable', ) self.main_loop_spawned = False self.args = args self.api_task_greenlets: List[gevent.Greenlet] = [] self.msg_aggregator = MessagesAggregator() self.greenlet_manager = GreenletManager( msg_aggregator=self.msg_aggregator) self.rotki_notifier = RotkiNotifier( greenlet_manager=self.greenlet_manager) self.msg_aggregator.rotki_notifier = self.rotki_notifier self.exchange_manager = ExchangeManager( msg_aggregator=self.msg_aggregator) # Initialize the GlobalDBHandler singleton. Has to be initialized BEFORE asset resolver GlobalDBHandler(data_dir=self.data_dir) self.data = DataHandler(self.data_dir, self.msg_aggregator) self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None) self.coingecko = Coingecko() self.icon_manager = IconManager(data_dir=self.data_dir, coingecko=self.coingecko) self.assets_updater = AssetsUpdater(self.msg_aggregator) # Initialize the Inquirer singleton Inquirer( data_dir=self.data_dir, cryptocompare=self.cryptocompare, coingecko=self.coingecko, ) self.task_manager: Optional[TaskManager] = None self.shutdown_event = gevent.event.Event() def reset_after_failed_account_creation_or_login(self) -> None: """If the account creation or login failed make sure that the rotki instance is clear Tricky instances are when after either failed premium credentials or user refusal to sync premium databases we relogged in """ self.cryptocompare.db = None def unlock_user( self, user: str, password: str, create_new: bool, sync_approval: Literal['yes', 'no', 'unknown'], premium_credentials: Optional[PremiumCredentials], initial_settings: Optional[ModifiableDBSettings] = None, sync_database: bool = True, ) -> None: """Unlocks an existing user or creates a new one if `create_new` is True May raise: - PremiumAuthenticationError if the password can't unlock the database. - AuthenticationError if premium_credentials are given and are invalid or can't authenticate with the server - DBUpgradeError if the rotki DB version is newer than the software or there is a DB upgrade and there is an error. - SystemPermissionError if the directory or DB file can not be accessed """ log.info( 'Unlocking user', user=user, create_new=create_new, sync_approval=sync_approval, sync_database=sync_database, initial_settings=initial_settings, ) # unlock or create the DB self.password = password self.user_directory = self.data.unlock(user, password, create_new, initial_settings) # Run the DB integrity check due to https://github.com/rotki/rotki/issues/3010 # TODO: Hopefully onece 3010 is handled this can go away self.greenlet_manager.spawn_and_track( after_seconds=None, task_name='user DB data integrity check', exception_is_error=False, method=self.data.db.ensure_data_integrity, ) self.data_importer = DataImporter(db=self.data.db) self.last_data_upload_ts = self.data.db.get_last_data_upload_ts() self.premium_sync_manager = PremiumSyncManager(data=self.data, password=password) # set the DB in the external services instances that need it self.cryptocompare.set_database(self.data.db) # Anything that was set above here has to be cleaned in case of failure in the next step # by reset_after_failed_account_creation_or_login() try: self.premium = self.premium_sync_manager.try_premium_at_start( given_premium_credentials=premium_credentials, username=user, create_new=create_new, sync_approval=sync_approval, sync_database=sync_database, ) except PremiumAuthenticationError: # Reraise it only if this is during the creation of a new account where # the premium credentials were given by the user if create_new: raise self.msg_aggregator.add_warning( 'Could not authenticate the rotki premium API keys found in the DB.' ' Has your subscription expired?', ) # else let's just continue. User signed in succesfully, but he just # has unauthenticable/invalid premium credentials remaining in his DB settings = self.get_settings() self.greenlet_manager.spawn_and_track( after_seconds=None, task_name='submit_usage_analytics', exception_is_error=False, method=maybe_submit_usage_analytics, data_dir=self.data_dir, should_submit=settings.submit_usage_analytics, ) self.etherscan = Etherscan(database=self.data.db, msg_aggregator=self.msg_aggregator) self.beaconchain = BeaconChain(database=self.data.db, msg_aggregator=self.msg_aggregator) eth_rpc_endpoint = settings.eth_rpc_endpoint # Initialize the price historian singleton PriceHistorian( data_directory=self.data_dir, cryptocompare=self.cryptocompare, coingecko=self.coingecko, ) PriceHistorian().set_oracles_order(settings.historical_price_oracles) exchange_credentials = self.data.db.get_exchange_credentials() self.exchange_manager.initialize_exchanges( exchange_credentials=exchange_credentials, database=self.data.db, ) # Initialize blockchain querying modules ethereum_manager = EthereumManager( ethrpc_endpoint=eth_rpc_endpoint, etherscan=self.etherscan, msg_aggregator=self.msg_aggregator, greenlet_manager=self.greenlet_manager, connect_at_start=ETHEREUM_NODES_TO_CONNECT_AT_START, ) kusama_manager = SubstrateManager( chain=SubstrateChain.KUSAMA, msg_aggregator=self.msg_aggregator, greenlet_manager=self.greenlet_manager, connect_at_start=KUSAMA_NODES_TO_CONNECT_AT_START, connect_on_startup=self._connect_ksm_manager_on_startup(), own_rpc_endpoint=settings.ksm_rpc_endpoint, ) polkadot_manager = SubstrateManager( chain=SubstrateChain.POLKADOT, msg_aggregator=self.msg_aggregator, greenlet_manager=self.greenlet_manager, connect_at_start=POLKADOT_NODES_TO_CONNECT_AT_START, connect_on_startup=self._connect_dot_manager_on_startup(), own_rpc_endpoint=settings.dot_rpc_endpoint, ) self.eth_transactions = EthTransactions(ethereum=ethereum_manager, database=self.data.db) self.covalent_avalanche = Covalent( database=self.data.db, msg_aggregator=self.msg_aggregator, chain_id=chains_id['avalanche'], ) avalanche_manager = AvalancheManager( avaxrpc_endpoint="https://api.avax.network/ext/bc/C/rpc", covalent=self.covalent_avalanche, msg_aggregator=self.msg_aggregator, ) Inquirer().inject_ethereum(ethereum_manager) uniswap_v2_oracle = UniswapV2Oracle(ethereum_manager) uniswap_v3_oracle = UniswapV3Oracle(ethereum_manager) saddle_oracle = SaddleOracle(ethereum_manager) Inquirer().add_defi_oracles( uniswap_v2=uniswap_v2_oracle, uniswap_v3=uniswap_v3_oracle, saddle=saddle_oracle, ) Inquirer().set_oracles_order(settings.current_price_oracles) self.chain_manager = ChainManager( blockchain_accounts=self.data.db.get_blockchain_accounts(), ethereum_manager=ethereum_manager, kusama_manager=kusama_manager, polkadot_manager=polkadot_manager, avalanche_manager=avalanche_manager, msg_aggregator=self.msg_aggregator, database=self.data.db, greenlet_manager=self.greenlet_manager, premium=self.premium, eth_modules=settings.active_modules, data_directory=self.data_dir, beaconchain=self.beaconchain, btc_derivation_gap_limit=settings.btc_derivation_gap_limit, ) self.evm_tx_decoder = EVMTransactionDecoder( database=self.data.db, ethereum_manager=ethereum_manager, eth_transactions=self.eth_transactions, msg_aggregator=self.msg_aggregator, ) self.evm_accounting_aggregator = EVMAccountingAggregator( ethereum_manager=ethereum_manager, msg_aggregator=self.msg_aggregator, ) self.accountant = Accountant( db=self.data.db, msg_aggregator=self.msg_aggregator, evm_accounting_aggregator=self.evm_accounting_aggregator, premium=self.premium, ) self.events_historian = EventsHistorian( user_directory=self.user_directory, db=self.data.db, msg_aggregator=self.msg_aggregator, exchange_manager=self.exchange_manager, chain_manager=self.chain_manager, evm_tx_decoder=self.evm_tx_decoder, eth_transactions=self.eth_transactions, ) self.task_manager = TaskManager( max_tasks_num=DEFAULT_MAX_TASKS_NUM, greenlet_manager=self.greenlet_manager, api_task_greenlets=self.api_task_greenlets, database=self.data.db, cryptocompare=self.cryptocompare, premium_sync_manager=self.premium_sync_manager, chain_manager=self.chain_manager, exchange_manager=self.exchange_manager, eth_transactions=self.eth_transactions, evm_tx_decoder=self.evm_tx_decoder, deactivate_premium=self.deactivate_premium_status, query_balances=self.query_balances, ) DataMigrationManager(self).maybe_migrate_data() self.greenlet_manager.spawn_and_track( after_seconds=5, task_name='periodically_query_icons_until_all_cached', exception_is_error=False, method=self.icon_manager.periodically_query_icons_until_all_cached, batch_size=ICONS_BATCH_SIZE, sleep_time_secs=ICONS_QUERY_SLEEP, ) self.user_is_logged_in = True log.debug('User unlocking complete') def _logout(self) -> None: if not self.user_is_logged_in: return user = self.data.username log.info( 'Logging out user', user=user, ) self.deactivate_premium_status() self.greenlet_manager.clear() del self.chain_manager self.exchange_manager.delete_all_exchanges() del self.accountant del self.events_historian del self.data_importer self.data.logout() self.password = '' self.cryptocompare.unset_database() # Make sure no messages leak to other user sessions self.msg_aggregator.consume_errors() self.msg_aggregator.consume_warnings() self.task_manager = None self.user_is_logged_in = False log.info( 'User successfully logged out', user=user, ) def logout(self) -> None: if self.task_manager is None: # no user logged in? return with self.task_manager.schedule_lock: self._logout() def set_premium_credentials(self, credentials: PremiumCredentials) -> None: """ Sets the premium credentials for rotki Raises PremiumAuthenticationError if the given key is rejected by the Rotkehlchen server """ log.info('Setting new premium credentials') if self.premium is not None: self.premium.set_credentials(credentials) else: self.premium = premium_create_and_verify(credentials) self.premium_sync_manager.premium = self.premium self.accountant.activate_premium_status(self.premium) self.chain_manager.activate_premium_status(self.premium) self.data.db.set_rotkehlchen_premium(credentials) def deactivate_premium_status(self) -> None: """Deactivate premium in the current session""" self.premium = None self.premium_sync_manager.premium = None self.accountant.deactivate_premium_status() self.chain_manager.deactivate_premium_status() def delete_premium_credentials(self) -> Tuple[bool, str]: """Deletes the premium credentials for rotki""" msg = '' success = self.data.db.del_rotkehlchen_premium() if success is False: msg = 'The database was unable to delete the Premium keys for the logged-in user' self.deactivate_premium_status() return success, msg def start(self) -> gevent.Greenlet: assert not self.main_loop_spawned, 'Tried to spawn the main loop twice' greenlet = gevent.spawn(self.main_loop) self.main_loop_spawned = True return greenlet def main_loop(self) -> None: """rotki main loop that fires often and runs the task manager's scheduler""" while self.shutdown_event.wait( timeout=MAIN_LOOP_SECS_DELAY) is not True: if self.task_manager is not None: self.task_manager.schedule() def get_blockchain_account_data( self, blockchain: SupportedBlockchain, ) -> Union[List[BlockchainAccountData], Dict[str, Any]]: account_data = self.data.db.get_blockchain_account_data(blockchain) if blockchain != SupportedBlockchain.BITCOIN: return account_data xpub_data = self.data.db.get_bitcoin_xpub_data() addresses_to_account_data = {x.address: x for x in account_data} address_to_xpub_mappings = self.data.db.get_addresses_to_xpub_mapping( list(addresses_to_account_data.keys()), # type: ignore ) xpub_mappings: Dict['XpubData', List[BlockchainAccountData]] = {} for address, xpub_entry in address_to_xpub_mappings.items(): if xpub_entry not in xpub_mappings: xpub_mappings[xpub_entry] = [] xpub_mappings[xpub_entry].append( addresses_to_account_data[address]) data: Dict[str, Any] = {'standalone': [], 'xpubs': []} # Add xpub data for xpub_entry in xpub_data: data_entry = xpub_entry.serialize() addresses = xpub_mappings.get(xpub_entry, None) data_entry['addresses'] = addresses if addresses and len( addresses) != 0 else None data['xpubs'].append(data_entry) # Add standalone addresses for account in account_data: if account.address not in address_to_xpub_mappings: data['standalone'].append(account) return data def add_blockchain_accounts( self, blockchain: SupportedBlockchain, account_data: List[BlockchainAccountData], ) -> BlockchainBalancesUpdate: """Adds new blockchain accounts Adds the accounts to the blockchain instance and queries them to get the updated balances. Also adds them in the DB May raise: - EthSyncError from modify_blockchain_account - InputError if the given accounts list is empty. - TagConstraintError if any of the given account data contain unknown tags. - RemoteError if an external service such as Etherscan is queried and there is a problem with its query. """ self.data.db.ensure_tags_exist( given_data=account_data, action='adding', data_type='blockchain accounts', ) address_type = blockchain.get_address_type() updated_balances = self.chain_manager.add_blockchain_accounts( blockchain=blockchain, accounts=[address_type(entry.address) for entry in account_data], ) self.data.db.add_blockchain_accounts( blockchain=blockchain, account_data=account_data, ) return updated_balances def edit_blockchain_accounts( self, blockchain: SupportedBlockchain, account_data: List[BlockchainAccountData], ) -> None: """Edits blockchain accounts Edits blockchain account data for the given accounts May raise: - InputError if the given accounts list is empty or if any of the accounts to edit do not exist. - TagConstraintError if any of the given account data contain unknown tags. """ # First check for validity of account data addresses if len(account_data) == 0: raise InputError( 'Empty list of blockchain account data to edit was given') accounts = [x.address for x in account_data] unknown_accounts = set(accounts).difference( self.chain_manager.accounts.get(blockchain)) if len(unknown_accounts) != 0: raise InputError( f'Tried to edit unknown {blockchain.value} ' f'accounts {",".join(unknown_accounts)}', ) self.data.db.ensure_tags_exist( given_data=account_data, action='editing', data_type='blockchain accounts', ) # Finally edit the accounts self.data.db.edit_blockchain_accounts( blockchain=blockchain, account_data=account_data, ) def remove_blockchain_accounts( self, blockchain: SupportedBlockchain, accounts: ListOfBlockchainAddresses, ) -> BlockchainBalancesUpdate: """Removes blockchain accounts Removes the accounts from the blockchain instance and queries them to get the updated balances. Also removes them from the DB May raise: - RemoteError if an external service such as Etherscan is queried and there is a problem with its query. - InputError if a non-existing account was given to remove """ balances_update = self.chain_manager.remove_blockchain_accounts( blockchain=blockchain, accounts=accounts, ) eth_addresses: List[ChecksumEthAddress] = cast( List[ChecksumEthAddress], accounts) if blockchain == SupportedBlockchain.ETHEREUM else [ ] # noqa: E501 with self.eth_transactions.wait_until_no_query_for(eth_addresses): self.data.db.remove_blockchain_accounts(blockchain, accounts) return balances_update def get_history_query_status(self) -> Dict[str, str]: if self.events_historian.progress < FVal('100'): processing_state = self.events_historian.processing_state_name progress = self.events_historian.progress / 2 elif self.accountant.first_processed_timestamp == -1: processing_state = 'Processing all retrieved historical events' progress = FVal(50) else: processing_state = 'Processing all retrieved historical events' # start_ts is min of the query start or the first action timestamp since action # processing can start well before query start to calculate cost basis start_ts = min( self.accountant.query_start_ts, self.accountant.first_processed_timestamp, ) diff = self.accountant.query_end_ts - start_ts progress = 50 + 100 * ( FVal(self.accountant.currently_processing_timestamp - start_ts) / FVal(diff) / 2) return { 'processing_state': str(processing_state), 'total_progress': str(progress) } def process_history( self, start_ts: Timestamp, end_ts: Timestamp, ) -> Tuple[int, str]: error_or_empty, events = self.events_historian.get_history( start_ts=start_ts, end_ts=end_ts, has_premium=self.premium is not None, ) report_id = self.accountant.process_history( start_ts=start_ts, end_ts=end_ts, events=events, ) return report_id, error_or_empty def query_balances( self, requested_save_data: bool = False, save_despite_errors: bool = False, timestamp: Timestamp = None, ignore_cache: bool = False, ) -> Dict[str, Any]: """Query all balances rotkehlchen can see. If requested_save_data is True then the data are always saved in the DB, if it is False then data are saved if self.data.should_save_balances() is True. If save_despite_errors is True then even if there is any error the snapshot will be saved. If timestamp is None then the current timestamp is used. If a timestamp is given then that is the time that the balances are going to be saved in the DB If ignore_cache is True then all underlying calls that have a cache ignore it Returns a dictionary with the queried balances. """ log.info( 'query_balances called', requested_save_data=requested_save_data, save_despite_errors=save_despite_errors, ) balances: Dict[str, Dict[Asset, Balance]] = {} problem_free = True for exchange in self.exchange_manager.iterate_exchanges(): exchange_balances, error_msg = exchange.query_balances( ignore_cache=ignore_cache) # If we got an error, disregard that exchange but make sure we don't save data if not isinstance(exchange_balances, dict): problem_free = False self.msg_aggregator.add_message( message_type=WSMessageType.BALANCE_SNAPSHOT_ERROR, data={ 'location': exchange.name, 'error': error_msg }, ) else: location_str = str(exchange.location) if location_str not in balances: balances[location_str] = exchange_balances else: # multiple exchange of same type. Combine balances balances[location_str] = combine_dicts( balances[location_str], exchange_balances, ) liabilities: Dict[Asset, Balance] try: blockchain_result = self.chain_manager.query_balances( blockchain=None, force_token_detection=ignore_cache, ignore_cache=ignore_cache, ) if len(blockchain_result.totals.assets) != 0: balances[str( Location.BLOCKCHAIN)] = blockchain_result.totals.assets liabilities = blockchain_result.totals.liabilities except (RemoteError, EthSyncError) as e: problem_free = False liabilities = {} log.error(f'Querying blockchain balances failed due to: {str(e)}') self.msg_aggregator.add_message( message_type=WSMessageType.BALANCE_SNAPSHOT_ERROR, data={ 'location': 'blockchain balances query', 'error': str(e) }, ) manually_tracked_liabilities = get_manually_tracked_balances( db=self.data.db, balance_type=BalanceType.LIABILITY, ) manual_liabilities_as_dict: DefaultDict[Asset, Balance] = defaultdict(Balance) for manual_liability in manually_tracked_liabilities: manual_liabilities_as_dict[ manual_liability.asset] += manual_liability.value liabilities = combine_dicts(liabilities, manual_liabilities_as_dict) # retrieve loopring balances if module is activated if self.chain_manager.get_module('loopring'): try: loopring_balances = self.chain_manager.get_loopring_balances() except RemoteError as e: problem_free = False self.msg_aggregator.add_message( message_type=WSMessageType.BALANCE_SNAPSHOT_ERROR, data={ 'location': 'loopring', 'error': str(e) }, ) else: if len(loopring_balances) != 0: balances[str(Location.LOOPRING)] = loopring_balances # retrieve nft balances if module is activated nfts = self.chain_manager.get_module('nfts') if nfts is not None: try: nft_mapping = nfts.get_balances( addresses=self.chain_manager.queried_addresses_for_module( 'nfts'), return_zero_values=False, ignore_cache=False, ) except RemoteError as e: log.error( f'At balance snapshot NFT balances query failed due to {str(e)}. Error ' f'is ignored and balance snapshot will still be saved.', ) else: if len(nft_mapping) != 0: if str(Location.BLOCKCHAIN) not in balances: balances[str(Location.BLOCKCHAIN)] = {} for _, nft_balances in nft_mapping.items(): for balance_entry in nft_balances: balances[str(Location.BLOCKCHAIN)][Asset( balance_entry['id'])] = Balance( amount=FVal(1), usd_value=balance_entry['usd_price'], ) balances = account_for_manually_tracked_asset_balances( db=self.data.db, balances=balances) # Calculate usd totals assets_total_balance: DefaultDict[Asset, Balance] = defaultdict(Balance) total_usd_per_location: Dict[str, FVal] = {} for location, asset_balance in balances.items(): total_usd_per_location[location] = ZERO for asset, balance in asset_balance.items(): assets_total_balance[asset] += balance total_usd_per_location[location] += balance.usd_value net_usd = sum((balance.usd_value for _, balance in assets_total_balance.items()), ZERO) liabilities_total_usd = sum( (liability.usd_value for _, liability in liabilities.items()), ZERO) # noqa: E501 net_usd -= liabilities_total_usd # Calculate location stats location_stats: Dict[str, Any] = {} for location, total_usd in total_usd_per_location.items(): if location == str(Location.BLOCKCHAIN): total_usd -= liabilities_total_usd percentage = (total_usd / net_usd).to_percentage() if net_usd != ZERO else '0%' location_stats[location] = { 'usd_value': total_usd, 'percentage_of_net_value': percentage, } # Calculate 'percentage_of_net_value' per asset assets_total_balance_as_dict: Dict[Asset, Dict[str, Any]] = { asset: balance.to_dict() for asset, balance in assets_total_balance.items() } liabilities_as_dict: Dict[Asset, Dict[str, Any]] = { asset: balance.to_dict() for asset, balance in liabilities.items() } for asset, balance_dict in assets_total_balance_as_dict.items(): percentage = (balance_dict['usd_value'] / net_usd).to_percentage( ) if net_usd != ZERO else '0%' # noqa: E501 assets_total_balance_as_dict[asset][ 'percentage_of_net_value'] = percentage for asset, balance_dict in liabilities_as_dict.items(): percentage = (balance_dict['usd_value'] / net_usd).to_percentage( ) if net_usd != ZERO else '0%' # noqa: E501 liabilities_as_dict[asset]['percentage_of_net_value'] = percentage # Compose balances response result_dict = { 'assets': assets_total_balance_as_dict, 'liabilities': liabilities_as_dict, 'location': location_stats, 'net_usd': net_usd, } allowed_to_save = requested_save_data or self.data.db.should_save_balances( ) if (problem_free or save_despite_errors) and allowed_to_save: if not timestamp: timestamp = Timestamp(int(time.time())) self.data.db.save_balances_data(data=result_dict, timestamp=timestamp) log.debug('query_balances data saved') else: log.debug( 'query_balances data not saved', allowed_to_save=allowed_to_save, problem_free=problem_free, save_despite_errors=save_despite_errors, ) return result_dict def set_settings(self, settings: ModifiableDBSettings) -> Tuple[bool, str]: """Tries to set new settings. Returns True in success or False with message if error""" if settings.eth_rpc_endpoint is not None: result, msg = self.chain_manager.set_eth_rpc_endpoint( settings.eth_rpc_endpoint) if not result: return False, msg if settings.ksm_rpc_endpoint is not None: result, msg = self.chain_manager.set_ksm_rpc_endpoint( settings.ksm_rpc_endpoint) if not result: return False, msg if settings.dot_rpc_endpoint is not None: result, msg = self.chain_manager.set_dot_rpc_endpoint( settings.dot_rpc_endpoint) if not result: return False, msg if settings.btc_derivation_gap_limit is not None: self.chain_manager.btc_derivation_gap_limit = settings.btc_derivation_gap_limit if settings.current_price_oracles is not None: Inquirer().set_oracles_order(settings.current_price_oracles) if settings.historical_price_oracles is not None: PriceHistorian().set_oracles_order( settings.historical_price_oracles) if settings.active_modules is not None: self.chain_manager.process_new_modules_list( settings.active_modules) self.data.db.set_settings(settings) return True, '' def get_settings(self) -> DBSettings: """Returns the db settings with a check whether premium is active or not""" db_settings = self.data.db.get_settings( have_premium=self.premium is not None) return db_settings def setup_exchange( self, name: str, location: Location, api_key: ApiKey, api_secret: ApiSecret, passphrase: Optional[str] = None, kraken_account_type: Optional['KrakenAccountType'] = None, PAIRS: Optional[List[str]] = None, # noqa: N803 ftx_subaccount: Optional[str] = None, ) -> Tuple[bool, str]: """ Setup a new exchange with an api key and an api secret and optionally a passphrase """ is_success, msg = self.exchange_manager.setup_exchange( name=name, location=location, api_key=api_key, api_secret=api_secret, database=self.data.db, passphrase=passphrase, ftx_subaccount=ftx_subaccount, PAIRS=PAIRS, ) if is_success: # Success, save the result in the DB self.data.db.add_exchange( name=name, location=location, api_key=api_key, api_secret=api_secret, passphrase=passphrase, kraken_account_type=kraken_account_type, PAIRS=PAIRS, ftx_subaccount=ftx_subaccount, ) return is_success, msg def remove_exchange(self, name: str, location: Location) -> Tuple[bool, str]: if self.exchange_manager.get_exchange(name=name, location=location) is None: return False, f'{str(location)} exchange {name} is not registered' self.exchange_manager.delete_exchange(name=name, location=location) # Success, remove it also from the DB self.data.db.remove_exchange(name=name, location=location) if self.exchange_manager.connected_exchanges.get(location) is None: # was last exchange of the location type. Delete used query ranges self.data.db.delete_used_query_range_for_exchange(location) return True, '' def query_periodic_data(self) -> Dict[str, Union[bool, Timestamp]]: """Query for frequently changing data""" result: Dict[str, Union[bool, Timestamp]] = {} if self.user_is_logged_in: result[ 'last_balance_save'] = self.data.db.get_last_balance_save_time( ) result[ 'eth_node_connection'] = self.chain_manager.ethereum.web3_mapping.get( NodeName.OWN, None) is not None # noqa : E501 result['last_data_upload_ts'] = Timestamp( self.premium_sync_manager.last_data_upload_ts) # noqa : E501 return result def shutdown(self) -> None: self.logout() self.shutdown_event.set() def _connect_ksm_manager_on_startup(self) -> bool: return bool(self.data.db.get_blockchain_accounts().ksm) def _connect_dot_manager_on_startup(self) -> bool: return bool(self.data.db.get_blockchain_accounts().dot) def create_oracle_cache( self, oracle: HistoricalPriceOracle, from_asset: Asset, to_asset: Asset, purge_old: bool, ) -> None: """Creates the cache of the given asset pair from the start of time until now for the given oracle. if purge_old is true then any old cache in memory and in a file is purged May raise: - RemoteError if there is a problem reaching the oracle - UnsupportedAsset if any of the two assets is not supported by the oracle """ if oracle != HistoricalPriceOracle.CRYPTOCOMPARE: return # only for cryptocompare for now self.cryptocompare.create_cache(from_asset, to_asset, purge_old)
class Rotkehlchen(object): def __init__(self, args): self.lock = Semaphore() self.lock.acquire() self.results_cache = {} self.connected_exchanges = [] logfilename = None if args.logtarget == 'file': logfilename = args.logfile loglevel = logging.DEBUG if args.loglevel == 'debug': loglevel = logging.DEBUG elif args.loglevel == 'info': loglevel = logging.INFO elif args.loglevel == 'warn': loglevel = logging.WARN elif args.loglevel == 'error': loglevel = logging.ERROR elif args.loglevel == 'critical': loglevel = logging.CRITICAL else: raise ValueError('Should never get here. Illegal log value') logging.basicConfig( filename=logfilename, filemode='w', level=loglevel, format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s', datefmt='%d/%m/%Y %H:%M:%S %Z', ) if not args.logfromothermodules: logging.getLogger('zerorpc').setLevel(logging.CRITICAL) logging.getLogger('zerorpc.channel').setLevel(logging.CRITICAL) logging.getLogger('urllib3').setLevel(logging.CRITICAL) logging.getLogger('urllib3.connectionpool').setLevel( logging.CRITICAL) self.sleep_secs = args.sleep_secs self.data_dir = args.data_dir self.args = args self.last_data_upload_ts = 0 self.poloniex = None self.kraken = None self.bittrex = None self.binance = None self.data = DataHandler(self.data_dir) self.lock.release() self.shutdown_event = gevent.event.Event() def initialize_exchanges(self, secret_data): # initialize exchanges for which we have keys and are not already initialized if self.kraken is None and 'kraken' in secret_data: self.kraken = Kraken( str.encode(secret_data['kraken']['api_key']), str.encode(secret_data['kraken']['api_secret']), self.data_dir) self.connected_exchanges.append('kraken') self.trades_historian.set_exchange('kraken', self.kraken) if self.poloniex is None and 'poloniex' in secret_data: self.poloniex = Poloniex( str.encode(secret_data['poloniex']['api_key']), str.encode(secret_data['poloniex']['api_secret']), self.cache_data_filename, self.inquirer, self.data_dir) self.connected_exchanges.append('poloniex') self.trades_historian.set_exchange('poloniex', self.poloniex) if self.bittrex is None and 'bittrex' in secret_data: self.bittrex = Bittrex( str.encode(secret_data['bittrex']['api_key']), str.encode(secret_data['bittrex']['api_secret']), self.inquirer, self.data_dir) self.connected_exchanges.append('bittrex') self.trades_historian.set_exchange('bittrex', self.bittrex) if self.binance is None and 'binance' in secret_data: self.binance = Binance( str.encode(secret_data['binance']['api_key']), str.encode(secret_data['binance']['api_secret']), self.inquirer, self.data_dir) self.connected_exchanges.append('binance') self.trades_historian.set_exchange('binance', self.binance) def try_premium_at_start(self, api_key, api_secret, create_new, sync_approval, user_dir): """Check if new user provided api pair or we already got one in the DB""" if api_key != '': self.premium, valid, empty_or_error = premium_create_and_verify( api_key, api_secret) if not valid: # At this point we are at a new user trying to create an account with # premium API keys and we failed. But a directory was created. Remove it. shutil.rmtree(user_dir) raise AuthenticationError( 'Could not verify keys for the new account. ' '{}'.format(empty_or_error)) else: # If we got premium initialize it and try to sync with the server premium_credentials = self.data.db.get_rotkehlchen_premium() if premium_credentials: api_key = premium_credentials[0] api_secret = premium_credentials[1] self.premium, valid, empty_or_error = premium_create_and_verify( api_key, api_secret) if not valid: logger.error( 'The API keys found in the Database are not valid. Perhaps ' 'they expired?') del self.premium return else: # no premium credentials in the DB return if self.can_sync_data_from_server(): if sync_approval == 'unknown' and not create_new: raise PermissionError( 'Rotkehlchen Server has newer version of your DB data. ' 'Should we replace local data with the server\'s?') elif sync_approval == 'yes' or sync_approval == 'unknown' and create_new: logger.debug('User approved data sync from server') if self.sync_data_from_server(): if create_new: # if we succesfully synced data from the server and this is # a new account, make sure the api keys are properly stored # in the DB self.data.db.set_rotkehlchen_premium( api_key, api_secret) else: logger.debug('Could sync data from server but user refused') def unlock_user(self, user, password, create_new, sync_approval, api_key, api_secret): # unlock or create the DB self.password = password user_dir = self.data.unlock(user, password, create_new) self.try_premium_at_start(api_key, api_secret, create_new, sync_approval, user_dir) secret_data = self.data.db.get_exchange_secrets() self.cache_data_filename = os.path.join(self.data_dir, 'cache_data.json') historical_data_start = self.data.historical_start_date() self.trades_historian = TradesHistorian( self.data_dir, self.data.db, self.data.get_eth_accounts(), historical_data_start, ) self.price_historian = PriceHistorian( self.data_dir, historical_data_start, ) self.accountant = Accountant(price_historian=self.price_historian, profit_currency=self.data.main_currency(), create_csv=True) self.inquirer = Inquirer(kraken=self.kraken) self.initialize_exchanges(secret_data) self.blockchain = Blockchain(self.data.db.get_blockchain_accounts(), self.data.eth_tokens, self.data.db.get_owned_tokens(), self.inquirer, self.args.ethrpc_port) def set_premium_credentials(self, api_key, api_secret): if hasattr(self, 'premium'): valid, empty_or_error = self.premium.set_credentials( api_key, api_secret) else: self.premium, valid, empty_or_error = premium_create_and_verify( api_key, api_secret) if valid: self.data.set_premium_credentials(api_key, api_secret) return True, '' return False, empty_or_error def maybe_upload_data_to_server(self): logger.debug('Maybe upload to server') # upload only if unlocked user has premium if not hasattr(self, 'premium'): return # upload only once per hour diff = ts_now() - self.last_data_upload_ts if diff > 3600: self.upload_data_to_server() def upload_data_to_server(self): logger.debug('upload to server -- start') data, our_hash = self.data.compress_and_encrypt_db(self.password) success, result_or_error = self.premium.query_last_data_metadata() if not success: logger.debug( 'upload to server -- query last metadata error: {}'.format( result_or_error)) return logger.debug("CAN_PUSH--> OURS: {} THEIRS: {}".format( our_hash, result_or_error['data_hash'])) if our_hash == result_or_error['data_hash']: logger.debug('upload to server -- same hash') # same hash -- no need to upload anything return our_last_write_ts = self.data.db.get_last_write_ts() if our_last_write_ts <= result_or_error['last_modify_ts']: # Server's DB was modified after our local DB logger.debug("CAN_PUSH -> 3") logger.debug( 'upload to server -- remote db more recent than local') return success, result_or_error = self.premium.upload_data( data, our_hash, our_last_write_ts, 'zlib') if not success: logger.debug( 'upload to server -- upload error: {}'.format(result_or_error)) return self.last_data_upload_ts = ts_now() logger.debug('upload to server -- success') def can_sync_data_from_server(self): logger.debug('sync data from server -- start') data, our_hash = self.data.compress_and_encrypt_db(self.password) success, result_or_error = self.premium.query_last_data_metadata() if not success: logger.debug( 'sync data from server-- error: {}'.format(result_or_error)) return False logger.debug("CAN_PULL--> OURS: {} THEIRS: {}".format( our_hash, result_or_error['data_hash'])) if our_hash == result_or_error['data_hash']: logger.debug('sync from server -- same hash') # same hash -- no need to get anything return False our_last_write_ts = self.data.db.get_last_write_ts() if our_last_write_ts >= result_or_error['last_modify_ts']: # Local DB is newer than Server DB logger.debug( 'sync from server -- local DB more recent than remote') return False return True def sync_data_from_server(self): success, error_or_result = self.premium.pull_data() if not success: logger.debug( 'sync from server -- pulling error {}'.format(error_or_result)) return False self.data.decompress_and_decrypt_db(self.password, error_or_result['data']) return True def start(self): return gevent.spawn(self.main_loop) def main_loop(self): while True and not self.shutdown_event.is_set(): logger.debug('Main loop start') if self.poloniex is not None: self.poloniex.main_logic() if self.kraken is not None: self.kraken.main_logic() self.maybe_upload_data_to_server() logger.debug('Main loop end') gevent.sleep(MAIN_LOOP_SECS_DELAY) def process_history(self, start_ts, end_ts): history, margin_history, loan_history, asset_movements, eth_transactions = self.trades_historian.get_history( start_ts= 0, # For entire history processing we need to have full history available end_ts=ts_now(), end_at_least_ts=end_ts) return self.accountant.process_history(start_ts, end_ts, history, margin_history, loan_history, asset_movements, eth_transactions) def query_fiat_balances(self): result = {} balances = self.data.get_fiat_balances() for currency, amount in balances.items(): amount = FVal(amount) usd_rate = query_fiat_pair(currency, 'USD') result[currency] = { 'amount': amount, 'usd_value': amount * usd_rate } return result def query_balances(self, save_data=False): balances = {} for exchange in self.connected_exchanges: balances[exchange] = getattr(self, exchange).query_balances() result = self.blockchain.query_balances()['totals'] if result != {}: balances['blockchain'] = result result = self.query_fiat_balances() if result != {}: balances['banks'] = result combined = combine_stat_dicts([v for k, v in balances.items()]) total_usd_per_location = [(k, dict_get_sumof(v, 'usd_value')) for k, v in balances.items()] # calculate net usd value net_usd = FVal(0) for k, v in combined.items(): net_usd += FVal(v['usd_value']) stats = {'location': {}, 'net_usd': net_usd} for entry in total_usd_per_location: name = entry[0] total = entry[1] stats['location'][name] = { 'usd_value': total, 'percentage_of_net_value': (total / net_usd).to_percentage(), } for k, v in combined.items(): combined[k]['percentage_of_net_value'] = (v['usd_value'] / net_usd).to_percentage() result_dict = merge_dicts(combined, stats) if save_data: self.data.save_balances_data(result_dict) # After adding it to the saved file we can overlay additional data that # is not required to be saved in the history file try: details = self.data.accountant.details for asset, (tax_free_amount, average_buy_value) in details.items(): if asset not in result_dict: continue result_dict[asset]['tax_free_amount'] = tax_free_amount result_dict[asset]['average_buy_value'] = average_buy_value current_price = result_dict[asset]['usd_value'] / result_dict[ asset]['amount'] if average_buy_value != FVal(0): result_dict[asset]['percent_change'] = ( ((current_price - average_buy_value) / average_buy_value) * 100) else: result_dict[asset]['percent_change'] = 'INF' except AttributeError: pass return result_dict def set_main_currency(self, currency): with self.lock: self.data.set_main_currency(currency, self.accountant) if currency != 'USD': self.usd_to_main_currency_rate = query_fiat_pair( 'USD', currency) def set_settings(self, settings): with self.lock: self.data.set_settings(settings) main_currency = settings['main_currency'] if main_currency != 'USD': self.usd_to_main_currency_rate = query_fiat_pair( 'USD', main_currency) def usd_to_main_currency(self, amount): main_currency = self.data.main_currency() if main_currency != 'USD' and not hasattr(self, 'usd_to_main_currency_rate'): self.usd_to_main_currency_rate = query_fiat_pair( 'USD', main_currency) return self.usd_to_main_currency_rate * amount def get_settings(self): return self.data.settings def setup_exchange(self, name, api_key, api_secret): if name not in SUPPORTED_EXCHANGES: return False, 'Attempted to register unsupported exchange {}'.format( name) if getattr(self, name) is not None: return False, 'Exchange {} is already registered'.format(name) secret_data = {} secret_data[name] = { 'api_key': api_key, 'api_secret': api_secret, } self.initialize_exchanges(secret_data) exchange = getattr(self, name) result, message = exchange.validate_api_key() if not result: self.delete_exchange_data(name) return False, message # Success, save the result in the DB self.data.db.add_exchange(name, api_key, api_secret) return True, '' def delete_exchange_data(self, name): self.connected_exchanges.remove(name) self.trades_historian.set_exchange(name, None) delattr(self, name) setattr(self, name, None) def remove_exchange(self, name): if getattr(self, name) is None: return False, 'Exchange {} is not registered'.format(name) self.delete_exchange_data(name) # Success, remove it also from the DB self.data.db.remove_exchange(name) return True, '' def shutdown(self): print("Shutting Down...") self.shutdown_event.set() def set(self, *args): if len(args) < 2: return ("ERROR: set requires at least two arguments but " "got: {}".format(args)) if args[0] == 'poloniex': resp = self.poloniex.set(*args[1:]) else: return "ERROR: Unrecognized first argument: {}".format(args[0]) self.save_data() return resp
def test_writting_fetching_external_trades(data_dir, username): msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) # add 2 trades and check they are in the DB otc_trade1 = { 'otc_timestamp': '10/03/2018 23:30', 'otc_pair': 'ETH_EUR', 'otc_type': 'buy', 'otc_amount': '10', 'otc_rate': '100', 'otc_fee': '0.001', 'otc_fee_currency': 'ETH', 'otc_link': 'a link', 'otc_notes': 'a note', } otc_trade2 = { 'otc_timestamp': '15/03/2018 23:35', 'otc_pair': 'ETH_EUR', 'otc_type': 'buy', 'otc_amount': '5', 'otc_rate': '100', 'otc_fee': '0.001', 'otc_fee_currency': 'ETH', 'otc_link': 'a link 2', 'otc_notes': 'a note 2', } trade1, _ = verify_otctrade_data(otc_trade1) trade2, _ = verify_otctrade_data(otc_trade2) result, _, = data.add_external_trade(otc_trade1) assert result result, _ = data.add_external_trade(otc_trade2) assert result result = data.get_external_trades() assert result[0] == trade1 assert result[1] == trade2 # query trades in period result = data.get_external_trades( from_ts=1520553600, # 09/03/2018 to_ts=1520726400, # 11/03/2018 ) assert len(result) == 1 # make sure id is there but do not compare it assert result[0] == trade1 # query trades only with to_ts result = data.get_external_trades( to_ts=1520726400, # 11/03/2018 ) assert len(result) == 1 # make sure id is there but do not compare it assert result[0] == trade1 # edit a trade and check the edit made it in the DB otc_trade1['otc_rate'] = '120' trade1_id = trade1.identifier otc_trade1['otc_id'] = trade1_id result, _ = data.edit_external_trade(otc_trade1) assert result result = data.get_external_trades() edited_trade1 = trade1._replace(rate=FVal(120)) assert result[0] == edited_trade1 assert result[1] == trade2 # edit a trade's time (or any other field which makes up the id) and see # that the ID changes later new_time_str = '10/03/2018 23:35' new_timestamp = 1520724900 otc_trade1['otc_timestamp'] = new_time_str otc_trade1['otc_id'] = edited_trade1.identifier result, _ = data.edit_external_trade(otc_trade1) assert result result = data.get_external_trades() edited_trade1 = trade1._replace(rate=FVal(120), timestamp=new_timestamp) assert result[0] == edited_trade1 assert result[1] == trade2 # Here trade is edited succesfully but let's also look in the DB to see # if the trade is indeed there with a new ID cursor = data.db.conn.cursor() results = cursor.execute( f'SELECT id, time, pair, amount FROM trades where id="{edited_trade1.identifier}";', ) results = results.fetchall() assert len(results) == 1 result = results[0] assert result[0] == edited_trade1.identifier assert result[1] == new_timestamp assert result[2] == otc_trade1['otc_pair'] assert result[3] == otc_trade1['otc_amount'] # try to edit a non-existing trade otc_trade1['otc_rate'] = '160' otc_trade1['otc_id'] = '5' result, _ = data.edit_external_trade(otc_trade1) assert not result otc_trade1['otc_rate'] = '120' otc_trade1['otc_id'] = trade1_id result = data.get_external_trades() assert result[0] == edited_trade1 assert result[1] == trade2 # try to delete non-existing trade result, _ = data.delete_external_trade('dasdasd') assert not result # delete an external trade result, _ = data.delete_external_trade(edited_trade1.identifier) result = data.get_external_trades() assert result[0] == trade2
def test_add_trades(data_dir, username, caplog): """Test that adding and retrieving trades from the DB works fine. Also duplicates should be ignored and an error returned """ msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) trade1 = Trade( timestamp=1451606400, location=Location.KRAKEN, base_asset=A_ETH, quote_asset=A_EUR, trade_type=TradeType.BUY, amount=FVal('1.1'), rate=FVal('10'), fee=Fee(FVal('0.01')), fee_currency=A_EUR, link='', notes='', ) trade2 = Trade( timestamp=1451607500, location=Location.BINANCE, base_asset=A_BTC, quote_asset=A_ETH, trade_type=TradeType.BUY, amount=FVal('0.00120'), rate=FVal('10'), fee=Fee(FVal('0.001')), fee_currency=A_ETH, link='', notes='', ) trade3 = Trade( timestamp=1451608600, location=Location.COINBASE, base_asset=A_BTC, quote_asset=A_ETH, trade_type=TradeType.SELL, amount=FVal('0.00120'), rate=FVal('1'), fee=Fee(FVal('0.001')), fee_currency=A_ETH, link='', notes='', ) # Add and retrieve the first 2 trades. All should be fine. data.db.add_trades([trade1, trade2]) errors = msg_aggregator.consume_errors() warnings = msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 0 returned_trades = data.db.get_trades() assert returned_trades == [trade1, trade2] # Add the last 2 trades. Since trade2 already exists in the DB it should be # ignored and a warning should be logged data.db.add_trades([trade2, trade3]) assert 'Did not add "buy trade with id a1ed19c8284940b4e59bdac941db2fd3c0ed004ddb10fdd3b9ef0a3a9b2c97bc' in caplog.text # noqa: E501 returned_trades = data.db.get_trades() assert returned_trades == [trade1, trade2, trade3]
class Rotkehlchen(): def __init__(self, args: argparse.Namespace) -> None: self.lock = Semaphore() self.lock.acquire() self.premium = None self.user_is_logged_in = False logfilename = None if args.logtarget == 'file': logfilename = args.logfile if args.loglevel == 'debug': loglevel = logging.DEBUG elif args.loglevel == 'info': loglevel = logging.INFO elif args.loglevel == 'warn': loglevel = logging.WARN elif args.loglevel == 'error': loglevel = logging.ERROR elif args.loglevel == 'critical': loglevel = logging.CRITICAL else: raise ValueError('Should never get here. Illegal log value') logging.basicConfig( filename=logfilename, filemode='w', level=loglevel, format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s', datefmt='%d/%m/%Y %H:%M:%S %Z', ) if not args.logfromothermodules: logging.getLogger('zerorpc').setLevel(logging.CRITICAL) logging.getLogger('zerorpc.channel').setLevel(logging.CRITICAL) logging.getLogger('urllib3').setLevel(logging.CRITICAL) logging.getLogger('urllib3.connectionpool').setLevel( logging.CRITICAL) self.sleep_secs = args.sleep_secs self.data_dir = args.data_dir self.args = args self.msg_aggregator = MessagesAggregator() self.exchange_manager = ExchangeManager( msg_aggregator=self.msg_aggregator) self.data = DataHandler(self.data_dir, self.msg_aggregator) # Initialize the Inquirer singleton Inquirer(data_dir=self.data_dir) self.lock.release() self.shutdown_event = gevent.event.Event() def unlock_user( self, user: str, password: str, create_new: bool, sync_approval: bool, api_key: ApiKey, api_secret: ApiSecret, ) -> None: """Unlocks an existing user or creates a new one if `create_new` is True""" log.info( 'Unlocking user', user=user, create_new=create_new, sync_approval=sync_approval, ) # unlock or create the DB self.password = password self.user_directory = self.data.unlock(user, password, create_new) self.data_importer = DataImporter(db=self.data.db) self.last_data_upload_ts = self.data.db.get_last_data_upload_ts() self.premium_sync_manager = PremiumSyncManager(data=self.data, password=password) try: self.premium = self.premium_sync_manager.try_premium_at_start( api_key=api_key, api_secret=api_secret, username=user, create_new=create_new, sync_approval=sync_approval, ) except AuthenticationError: # It means that our credentials were not accepted by the server # or some other error happened pass settings = self.data.db.get_settings() historical_data_start = settings['historical_data_start'] # TODO: Once settings returns a named tuple this should go away msg = 'setting historical_data_start should be a string' assert isinstance(historical_data_start, str), msg eth_rpc_endpoint = settings['eth_rpc_endpoint'] msg = 'setting eth_rpc_endpoint should be a string' assert isinstance(eth_rpc_endpoint, str), msg self.trades_historian = TradesHistorian( user_directory=self.user_directory, db=self.data.db, eth_accounts=self.data.get_eth_accounts(), msg_aggregator=self.msg_aggregator, exchange_manager=self.exchange_manager, ) # Initialize the price historian singleton PriceHistorian( data_directory=self.data_dir, history_date_start=historical_data_start, cryptocompare=Cryptocompare(data_directory=self.data_dir), ) db_settings = self.data.db.get_settings() # TODO: Once settings returns a named tuple these should go away crypto2crypto = db_settings['include_crypto2crypto'] msg = 'settings include_crypto2crypto should be a bool' assert isinstance(crypto2crypto, bool), msg taxfree_after_period = db_settings['taxfree_after_period'] msg = 'settings taxfree_after_period should be an int' assert isinstance(taxfree_after_period, int), msg include_gas_costs = db_settings['include_gas_costs'] msg = 'settings include_gas_costs should be a bool' assert isinstance(include_gas_costs, bool), msg self.accountant = Accountant( profit_currency=self.data.main_currency(), user_directory=self.user_directory, msg_aggregator=self.msg_aggregator, create_csv=True, ignored_assets=self.data.db.get_ignored_assets(), include_crypto2crypto=crypto2crypto, taxfree_after_period=taxfree_after_period, include_gas_costs=include_gas_costs, ) # Initialize the rotkehlchen logger LoggingSettings(anonymized_logs=db_settings['anonymized_logs']) exchange_credentials = self.data.db.get_exchange_credentials() self.exchange_manager.initialize_exchanges( exchange_credentials=exchange_credentials, database=self.data.db, ) ethchain = Ethchain(eth_rpc_endpoint) self.blockchain = Blockchain( blockchain_accounts=self.data.db.get_blockchain_accounts(), owned_eth_tokens=self.data.db.get_owned_tokens(), ethchain=ethchain, msg_aggregator=self.msg_aggregator, ) self.user_is_logged_in = True def logout(self) -> None: if not self.user_is_logged_in: return user = self.data.username log.info( 'Logging out user', user=user, ) del self.blockchain self.exchange_manager.delete_all_exchanges() # Reset rotkehlchen logger to default LoggingSettings(anonymized_logs=DEFAULT_ANONYMIZED_LOGS) del self.accountant del self.trades_historian del self.data_importer if self.premium is not None: # For some reason mypy does not see that self.premium is set del self.premium # type: ignore self.data.logout() self.password = '' self.user_is_logged_in = False log.info( 'User successfully logged out', user=user, ) def set_premium_credentials(self, api_key: ApiKey, api_secret: ApiSecret) -> None: """ Raises IncorrectApiKeyFormat if the given key is not in a proper format Raises AuthenticationError if the given key is rejected by the Rotkehlchen server """ log.info('Setting new premium credentials') if self.premium is not None: # For some reason mypy does not see that self.premium is set self.premium.set_credentials(api_key, api_secret) # type: ignore else: self.premium = premium_create_and_verify(api_key, api_secret) self.data.set_premium_credentials(api_key, api_secret) def start(self) -> None: return gevent.spawn(self.main_loop) def main_loop(self) -> None: while self.shutdown_event.wait(MAIN_LOOP_SECS_DELAY) is not True: log.debug('Main loop start') self.premium_sync_manager.maybe_upload_data_to_server() log.debug('Main loop end') def add_blockchain_account( self, blockchain: SupportedBlockchain, account: BlockchainAddress, ) -> Dict: try: new_data = self.blockchain.add_blockchain_account( blockchain, account) except (InputError, EthSyncError) as e: return simple_result(False, str(e)) self.data.add_blockchain_account(blockchain, account) return accounts_result(new_data['per_account'], new_data['totals']) def remove_blockchain_account( self, blockchain: SupportedBlockchain, account: BlockchainAddress, ) -> Dict[str, Any]: try: new_data = self.blockchain.remove_blockchain_account( blockchain, account) except (InputError, EthSyncError) as e: return simple_result(False, str(e)) self.data.remove_blockchain_account(blockchain, account) return accounts_result(new_data['per_account'], new_data['totals']) def add_owned_eth_tokens(self, tokens: List[str]) -> Dict[str, Any]: ethereum_tokens = [ EthereumToken(identifier=identifier) for identifier in tokens ] try: new_data = self.blockchain.track_new_tokens(ethereum_tokens) except (InputError, EthSyncError) as e: return simple_result(False, str(e)) self.data.write_owned_eth_tokens(self.blockchain.owned_eth_tokens) return accounts_result(new_data['per_account'], new_data['totals']) def remove_owned_eth_tokens(self, tokens: List[str]) -> Dict[str, Any]: ethereum_tokens = [ EthereumToken(identifier=identifier) for identifier in tokens ] try: new_data = self.blockchain.remove_eth_tokens(ethereum_tokens) except InputError as e: return simple_result(False, str(e)) self.data.write_owned_eth_tokens(self.blockchain.owned_eth_tokens) return accounts_result(new_data['per_account'], new_data['totals']) def process_history( self, start_ts: Timestamp, end_ts: Timestamp, ) -> Tuple[Dict[str, Any], str]: ( error_or_empty, history, loan_history, asset_movements, eth_transactions, ) = self.trades_historian.get_history( # For entire history processing we need to have full history available start_ts=Timestamp(0), end_ts=ts_now(), ) result = self.accountant.process_history( start_ts=start_ts, end_ts=end_ts, trade_history=history, loan_history=loan_history, asset_movements=asset_movements, eth_transactions=eth_transactions, ) return result, error_or_empty def query_fiat_balances(self) -> Dict[Asset, Dict[str, FVal]]: result = {} balances = self.data.get_fiat_balances() for currency, str_amount in balances.items(): amount = FVal(str_amount) usd_rate = Inquirer().query_fiat_pair(currency, A_USD) result[currency] = { 'amount': amount, 'usd_value': amount * usd_rate, } return result def query_balances( self, requested_save_data: bool = False, timestamp: Timestamp = None, ) -> Dict[str, Any]: """Query all balances rotkehlchen can see. If requested_save_data is True then the data are saved in the DB. If timestamp is None then the current timestamp is used. If a timestamp is given then that is the time that the balances are going to be saved in the DB Returns a dictionary with the queried balances. """ log.info('query_balances called', requested_save_data=requested_save_data) balances = {} problem_free = True for _, exchange in self.exchange_manager.connected_exchanges.items(): exchange_balances, _ = exchange.query_balances() # If we got an error, disregard that exchange but make sure we don't save data if not isinstance(exchange_balances, dict): problem_free = False else: balances[exchange.name] = exchange_balances result, error_or_empty = self.blockchain.query_balances() if error_or_empty == '': balances['blockchain'] = result['totals'] else: problem_free = False result = self.query_fiat_balances() if result != {}: balances['banks'] = result combined = combine_stat_dicts([v for k, v in balances.items()]) total_usd_per_location = [(k, dict_get_sumof(v, 'usd_value')) for k, v in balances.items()] # calculate net usd value net_usd = FVal(0) for _, v in combined.items(): net_usd += FVal(v['usd_value']) stats: Dict[str, Any] = { 'location': {}, 'net_usd': net_usd, } for entry in total_usd_per_location: name = entry[0] total = entry[1] if net_usd != FVal(0): percentage = (total / net_usd).to_percentage() else: percentage = '0%' stats['location'][name] = { 'usd_value': total, 'percentage_of_net_value': percentage, } for k, v in combined.items(): if net_usd != FVal(0): percentage = (v['usd_value'] / net_usd).to_percentage() else: percentage = '0%' combined[k]['percentage_of_net_value'] = percentage result_dict = merge_dicts(combined, stats) allowed_to_save = requested_save_data or self.data.should_save_balances( ) if problem_free and allowed_to_save: if not timestamp: timestamp = Timestamp(int(time.time())) self.data.save_balances_data(data=result_dict, timestamp=timestamp) log.debug('query_balances data saved') else: log.debug( 'query_balances data not saved', allowed_to_save=allowed_to_save, problem_free=problem_free, ) # After adding it to the saved file we can overlay additional data that # is not required to be saved in the history file try: details = self.accountant.events.details for asset, (tax_free_amount, average_buy_value) in details.items(): if asset not in result_dict: continue result_dict[asset]['tax_free_amount'] = tax_free_amount result_dict[asset]['average_buy_value'] = average_buy_value current_price = result_dict[asset]['usd_value'] / result_dict[ asset]['amount'] if average_buy_value != FVal(0): result_dict[asset]['percent_change'] = ( ((current_price - average_buy_value) / average_buy_value) * 100) else: result_dict[asset]['percent_change'] = 'INF' except AttributeError: pass return result_dict def set_main_currency(self, currency_string: str) -> Tuple[bool, str]: """Takes a currency string from the API and sets it as the main currency for rotki Returns True and empty string for success and False and error string for error """ try: currency = Asset(currency_string) except UnknownAsset: msg = f'An unknown asset {currency_string} was given for main currency' log.critical(msg) return False, msg if not currency.is_fiat(): msg = f'A non-fiat asset {currency_string} was given for main currency' log.critical(msg) return False, msg fiat_currency = FiatAsset(currency.identifier) with self.lock: self.data.set_main_currency(fiat_currency, self.accountant) if currency != A_USD: self.usd_to_main_currency_rate = Inquirer().query_fiat_pair( A_USD, currency) return True, '' def set_settings(self, settings: Dict[str, Any]) -> Tuple[bool, str]: log.info('Add new settings') message = '' with self.lock: if 'eth_rpc_endpoint' in settings: result, msg = self.blockchain.set_eth_rpc_endpoint( settings['eth_rpc_endpoint']) if not result: # Don't save it in the DB del settings['eth_rpc_endpoint'] message += "\nEthereum RPC endpoint not set: " + msg if 'main_currency' in settings: given_symbol = settings['main_currency'] try: main_currency = Asset(given_symbol) except UnknownAsset: return False, f'Unknown fiat currency {given_symbol} provided' except DeserializationError: return False, 'Non string type given for fiat currency' if not main_currency.is_fiat(): msg = ( f'Provided symbol for main currency {given_symbol} is ' f'not a fiat currency') return False, msg if main_currency != A_USD: self.usd_to_main_currency_rate = Inquirer( ).query_fiat_pair( A_USD, main_currency, ) res, msg = self.accountant.customize(settings) if not res: message += '\n' + msg return False, message _, msg, = self.data.set_settings(settings, self.accountant) if msg != '': message += '\n' + msg # Always return success here but with a message return True, message def setup_exchange( self, name: str, api_key: str, api_secret: str, ) -> Tuple[bool, str]: """ Setup a new exchange with an api key and an api secret By default the api keys are always validated unless validate is False. """ is_success, msg = self.exchange_manager.setup_exchange( name=name, api_key=api_key, api_secret=api_secret, database=self.data.db, ) if is_success: # Success, save the result in the DB self.data.db.add_exchange(name, api_key, api_secret) return is_success, msg def remove_exchange(self, name: str) -> Tuple[bool, str]: if not self.exchange_manager.has_exchange(name): return False, 'Exchange {} is not registered'.format(name) self.exchange_manager.delete_exchange(name) # Success, remove it also from the DB self.data.db.remove_exchange(name) return True, '' def query_periodic_data(self) -> Dict[str, Union[bool, Timestamp]]: """Query for frequently changing data""" result: Dict[str, Union[bool, Timestamp]] = {} if self.user_is_logged_in: result[ 'last_balance_save'] = self.data.db.get_last_balance_save_time( ) result['eth_node_connection'] = self.blockchain.ethchain.connected result[ 'history_process_start_ts'] = self.accountant.started_processing_timestamp result[ 'history_process_current_ts'] = self.accountant.currently_processing_timestamp return result def shutdown(self) -> None: self.logout() self.shutdown_event.set()
class Rotkehlchen(): def __init__(self, args): self.lock = Semaphore() self.lock.acquire() self.results_cache: ResultCache = dict() self.premium = None self.connected_exchanges = [] self.user_is_logged_in = False logfilename = None if args.logtarget == 'file': logfilename = args.logfile if args.loglevel == 'debug': loglevel = logging.DEBUG elif args.loglevel == 'info': loglevel = logging.INFO elif args.loglevel == 'warn': loglevel = logging.WARN elif args.loglevel == 'error': loglevel = logging.ERROR elif args.loglevel == 'critical': loglevel = logging.CRITICAL else: raise ValueError('Should never get here. Illegal log value') logging.basicConfig( filename=logfilename, filemode='w', level=loglevel, format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s', datefmt='%d/%m/%Y %H:%M:%S %Z', ) if not args.logfromothermodules: logging.getLogger('zerorpc').setLevel(logging.CRITICAL) logging.getLogger('zerorpc.channel').setLevel(logging.CRITICAL) logging.getLogger('urllib3').setLevel(logging.CRITICAL) logging.getLogger('urllib3.connectionpool').setLevel( logging.CRITICAL) self.sleep_secs = args.sleep_secs self.data_dir = args.data_dir self.args = args self.poloniex = None self.kraken = None self.bittrex = None self.bitmex = None self.binance = None self.msg_aggregator = MessagesAggregator() self.data = DataHandler(self.data_dir, self.msg_aggregator) # Initialize the Inquirer singleton Inquirer(data_dir=self.data_dir) self.lock.release() self.shutdown_event = gevent.event.Event() def initialize_exchanges(self, secret_data): # initialize exchanges for which we have keys and are not already initialized if self.kraken is None and 'kraken' in secret_data: self.kraken = Kraken( api_key=str.encode(secret_data['kraken']['api_key']), secret=str.encode(secret_data['kraken']['api_secret']), user_directory=self.user_directory, msg_aggregator=self.msg_aggregator, usd_eur_price=Inquirer().query_fiat_pair(S_EUR, A_USD), ) self.connected_exchanges.append('kraken') self.trades_historian.set_exchange('kraken', self.kraken) if self.poloniex is None and 'poloniex' in secret_data: self.poloniex = Poloniex( api_key=str.encode(secret_data['poloniex']['api_key']), secret=str.encode(secret_data['poloniex']['api_secret']), user_directory=self.user_directory, msg_aggregator=self.msg_aggregator, ) self.connected_exchanges.append('poloniex') self.trades_historian.set_exchange('poloniex', self.poloniex) if self.bittrex is None and 'bittrex' in secret_data: self.bittrex = Bittrex( api_key=str.encode(secret_data['bittrex']['api_key']), secret=str.encode(secret_data['bittrex']['api_secret']), user_directory=self.user_directory, msg_aggregator=self.msg_aggregator, ) self.connected_exchanges.append('bittrex') self.trades_historian.set_exchange('bittrex', self.bittrex) if self.binance is None and 'binance' in secret_data: self.binance = Binance( api_key=str.encode(secret_data['binance']['api_key']), secret=str.encode(secret_data['binance']['api_secret']), data_dir=self.user_directory, msg_aggregator=self.msg_aggregator, ) self.connected_exchanges.append('binance') self.trades_historian.set_exchange('binance', self.binance) if self.bitmex is None and 'bitmex' in secret_data: self.bitmex = Bitmex( api_key=str.encode(secret_data['bitmex']['api_key']), secret=str.encode(secret_data['bitmex']['api_secret']), user_directory=self.user_directory, ) self.connected_exchanges.append('bitmex') self.trades_historian.set_exchange('bitmex', self.bitmex) def remove_all_exchanges(self): if self.kraken is not None: self.delete_exchange_data('kraken') if self.poloniex is not None: self.delete_exchange_data('poloniex') if self.bittrex is not None: self.delete_exchange_data('bittrex') if self.binance is not None: self.delete_exchange_data('binance') if self.bitmex is not None: self.delete_exchange_data('bitmex') def try_premium_at_start( self, api_key: ApiKey, api_secret: ApiSecret, username: str, create_new: bool, sync_approval: bool, ) -> None: """Check if new user provided api pair or we already got one in the DB""" if api_key != '': assert create_new, 'We should never get here for an already existing account' try: self.premium = premium_create_and_verify(api_key, api_secret) except (IncorrectApiKeyFormat, AuthenticationError) as e: log.error('Given API key is invalid') # At this point we are at a new user trying to create an account with # premium API keys and we failed. But a directory was created. Remove it. # But create a backup of it in case something went really wrong # and the directory contained data we did not want to lose shutil.move( self.user_directory, os.path.join( self.data_dir, f'auto_backup_{username}_{ts_now()}', ), ) shutil.rmtree(self.user_directory) raise AuthenticationError( 'Could not verify keys for the new account. ' '{}'.format(str(e)), ) # else, if we got premium data in the DB initialize it and try to sync with the server premium_credentials = self.data.db.get_rotkehlchen_premium() if premium_credentials: assert not create_new, 'We should never get here for a new account' api_key = premium_credentials[0] api_secret = premium_credentials[1] try: self.premium = premium_create_and_verify(api_key, api_secret) except (IncorrectApiKeyFormat, AuthenticationError) as e: log.error( f'Could not authenticate with the rotkehlchen server with ' f'the API keys found in the Database. Error: {str(e)}', ) del self.premium self.premium = None if not self.premium: return if self.can_sync_data_from_server(): if sync_approval == 'unknown' and not create_new: log.info('DB data at server newer than local') raise RotkehlchenPermissionError( 'Rotkehlchen Server has newer version of your DB data. ' 'Should we replace local data with the server\'s?', ) elif sync_approval == 'yes' or sync_approval == 'unknown' and create_new: log.info('User approved data sync from server') if self.sync_data_from_server(): if create_new: # if we successfully synced data from the server and this is # a new account, make sure the api keys are properly stored # in the DB self.data.db.set_rotkehlchen_premium( api_key, api_secret) else: log.debug('Could sync data from server but user refused') def unlock_user( self, user: str, password: str, create_new: bool, sync_approval: bool, api_key: ApiKey, api_secret: ApiSecret, ) -> None: """Unlocks an existing user or creates a new one if `create_new` is True""" log.info( 'Unlocking user', user=user, create_new=create_new, sync_approval=sync_approval, ) # unlock or create the DB self.password = password self.user_directory = self.data.unlock(user, password, create_new) self.last_data_upload_ts = self.data.db.get_last_data_upload_ts() self.try_premium_at_start( api_key=api_key, api_secret=api_secret, username=user, create_new=create_new, sync_approval=sync_approval, ) secret_data = self.data.db.get_exchange_secrets() settings = self.data.db.get_settings() historical_data_start = settings['historical_data_start'] eth_rpc_endpoint = settings['eth_rpc_endpoint'] self.trades_historian = TradesHistorian( user_directory=self.user_directory, db=self.data.db, eth_accounts=self.data.get_eth_accounts(), msg_aggregator=self.msg_aggregator, ) # Initialize the price historian singleton PriceHistorian( data_directory=self.data_dir, history_date_start=historical_data_start, cryptocompare=Cryptocompare(data_directory=self.data_dir), ) db_settings = self.data.db.get_settings() self.accountant = Accountant( profit_currency=self.data.main_currency(), user_directory=self.user_directory, msg_aggregator=self.msg_aggregator, create_csv=True, ignored_assets=self.data.db.get_ignored_assets(), include_crypto2crypto=db_settings['include_crypto2crypto'], taxfree_after_period=db_settings['taxfree_after_period'], include_gas_costs=db_settings['include_gas_costs'], ) # Initialize the rotkehlchen logger LoggingSettings(anonymized_logs=db_settings['anonymized_logs']) self.initialize_exchanges(secret_data) ethchain = Ethchain(eth_rpc_endpoint) self.blockchain = Blockchain( blockchain_accounts=self.data.db.get_blockchain_accounts(), owned_eth_tokens=self.data.db.get_owned_tokens(), ethchain=ethchain, msg_aggregator=self.msg_aggregator, ) self.user_is_logged_in = True def logout(self) -> None: if not self.user_is_logged_in: return user = self.data.username log.info( 'Logging out user', user=user, ) del self.blockchain self.remove_all_exchanges() # Reset rotkehlchen logger to default LoggingSettings(anonymized_logs=DEFAULT_ANONYMIZED_LOGS) del self.accountant del self.trades_historian if self.premium is not None: del self.premium self.data.logout() self.password = '' self.user_is_logged_in = False log.info( 'User successfully logged out', user=user, ) def set_premium_credentials(self, api_key: ApiKey, api_secret: ApiSecret) -> None: """ Raises IncorrectApiKeyFormat if the given key is not in a proper format Raises AuthenticationError if the given key is rejected by the Rotkehlchen server """ log.info('Setting new premium credentials') if self.premium is not None: self.premium.set_credentials(api_key, api_secret) else: self.premium = premium_create_and_verify(api_key, api_secret) self.data.set_premium_credentials(api_key, api_secret) def maybe_upload_data_to_server(self) -> None: # upload only if unlocked user has premium if self.premium is None: return # upload only once per hour diff = ts_now() - self.last_data_upload_ts if diff > 3600: self.upload_data_to_server() def upload_data_to_server(self) -> None: log.debug('upload to server -- start') data, our_hash = self.data.compress_and_encrypt_db(self.password) try: result = self.premium.query_last_data_metadata() except RemoteError as e: log.debug( 'upload to server -- query last metadata failed', error=str(e), ) return log.debug( 'CAN_PUSH', ours=our_hash, theirs=result['data_hash'], ) if our_hash == result['data_hash']: log.debug('upload to server -- same hash') # same hash -- no need to upload anything return our_last_write_ts = self.data.db.get_last_write_ts() if our_last_write_ts <= result['last_modify_ts']: # Server's DB was modified after our local DB log.debug("CAN_PUSH -> 3") log.debug('upload to server -- remote db more recent than local') return try: self.premium.upload_data( data_blob=data, our_hash=our_hash, last_modify_ts=our_last_write_ts, compression_type='zlib', ) except RemoteError as e: log.debug('upload to server -- upload error', error=str(e)) return # update the last data upload value self.last_data_upload_ts = ts_now() self.data.db.update_last_data_upload_ts(self.last_data_upload_ts) log.debug('upload to server -- success') def can_sync_data_from_server(self) -> bool: log.debug('sync data from server -- start') _, our_hash = self.data.compress_and_encrypt_db(self.password) try: result = self.premium.query_last_data_metadata() except RemoteError as e: log.debug('sync data from server failed', error=str(e)) return False if not self.data.db.get_premium_sync(): return False log.debug( 'CAN_PULL', ours=our_hash, theirs=result['data_hash'], ) if our_hash == result['data_hash']: log.debug('sync from server -- same hash') # same hash -- no need to get anything return False our_last_write_ts = self.data.db.get_last_write_ts() if our_last_write_ts >= result['last_modify_ts']: # Local DB is newer than Server DB log.debug('sync from server -- local DB more recent than remote') return False return True def sync_data_from_server(self) -> bool: try: result = self.premium.pull_data() except RemoteError as e: log.debug('sync from server -- pulling failed.', error=str(e)) return False self.data.decompress_and_decrypt_db(self.password, result['data']) return True def start(self): return gevent.spawn(self.main_loop) def main_loop(self): while self.shutdown_event.wait(MAIN_LOOP_SECS_DELAY) is not True: log.debug('Main loop start') if self.poloniex is not None: self.poloniex.main_logic() if self.kraken is not None: self.kraken.main_logic() self.maybe_upload_data_to_server() log.debug('Main loop end') def add_blockchain_account( self, blockchain: SupportedBlockchain, account: BlockchainAddress, ) -> Dict: try: new_data = self.blockchain.add_blockchain_account( blockchain, account) except (InputError, EthSyncError) as e: return simple_result(False, str(e)) self.data.add_blockchain_account(blockchain, account) return accounts_result(new_data['per_account'], new_data['totals']) def remove_blockchain_account( self, blockchain: SupportedBlockchain, account: BlockchainAddress, ): try: new_data = self.blockchain.remove_blockchain_account( blockchain, account) except (InputError, EthSyncError) as e: return simple_result(False, str(e)) self.data.remove_blockchain_account(blockchain, account) return accounts_result(new_data['per_account'], new_data['totals']) def add_owned_eth_tokens(self, tokens: List[str]): ethereum_tokens = [ EthereumToken(identifier=identifier) for identifier in tokens ] try: new_data = self.blockchain.track_new_tokens(ethereum_tokens) except (InputError, EthSyncError) as e: return simple_result(False, str(e)) self.data.write_owned_eth_tokens(self.blockchain.owned_eth_tokens) return accounts_result(new_data['per_account'], new_data['totals']) def remove_owned_eth_tokens(self, tokens: List[str]): ethereum_tokens = [ EthereumToken(identifier=identifier) for identifier in tokens ] try: new_data = self.blockchain.remove_eth_tokens(ethereum_tokens) except InputError as e: return simple_result(False, str(e)) self.data.write_owned_eth_tokens(self.blockchain.owned_eth_tokens) return accounts_result(new_data['per_account'], new_data['totals']) def process_history(self, start_ts, end_ts): ( error_or_empty, history, margin_history, loan_history, asset_movements, eth_transactions, ) = self.trades_historian.get_history( start_ts= 0, # For entire history processing we need to have full history available end_ts=ts_now(), end_at_least_ts=end_ts, ) result = self.accountant.process_history( start_ts, end_ts, history, margin_history, loan_history, asset_movements, eth_transactions, ) return result, error_or_empty def query_fiat_balances(self): log.info('query_fiat_balances called') result = {} balances = self.data.get_fiat_balances() for currency, amount in balances.items(): amount = FVal(amount) usd_rate = Inquirer().query_fiat_pair(currency, 'USD') result[currency] = { 'amount': amount, 'usd_value': amount * usd_rate, } return result def query_balances( self, requested_save_data: bool = False, timestamp: Timestamp = None, ) -> Dict[str, Any]: """Query all balances rotkehlchen can see. If requested_save_data is True then the data are saved in the DB. If timestamp is None then the current timestamp is used. If a timestamp is given then that is the time that the balances are going to be saved in the DB Returns a dictionary with the queried balances. """ log.info('query_balances called', requested_save_data=requested_save_data) balances = {} problem_free = True for exchange in self.connected_exchanges: exchange_balances, _ = getattr(self, exchange).query_balances() # If we got an error, disregard that exchange but make sure we don't save data if not isinstance(exchange_balances, dict): problem_free = False else: balances[exchange] = exchange_balances result, error_or_empty = self.blockchain.query_balances() if error_or_empty == '': balances['blockchain'] = result['totals'] else: problem_free = False result = self.query_fiat_balances() if result != {}: balances['banks'] = result combined = combine_stat_dicts([v for k, v in balances.items()]) total_usd_per_location = [(k, dict_get_sumof(v, 'usd_value')) for k, v in balances.items()] # calculate net usd value net_usd = FVal(0) for _, v in combined.items(): net_usd += FVal(v['usd_value']) stats: Dict[str, Any] = { 'location': {}, 'net_usd': net_usd, } for entry in total_usd_per_location: name = entry[0] total = entry[1] if net_usd != FVal(0): percentage = (total / net_usd).to_percentage() else: percentage = '0%' stats['location'][name] = { 'usd_value': total, 'percentage_of_net_value': percentage, } for k, v in combined.items(): if net_usd != FVal(0): percentage = (v['usd_value'] / net_usd).to_percentage() else: percentage = '0%' combined[k]['percentage_of_net_value'] = percentage result_dict = merge_dicts(combined, stats) allowed_to_save = requested_save_data or self.data.should_save_balances( ) if problem_free and allowed_to_save: if not timestamp: timestamp = Timestamp(int(time.time())) self.data.save_balances_data(data=result_dict, timestamp=timestamp) log.debug('query_balances data saved') else: log.debug( 'query_balances data not saved', allowed_to_save=allowed_to_save, problem_free=problem_free, ) # After adding it to the saved file we can overlay additional data that # is not required to be saved in the history file try: details = self.data.accountant.details for asset, (tax_free_amount, average_buy_value) in details.items(): if asset not in result_dict: continue result_dict[asset]['tax_free_amount'] = tax_free_amount result_dict[asset]['average_buy_value'] = average_buy_value current_price = result_dict[asset]['usd_value'] / result_dict[ asset]['amount'] if average_buy_value != FVal(0): result_dict[asset]['percent_change'] = ( ((current_price - average_buy_value) / average_buy_value) * 100) else: result_dict[asset]['percent_change'] = 'INF' except AttributeError: pass return result_dict def set_main_currency(self, currency): with self.lock: self.data.set_main_currency(currency, self.accountant) if currency != 'USD': self.usd_to_main_currency_rate = Inquirer().query_fiat_pair( 'USD', currency) def set_settings(self, settings): log.info('Add new settings') message = '' with self.lock: if 'eth_rpc_endpoint' in settings: result, msg = self.blockchain.set_eth_rpc_endpoint( settings['eth_rpc_endpoint']) if not result: # Don't save it in the DB del settings['eth_rpc_endpoint'] message += "\nEthereum RPC endpoint not set: " + msg if 'main_currency' in settings: given_symbol = settings['main_currency'] try: main_currency = Asset(given_symbol) except UnknownAsset: return False, f'Unknown fiat currency {given_symbol} provided' if not main_currency.is_fiat(): msg = ( f'Provided symbol for main currency {given_symbol} is ' f'not a fiat currency') return False, msg if main_currency != A_USD: self.usd_to_main_currency_rate = Inquirer( ).query_fiat_pair( 'USD', main_currency.identifier, ) res, msg = self.accountant.customize(settings) if not res: message += '\n' + msg return False, message _, msg, = self.data.set_settings(settings, self.accountant) if msg != '': message += '\n' + msg # Always return success here but with a message return True, message def setup_exchange( self, name: str, api_key: ApiKey, api_secret: ApiSecret, ) -> Tuple[bool, str]: """ Setup a new exchange with an api key and an api secret By default the api keys are always validated unless validate is False. """ log.info('setup_exchange', name=name) if name not in SUPPORTED_EXCHANGES: return False, 'Attempted to register unsupported exchange {}'.format( name) if getattr(self, name) is not None: return False, 'Exchange {} is already registered'.format(name) secret_data = {} secret_data[name] = { 'api_key': api_key, 'api_secret': api_secret, } self.initialize_exchanges(secret_data) exchange = getattr(self, name) result, message = exchange.validate_api_key() if not result: log.error( 'Failed to validate API key for exchange', name=name, error=message, ) self.delete_exchange_data(name) return False, message # Success, save the result in the DB self.data.db.add_exchange(name, api_key, api_secret) return True, '' def delete_exchange_data(self, name): self.connected_exchanges.remove(name) self.trades_historian.set_exchange(name, None) delattr(self, name) setattr(self, name, None) def remove_exchange(self, name): if getattr(self, name) is None: return False, 'Exchange {} is not registered'.format(name) self.delete_exchange_data(name) # Success, remove it also from the DB self.data.db.remove_exchange(name) return True, '' def query_periodic_data(self) -> Dict[str, Union[bool, Timestamp]]: """Query for frequently changing data""" result = {} if self.user_is_logged_in: result[ 'last_balance_save'] = self.data.db.get_last_balance_save_time( ) result['eth_node_connection'] = self.blockchain.ethchain.connected result[ 'history_process_current_ts'] = self.accountant.currently_processed_timestamp return result def shutdown(self): self.logout() self.shutdown_event.set()
class Rotkehlchen(): def __init__(self, args: argparse.Namespace) -> None: """Initialize the Rotkehlchen object This runs during backend initialization so it should be as light as possible. May Raise: - SystemPermissionError if the given data directory's permissions are not correct. """ # Can also be None after unlock if premium credentials did not # authenticate or premium server temporarily offline self.premium: Optional[Premium] = None self.user_is_logged_in: bool = False configure_logging(args) self.sleep_secs = args.sleep_secs if args.data_dir is None: self.data_dir = default_data_directory() else: self.data_dir = Path(args.data_dir) self.data_dir.mkdir(parents=True, exist_ok=True) if not os.access(self.data_dir, os.W_OK | os.R_OK): raise SystemPermissionError( f'The given data directory {self.data_dir} is not readable or writable', ) self.main_loop_spawned = False self.args = args self.api_task_greenlets: List[gevent.Greenlet] = [] self.msg_aggregator = MessagesAggregator() self.greenlet_manager = GreenletManager( msg_aggregator=self.msg_aggregator) self.exchange_manager = ExchangeManager( msg_aggregator=self.msg_aggregator) # Initialize the GlobalDBHandler singleton. Has to be initialized BEFORE asset resolver GlobalDBHandler(data_dir=self.data_dir) self.data = DataHandler(self.data_dir, self.msg_aggregator) self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None) self.coingecko = Coingecko() self.icon_manager = IconManager(data_dir=self.data_dir, coingecko=self.coingecko) self.assets_updater = AssetsUpdater(self.msg_aggregator) # Initialize the Inquirer singleton Inquirer( data_dir=self.data_dir, cryptocompare=self.cryptocompare, coingecko=self.coingecko, ) # Keeps how many trades we have found per location. Used for free user limiting self.actions_per_location: Dict[str, Dict[Location, int]] = { 'trade': defaultdict(int), 'asset_movement': defaultdict(int), } self.task_manager: Optional[TaskManager] = None self.shutdown_event = gevent.event.Event() def reset_after_failed_account_creation_or_login(self) -> None: """If the account creation or login failed make sure that the rotki instance is clear Tricky instances are when after either failed premium credentials or user refusal to sync premium databases we relogged in """ self.cryptocompare.db = None def unlock_user( self, user: str, password: str, create_new: bool, sync_approval: Literal['yes', 'no', 'unknown'], premium_credentials: Optional[PremiumCredentials], initial_settings: Optional[ModifiableDBSettings] = None, ) -> None: """Unlocks an existing user or creates a new one if `create_new` is True May raise: - PremiumAuthenticationError if the password can't unlock the database. - AuthenticationError if premium_credentials are given and are invalid or can't authenticate with the server - DBUpgradeError if the rotki DB version is newer than the software or there is a DB upgrade and there is an error. - SystemPermissionError if the directory or DB file can not be accessed """ log.info( 'Unlocking user', user=user, create_new=create_new, sync_approval=sync_approval, initial_settings=initial_settings, ) # unlock or create the DB self.password = password self.user_directory = self.data.unlock(user, password, create_new, initial_settings) self.data_importer = DataImporter(db=self.data.db) self.last_data_upload_ts = self.data.db.get_last_data_upload_ts() self.premium_sync_manager = PremiumSyncManager(data=self.data, password=password) # set the DB in the external services instances that need it self.cryptocompare.set_database(self.data.db) # Anything that was set above here has to be cleaned in case of failure in the next step # by reset_after_failed_account_creation_or_login() try: self.premium = self.premium_sync_manager.try_premium_at_start( given_premium_credentials=premium_credentials, username=user, create_new=create_new, sync_approval=sync_approval, ) except PremiumAuthenticationError: # Reraise it only if this is during the creation of a new account where # the premium credentials were given by the user if create_new: raise self.msg_aggregator.add_warning( 'Could not authenticate the rotki premium API keys found in the DB.' ' Has your subscription expired?', ) # else let's just continue. User signed in succesfully, but he just # has unauthenticable/invalid premium credentials remaining in his DB settings = self.get_settings() self.greenlet_manager.spawn_and_track( after_seconds=None, task_name='submit_usage_analytics', exception_is_error=False, method=maybe_submit_usage_analytics, should_submit=settings.submit_usage_analytics, ) self.etherscan = Etherscan(database=self.data.db, msg_aggregator=self.msg_aggregator) self.beaconchain = BeaconChain(database=self.data.db, msg_aggregator=self.msg_aggregator) eth_rpc_endpoint = settings.eth_rpc_endpoint # Initialize the price historian singleton PriceHistorian( data_directory=self.data_dir, cryptocompare=self.cryptocompare, coingecko=self.coingecko, ) PriceHistorian().set_oracles_order(settings.historical_price_oracles) self.accountant = Accountant( db=self.data.db, user_directory=self.user_directory, msg_aggregator=self.msg_aggregator, create_csv=True, premium=self.premium, ) exchange_credentials = self.data.db.get_exchange_credentials() self.exchange_manager.initialize_exchanges( exchange_credentials=exchange_credentials, database=self.data.db, ) # Initialize blockchain querying modules ethereum_manager = EthereumManager( ethrpc_endpoint=eth_rpc_endpoint, etherscan=self.etherscan, database=self.data.db, msg_aggregator=self.msg_aggregator, greenlet_manager=self.greenlet_manager, connect_at_start=ETHEREUM_NODES_TO_CONNECT_AT_START, ) kusama_manager = SubstrateManager( chain=SubstrateChain.KUSAMA, msg_aggregator=self.msg_aggregator, greenlet_manager=self.greenlet_manager, connect_at_start=KUSAMA_NODES_TO_CONNECT_AT_START, connect_on_startup=self._connect_ksm_manager_on_startup(), own_rpc_endpoint=settings.ksm_rpc_endpoint, ) Inquirer().inject_ethereum(ethereum_manager) Inquirer().set_oracles_order(settings.current_price_oracles) self.chain_manager = ChainManager( blockchain_accounts=self.data.db.get_blockchain_accounts(), ethereum_manager=ethereum_manager, kusama_manager=kusama_manager, msg_aggregator=self.msg_aggregator, database=self.data.db, greenlet_manager=self.greenlet_manager, premium=self.premium, eth_modules=settings.active_modules, data_directory=self.data_dir, beaconchain=self.beaconchain, btc_derivation_gap_limit=settings.btc_derivation_gap_limit, ) self.events_historian = EventsHistorian( user_directory=self.user_directory, db=self.data.db, msg_aggregator=self.msg_aggregator, exchange_manager=self.exchange_manager, chain_manager=self.chain_manager, ) self.task_manager = TaskManager( max_tasks_num=DEFAULT_MAX_TASKS_NUM, greenlet_manager=self.greenlet_manager, api_task_greenlets=self.api_task_greenlets, database=self.data.db, cryptocompare=self.cryptocompare, premium_sync_manager=self.premium_sync_manager, chain_manager=self.chain_manager, exchange_manager=self.exchange_manager, ) self.greenlet_manager.spawn_and_track( after_seconds=5, task_name='periodically_query_icons_until_all_cached', exception_is_error=False, method=self.icon_manager.periodically_query_icons_until_all_cached, batch_size=ICONS_BATCH_SIZE, sleep_time_secs=ICONS_QUERY_SLEEP, ) self.user_is_logged_in = True log.debug('User unlocking complete') def logout(self) -> None: if not self.user_is_logged_in: return user = self.data.username log.info( 'Logging out user', user=user, ) self.deactivate_premium_status() self.greenlet_manager.clear() del self.chain_manager self.exchange_manager.delete_all_exchanges() del self.accountant del self.events_historian del self.data_importer self.data.logout() self.password = '' self.cryptocompare.unset_database() # Make sure no messages leak to other user sessions self.msg_aggregator.consume_errors() self.msg_aggregator.consume_warnings() self.task_manager = None self.user_is_logged_in = False log.info( 'User successfully logged out', user=user, ) def set_premium_credentials(self, credentials: PremiumCredentials) -> None: """ Sets the premium credentials for rotki Raises PremiumAuthenticationError if the given key is rejected by the Rotkehlchen server """ log.info('Setting new premium credentials') if self.premium is not None: self.premium.set_credentials(credentials) else: self.premium = premium_create_and_verify(credentials) self.premium_sync_manager.premium = self.premium self.accountant.premium = self.premium self.data.db.set_rotkehlchen_premium(credentials) def delete_premium_credentials(self) -> Tuple[bool, str]: """Deletes the premium credentials for rotki""" msg = '' success = self.data.db.del_rotkehlchen_premium() if success is False: msg = 'The database was unable to delete the Premium keys for the logged-in user' self.deactivate_premium_status() return success, msg def deactivate_premium_status(self) -> None: """Deactivate premium in the current session""" self.premium = None self.premium_sync_manager.premium = None self.chain_manager.deactivate_premium_status() self.accountant.deactivate_premium_status() def start(self) -> gevent.Greenlet: assert not self.main_loop_spawned, 'Tried to spawn the main loop twice' greenlet = gevent.spawn(self.main_loop) self.main_loop_spawned = True return greenlet def main_loop(self) -> None: """rotki main loop that fires often and runs the task manager's scheduler""" while self.shutdown_event.wait( timeout=MAIN_LOOP_SECS_DELAY) is not True: if self.task_manager is not None: self.task_manager.schedule() def get_blockchain_account_data( self, blockchain: SupportedBlockchain, ) -> Union[List[BlockchainAccountData], Dict[str, Any]]: account_data = self.data.db.get_blockchain_account_data(blockchain) if blockchain != SupportedBlockchain.BITCOIN: return account_data xpub_data = self.data.db.get_bitcoin_xpub_data() addresses_to_account_data = {x.address: x for x in account_data} address_to_xpub_mappings = self.data.db.get_addresses_to_xpub_mapping( list(addresses_to_account_data.keys()), # type: ignore ) xpub_mappings: Dict['XpubData', List[BlockchainAccountData]] = {} for address, xpub_entry in address_to_xpub_mappings.items(): if xpub_entry not in xpub_mappings: xpub_mappings[xpub_entry] = [] xpub_mappings[xpub_entry].append( addresses_to_account_data[address]) data: Dict[str, Any] = {'standalone': [], 'xpubs': []} # Add xpub data for xpub_entry in xpub_data: data_entry = xpub_entry.serialize() addresses = xpub_mappings.get(xpub_entry, None) data_entry['addresses'] = addresses if addresses and len( addresses) != 0 else None data['xpubs'].append(data_entry) # Add standalone addresses for account in account_data: if account.address not in address_to_xpub_mappings: data['standalone'].append(account) return data def add_blockchain_accounts( self, blockchain: SupportedBlockchain, account_data: List[BlockchainAccountData], ) -> BlockchainBalancesUpdate: """Adds new blockchain accounts Adds the accounts to the blockchain instance and queries them to get the updated balances. Also adds them in the DB May raise: - EthSyncError from modify_blockchain_account - InputError if the given accounts list is empty. - TagConstraintError if any of the given account data contain unknown tags. - RemoteError if an external service such as Etherscan is queried and there is a problem with its query. """ self.data.db.ensure_tags_exist( given_data=account_data, action='adding', data_type='blockchain accounts', ) address_type = blockchain.get_address_type() updated_balances = self.chain_manager.add_blockchain_accounts( blockchain=blockchain, accounts=[address_type(entry.address) for entry in account_data], ) self.data.db.add_blockchain_accounts( blockchain=blockchain, account_data=account_data, ) return updated_balances def edit_blockchain_accounts( self, blockchain: SupportedBlockchain, account_data: List[BlockchainAccountData], ) -> None: """Edits blockchain accounts Edits blockchain account data for the given accounts May raise: - InputError if the given accounts list is empty or if any of the accounts to edit do not exist. - TagConstraintError if any of the given account data contain unknown tags. """ # First check for validity of account data addresses if len(account_data) == 0: raise InputError( 'Empty list of blockchain account data to edit was given') accounts = [x.address for x in account_data] unknown_accounts = set(accounts).difference( self.chain_manager.accounts.get(blockchain)) if len(unknown_accounts) != 0: raise InputError( f'Tried to edit unknown {blockchain.value} ' f'accounts {",".join(unknown_accounts)}', ) self.data.db.ensure_tags_exist( given_data=account_data, action='editing', data_type='blockchain accounts', ) # Finally edit the accounts self.data.db.edit_blockchain_accounts( blockchain=blockchain, account_data=account_data, ) def remove_blockchain_accounts( self, blockchain: SupportedBlockchain, accounts: ListOfBlockchainAddresses, ) -> BlockchainBalancesUpdate: """Removes blockchain accounts Removes the accounts from the blockchain instance and queries them to get the updated balances. Also removes them from the DB May raise: - RemoteError if an external service such as Etherscan is queried and there is a problem with its query. - InputError if a non-existing account was given to remove """ balances_update = self.chain_manager.remove_blockchain_accounts( blockchain=blockchain, accounts=accounts, ) self.data.db.remove_blockchain_accounts(blockchain, accounts) return balances_update def get_history_query_status(self) -> Dict[str, str]: if self.events_historian.progress < FVal('100'): processing_state = self.events_historian.processing_state_name progress = self.events_historian.progress / 2 elif self.accountant.first_processed_timestamp == -1: processing_state = 'Processing all retrieved historical events' progress = FVal(50) else: processing_state = 'Processing all retrieved historical events' # start_ts is min of the query start or the first action timestamp since action # processing can start well before query start to calculate cost basis start_ts = min( self.accountant.events.query_start_ts, self.accountant.first_processed_timestamp, ) diff = self.accountant.events.query_end_ts - start_ts progress = 50 + 100 * ( FVal(self.accountant.currently_processing_timestamp - start_ts) / FVal(diff) / 2) return { 'processing_state': str(processing_state), 'total_progress': str(progress) } def process_history( self, start_ts: Timestamp, end_ts: Timestamp, ) -> Tuple[Dict[str, Any], str]: ( error_or_empty, history, loan_history, asset_movements, eth_transactions, defi_events, ledger_actions, ) = self.events_historian.get_history( start_ts=start_ts, end_ts=end_ts, has_premium=self.premium is not None, ) result = self.accountant.process_history( start_ts=start_ts, end_ts=end_ts, trade_history=history, loan_history=loan_history, asset_movements=asset_movements, eth_transactions=eth_transactions, defi_events=defi_events, ledger_actions=ledger_actions, ) return result, error_or_empty @overload def _apply_actions_limit( self, location: Location, action_type: Literal['trade'], location_actions: TRADES_LIST, all_actions: TRADES_LIST, ) -> TRADES_LIST: ... @overload def _apply_actions_limit( self, location: Location, action_type: Literal['asset_movement'], location_actions: List[AssetMovement], all_actions: List[AssetMovement], ) -> List[AssetMovement]: ... def _apply_actions_limit( self, location: Location, action_type: Literal['trade', 'asset_movement'], location_actions: Union[TRADES_LIST, List[AssetMovement]], all_actions: Union[TRADES_LIST, List[AssetMovement]], ) -> Union[TRADES_LIST, List[AssetMovement]]: """Take as many actions from location actions and add them to all actions as the limit permits Returns the modified (or not) all_actions """ # If we are already at or above the limit return current actions disregarding this location actions_mapping = self.actions_per_location[action_type] current_num_actions = sum(x for _, x in actions_mapping.items()) limit = LIMITS_MAPPING[action_type] if current_num_actions >= limit: return all_actions # Find out how many more actions can we return, and depending on that get # the number of actions from the location actions and add them to the total remaining_num_actions = limit - current_num_actions if remaining_num_actions < 0: remaining_num_actions = 0 num_actions_to_take = min(len(location_actions), remaining_num_actions) actions_mapping[location] = num_actions_to_take all_actions.extend( location_actions[0:num_actions_to_take]) # type: ignore return all_actions def query_trades( self, from_ts: Timestamp, to_ts: Timestamp, location: Optional[Location], only_cache: bool, ) -> TRADES_LIST: """Queries trades for the given location and time range. If no location is given then all external, all exchange and DEX trades are queried. If only_cache is given then only trades cached in the DB are returned. No service is queried. DEX Trades are queried only if the user has premium If the user does not have premium then a trade limit is applied. May raise: - RemoteError: If there are problems connecting to any of the remote exchanges """ trades: TRADES_LIST if location is not None: # clear the trades queried for this location self.actions_per_location['trade'][location] = 0 trades = self.query_location_trades(from_ts, to_ts, location, only_cache) else: for given_location in ALL_SUPPORTED_EXCHANGES + [ Location.EXTERNAL ]: # clear the trades queried for this location self.actions_per_location['trade'][given_location] = 0 trades = self.query_location_trades(from_ts, to_ts, Location.EXTERNAL, only_cache) # Look for trades that might be imported from CSV files for csv_location in EXTERNAL_EXCHANGES: trades.extend( self.query_location_trades( from_ts=from_ts, to_ts=to_ts, location=csv_location, only_cache=only_cache, )) for exchange in self.exchange_manager.iterate_exchanges(): all_set = {x.identifier for x in trades} exchange_trades = exchange.query_trade_history( start_ts=from_ts, end_ts=to_ts, only_cache=only_cache, ) # TODO: Really dirty. Figure out a better way. # Since some of the trades may already be in the DB if multiple # keys are used for a single exchange. exchange_trades = [ x for x in exchange_trades if x.identifier not in all_set ] if self.premium is None: trades = self._apply_actions_limit( location=exchange.location, action_type='trade', location_actions=exchange_trades, all_actions=trades, ) else: trades.extend(exchange_trades) # for all trades we also need the trades from the amm protocols if self.premium is not None: for amm_location in AMMTradeLocations: amm_module_name = cast(AMMTRADE_LOCATION_NAMES, str(amm_location)) amm_module = self.chain_manager.get_module(amm_module_name) if amm_module is not None: trades.extend( amm_module.get_trades( addresses=self.chain_manager. queried_addresses_for_module( amm_module_name), # noqa: E501 from_timestamp=from_ts, to_timestamp=to_ts, only_cache=only_cache, ), ) # return trades with most recent first trades.sort(key=lambda x: x.timestamp, reverse=True) return trades def query_location_trades( self, from_ts: Timestamp, to_ts: Timestamp, location: Location, only_cache: bool, ) -> TRADES_LIST: location_trades: TRADES_LIST if location in EXTERNAL_LOCATION: location_trades = self.data.db.get_trades( # type: ignore # list invariance from_ts=from_ts, to_ts=to_ts, location=location, ) elif location in AMMTradeLocations: if self.premium is not None: amm_module_name = cast(AMMTRADE_LOCATION_NAMES, str(location)) amm_module = self.chain_manager.get_module(amm_module_name) if amm_module is not None: location_trades = amm_module.get_trades( # type: ignore # list invariance addresses=self.chain_manager. queried_addresses_for_module(amm_module_name), from_timestamp=from_ts, to_timestamp=to_ts, only_cache=only_cache, ) else: # should only be an exchange exchanges_list = self.exchange_manager.connected_exchanges.get( location) if exchanges_list is None: logger.warning( f'Tried to query trades from {str(location)} which is either not an ' f'exchange or not an exchange the user has connected to', ) return [] location_trades = [] for exchange in exchanges_list: all_set = {x.identifier for x in location_trades} new_trades = exchange.query_trade_history( start_ts=from_ts, end_ts=to_ts, only_cache=only_cache, ) # TODO: Really dirty. Figure out a better way. # Since some of the trades may already be in the DB if multiple # keys are used for a single exchange. new_trades = [ x for x in new_trades if x.identifier not in all_set ] location_trades.extend(new_trades) trades: TRADES_LIST = [] if self.premium is None: trades = self._apply_actions_limit( location=location, action_type='trade', location_actions=location_trades, all_actions=trades, ) else: trades = location_trades return trades def query_balances( self, requested_save_data: bool = False, timestamp: Timestamp = None, ignore_cache: bool = False, ) -> Dict[str, Any]: """Query all balances rotkehlchen can see. If requested_save_data is True then the data are always saved in the DB, if it is False then data are saved if self.data.should_save_balances() is True. If timestamp is None then the current timestamp is used. If a timestamp is given then that is the time that the balances are going to be saved in the DB If ignore_cache is True then all underlying calls that have a cache ignore it Returns a dictionary with the queried balances. """ log.info('query_balances called', requested_save_data=requested_save_data) balances: Dict[str, Dict[Asset, Balance]] = {} problem_free = True for exchange in self.exchange_manager.iterate_exchanges(): exchange_balances, _ = exchange.query_balances( ignore_cache=ignore_cache) # If we got an error, disregard that exchange but make sure we don't save data if not isinstance(exchange_balances, dict): problem_free = False else: location_str = str(exchange.location) if location_str not in balances: balances[location_str] = exchange_balances else: # multiple exchange of same type. Combine balances balances[location_str] = combine_dicts( balances[location_str], exchange_balances, ) liabilities: Dict[Asset, Balance] try: blockchain_result = self.chain_manager.query_balances( blockchain=None, force_token_detection=ignore_cache, ignore_cache=ignore_cache, ) if len(blockchain_result.totals.assets) != 0: balances[str( Location.BLOCKCHAIN)] = blockchain_result.totals.assets liabilities = blockchain_result.totals.liabilities except (RemoteError, EthSyncError) as e: problem_free = False liabilities = {} log.error(f'Querying blockchain balances failed due to: {str(e)}') # retrieve loopring balances if module is activated if self.chain_manager.get_module('loopring'): loopring_balances = self.chain_manager.get_loopring_balances() if len(loopring_balances) != 0: balances[str(Location.LOOPRING)] = loopring_balances balances = account_for_manually_tracked_balances(db=self.data.db, balances=balances) # Calculate usd totals assets_total_balance: DefaultDict[Asset, Balance] = defaultdict(Balance) total_usd_per_location: Dict[str, FVal] = {} for location, asset_balance in balances.items(): total_usd_per_location[location] = ZERO for asset, balance in asset_balance.items(): assets_total_balance[asset] += balance total_usd_per_location[location] += balance.usd_value net_usd = sum((balance.usd_value for _, balance in assets_total_balance.items()), ZERO) liabilities_total_usd = sum( (liability.usd_value for _, liability in liabilities.items()), ZERO) # noqa: E501 net_usd -= liabilities_total_usd # Calculate location stats location_stats: Dict[str, Any] = {} for location, total_usd in total_usd_per_location.items(): if location == str(Location.BLOCKCHAIN): total_usd -= liabilities_total_usd percentage = (total_usd / net_usd).to_percentage() if net_usd != ZERO else '0%' location_stats[location] = { 'usd_value': total_usd, 'percentage_of_net_value': percentage, } # Calculate 'percentage_of_net_value' per asset assets_total_balance_as_dict: Dict[Asset, Dict[str, Any]] = { asset: balance.to_dict() for asset, balance in assets_total_balance.items() } liabilities_as_dict: Dict[Asset, Dict[str, Any]] = { asset: balance.to_dict() for asset, balance in liabilities.items() } for asset, balance_dict in assets_total_balance_as_dict.items(): percentage = (balance_dict['usd_value'] / net_usd).to_percentage( ) if net_usd != ZERO else '0%' # noqa: E501 assets_total_balance_as_dict[asset][ 'percentage_of_net_value'] = percentage for asset, balance_dict in liabilities_as_dict.items(): percentage = (balance_dict['usd_value'] / net_usd).to_percentage( ) if net_usd != ZERO else '0%' # noqa: E501 liabilities_as_dict[asset]['percentage_of_net_value'] = percentage # Compose balances response result_dict = { 'assets': assets_total_balance_as_dict, 'liabilities': liabilities_as_dict, 'location': location_stats, 'net_usd': net_usd, } allowed_to_save = requested_save_data or self.data.should_save_balances( ) if problem_free and allowed_to_save: if not timestamp: timestamp = Timestamp(int(time.time())) self.data.db.save_balances_data(data=result_dict, timestamp=timestamp) log.debug('query_balances data saved') else: log.debug( 'query_balances data not saved', allowed_to_save=allowed_to_save, problem_free=problem_free, ) return result_dict def _query_and_populate_exchange_asset_movements( self, from_ts: Timestamp, to_ts: Timestamp, all_movements: List[AssetMovement], exchange: Union[ExchangeInterface, Location], only_cache: bool, ) -> List[AssetMovement]: """Queries exchange for asset movements and adds it to all_movements""" all_set = {x.identifier for x in all_movements} if isinstance(exchange, ExchangeInterface): location = exchange.location location_movements = exchange.query_deposits_withdrawals( start_ts=from_ts, end_ts=to_ts, only_cache=only_cache, ) # TODO: Really dirty. Figure out a better way. # Since some of the asset movements may already be in the DB if multiple # keys are used for a single exchange. location_movements = [ x for x in location_movements if x.identifier not in all_set ] else: assert isinstance(exchange, Location), 'only a location should make it here' assert exchange in EXTERNAL_EXCHANGES, 'only csv supported exchanges should get here' # noqa : E501 location = exchange # We might have no exchange information but CSV imported information self.actions_per_location['asset_movement'][location] = 0 location_movements = self.data.db.get_asset_movements( from_ts=from_ts, to_ts=to_ts, location=location, ) movements: List[AssetMovement] = [] if self.premium is None: movements = self._apply_actions_limit( location=location, action_type='asset_movement', location_actions=location_movements, all_actions=all_movements, ) else: all_movements.extend(location_movements) movements = all_movements return movements def query_asset_movements( self, from_ts: Timestamp, to_ts: Timestamp, location: Optional[Location], only_cache: bool, ) -> List[AssetMovement]: """Queries AssetMovements for the given location and time range. If no location is given then all exchange asset movements are queried. If only_cache is True then only what is already in the DB is returned. If the user does not have premium then a limit is applied. May raise: - RemoteError: If there are problems connecting to any of the remote exchanges """ movements: List[AssetMovement] = [] if location is not None: # clear the asset movements queried for this exchange self.actions_per_location['asset_movement'][location] = 0 if location in EXTERNAL_EXCHANGES: movements = self._query_and_populate_exchange_asset_movements( from_ts=from_ts, to_ts=to_ts, all_movements=movements, exchange=location, only_cache=only_cache, ) else: exchanges_list = self.exchange_manager.connected_exchanges.get( location) if exchanges_list is None: logger.warning( f'Tried to query deposits/withdrawals from {str(location)} which is ' f'either not an exchange or not an exchange the user has connected to', ) return [] # clear the asset movements queried for this exchange self.actions_per_location['asset_movement'][location] = 0 for exchange in exchanges_list: self._query_and_populate_exchange_asset_movements( from_ts=from_ts, to_ts=to_ts, all_movements=movements, exchange=exchange, only_cache=only_cache, ) else: for exchange_location in ALL_SUPPORTED_EXCHANGES: # clear the asset movements queried for this exchange self.actions_per_location['asset_movement'][ exchange_location] = 0 # we may have DB entries due to csv import from supported locations for external_location in EXTERNAL_EXCHANGES: movements = self._query_and_populate_exchange_asset_movements( from_ts=from_ts, to_ts=to_ts, all_movements=movements, exchange=external_location, only_cache=only_cache, ) for exchange in self.exchange_manager.iterate_exchanges(): self._query_and_populate_exchange_asset_movements( from_ts=from_ts, to_ts=to_ts, all_movements=movements, exchange=exchange, only_cache=only_cache, ) # return movements with most recent first movements.sort(key=lambda x: x.timestamp, reverse=True) return movements def set_settings(self, settings: ModifiableDBSettings) -> Tuple[bool, str]: """Tries to set new settings. Returns True in success or False with message if error""" if settings.eth_rpc_endpoint is not None: result, msg = self.chain_manager.set_eth_rpc_endpoint( settings.eth_rpc_endpoint) if not result: return False, msg if settings.ksm_rpc_endpoint is not None: result, msg = self.chain_manager.set_ksm_rpc_endpoint( settings.ksm_rpc_endpoint) if not result: return False, msg if settings.btc_derivation_gap_limit is not None: self.chain_manager.btc_derivation_gap_limit = settings.btc_derivation_gap_limit if settings.current_price_oracles is not None: Inquirer().set_oracles_order(settings.current_price_oracles) if settings.historical_price_oracles is not None: PriceHistorian().set_oracles_order( settings.historical_price_oracles) if settings.active_modules is not None: self.chain_manager.process_new_modules_list( settings.active_modules) self.data.db.set_settings(settings) return True, '' def get_settings(self) -> DBSettings: """Returns the db settings with a check whether premium is active or not""" db_settings = self.data.db.get_settings( have_premium=self.premium is not None) return db_settings def setup_exchange( self, name: str, location: Location, api_key: ApiKey, api_secret: ApiSecret, passphrase: Optional[str] = None, kraken_account_type: Optional['KrakenAccountType'] = None, binance_markets: Optional[List[str]] = None, ftx_subaccount_name: Optional[str] = None, ) -> Tuple[bool, str]: """ Setup a new exchange with an api key and an api secret and optionally a passphrase """ is_success, msg = self.exchange_manager.setup_exchange( name=name, location=location, api_key=api_key, api_secret=api_secret, database=self.data.db, passphrase=passphrase, ftx_subaccount_name=ftx_subaccount_name, ) if is_success: # Success, save the result in the DB self.data.db.add_exchange( name=name, location=location, api_key=api_key, api_secret=api_secret, passphrase=passphrase, kraken_account_type=kraken_account_type, binance_markets=binance_markets, ftx_subaccount_name=ftx_subaccount_name, ) return is_success, msg def remove_exchange(self, name: str, location: Location) -> Tuple[bool, str]: if self.exchange_manager.get_exchange(name=name, location=location) is None: return False, f'{str(location)} exchange {name} is not registered' self.exchange_manager.delete_exchange(name=name, location=location) # Success, remove it also from the DB self.data.db.remove_exchange(name=name, location=location) if self.exchange_manager.connected_exchanges.get(location) is None: # was last exchange of the location type. Delete used query ranges self.data.db.delete_used_query_range_for_exchange(location) return True, '' def query_periodic_data(self) -> Dict[str, Union[bool, Timestamp]]: """Query for frequently changing data""" result: Dict[str, Union[bool, Timestamp]] = {} if self.user_is_logged_in: result[ 'last_balance_save'] = self.data.db.get_last_balance_save_time( ) result[ 'eth_node_connection'] = self.chain_manager.ethereum.web3_mapping.get( NodeName.OWN, None) is not None # noqa : E501 result['last_data_upload_ts'] = Timestamp( self.premium_sync_manager.last_data_upload_ts) # noqa : E501 return result def shutdown(self) -> None: self.logout() self.shutdown_event.set() def _connect_ksm_manager_on_startup(self) -> bool: return bool(self.data.db.get_blockchain_accounts().ksm) def create_oracle_cache( self, oracle: HistoricalPriceOracle, from_asset: Asset, to_asset: Asset, purge_old: bool, ) -> None: """Creates the cache of the given asset pair from the start of time until now for the given oracle. if purge_old is true then any old cache in memory and in a file is purged May raise: - RemoteError if there is a problem reaching the oracle - UnsupportedAsset if any of the two assets is not supported by the oracle """ if oracle != HistoricalPriceOracle.CRYPTOCOMPARE: return # only for cryptocompare for now self.cryptocompare.create_cache(from_asset, to_asset, purge_old)
def test_add_trades(data_dir, username): """Test that adding and retrieving trades from the DB works fine. Also duplicates should be ignored and an error returned """ msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) trade1 = Trade( timestamp=1451606400, location=Location.KRAKEN, pair='ETH_EUR', trade_type=TradeType.BUY, amount=FVal('1.1'), rate=FVal('10'), fee=Fee(FVal('0.01')), fee_currency=A_EUR, link='', notes='', ) trade2 = Trade( timestamp=1451607500, location=Location.BINANCE, pair='BTC_ETH', trade_type=TradeType.BUY, amount=FVal('0.00120'), rate=FVal('10'), fee=Fee(FVal('0.001')), fee_currency=A_ETH, link='', notes='', ) trade3 = Trade( timestamp=1451608600, location=Location.COINBASE, pair='BTC_ETH', trade_type=TradeType.SELL, amount=FVal('0.00120'), rate=FVal('1'), fee=Fee(FVal('0.001')), fee_currency=A_ETH, link='', notes='', ) # Add and retrieve the first 2 trades. All should be fine. data.db.add_trades([trade1, trade2]) errors = msg_aggregator.consume_errors() warnings = msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 0 returned_trades = data.db.get_trades() assert returned_trades == [trade1, trade2] # Add the last 2 trades. Since trade2 already exists in the DB it should be # ignored and a warning should be shown data.db.add_trades([trade2, trade3]) errors = msg_aggregator.consume_errors() warnings = msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 1 returned_trades = data.db.get_trades() assert returned_trades == [trade1, trade2, trade3]
def populate_db_and_check_for_asset_renaming( cursor: Cursor, data: DataHandler, data_dir: Path, msg_aggregator: MessagesAggregator, username: str, to_rename_asset: str, renamed_asset: Asset, target_version: int, ): # Manually input data to the affected tables. # timed_balances, multisettings and (external) trades # At this time point we only have occurence of the to_rename_asset cursor.execute( 'INSERT INTO timed_balances(' ' time, currency, amount, usd_value) ' ' VALUES(?, ?, ?, ?)', ('1557499129', to_rename_asset, '10.1', '150'), ) # But add a time point where we got both to_rename_asset and # renamed_asset. This is to test merging if renaming falls in time where # both new and old asset had entries cursor.execute( 'INSERT INTO timed_balances(' ' time, currency, amount, usd_value) ' ' VALUES(?, ?, ?, ?)', ('1558499129', to_rename_asset, '1.1', '15'), ) cursor.execute( 'INSERT INTO timed_balances(' ' time, currency, amount, usd_value) ' ' VALUES(?, ?, ?, ?)', ('1558499129', renamed_asset.identifier, '2.2', '25'), ) # Add one different asset for control test cursor.execute( 'INSERT INTO timed_balances(' ' time, currency, amount, usd_value) ' ' VALUES(?, ?, ?, ?)', ('1556392121', 'ETH', '5.5', '245'), ) # Also populate an ignored assets entry cursor.execute( 'INSERT INTO multisettings(name, value) VALUES(?, ?)', ('ignored_asset', to_rename_asset), ) cursor.execute( 'INSERT INTO multisettings(name, value) VALUES(?, ?)', ('ignored_asset', 'RDN'), ) # Finally include it in some trades cursor.execute( 'INSERT INTO trades(' ' time,' ' location,' ' pair,' ' type,' ' amount,' ' rate,' ' fee,' ' fee_currency,' ' link,' ' notes)' 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', ( 1543298883, 'external', 'ETH_EUR', 'buy', '100', '0.5', '0.1', 'EUR', '', '', ), ) cursor.execute( 'INSERT INTO trades(' ' time,' ' location,' ' pair,' ' type,' ' amount,' ' rate,' ' fee,' ' fee_currency,' ' link,' ' notes)' 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', ( 1563298883, 'kraken', f'{to_rename_asset}_EUR', 'buy', '100', '0.5', '0.1', to_rename_asset, '', '', ), ) cursor.execute( 'INSERT INTO trades(' ' time,' ' location,' ' pair,' ' type,' ' amount,' ' rate,' ' fee,' ' fee_currency,' ' link,' ' notes)' 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', ( 1564218181, 'binance', f'{to_rename_asset}_EUR', 'buy', '100', '0.5', '0.1', 'BNB', '', '', ), ) data.db.conn.commit() # now relogin and check that all tables have appropriate data del data data = DataHandler(data_dir, msg_aggregator) with creation_patch, target_patch(target_version=target_version): data.unlock(username, '123', create_new=False) # Check that owned and ignored assets reflect the new state ignored_assets = data.db.get_ignored_assets() assert A_RDN in ignored_assets assert renamed_asset in ignored_assets owned_assets = data.db.query_owned_assets() assert A_ETH in owned_assets assert renamed_asset in owned_assets # Make sure that the merging of both new and old name entry in same timestamp works timed_balances = data.db.query_timed_balances( from_ts=Timestamp(0), to_ts=Timestamp(2556392121), asset=renamed_asset, ) assert len(timed_balances) == 2 assert timed_balances[0].time == 1557499129 assert timed_balances[0].amount == '10.1' assert timed_balances[0].usd_value == '150' assert timed_balances[1].time == 1558499129 assert timed_balances[1].amount == '3.3' assert timed_balances[1].usd_value == '40' # Assert that trades got renamed properly cursor = data.db.conn.cursor() query = ('SELECT id,' ' time,' ' location,' ' pair,' ' type,' ' amount,' ' rate,' ' fee,' ' fee_currency,' ' link,' ' notes FROM trades ORDER BY time ASC;') results = cursor.execute(query) trades = [] for result in results: trades.append({ 'id': result[0], 'timestamp': result[1], 'location': result[2], 'pair': result[3], 'trade_type': result[4], 'amount': result[5], 'rate': result[6], 'fee': result[7], 'fee_currency': result[8], 'link': result[9], 'notes': result[10], }) assert len(trades) == 3 assert trades[0]['fee_currency'] == 'EUR' assert trades[0]['pair'] == 'ETH_EUR' assert trades[1]['fee_currency'] == renamed_asset.identifier assert trades[1]['pair'] == f'{renamed_asset.identifier}_EUR' assert trades[2]['pair'] == f'{renamed_asset.identifier}_EUR' assert data.db.get_version() == target_version
def test_add_ethereum_transactions(data_dir, username): """Test that adding and retrieving ethereum transactions from the DB works fine. Also duplicates should be ignored and an error returned """ msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) tx1 = EthereumTransaction( tx_hash=b'1', timestamp=Timestamp(1451606400), block_number=1, from_address=ETH_ADDRESS1, to_address=ETH_ADDRESS3, value=FVal('2000000'), gas=FVal('5000000'), gas_price=FVal('2000000000'), gas_used=FVal('25000000'), input_data=MOCK_INPUT_DATA, nonce=1, ) tx2 = EthereumTransaction( tx_hash=b'2', timestamp=Timestamp(1451706400), block_number=3, from_address=ETH_ADDRESS2, to_address=ETH_ADDRESS3, value=FVal('4000000'), gas=FVal('5000000'), gas_price=FVal('2000000000'), gas_used=FVal('25000000'), input_data=MOCK_INPUT_DATA, nonce=1, ) tx3 = EthereumTransaction( tx_hash=b'3', timestamp=Timestamp(1452806400), block_number=5, from_address=ETH_ADDRESS3, to_address=ETH_ADDRESS1, value=FVal('1000000'), gas=FVal('5000000'), gas_price=FVal('2000000000'), gas_used=FVal('25000000'), input_data=MOCK_INPUT_DATA, nonce=3, ) # Add and retrieve the first 2 margins. All should be fine. data.db.add_ethereum_transactions([tx1, tx2], from_etherscan=True) errors = msg_aggregator.consume_errors() warnings = msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 0 returned_transactions = data.db.get_ethereum_transactions() assert returned_transactions == [tx1, tx2] # Add the last 2 transactions. Since tx2 already exists in the DB it should be # ignored (no errors shown for attempting to add already existing transaction) data.db.add_ethereum_transactions([tx2, tx3], from_etherscan=True) errors = msg_aggregator.consume_errors() warnings = msg_aggregator.consume_warnings() assert len(errors) == 0 assert len(warnings) == 0 returned_transactions = data.db.get_ethereum_transactions() assert returned_transactions == [tx1, tx2, tx3]
def test_writting_fetching_data(data_dir, username): msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) tokens = [A_GNO, A_RDN] data.write_owned_eth_tokens(tokens) result = data.db.get_owned_tokens() assert set(tokens) == set(result) data.add_blockchain_account(SupportedBlockchain.BITCOIN, '1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS') data.add_blockchain_account( SupportedBlockchain.ETHEREUM, '0xd36029d76af6fE4A356528e4Dc66B2C18123597D', ) # Add a non checksummed address data.add_blockchain_account( SupportedBlockchain.ETHEREUM, '0x80b369799104a47e98a553f3329812a44a7facdc', ) accounts = data.db.get_blockchain_accounts() assert isinstance(accounts, BlockchainAccounts) assert accounts.btc == ['1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS'] # See that after addition the address has been checksummed assert set(accounts.eth) == set([ '0xd36029d76af6fE4A356528e4Dc66B2C18123597D', to_checksum_address('0x80b369799104a47e98a553f3329812a44a7facdc'), ]) # Add existing account should fail with pytest.raises(sqlcipher.IntegrityError): # pylint: disable=no-member data.add_blockchain_account( SupportedBlockchain.ETHEREUM, '0xd36029d76af6fE4A356528e4Dc66B2C18123597D', ) # Remove non-existing account with pytest.raises(InputError): data.remove_blockchain_account( SupportedBlockchain.ETHEREUM, '0x136029d76af6fE4A356528e4Dc66B2C18123597D', ) # Remove existing account data.remove_blockchain_account( SupportedBlockchain.ETHEREUM, '0xd36029d76af6fE4A356528e4Dc66B2C18123597D', ) accounts = data.db.get_blockchain_accounts() assert accounts.eth == [ to_checksum_address('0x80b369799104a47e98a553f3329812a44a7facdc') ] result, _ = data.add_ignored_asset('DAO') assert result result, _ = data.add_ignored_asset('DOGE') assert result result, _ = data.add_ignored_asset('DOGE') assert not result # Test adding non existing asset result, msg = data.add_ignored_asset('dsajdhskajdad') assert not result assert 'for ignoring is not known/supported' in msg ignored_assets = data.db.get_ignored_assets() assert all([isinstance(asset, Asset) for asset in ignored_assets]) assert set(ignored_assets) == set([A_DAO, A_DOGE]) # Test removing asset that is not in the list result, msg = data.remove_ignored_asset('RDN') assert 'not in ignored assets' in msg # Test removing non existing asset result, msg = data.remove_ignored_asset('dshajdhsjkahdjssad') assert 'is not known/supported' in msg assert not result result, _ = data.remove_ignored_asset('DOGE') assert result assert data.db.get_ignored_assets() == ['DAO'] # With nothing inserted in settings make sure default values are returned result = data.db.get_settings() last_write_diff = ts_now() - result['last_write_ts'] # make sure last_write was within 3 secs assert last_write_diff >= 0 and last_write_diff < 3 del result['last_write_ts'] assert result == { 'historical_data_start': DEFAULT_START_DATE, 'eth_rpc_endpoint': 'http://localhost:8545', 'ui_floating_precision': DEFAULT_UI_FLOATING_PRECISION, 'db_version': ROTKEHLCHEN_DB_VERSION, 'include_crypto2crypto': DEFAULT_INCLUDE_CRYPTO2CRYPTO, 'include_gas_costs': DEFAULT_INCLUDE_GAS_COSTS, 'taxfree_after_period': YEAR_IN_SECONDS, 'balance_save_frequency': DEFAULT_BALANCE_SAVE_FREQUENCY, 'last_balance_save': 0, 'main_currency': DEFAULT_MAIN_CURRENCY, 'anonymized_logs': DEFAULT_ANONYMIZED_LOGS, 'date_display_format': DEFAULT_DATE_DISPLAY_FORMAT, 'last_data_upload_ts': 0, 'premium_should_sync': False, } # Check setting non-existing settings. Should be ignored success, msg = data.set_settings({'nonexisting_setting': 1}, accountant=None) assert success assert msg != '' and 'nonexisting_setting' in msg _, msg = data.set_settings( { 'nonexisting_setting': 1, 'eth_rpc_endpoint': 'http://localhost:8555', 'ui_floating_precision': 3, }, accountant=None) assert msg != '' and 'nonexisting_setting' in msg # Now check nothing funny made it in the db result = data.db.get_settings() assert result['eth_rpc_endpoint'] == 'http://localhost:8555' assert result['ui_floating_precision'] == 3 assert 'nonexisting_setting' not in result
class Rotkehlchen(): def __init__(self, args: argparse.Namespace) -> None: """Initialize the Rotkehlchen object May Raise: - SystemPermissionError if the given data directory's permissions are not correct. """ self.lock = Semaphore() self.lock.acquire() # Can also be None after unlock if premium credentials did not # authenticate or premium server temporarily offline self.premium: Optional[Premium] = None self.user_is_logged_in: bool = False configure_logging(args) self.sleep_secs = args.sleep_secs if args.data_dir is None: self.data_dir = default_data_directory() else: self.data_dir = Path(args.data_dir) if not os.access(self.data_dir, os.W_OK | os.R_OK): raise SystemPermissionError( f'The given data directory {self.data_dir} is not readable or writable', ) self.args = args self.msg_aggregator = MessagesAggregator() self.greenlet_manager = GreenletManager(msg_aggregator=self.msg_aggregator) self.exchange_manager = ExchangeManager(msg_aggregator=self.msg_aggregator) # Initialize the AssetResolver singleton AssetResolver(data_directory=self.data_dir) self.data = DataHandler(self.data_dir, self.msg_aggregator) self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None) self.coingecko = Coingecko() self.icon_manager = IconManager(data_dir=self.data_dir, coingecko=self.coingecko) self.greenlet_manager.spawn_and_track( after_seconds=None, task_name='periodically_query_icons_until_all_cached', method=self.icon_manager.periodically_query_icons_until_all_cached, batch_size=ICONS_BATCH_SIZE, sleep_time_secs=ICONS_QUERY_SLEEP, ) # Initialize the Inquirer singleton Inquirer( data_dir=self.data_dir, cryptocompare=self.cryptocompare, coingecko=self.coingecko, ) # Keeps how many trades we have found per location. Used for free user limiting self.actions_per_location: Dict[str, Dict[Location, int]] = { 'trade': defaultdict(int), 'asset_movement': defaultdict(int), } self.lock.release() self.shutdown_event = gevent.event.Event() def reset_after_failed_account_creation_or_login(self) -> None: """If the account creation or login failed make sure that the Rotki instance is clear Tricky instances are when after either failed premium credentials or user refusal to sync premium databases we relogged in. """ self.cryptocompare.db = None def unlock_user( self, user: str, password: str, create_new: bool, sync_approval: Literal['yes', 'no', 'unknown'], premium_credentials: Optional[PremiumCredentials], initial_settings: Optional[ModifiableDBSettings] = None, ) -> None: """Unlocks an existing user or creates a new one if `create_new` is True May raise: - PremiumAuthenticationError if the password can't unlock the database. - AuthenticationError if premium_credentials are given and are invalid or can't authenticate with the server - DBUpgradeError if the rotki DB version is newer than the software or there is a DB upgrade and there is an error. - SystemPermissionError if the directory or DB file can not be accessed """ log.info( 'Unlocking user', user=user, create_new=create_new, sync_approval=sync_approval, initial_settings=initial_settings, ) # unlock or create the DB self.password = password self.user_directory = self.data.unlock(user, password, create_new, initial_settings) self.data_importer = DataImporter(db=self.data.db) self.last_data_upload_ts = self.data.db.get_last_data_upload_ts() self.premium_sync_manager = PremiumSyncManager(data=self.data, password=password) # set the DB in the external services instances that need it self.cryptocompare.set_database(self.data.db) # Anything that was set above here has to be cleaned in case of failure in the next step # by reset_after_failed_account_creation_or_login() try: self.premium = self.premium_sync_manager.try_premium_at_start( given_premium_credentials=premium_credentials, username=user, create_new=create_new, sync_approval=sync_approval, ) except PremiumAuthenticationError: # Reraise it only if this is during the creation of a new account where # the premium credentials were given by the user if create_new: raise self.msg_aggregator.add_error( 'Tried to synchronize the database from remote but the local password ' 'does not match the one the remote DB has. Please change the password ' 'to be the same as the password of the account you want to sync from ', ) # else let's just continue. User signed in succesfully, but he just # has unauthenticable/invalid premium credentials remaining in his DB settings = self.get_settings() self.greenlet_manager.spawn_and_track( after_seconds=None, task_name='submit_usage_analytics', method=maybe_submit_usage_analytics, should_submit=settings.submit_usage_analytics, ) self.etherscan = Etherscan(database=self.data.db, msg_aggregator=self.msg_aggregator) self.beaconchain = BeaconChain(database=self.data.db, msg_aggregator=self.msg_aggregator) historical_data_start = settings.historical_data_start eth_rpc_endpoint = settings.eth_rpc_endpoint # Initialize the price historian singleton PriceHistorian( data_directory=self.data_dir, history_date_start=historical_data_start, cryptocompare=self.cryptocompare, ) self.accountant = Accountant( db=self.data.db, user_directory=self.user_directory, msg_aggregator=self.msg_aggregator, create_csv=True, ) # Initialize the rotkehlchen logger LoggingSettings(anonymized_logs=settings.anonymized_logs) exchange_credentials = self.data.db.get_exchange_credentials() self.exchange_manager.initialize_exchanges( exchange_credentials=exchange_credentials, database=self.data.db, ) # Initialize blockchain querying modules ethereum_manager = EthereumManager( ethrpc_endpoint=eth_rpc_endpoint, etherscan=self.etherscan, database=self.data.db, msg_aggregator=self.msg_aggregator, greenlet_manager=self.greenlet_manager, connect_at_start=ETHEREUM_NODES_TO_CONNECT_AT_START, ) Inquirer().inject_ethereum(ethereum_manager) self.chain_manager = ChainManager( blockchain_accounts=self.data.db.get_blockchain_accounts(), ethereum_manager=ethereum_manager, msg_aggregator=self.msg_aggregator, database=self.data.db, greenlet_manager=self.greenlet_manager, premium=self.premium, eth_modules=settings.active_modules, data_directory=self.data_dir, beaconchain=self.beaconchain, ) self.trades_historian = TradesHistorian( user_directory=self.user_directory, db=self.data.db, msg_aggregator=self.msg_aggregator, exchange_manager=self.exchange_manager, chain_manager=self.chain_manager, ) self.user_is_logged_in = True log.debug('User unlocking complete') def logout(self) -> None: if not self.user_is_logged_in: return user = self.data.username log.info( 'Logging out user', user=user, ) self.greenlet_manager.clear() del self.chain_manager self.exchange_manager.delete_all_exchanges() # Reset rotkehlchen logger to default LoggingSettings(anonymized_logs=DEFAULT_ANONYMIZED_LOGS) del self.accountant del self.trades_historian del self.data_importer if self.premium is not None: del self.premium self.data.logout() self.password = '' self.cryptocompare.unset_database() # Make sure no messages leak to other user sessions self.msg_aggregator.consume_errors() self.msg_aggregator.consume_warnings() self.user_is_logged_in = False log.info( 'User successfully logged out', user=user, ) def set_premium_credentials(self, credentials: PremiumCredentials) -> None: """ Sets the premium credentials for Rotki Raises PremiumAuthenticationError if the given key is rejected by the Rotkehlchen server """ log.info('Setting new premium credentials') if self.premium is not None: self.premium.set_credentials(credentials) else: self.premium = premium_create_and_verify(credentials) self.data.db.set_rotkehlchen_premium(credentials) def delete_premium_credentials(self) -> Tuple[bool, str]: """Deletes the premium credentials for Rotki""" msg = '' success = self.data.db.del_rotkehlchen_premium() if success is False: msg = 'The database was unable to delete the Premium keys for the logged-in user' self.deactivate_premium_status() return success, msg def deactivate_premium_status(self) -> None: """Deactivate premium in the current session""" self.premium = None self.premium_sync_manager.premium = None self.chain_manager.deactivate_premium_status() def start(self) -> gevent.Greenlet: return gevent.spawn(self.main_loop) def main_loop(self) -> None: """Rotki main loop that fires often and manages many different tasks Each task remembers the last time it run successfully and know how often it should run. So each task manages itself. """ # super hacky -- organize better when recurring tasks are implemented # https://github.com/rotki/rotki/issues/1106 xpub_derivation_scheduled = False while self.shutdown_event.wait(MAIN_LOOP_SECS_DELAY) is not True: if self.user_is_logged_in: log.debug('Main loop start') self.premium_sync_manager.maybe_upload_data_to_server() if not xpub_derivation_scheduled: # 1 minute in the app's startup try to derive new xpub addresses self.greenlet_manager.spawn_and_track( after_seconds=60.0, task_name='Derive new xpub addresses', method=XpubManager(self.chain_manager).check_for_new_xpub_addresses, ) xpub_derivation_scheduled = True log.debug('Main loop end') def get_blockchain_account_data( self, blockchain: SupportedBlockchain, ) -> Union[List[BlockchainAccountData], Dict[str, Any]]: account_data = self.data.db.get_blockchain_account_data(blockchain) if blockchain != SupportedBlockchain.BITCOIN: return account_data xpub_data = self.data.db.get_bitcoin_xpub_data() addresses_to_account_data = {x.address: x for x in account_data} address_to_xpub_mappings = self.data.db.get_addresses_to_xpub_mapping( list(addresses_to_account_data.keys()), # type: ignore ) xpub_mappings: Dict['XpubData', List[BlockchainAccountData]] = {} for address, xpub_entry in address_to_xpub_mappings.items(): if xpub_entry not in xpub_mappings: xpub_mappings[xpub_entry] = [] xpub_mappings[xpub_entry].append(addresses_to_account_data[address]) data: Dict[str, Any] = {'standalone': [], 'xpubs': []} # Add xpub data for xpub_entry in xpub_data: data_entry = xpub_entry.serialize() addresses = xpub_mappings.get(xpub_entry, None) data_entry['addresses'] = addresses if addresses and len(addresses) != 0 else None data['xpubs'].append(data_entry) # Add standalone addresses for account in account_data: if account.address not in address_to_xpub_mappings: data['standalone'].append(account) return data def add_blockchain_accounts( self, blockchain: SupportedBlockchain, account_data: List[BlockchainAccountData], ) -> BlockchainBalancesUpdate: """Adds new blockchain accounts Adds the accounts to the blockchain instance and queries them to get the updated balances. Also adds them in the DB May raise: - EthSyncError from modify_blockchain_account - InputError if the given accounts list is empty. - TagConstraintError if any of the given account data contain unknown tags. - RemoteError if an external service such as Etherscan is queried and there is a problem with its query. """ self.data.db.ensure_tags_exist( given_data=account_data, action='adding', data_type='blockchain accounts', ) address_type = blockchain.get_address_type() updated_balances = self.chain_manager.add_blockchain_accounts( blockchain=blockchain, accounts=[address_type(entry.address) for entry in account_data], ) self.data.db.add_blockchain_accounts( blockchain=blockchain, account_data=account_data, ) return updated_balances def edit_blockchain_accounts( self, blockchain: SupportedBlockchain, account_data: List[BlockchainAccountData], ) -> None: """Edits blockchain accounts Edits blockchain account data for the given accounts May raise: - InputError if the given accounts list is empty or if any of the accounts to edit do not exist. - TagConstraintError if any of the given account data contain unknown tags. """ # First check for validity of account data addresses if len(account_data) == 0: raise InputError('Empty list of blockchain account data to edit was given') accounts = [x.address for x in account_data] unknown_accounts = set(accounts).difference(self.chain_manager.accounts.get(blockchain)) if len(unknown_accounts) != 0: raise InputError( f'Tried to edit unknown {blockchain.value} ' f'accounts {",".join(unknown_accounts)}', ) self.data.db.ensure_tags_exist( given_data=account_data, action='editing', data_type='blockchain accounts', ) # Finally edit the accounts self.data.db.edit_blockchain_accounts( blockchain=blockchain, account_data=account_data, ) return None def remove_blockchain_accounts( self, blockchain: SupportedBlockchain, accounts: ListOfBlockchainAddresses, ) -> BlockchainBalancesUpdate: """Removes blockchain accounts Removes the accounts from the blockchain instance and queries them to get the updated balances. Also removes them from the DB May raise: - RemoteError if an external service such as Etherscan is queried and there is a problem with its query. - InputError if a non-existing account was given to remove """ balances_update = self.chain_manager.remove_blockchain_accounts( blockchain=blockchain, accounts=accounts, ) self.data.db.remove_blockchain_accounts(blockchain, accounts) return balances_update def process_history( self, start_ts: Timestamp, end_ts: Timestamp, ) -> Tuple[Dict[str, Any], str]: ( error_or_empty, history, loan_history, asset_movements, eth_transactions, defi_events, ) = self.trades_historian.get_history( start_ts=start_ts, end_ts=end_ts, has_premium=self.premium is not None, ) result = self.accountant.process_history( start_ts=start_ts, end_ts=end_ts, trade_history=history, loan_history=loan_history, asset_movements=asset_movements, eth_transactions=eth_transactions, defi_events=defi_events, ) return result, error_or_empty @overload def _apply_actions_limit( self, location: Location, action_type: Literal['trade'], location_actions: TRADES_LIST, all_actions: TRADES_LIST, ) -> TRADES_LIST: ... @overload def _apply_actions_limit( self, location: Location, action_type: Literal['asset_movement'], location_actions: List[AssetMovement], all_actions: List[AssetMovement], ) -> List[AssetMovement]: ... def _apply_actions_limit( self, location: Location, action_type: Literal['trade', 'asset_movement'], location_actions: Union[TRADES_LIST, List[AssetMovement]], all_actions: Union[TRADES_LIST, List[AssetMovement]], ) -> Union[TRADES_LIST, List[AssetMovement]]: """Take as many actions from location actions and add them to all actions as the limit permits Returns the modified (or not) all_actions """ # If we are already at or above the limit return current actions disregarding this location actions_mapping = self.actions_per_location[action_type] current_num_actions = sum(x for _, x in actions_mapping.items()) limit = LIMITS_MAPPING[action_type] if current_num_actions >= limit: return all_actions # Find out how many more actions can we return, and depending on that get # the number of actions from the location actions and add them to the total remaining_num_actions = limit - current_num_actions if remaining_num_actions < 0: remaining_num_actions = 0 num_actions_to_take = min(len(location_actions), remaining_num_actions) actions_mapping[location] = num_actions_to_take all_actions.extend(location_actions[0:num_actions_to_take]) # type: ignore return all_actions def query_trades( self, from_ts: Timestamp, to_ts: Timestamp, location: Optional[Location], ) -> TRADES_LIST: """Queries trades for the given location and time range. If no location is given then all external, all exchange and DEX trades are queried. DEX Trades are queried only if the user has premium If the user does not have premium then a trade limit is applied. May raise: - RemoteError: If there are problems connecting to any of the remote exchanges """ trades: TRADES_LIST if location is not None: trades = self.query_location_trades(from_ts, to_ts, location) else: trades = self.query_location_trades(from_ts, to_ts, Location.EXTERNAL) # crypto.com is not an API key supported exchange but user can import from CSV trades.extend(self.query_location_trades(from_ts, to_ts, Location.CRYPTOCOM)) for name, exchange in self.exchange_manager.connected_exchanges.items(): exchange_trades = exchange.query_trade_history(start_ts=from_ts, end_ts=to_ts) if self.premium is None: trades = self._apply_actions_limit( location=deserialize_location(name), action_type='trade', location_actions=exchange_trades, all_actions=trades, ) else: trades.extend(exchange_trades) # for all trades we also need uniswap trades if self.premium is not None: uniswap = self.chain_manager.uniswap if uniswap is not None: trades.extend( uniswap.get_trades( addresses=self.chain_manager.queried_addresses_for_module('uniswap'), from_timestamp=from_ts, to_timestamp=to_ts, ), ) # return trades with most recent first trades.sort(key=lambda x: x.timestamp, reverse=True) return trades def query_location_trades( self, from_ts: Timestamp, to_ts: Timestamp, location: Location, ) -> TRADES_LIST: # clear the trades queried for this location self.actions_per_location['trade'][location] = 0 location_trades: TRADES_LIST if location in (Location.EXTERNAL, Location.CRYPTOCOM): location_trades = self.data.db.get_trades( # type: ignore # list invariance from_ts=from_ts, to_ts=to_ts, location=location, ) elif location == Location.UNISWAP: if self.premium is not None: uniswap = self.chain_manager.uniswap if uniswap is not None: location_trades = uniswap.get_trades( # type: ignore # list invariance addresses=self.chain_manager.queried_addresses_for_module('uniswap'), from_timestamp=from_ts, to_timestamp=to_ts, ) else: # should only be an exchange exchange = self.exchange_manager.get(str(location)) if not exchange: logger.warn( f'Tried to query trades from {location} which is either not an ' f'exchange or not an exchange the user has connected to', ) return [] location_trades = exchange.query_trade_history(start_ts=from_ts, end_ts=to_ts) trades: TRADES_LIST = [] if self.premium is None: trades = self._apply_actions_limit( location=location, action_type='trade', location_actions=location_trades, all_actions=trades, ) else: trades = location_trades return trades def query_balances( self, requested_save_data: bool = False, timestamp: Timestamp = None, ignore_cache: bool = False, ) -> Dict[str, Any]: """Query all balances rotkehlchen can see. If requested_save_data is True then the data are always saved in the DB, if it is False then data are saved if self.data.should_save_balances() is True. If timestamp is None then the current timestamp is used. If a timestamp is given then that is the time that the balances are going to be saved in the DB If ignore_cache is True then all underlying calls that have a cache ignore it Returns a dictionary with the queried balances. """ log.info('query_balances called', requested_save_data=requested_save_data) balances = {} problem_free = True for _, exchange in self.exchange_manager.connected_exchanges.items(): exchange_balances, _ = exchange.query_balances(ignore_cache=ignore_cache) # If we got an error, disregard that exchange but make sure we don't save data if not isinstance(exchange_balances, dict): problem_free = False else: balances[exchange.name] = exchange_balances try: blockchain_result = self.chain_manager.query_balances( blockchain=None, force_token_detection=ignore_cache, ignore_cache=ignore_cache, ) serialized_chain_result = blockchain_result.totals.to_dict() balances['blockchain'] = serialized_chain_result['assets'] except (RemoteError, EthSyncError) as e: problem_free = False log.error(f'Querying blockchain balances failed due to: {str(e)}') balances = account_for_manually_tracked_balances(db=self.data.db, balances=balances) combined = combine_stat_dicts([v for k, v in balances.items()]) total_usd_per_location = [(k, dict_get_sumof(v, 'usd_value')) for k, v in balances.items()] liabilities = serialized_chain_result['liabilities'] # atm liabilities only on chain # calculate net usd value net_usd = ZERO for _, v in combined.items(): net_usd += FVal(v['usd_value']) # subtract liabilities liabilities_total_usd = sum(x['usd_value'] for _, x in liabilities.items()) net_usd -= liabilities_total_usd stats: Dict[str, Any] = { 'location': { }, 'net_usd': net_usd, } for entry in total_usd_per_location: name = entry[0] total = entry[1] if name == 'blockchain': # blockchain is the only location with liabilities atm total -= liabilities_total_usd if net_usd != ZERO: percentage = (total / net_usd).to_percentage() else: percentage = '0%' stats['location'][name] = { 'usd_value': total, 'percentage_of_net_value': percentage, } for k, v in combined.items(): if net_usd != ZERO: percentage = (v['usd_value'] / net_usd).to_percentage() else: percentage = '0%' combined[k]['percentage_of_net_value'] = percentage for k, v in liabilities.items(): if net_usd != ZERO: percentage = (v['usd_value'] / net_usd).to_percentage() else: percentage = '0%' liabilities[k]['percentage_of_net_value'] = percentage balance_sheet = { 'assets': combined, 'liabilities': liabilities, } result_dict = merge_dicts(balance_sheet, stats) allowed_to_save = requested_save_data or self.data.should_save_balances() if problem_free and allowed_to_save: if not timestamp: timestamp = Timestamp(int(time.time())) self.data.db.save_balances_data(data=result_dict, timestamp=timestamp) log.debug('query_balances data saved') else: log.debug( 'query_balances data not saved', allowed_to_save=allowed_to_save, problem_free=problem_free, ) return result_dict def _query_exchange_asset_movements( self, from_ts: Timestamp, to_ts: Timestamp, all_movements: List[AssetMovement], exchange: Union[ExchangeInterface, Location], ) -> List[AssetMovement]: if isinstance(exchange, ExchangeInterface): location = deserialize_location(exchange.name) # clear the asset movements queried for this exchange self.actions_per_location['asset_movement'][location] = 0 location_movements = exchange.query_deposits_withdrawals( start_ts=from_ts, end_ts=to_ts, ) else: assert isinstance(exchange, Location), 'only a location should make it here' assert exchange == Location.CRYPTOCOM, 'only cryptocom should make it here' location = exchange # cryptocom has no exchange integration but we may have DB entries self.actions_per_location['asset_movement'][location] = 0 location_movements = self.data.db.get_asset_movements( from_ts=from_ts, to_ts=to_ts, location=str(location), ) movements: List[AssetMovement] = [] if self.premium is None: movements = self._apply_actions_limit( location=location, action_type='asset_movement', location_actions=location_movements, all_actions=all_movements, ) else: all_movements.extend(location_movements) movements = all_movements return movements def query_asset_movements( self, from_ts: Timestamp, to_ts: Timestamp, location: Optional[Location], ) -> List[AssetMovement]: """Queries AssetMovements for the given location and time range. If no location is given then all exchange asset movements are queried. If the user does not have premium then a limit is applied. May raise: - RemoteError: If there are problems connecting to any of the remote exchanges """ movements: List[AssetMovement] = [] if location is not None: if location == Location.CRYPTOCOM: movements = self._query_exchange_asset_movements( from_ts=from_ts, to_ts=to_ts, all_movements=movements, exchange=Location.CRYPTOCOM, ) else: exchange = self.exchange_manager.get(str(location)) if not exchange: logger.warn( f'Tried to query deposits/withdrawals from {location} which is either ' f'not at exchange or not an exchange the user has connected to', ) return [] movements = self._query_exchange_asset_movements( from_ts=from_ts, to_ts=to_ts, all_movements=movements, exchange=exchange, ) else: # cryptocom has no exchange integration but we may have DB entries due to csv import movements = self._query_exchange_asset_movements( from_ts=from_ts, to_ts=to_ts, all_movements=movements, exchange=Location.CRYPTOCOM, ) for _, exchange in self.exchange_manager.connected_exchanges.items(): self._query_exchange_asset_movements( from_ts=from_ts, to_ts=to_ts, all_movements=movements, exchange=exchange, ) # return movements with most recent first movements.sort(key=lambda x: x.timestamp, reverse=True) return movements def set_settings(self, settings: ModifiableDBSettings) -> Tuple[bool, str]: """Tries to set new settings. Returns True in success or False with message if error""" with self.lock: if settings.eth_rpc_endpoint is not None: result, msg = self.chain_manager.set_eth_rpc_endpoint(settings.eth_rpc_endpoint) if not result: return False, msg if settings.kraken_account_type is not None: kraken = self.exchange_manager.get('kraken') if kraken: kraken.set_account_type(settings.kraken_account_type) # type: ignore self.data.db.set_settings(settings) return True, '' def get_settings(self) -> DBSettings: """Returns the db settings with a check whether premium is active or not""" db_settings = self.data.db.get_settings(have_premium=self.premium is not None) return db_settings def setup_exchange( self, name: str, api_key: ApiKey, api_secret: ApiSecret, passphrase: Optional[str] = None, ) -> Tuple[bool, str]: """ Setup a new exchange with an api key and an api secret and optionally a passphrase By default the api keys are always validated unless validate is False. """ is_success, msg = self.exchange_manager.setup_exchange( name=name, api_key=api_key, api_secret=api_secret, database=self.data.db, passphrase=passphrase, ) if is_success: # Success, save the result in the DB self.data.db.add_exchange(name, api_key, api_secret, passphrase=passphrase) return is_success, msg def remove_exchange(self, name: str) -> Tuple[bool, str]: if not self.exchange_manager.has_exchange(name): return False, 'Exchange {} is not registered'.format(name) self.exchange_manager.delete_exchange(name) # Success, remove it also from the DB self.data.db.remove_exchange(name) self.data.db.delete_used_query_range_for_exchange(name) return True, '' def query_periodic_data(self) -> Dict[str, Union[bool, Timestamp]]: """Query for frequently changing data""" result: Dict[str, Union[bool, Timestamp]] = {} if self.user_is_logged_in: result['last_balance_save'] = self.data.db.get_last_balance_save_time() result['eth_node_connection'] = self.chain_manager.ethereum.web3_mapping.get(NodeName.OWN, None) is not None # noqa : E501 result['history_process_start_ts'] = self.accountant.started_processing_timestamp result['history_process_current_ts'] = self.accountant.currently_processing_timestamp result['last_data_upload_ts'] = Timestamp(self.premium_sync_manager.last_data_upload_ts) # noqa : E501 return result def shutdown(self) -> None: self.logout() self.shutdown_event.set()
def test_writting_fetching_external_trades(data_dir, username): msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) # add 2 trades and check they are in the DB trade1 = { 'otc_timestamp': '10/03/2018 23:30', 'otc_pair': 'ETH_EUR', 'otc_type': 'buy', 'otc_amount': '10', 'otc_rate': '100', 'otc_fee': '0.001', 'otc_fee_currency': 'ETH', 'otc_link': 'a link', 'otc_notes': 'a note', } trade2 = { 'otc_timestamp': '15/03/2018 23:35', 'otc_pair': 'ETH_EUR', 'otc_type': 'buy', 'otc_amount': '5', 'otc_rate': '100', 'otc_fee': '0.001', 'otc_fee_currency': 'ETH', 'otc_link': 'a link 2', 'otc_notes': 'a note 2', } result, _, = data.add_external_trade(trade1) assert result result, _ = data.add_external_trade(trade2) assert result result = data.get_external_trades() del result[0]['id'] assert result[0] == from_otc_trade(trade1) del result[1]['id'] assert result[1] == from_otc_trade(trade2) # query trades in period result = data.get_external_trades( from_ts=1520553600, # 09/03/2018 to_ts=1520726400, # 11/03/2018 ) assert len(result) == 1 del result[0]['id'] assert result[0] == from_otc_trade(trade1) # query trades only with to_ts result = data.get_external_trades( to_ts=1520726400, # 11/03/2018 ) assert len(result) == 1 del result[0]['id'] assert result[0] == from_otc_trade(trade1) # edit a trade and check the edit made it in the DB trade1['otc_rate'] = '120' trade1['otc_id'] = 1 result, _ = data.edit_external_trade(trade1) assert result result = data.get_external_trades() assert result[0] == from_otc_trade(trade1) del result[1]['id'] assert result[1] == from_otc_trade(trade2) # try to edit a non-existing trade trade1['otc_rate'] = '160' trade1['otc_id'] = 5 result, _ = data.edit_external_trade(trade1) assert not result trade1['otc_rate'] = '120' trade1['otc_id'] = 1 result = data.get_external_trades() assert result[0] == from_otc_trade(trade1) del result[1]['id'] assert result[1] == from_otc_trade(trade2) # try to delete non-existing trade result, _ = data.delete_external_trade(6) assert not result # delete an external trade result, _ = data.delete_external_trade(1) result = data.get_external_trades() del result[0]['id'] assert result[0] == from_otc_trade(trade2)
def __init__(self, args: argparse.Namespace) -> None: """Initialize the Rotkehlchen object May Raise: - SystemPermissionError if the given data directory's permissions are not correct. """ self.lock = Semaphore() self.lock.acquire() # Can also be None after unlock if premium credentials did not # authenticate or premium server temporarily offline self.premium: Optional[Premium] = None self.user_is_logged_in = False logfilename = None if args.logtarget == 'file': logfilename = args.logfile if args.loglevel == 'debug': loglevel = logging.DEBUG elif args.loglevel == 'info': loglevel = logging.INFO elif args.loglevel == 'warn': loglevel = logging.WARN elif args.loglevel == 'error': loglevel = logging.ERROR elif args.loglevel == 'critical': loglevel = logging.CRITICAL else: raise AssertionError('Should never get here. Illegal log value') logging.basicConfig( filename=logfilename, filemode='w', level=loglevel, format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s', datefmt='%d/%m/%Y %H:%M:%S %Z', ) if not args.logfromothermodules: logging.getLogger('urllib3').setLevel(logging.CRITICAL) logging.getLogger('urllib3.connectionpool').setLevel( logging.CRITICAL) self.sleep_secs = args.sleep_secs self.data_dir = args.data_dir if not os.access(self.data_dir, os.W_OK | os.R_OK): raise SystemPermissionError( f'The given data directory {self.data_dir} is not readable or writable', ) self.args = args self.msg_aggregator = MessagesAggregator() self.greenlet_manager = GreenletManager( msg_aggregator=self.msg_aggregator) self.exchange_manager = ExchangeManager( msg_aggregator=self.msg_aggregator) self.all_eth_tokens = AssetResolver().get_all_eth_tokens() self.data = DataHandler(self.data_dir, self.msg_aggregator) self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None) # Initialize the Inquirer singleton Inquirer(data_dir=self.data_dir, cryptocompare=self.cryptocompare) self.lock.release() self.shutdown_event = gevent.event.Event()
def test_get_latest_location_value_distribution(data_dir, username): msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) data.unlock(username, '123', create_new=True) locations = [ LocationData( time=Timestamp(1451606400), location='kraken', usd_value='100', ), LocationData( time=Timestamp(1451606400), location='banks', usd_value='1000', ), LocationData( time=Timestamp(1461606500), location='poloniex', usd_value='50', ), LocationData( time=Timestamp(1461606500), location='kraken', usd_value='200', ), LocationData( time=Timestamp(1461606500), location='banks', usd_value='50000', ), LocationData( time=Timestamp(1491607800), location='poloniex', usd_value='100', ), LocationData( time=Timestamp(1491607800), location='kraken', usd_value='2000', ), LocationData( time=Timestamp(1491607800), location='banks', usd_value='10000', ), LocationData( time=Timestamp(1491607800), location='blockchain', usd_value='200000', ), ] data.db.add_multiple_location_data(locations) distribution = data.db.get_latest_location_value_distribution() assert len(distribution) == 4 assert all(entry.time == Timestamp(1491607800) for entry in distribution) assert distribution[0].location == 'banks' assert distribution[0].usd_value == '10000' assert distribution[1].location == 'blockchain' assert distribution[1].usd_value == '200000' assert distribution[2].location == 'kraken' assert distribution[2].usd_value == '2000' assert distribution[3].location == 'poloniex' assert distribution[3].usd_value == '100'