def unlock_user(self, user, password, create_new, sync_approval, api_key, api_secret): # unlock or create the DB self.password = password user_dir = self.data.unlock(user, password, create_new) self.try_premium_at_start(api_key, api_secret, create_new, sync_approval, user_dir) secret_data = self.data.db.get_exchange_secrets() self.cache_data_filename = os.path.join(self.data_dir, 'cache_data.json') historical_data_start = self.data.historical_start_date() self.trades_historian = TradesHistorian( self.data_dir, self.data.db, self.data.get_eth_accounts(), historical_data_start, ) self.price_historian = PriceHistorian( self.data_dir, historical_data_start, ) self.accountant = Accountant(price_historian=self.price_historian, profit_currency=self.data.main_currency(), create_csv=True) self.inquirer = Inquirer(kraken=self.kraken) self.initialize_exchanges(secret_data) self.blockchain = Blockchain(self.data.db.get_blockchain_accounts(), self.data.eth_tokens, self.data.db.get_owned_tokens(), self.inquirer, self.args.ethrpc_port)
def accountant(price_historian, profit_currency, accounting_data_dir, accounting_create_csv, accounting_ignored_assets): return Accountant(price_historian=price_historian, profit_currency=profit_currency, user_directory=accounting_data_dir, create_csv=accounting_create_csv, ignored_assets=accounting_ignored_assets)
def init_accounting_tests(history_list, margin_list, start_ts, end_ts): price_historian = PriceHistorian('/home/lefteris/.rotkehlchen') accountant = Accountant( price_historian=price_historian, profit_currency='EUR', create_csv=False ) accountant.process_history( start_ts=start_ts, end_ts=end_ts, trade_history=trades_from_dictlist(history_list, start_ts, end_ts), margin_history=trades_from_dictlist(margin_list, start_ts, end_ts), loan_history=list(), asset_movements=list(), eth_transactions=list() ) return accountant
def init_accounting_tests(history_list, margin_list, start_ts, end_ts): # TODO: This should become part of test fixtures. Also HAS to be deleted at teardown user_directory = mkdtemp() price_historian = PriceHistorian(user_directory) accountant = Accountant(price_historian=price_historian, profit_currency='EUR', user_directory=user_directory, create_csv=False, ignored_assets=[]) accountant.process_history( start_ts=start_ts, end_ts=end_ts, trade_history=trades_from_dictlist(history_list, start_ts, end_ts), margin_history=trades_from_dictlist(margin_list, start_ts, end_ts), loan_history=list(), asset_movements=list(), eth_transactions=list()) return accountant
class Rotkehlchen(object): def __init__(self, args): self.lock = Semaphore() self.lock.acquire() self.results_cache = {} self.connected_exchanges = [] logfilename = None if args.logtarget == 'file': logfilename = args.logfile loglevel = logging.DEBUG if args.loglevel == 'debug': loglevel = logging.DEBUG elif args.loglevel == 'info': loglevel = logging.INFO elif args.loglevel == 'warn': loglevel = logging.WARN elif args.loglevel == 'error': loglevel = logging.ERROR elif args.loglevel == 'critical': loglevel = logging.CRITICAL else: raise ValueError('Should never get here. Illegal log value') logging.basicConfig( filename=logfilename, filemode='w', level=loglevel, format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s', datefmt='%d/%m/%Y %H:%M:%S %Z', ) if not args.logfromothermodules: logging.getLogger('zerorpc').setLevel(logging.CRITICAL) logging.getLogger('zerorpc.channel').setLevel(logging.CRITICAL) logging.getLogger('urllib3').setLevel(logging.CRITICAL) logging.getLogger('urllib3.connectionpool').setLevel( logging.CRITICAL) self.sleep_secs = args.sleep_secs self.data_dir = args.data_dir self.args = args self.last_data_upload_ts = 0 self.poloniex = None self.kraken = None self.bittrex = None self.binance = None self.data = DataHandler(self.data_dir) self.lock.release() self.shutdown_event = gevent.event.Event() def initialize_exchanges(self, secret_data): # initialize exchanges for which we have keys and are not already initialized if self.kraken is None and 'kraken' in secret_data: self.kraken = Kraken( str.encode(secret_data['kraken']['api_key']), str.encode(secret_data['kraken']['api_secret']), self.data_dir) self.connected_exchanges.append('kraken') self.trades_historian.set_exchange('kraken', self.kraken) if self.poloniex is None and 'poloniex' in secret_data: self.poloniex = Poloniex( str.encode(secret_data['poloniex']['api_key']), str.encode(secret_data['poloniex']['api_secret']), self.cache_data_filename, self.inquirer, self.data_dir) self.connected_exchanges.append('poloniex') self.trades_historian.set_exchange('poloniex', self.poloniex) if self.bittrex is None and 'bittrex' in secret_data: self.bittrex = Bittrex( str.encode(secret_data['bittrex']['api_key']), str.encode(secret_data['bittrex']['api_secret']), self.inquirer, self.data_dir) self.connected_exchanges.append('bittrex') self.trades_historian.set_exchange('bittrex', self.bittrex) if self.binance is None and 'binance' in secret_data: self.binance = Binance( str.encode(secret_data['binance']['api_key']), str.encode(secret_data['binance']['api_secret']), self.inquirer, self.data_dir) self.connected_exchanges.append('binance') self.trades_historian.set_exchange('binance', self.binance) def try_premium_at_start(self, api_key, api_secret, create_new, sync_approval, user_dir): """Check if new user provided api pair or we already got one in the DB""" if api_key != '': self.premium, valid, empty_or_error = premium_create_and_verify( api_key, api_secret) if not valid: # At this point we are at a new user trying to create an account with # premium API keys and we failed. But a directory was created. Remove it. shutil.rmtree(user_dir) raise AuthenticationError( 'Could not verify keys for the new account. ' '{}'.format(empty_or_error)) else: # If we got premium initialize it and try to sync with the server premium_credentials = self.data.db.get_rotkehlchen_premium() if premium_credentials: api_key = premium_credentials[0] api_secret = premium_credentials[1] self.premium, valid, empty_or_error = premium_create_and_verify( api_key, api_secret) if not valid: logger.error( 'The API keys found in the Database are not valid. Perhaps ' 'they expired?') del self.premium return else: # no premium credentials in the DB return if self.can_sync_data_from_server(): if sync_approval == 'unknown' and not create_new: raise PermissionError( 'Rotkehlchen Server has newer version of your DB data. ' 'Should we replace local data with the server\'s?') elif sync_approval == 'yes' or sync_approval == 'unknown' and create_new: logger.debug('User approved data sync from server') if self.sync_data_from_server(): if create_new: # if we succesfully synced data from the server and this is # a new account, make sure the api keys are properly stored # in the DB self.data.db.set_rotkehlchen_premium( api_key, api_secret) else: logger.debug('Could sync data from server but user refused') def unlock_user(self, user, password, create_new, sync_approval, api_key, api_secret): # unlock or create the DB self.password = password user_dir = self.data.unlock(user, password, create_new) self.try_premium_at_start(api_key, api_secret, create_new, sync_approval, user_dir) secret_data = self.data.db.get_exchange_secrets() self.cache_data_filename = os.path.join(self.data_dir, 'cache_data.json') historical_data_start = self.data.historical_start_date() self.trades_historian = TradesHistorian( self.data_dir, self.data.db, self.data.get_eth_accounts(), historical_data_start, ) self.price_historian = PriceHistorian( self.data_dir, historical_data_start, ) self.accountant = Accountant( price_historian=self.price_historian, profit_currency=self.data.main_currency(), user_directory=user_dir, create_csv=True, ignored_assets=self.data.db.get_ignored_assets()) self.inquirer = Inquirer(kraken=self.kraken) self.initialize_exchanges(secret_data) self.blockchain = Blockchain(self.data.db.get_blockchain_accounts(), self.data.eth_tokens, self.data.db.get_owned_tokens(), self.inquirer, self.args.ethrpc_port) def set_premium_credentials(self, api_key, api_secret): if hasattr(self, 'premium'): valid, empty_or_error = self.premium.set_credentials( api_key, api_secret) else: self.premium, valid, empty_or_error = premium_create_and_verify( api_key, api_secret) if valid: self.data.set_premium_credentials(api_key, api_secret) return True, '' return False, empty_or_error def maybe_upload_data_to_server(self): logger.debug('Maybe upload to server') # upload only if unlocked user has premium if not hasattr(self, 'premium'): return # upload only once per hour diff = ts_now() - self.last_data_upload_ts if diff > 3600: self.upload_data_to_server() def upload_data_to_server(self): logger.debug('upload to server -- start') data, our_hash = self.data.compress_and_encrypt_db(self.password) success, result_or_error = self.premium.query_last_data_metadata() if not success: logger.debug( 'upload to server -- query last metadata error: {}'.format( result_or_error)) return logger.debug("CAN_PUSH--> OURS: {} THEIRS: {}".format( our_hash, result_or_error['data_hash'])) if our_hash == result_or_error['data_hash']: logger.debug('upload to server -- same hash') # same hash -- no need to upload anything return our_last_write_ts = self.data.db.get_last_write_ts() if our_last_write_ts <= result_or_error['last_modify_ts']: # Server's DB was modified after our local DB logger.debug("CAN_PUSH -> 3") logger.debug( 'upload to server -- remote db more recent than local') return success, result_or_error = self.premium.upload_data( data, our_hash, our_last_write_ts, 'zlib') if not success: logger.debug( 'upload to server -- upload error: {}'.format(result_or_error)) return self.last_data_upload_ts = ts_now() logger.debug('upload to server -- success') def can_sync_data_from_server(self): logger.debug('sync data from server -- start') data, our_hash = self.data.compress_and_encrypt_db(self.password) success, result_or_error = self.premium.query_last_data_metadata() if not success: logger.debug( 'sync data from server-- error: {}'.format(result_or_error)) return False logger.debug("CAN_PULL--> OURS: {} THEIRS: {}".format( our_hash, result_or_error['data_hash'])) if our_hash == result_or_error['data_hash']: logger.debug('sync from server -- same hash') # same hash -- no need to get anything return False our_last_write_ts = self.data.db.get_last_write_ts() if our_last_write_ts >= result_or_error['last_modify_ts']: # Local DB is newer than Server DB logger.debug( 'sync from server -- local DB more recent than remote') return False return True def sync_data_from_server(self): success, error_or_result = self.premium.pull_data() if not success: logger.debug( 'sync from server -- pulling error {}'.format(error_or_result)) return False self.data.decompress_and_decrypt_db(self.password, error_or_result['data']) return True def start(self): return gevent.spawn(self.main_loop) def main_loop(self): while True and not self.shutdown_event.is_set(): logger.debug('Main loop start') if self.poloniex is not None: self.poloniex.main_logic() if self.kraken is not None: self.kraken.main_logic() self.maybe_upload_data_to_server() logger.debug('Main loop end') gevent.sleep(MAIN_LOOP_SECS_DELAY) def process_history(self, start_ts, end_ts): history, margin_history, loan_history, asset_movements, eth_transactions = self.trades_historian.get_history( start_ts= 0, # For entire history processing we need to have full history available end_ts=ts_now(), end_at_least_ts=end_ts) return self.accountant.process_history(start_ts, end_ts, history, margin_history, loan_history, asset_movements, eth_transactions) def query_fiat_balances(self): result = {} balances = self.data.get_fiat_balances() for currency, amount in balances.items(): amount = FVal(amount) usd_rate = query_fiat_pair(currency, 'USD') result[currency] = { 'amount': amount, 'usd_value': amount * usd_rate } return result def query_balances(self, save_data=False): balances = {} for exchange in self.connected_exchanges: balances[exchange] = getattr(self, exchange).query_balances() result = self.blockchain.query_balances()['totals'] if result != {}: balances['blockchain'] = result result = self.query_fiat_balances() if result != {}: balances['banks'] = result combined = combine_stat_dicts([v for k, v in balances.items()]) total_usd_per_location = [(k, dict_get_sumof(v, 'usd_value')) for k, v in balances.items()] # calculate net usd value net_usd = FVal(0) for k, v in combined.items(): net_usd += FVal(v['usd_value']) stats = {'location': {}, 'net_usd': net_usd} for entry in total_usd_per_location: name = entry[0] total = entry[1] stats['location'][name] = { 'usd_value': total, 'percentage_of_net_value': (total / net_usd).to_percentage(), } for k, v in combined.items(): combined[k]['percentage_of_net_value'] = (v['usd_value'] / net_usd).to_percentage() result_dict = merge_dicts(combined, stats) if save_data: self.data.save_balances_data(result_dict) # After adding it to the saved file we can overlay additional data that # is not required to be saved in the history file try: details = self.data.accountant.details for asset, (tax_free_amount, average_buy_value) in details.items(): if asset not in result_dict: continue result_dict[asset]['tax_free_amount'] = tax_free_amount result_dict[asset]['average_buy_value'] = average_buy_value current_price = result_dict[asset]['usd_value'] / result_dict[ asset]['amount'] if average_buy_value != FVal(0): result_dict[asset]['percent_change'] = ( ((current_price - average_buy_value) / average_buy_value) * 100) else: result_dict[asset]['percent_change'] = 'INF' except AttributeError: pass return result_dict def set_main_currency(self, currency): with self.lock: self.data.set_main_currency(currency, self.accountant) if currency != 'USD': self.usd_to_main_currency_rate = query_fiat_pair( 'USD', currency) def set_settings(self, settings): with self.lock: self.data.set_settings(settings) main_currency = settings['main_currency'] if main_currency != 'USD': self.usd_to_main_currency_rate = query_fiat_pair( 'USD', main_currency) def usd_to_main_currency(self, amount): main_currency = self.data.main_currency() if main_currency != 'USD' and not hasattr(self, 'usd_to_main_currency_rate'): self.usd_to_main_currency_rate = query_fiat_pair( 'USD', main_currency) return self.usd_to_main_currency_rate * amount def get_settings(self): return self.data.settings def setup_exchange(self, name, api_key, api_secret): if name not in SUPPORTED_EXCHANGES: return False, 'Attempted to register unsupported exchange {}'.format( name) if getattr(self, name) is not None: return False, 'Exchange {} is already registered'.format(name) secret_data = {} secret_data[name] = { 'api_key': api_key, 'api_secret': api_secret, } self.initialize_exchanges(secret_data) exchange = getattr(self, name) result, message = exchange.validate_api_key() if not result: self.delete_exchange_data(name) return False, message # Success, save the result in the DB self.data.db.add_exchange(name, api_key, api_secret) return True, '' def delete_exchange_data(self, name): self.connected_exchanges.remove(name) self.trades_historian.set_exchange(name, None) delattr(self, name) setattr(self, name, None) def remove_exchange(self, name): if getattr(self, name) is None: return False, 'Exchange {} is not registered'.format(name) self.delete_exchange_data(name) # Success, remove it also from the DB self.data.db.remove_exchange(name) return True, '' def shutdown(self): print("Shutting Down...") self.shutdown_event.set() def set(self, *args): if len(args) < 2: return ("ERROR: set requires at least two arguments but " "got: {}".format(args)) if args[0] == 'poloniex': resp = self.poloniex.set(*args[1:]) else: return "ERROR: Unrecognized first argument: {}".format(args[0]) self.save_data() return resp