def accounting_history_process( accountant, start_ts: Timestamp, end_ts: Timestamp, history_list: List[Dict], margin_list: List[MarginPosition] = None, loans_list: List[Dict] = None, asset_movements_list: List[Dict] = None, eth_transaction_list: List[Dict] = None, ) -> Dict[str, Any]: # For filtering the taxable actions list we start with 0 ts so that we have the # full history available trade_history = trades_from_dictlist( given_trades=history_list, start_ts=0, end_ts=end_ts, location='accounting_history_process for tests', msg_aggregator=accountant.msg_aggregator, ) # if present, append margin positions to trade history if margin_list: trade_history.extend(margin_list) asset_movements = list() if asset_movements_list: asset_movements = asset_movements_from_dictlist( given_data=asset_movements_list, start_ts=0, end_ts=end_ts, ) loan_history = list() if loans_list: loan_history = process_polo_loans( msg_aggregator=accountant.msg_aggregator, data=loans_list, start_ts=0, end_ts=end_ts, ) eth_transactions = list() if eth_transaction_list: eth_transactions = transactions_from_dictlist( given_transactions=eth_transaction_list, start_ts=0, end_ts=end_ts, ) result = accountant.process_history( start_ts=start_ts, end_ts=end_ts, trade_history=trade_history, loan_history=loan_history, asset_movements=asset_movements, eth_transactions=eth_transactions, ) return result
def accounting_history_process( accountant, start_ts: Timestamp, end_ts: Timestamp, history_list: List[Dict], margin_list: List[Dict] = None, loans_list: List[Dict] = None, asset_movements_list: List[Dict] = None, eth_transaction_list: List[Dict] = None, ) -> Dict[str, Any]: # For filtering the taxable actions list we start with 0 ts so that we have the # full history available trade_history = trades_from_dictlist( given_trades=history_list, start_ts=0, end_ts=end_ts, location='accounting_history_process for tests', msg_aggregator=accountant.msg_aggregator, ) margin_history = [] if not margin_list else margin_list loan_history = [] if not loans_list else loans_list asset_movements = list() if asset_movements_list: asset_movements = asset_movements_from_dictlist( given_data=asset_movements_list, start_ts=0, end_ts=end_ts, ) eth_transactions = list() if eth_transaction_list: eth_transactions = transactions_from_dictlist( given_transactions=eth_transaction_list, start_ts=0, end_ts=end_ts, ) result = accountant.process_history( start_ts=start_ts, end_ts=end_ts, trade_history=trade_history, margin_history=margin_history, loan_history=loan_history, asset_movements=asset_movements, eth_transactions=eth_transactions, ) return result
def get_history(self, start_ts, end_ts, end_at_least_ts=None): """Gets or creates trades and loans history from start_ts to end_ts or if `end_at_least` is given and we have a cache history which satisfies it we return the cache """ if end_at_least_ts is None: end_at_least_ts = end_ts historyfile_path = os.path.join(self.data_directory, TRADES_HISTORYFILE) if os.path.isfile(historyfile_path): with open(historyfile_path, 'r') as infile: try: history_json_data = rlk_jsonloads(infile.read()) except: pass all_history_okay = data_up_todate(history_json_data, start_ts, end_at_least_ts) poloniex_history_okay = True if self.poloniex is not None: poloniex_history_okay = self.poloniex.check_trades_cache( start_ts, end_at_least_ts ) is not None kraken_history_okay = True if self.kraken is not None: kraken_history_okay = self.kraken.check_trades_cache( start_ts, end_at_least_ts ) is not None bittrex_history_okay = True if self.bittrex is not None: bittrex_history_okay = self.bittrex.check_trades_cache( start_ts, end_at_least_ts ) is not None binance_history_okay = True if self.binance is not None: binance_history_okay = self.binance.check_trades_cache( start_ts, end_at_least_ts ) is not None if not self.read_manual_margin_positions: marginfile_path = os.path.join(self.data_directory, MARGIN_HISTORYFILE) margin_file_contents = get_jsonfile_contents_or_empty_dict(marginfile_path) margin_history_is_okay = data_up_todate( margin_file_contents, start_ts, end_at_least_ts ) else: margin_history_is_okay = True margin_file_contents = do_read_manual_margin_positions( self.data_directory ) loansfile_path = os.path.join(self.data_directory, LOANS_HISTORYFILE) loan_file_contents = get_jsonfile_contents_or_empty_dict(loansfile_path) loan_history_is_okay = data_up_todate( loan_file_contents, start_ts, end_at_least_ts ) assetmovementsfile_path = os.path.join( self.data_directory, ASSETMOVEMENTS_HISTORYFILE ) asset_movements_contents = get_jsonfile_contents_or_empty_dict( assetmovementsfile_path ) asset_movements_history_is_okay = data_up_todate( asset_movements_contents, start_ts, end_at_least_ts ) eth_tx_log_path = os.path.join(self.data_directory, ETHEREUM_TX_LOGFILE) eth_tx_log_contents = get_jsonfile_contents_or_empty_dict(eth_tx_log_path) eth_tx_log_history_history_is_okay = data_up_todate( eth_tx_log_contents, start_ts, end_at_least_ts ) if ( all_history_okay and poloniex_history_okay and kraken_history_okay and bittrex_history_okay and binance_history_okay and margin_history_is_okay and loan_history_is_okay and asset_movements_history_is_okay and eth_tx_log_history_history_is_okay): history_trades = trades_from_dictlist( history_json_data['data'], start_ts, end_ts ) if not self.read_manual_margin_positions: margin_trades = trades_from_dictlist( margin_file_contents['data'], start_ts, end_ts ) else: margin_trades = margin_file_contents eth_transactions = transactions_from_dictlist( eth_tx_log_contents['data'], start_ts, end_ts ) asset_movements = asset_movements_from_dictlist( asset_movements_contents['data'], start_ts, end_ts ) history_trades = include_external_trades( self.db, start_ts, end_ts, history_trades ) # make sure that this is the same as what is returned # from create_history return ( history_trades, margin_trades, loan_file_contents['data'], asset_movements, eth_transactions ) return self.create_history(start_ts, end_ts, end_at_least_ts)
def get_cached_history(self, start_ts, end_ts, end_at_least_ts=None): """Gets all the cached history data instead of querying all external sources to create the history through create_history() Can raise: - HistoryCacheInvalid: If any of the cache files are corrupt in any way, missing or do not cover the given time range """ if end_at_least_ts is None: end_at_least_ts = end_ts historyfile_path = os.path.join(self.user_directory, TRADES_HISTORYFILE) if not os.path.isfile(historyfile_path): raise HistoryCacheInvalid() with open(historyfile_path, 'r') as infile: try: history_json_data = rlk_jsonloads(infile.read()) except JSONDecodeError: pass if not data_up_todate(history_json_data, start_ts, end_at_least_ts): raise HistoryCacheInvalid('Historical trades cache invalid') try: history_trades = trades_from_dictlist( given_trades=history_json_data['data'], start_ts=start_ts, end_ts=end_ts, location='historical trades', msg_aggregator=self.msg_aggregator, ) except KeyError: raise HistoryCacheInvalid('Historical trades cache invalid') history_trades = maybe_add_external_trades_to_history( db=self.db, start_ts=start_ts, end_ts=end_ts, history=history_trades, msg_aggregator=self.msg_aggregator, ) kraken_okay = (self.kraken is None or self.kraken.check_trades_cache( start_ts, end_at_least_ts, ) is not None) if not kraken_okay: raise HistoryCacheInvalid('Kraken cache is invalid') bittrex_okay = (self.bittrex is None or self.bittrex.check_trades_cache( start_ts, end_at_least_ts, ) is not None) if not bittrex_okay: raise HistoryCacheInvalid('Bittrex cache is invalid') binance_okay = (self.binance is None or self.binance.check_trades_cache( start_ts, end_at_least_ts, ) is not None) if not binance_okay: raise HistoryCacheInvalid('Binance cache is invalid') bitmex_okay = (self.bitmex is None or self.bitmex.check_trades_cache( start_ts, end_at_least_ts, ) is not None) if not bitmex_okay: raise HistoryCacheInvalid('Bitmex cache is invalid') # Poloniex specific loan_data = [] if self.poloniex: if not self.poloniex.check_trades_cache(start_ts, end_at_least_ts): raise HistoryCacheInvalid('Poloniex cache is invalid') loansfile_path = os.path.join(self.user_directory, LOANS_HISTORYFILE) loan_file_contents = get_jsonfile_contents_or_empty_dict( loansfile_path) loan_history_is_okay = data_up_todate( loan_file_contents, start_ts, end_at_least_ts, ) if not loan_history_is_okay: raise HistoryCacheInvalid('Poloniex loan cache is invalid') loan_data = loan_file_contents['data'] # margin positions that have been manually input if not self.read_manual_margin_positions: marginfile_path = os.path.join(self.user_directory, MARGIN_HISTORYFILE) margin_file_contents = get_jsonfile_contents_or_empty_dict( marginfile_path) margin_history_is_okay = data_up_todate( margin_file_contents, start_ts, end_at_least_ts, ) if not margin_history_is_okay: raise HistoryCacheInvalid('Margin Positions cache is invalid') try: margin_trades = trades_from_dictlist( given_trades=margin_file_contents['data'], start_ts=start_ts, end_ts=end_ts, location='Margin position trades', msg_aggregator=self.msg_aggregator, ) except KeyError: raise HistoryCacheInvalid('Margin Positions cache is invalid') else: margin_trades = do_read_manual_margin_positions( self.user_directory, ) asset_movements = self._get_cached_asset_movements( start_ts=start_ts, end_ts=end_ts, end_at_least_ts=end_at_least_ts, ) eth_tx_log_path = os.path.join(self.user_directory, ETHEREUM_TX_LOGFILE) eth_tx_log_contents = get_jsonfile_contents_or_empty_dict( eth_tx_log_path) eth_tx_log_history_is_okay = data_up_todate( eth_tx_log_contents, start_ts, end_at_least_ts, ) if not eth_tx_log_history_is_okay: raise HistoryCacheInvalid('Ethereum transactions cache is invalid') try: eth_transactions = transactions_from_dictlist( eth_tx_log_contents['data'], start_ts, end_ts, ) except KeyError: raise HistoryCacheInvalid('Ethereum transactions cache is invalid') # make sure that this is the same as what is returned # from create_history, except for the first argument return ( history_trades, margin_trades, loan_data, asset_movements, eth_transactions, )
def get_cached_history(self, start_ts, end_ts, end_at_least_ts=None): """Gets all the cached history data instead of querying all external sources to create the history through create_history() Can raise: - HistoryCacheInvalid: If any of the cache files are corrupt in any way, missing or do not cover the given time range """ if end_at_least_ts is None: end_at_least_ts = end_ts historyfile_path = os.path.join(self.user_directory, TRADES_HISTORYFILE) if not os.path.isfile(historyfile_path): raise HistoryCacheInvalid() with open(historyfile_path, 'r') as infile: try: history_json_data = rlk_jsonloads(infile.read()) except JSONDecodeError: pass if not data_up_todate(history_json_data, start_ts, end_at_least_ts): raise HistoryCacheInvalid('Historical trades cache invalid') try: history_trades = trades_from_dictlist( given_trades=history_json_data['data'], start_ts=start_ts, end_ts=end_ts, location='historical trades', msg_aggregator=self.msg_aggregator, ) except (KeyError, DeserializationError): raise HistoryCacheInvalid('Historical trades cache invalid') history_trades = maybe_add_external_trades_to_history( db=self.db, start_ts=start_ts, end_ts=end_ts, history=history_trades, msg_aggregator=self.msg_aggregator, ) # Check the cache of each exchange poloniex = None for _, exchange in self.exchange_manager.connected_exchanges.items(): if exchange.name == 'poloniex': poloniex = exchange if not exchange.check_trades_cache(start_ts, end_at_least_ts): raise HistoryCacheInvalid(f'{exchange.name} cache is invalid') # Poloniex specific loan_data = [] if poloniex: loansfile_path = os.path.join(self.user_directory, LOANS_HISTORYFILE) loan_file_contents = get_jsonfile_contents_or_empty_dict( loansfile_path) loan_history_is_okay = data_up_todate( loan_file_contents, start_ts, end_at_least_ts, ) if not loan_history_is_okay: raise HistoryCacheInvalid('Poloniex loan cache is invalid') loan_data = loan_file_contents['data'] asset_movements = self._get_cached_asset_movements( start_ts=start_ts, end_ts=end_ts, end_at_least_ts=end_at_least_ts, ) eth_tx_log_path = os.path.join(self.user_directory, ETHEREUM_TX_LOGFILE) eth_tx_log_contents = get_jsonfile_contents_or_empty_dict( eth_tx_log_path) eth_tx_log_history_is_okay = data_up_todate( eth_tx_log_contents, start_ts, end_at_least_ts, ) if not eth_tx_log_history_is_okay: raise HistoryCacheInvalid('Ethereum transactions cache is invalid') try: eth_transactions = transactions_from_dictlist( eth_tx_log_contents['data'], start_ts, end_ts, ) except KeyError: raise HistoryCacheInvalid('Ethereum transactions cache is invalid') # make sure that this is the same as what is returned # from create_history, except for the first argument return ( history_trades, loan_data, asset_movements, eth_transactions, )