def create_history(self, start_ts, end_ts, end_at_least_ts): """Creates trades and loans history from start_ts to end_ts or if `end_at_least` is given and we have a cache history for that particular source which satisfies it we return the cache """ # start creating the all trades history list history = list() asset_movements = list() if self.kraken is not None: kraken_history = self.kraken.query_trade_history( start_ts=start_ts, end_ts=end_ts, end_at_least_ts=end_at_least_ts ) for trade in kraken_history: history.append(trade_from_kraken(trade)) kraken_asset_movements = self.kraken.query_deposits_withdrawals( start_ts=start_ts, end_ts=end_ts, end_at_least_ts=end_at_least_ts, ) asset_movements.extend(kraken_asset_movements) if self.poloniex is not None: polo_history = self.poloniex.query_trade_history( start_ts=start_ts, end_ts=end_ts, end_at_least_ts=end_at_least_ts ) poloniex_margin_trades = list() for pair, trades in polo_history.items(): for trade in trades: category = trade['category'] if category == 'exchange' or category == 'settlement': history.append(trade_from_poloniex(trade, pair)) elif category == 'marginTrade': if not self.read_manual_margin_positions: poloniex_margin_trades.append(trade_from_poloniex(trade, pair)) else: raise ValueError("Unexpected poloniex trade category: {}".format(category)) if self.read_manual_margin_positions: # Just read the manual positions log and make virtual trades that # correspond to the profits assert poloniex_margin_trades == list(), ( "poloniex margin trades list should be empty here" ) poloniex_margin_trades = do_read_manual_margin_positions( self.data_directory ) else: poloniex_margin_trades.sort(key=lambda trade: trade.timestamp) poloniex_margin_trades = limit_trade_list_to_period( poloniex_margin_trades, start_ts, end_ts ) polo_loans = self.poloniex.query_loan_history( start_ts=start_ts, end_ts=end_ts, end_at_least_ts=end_at_least_ts, from_csv=True ) polo_loans = process_polo_loans(polo_loans, start_ts, end_ts) polo_asset_movements = self.poloniex.query_deposits_withdrawals( start_ts=start_ts, end_ts=end_ts, end_at_least_ts=end_at_least_ts, ) asset_movements.extend(polo_asset_movements) if self.bittrex is not None: bittrex_history = self.bittrex.query_trade_history( start_ts=start_ts, end_ts=end_ts, end_at_least_ts=end_at_least_ts ) for trade in bittrex_history: history.append(trade_from_bittrex(trade)) if self.binance is not None: binance_history = self.binance.query_trade_history( start_ts=start_ts, end_ts=end_ts, end_at_least_ts=end_at_least_ts ) for trade in binance_history: history.append(trade_from_binance(trade)) eth_transactions = query_etherscan_for_transactions(self.eth_accounts) # We sort it here ... but when accounting runs through the entire actions list, # it resorts, so unless the fact that we sort is used somewhere else too, perhaps # we can skip it? history.sort(key=lambda trade: trade.timestamp) history = limit_trade_list_to_period(history, start_ts, end_ts) # Write to files historyfile_path = os.path.join(self.data_directory, TRADES_HISTORYFILE) write_tupledata_history_in_file(history, historyfile_path, start_ts, end_ts) if not self.read_manual_margin_positions: marginfile_path = os.path.join(self.data_directory, MARGIN_HISTORYFILE) write_tupledata_history_in_file( poloniex_margin_trades, marginfile_path, start_ts, end_ts ) loansfile_path = os.path.join(self.data_directory, LOANS_HISTORYFILE) write_history_data_in_file(polo_loans, loansfile_path, start_ts, end_ts) assetmovementsfile_path = os.path.join(self.data_directory, ASSETMOVEMENTS_HISTORYFILE) write_tupledata_history_in_file(asset_movements, assetmovementsfile_path, start_ts, end_ts) eth_tx_log_path = os.path.join(self.data_directory, ETHEREUM_TX_LOGFILE) write_tupledata_history_in_file(eth_transactions, eth_tx_log_path, start_ts, end_ts) # After writting everything to files include the external trades in the history history = include_external_trades(self.db, start_ts, end_ts, history) return history, poloniex_margin_trades, polo_loans, asset_movements, eth_transactions
def create_history(self, start_ts: Timestamp, end_ts: Timestamp, end_at_least_ts: Timestamp): """Creates trades and loans history from start_ts to end_ts or if `end_at_least` is given and we have a cache history for that particular source which satisfies it we return the cache """ log.info( 'Starting trade history creation', start_ts=start_ts, end_ts=end_ts, end_at_least_ts=end_at_least_ts, ) # start creating the all trades history list history: List[Union[Trade, MarginPosition]] = list() asset_movements = list() polo_loans = list() empty_or_error = '' def populate_history_cb( result_history: Union[List[Trade], List[MarginPosition]], result_asset_movements: List[AssetMovement], exchange_specific_data: Any, ) -> None: """This callback will run for succesfull exchange history query""" history.extend(result_history) asset_movements.extend(result_asset_movements) if exchange_specific_data: # This can only be poloniex at the moment polo_loans_data = exchange_specific_data polo_loans.extend( process_polo_loans( msg_aggregator=self.msg_aggregator, data=polo_loans_data, start_ts=start_ts, end_ts=end_ts, )) loansfile_path = os.path.join(self.user_directory, LOANS_HISTORYFILE) write_history_data_in_file(polo_loans, loansfile_path, start_ts, end_ts) def fail_history_cb(error_msg: str) -> None: """This callback will run for failure in exchange history query""" nonlocal empty_or_error empty_or_error += '\n' + error_msg for _, exchange in self.exchange_manager.connected_exchanges.items(): exchange.query_history_with_callbacks( start_ts=start_ts, end_ts=end_ts, end_at_least_ts=end_at_least_ts, success_callback=populate_history_cb, fail_callback=fail_history_cb, ) try: eth_transactions = query_etherscan_for_transactions( self.eth_accounts) except RemoteError as e: empty_or_error += '\n' + str(e) # We sort it here ... but when accounting runs through the entire actions list, # it resorts, so unless the fact that we sort is used somewhere else too, perhaps # we can skip it? history.sort(key=lambda trade: action_get_timestamp(trade)) history = limit_trade_list_to_period(history, start_ts, end_ts) # Write to files historyfile_path = os.path.join(self.user_directory, TRADES_HISTORYFILE) write_tupledata_history_in_file(history, historyfile_path, start_ts, end_ts) assetmovementsfile_path = os.path.join(self.user_directory, ASSETMOVEMENTS_HISTORYFILE) write_tupledata_history_in_file(asset_movements, assetmovementsfile_path, start_ts, end_ts) eth_tx_log_path = os.path.join(self.user_directory, ETHEREUM_TX_LOGFILE) write_tupledata_history_in_file(eth_transactions, eth_tx_log_path, start_ts, end_ts) # After writting everything to files include the external trades in the history history = maybe_add_external_trades_to_history( db=self.db, start_ts=start_ts, end_ts=end_ts, history=history, msg_aggregator=self.msg_aggregator, ) return ( empty_or_error, history, polo_loans, asset_movements, eth_transactions, )
def create_history(self, start_ts, end_ts, end_at_least_ts): """Creates trades and loans history from start_ts to end_ts or if `end_at_least` is given and we have a cache history for that particular source which satisfies it we return the cache """ log.info( 'Starting trade history creation', start_ts=start_ts, end_ts=end_ts, end_at_least_ts=end_at_least_ts, ) # start creating the all trades history list history = list() asset_movements = list() empty_or_error = '' if self.kraken is not None: try: kraken_history = self.kraken.query_trade_history( start_ts=start_ts, end_ts=end_ts, end_at_least_ts=end_at_least_ts, ) history.extend(kraken_history) kraken_asset_movements = self.kraken.query_deposits_withdrawals( start_ts=start_ts, end_ts=end_ts, end_at_least_ts=end_at_least_ts, ) asset_movements.extend(kraken_asset_movements) except RemoteError as e: empty_or_error += '\n' + str(e) poloniex_query_error = False poloniex_margin_trades = [] polo_loans = [] try: ( history, asset_movements, poloniex_margin_trades, polo_loans, ) = self.query_poloniex_history( history, asset_movements, start_ts, end_ts, end_at_least_ts, ) except RemoteError as e: empty_or_error += '\n' + str(e) poloniex_query_error = True if self.bittrex is not None: try: bittrex_history = self.bittrex.query_trade_history( start_ts=start_ts, end_ts=end_ts, end_at_least_ts=end_at_least_ts, ) history.extend(bittrex_history) except RemoteError as e: empty_or_error += '\n' + str(e) if self.bitmex is not None: try: bitmex_history = self.bitmex.query_trade_history( start_ts=start_ts, end_ts=end_ts, end_at_least_ts=end_at_least_ts, ) for trade in bitmex_history: history.append(trade_from_bitmex(trade)) bitmex_asset_movements = self.bitmex.query_deposits_withdrawals( start_ts=start_ts, end_ts=end_ts, end_at_least_ts=end_at_least_ts, ) asset_movements.extend(bitmex_asset_movements) except RemoteError as e: empty_or_error += '\n' + str(e) if self.binance is not None: try: binance_history = self.binance.query_trade_history( start_ts=start_ts, end_ts=end_ts, end_at_least_ts=end_at_least_ts, ) history.extend(binance_history) except RemoteError as e: empty_or_error += '\n' + str(e) try: eth_transactions = query_etherscan_for_transactions( self.eth_accounts) except RemoteError as e: empty_or_error += '\n' + str(e) # We sort it here ... but when accounting runs through the entire actions list, # it resorts, so unless the fact that we sort is used somewhere else too, perhaps # we can skip it? history.sort(key=lambda trade: trade.timestamp) history = limit_trade_list_to_period(history, start_ts, end_ts) # Write to files historyfile_path = os.path.join(self.user_directory, TRADES_HISTORYFILE) write_tupledata_history_in_file(history, historyfile_path, start_ts, end_ts) if self.poloniex is not None: if not self.read_manual_margin_positions and not poloniex_query_error: marginfile_path = os.path.join(self.user_directory, MARGIN_HISTORYFILE) write_tupledata_history_in_file( poloniex_margin_trades, marginfile_path, start_ts, end_ts, ) if not poloniex_query_error: loansfile_path = os.path.join(self.user_directory, LOANS_HISTORYFILE) write_history_data_in_file(polo_loans, loansfile_path, start_ts, end_ts) assetmovementsfile_path = os.path.join(self.user_directory, ASSETMOVEMENTS_HISTORYFILE) write_tupledata_history_in_file(asset_movements, assetmovementsfile_path, start_ts, end_ts) eth_tx_log_path = os.path.join(self.user_directory, ETHEREUM_TX_LOGFILE) write_tupledata_history_in_file(eth_transactions, eth_tx_log_path, start_ts, end_ts) # After writting everything to files include the external trades in the history history = maybe_add_external_trades_to_history( db=self.db, start_ts=start_ts, end_ts=end_ts, history=history, msg_aggregator=self.msg_aggregator, ) return ( empty_or_error, history, poloniex_margin_trades, polo_loans, asset_movements, eth_transactions, )
def create_history(self, start_ts: Timestamp, end_ts: Timestamp) -> HistoryResult: """Creates trades and loans history from start_ts to end_ts""" log.info( 'Starting trade history creation', start_ts=start_ts, end_ts=end_ts, ) # start creating the all trades history list history: List[Union[Trade, MarginPosition]] = list() asset_movements = list() polo_loans = list() empty_or_error = '' def populate_history_cb( trades_history: List[Trade], margin_history: List[MarginPosition], result_asset_movements: List[AssetMovement], exchange_specific_data: Any, ) -> None: """This callback will run for succesfull exchange history query""" history.extend(trades_history) history.extend(margin_history) asset_movements.extend(result_asset_movements) if exchange_specific_data: # This can only be poloniex at the moment polo_loans_data = exchange_specific_data polo_loans.extend(process_polo_loans( msg_aggregator=self.msg_aggregator, data=polo_loans_data, start_ts=start_ts, end_ts=end_ts, )) def fail_history_cb(error_msg: str) -> None: """This callback will run for failure in exchange history query""" nonlocal empty_or_error empty_or_error += '\n' + error_msg for _, exchange in self.exchange_manager.connected_exchanges.items(): exchange.query_history_with_callbacks( start_ts=start_ts, end_ts=end_ts, success_callback=populate_history_cb, fail_callback=fail_history_cb, ) # TODO: Also save those in the DB (?). We used to have a cache but if anything # makes sense is to save them in the DB. Also realized the old cache broke # if more accounts were added. try: eth_transactions = query_etherscan_for_transactions( db=self.db, msg_aggregator=self.msg_aggregator, from_ts=start_ts, to_ts=end_ts, ) except RemoteError as e: empty_or_error += '\n' + str(e) # We sort it here ... but when accounting runs through the entire actions list, # it resorts, so unless the fact that we sort is used somewhere else too, perhaps # we can skip it? history.sort(key=lambda trade: action_get_timestamp(trade)) history = limit_trade_list_to_period(history, start_ts, end_ts) # Include the external trades in the history external_trades = self.db.get_trades( from_ts=start_ts, to_ts=end_ts, location=Location.EXTERNAL, ) history.extend(external_trades) history.sort(key=lambda trade: action_get_timestamp(trade)) return ( empty_or_error, history, polo_loans, asset_movements, eth_transactions, )