def test_find_min_time_frame(): assert find_min_time_frame([ TimeFrames.FOUR_HOURS, TimeFrames.ONE_DAY, TimeFrames.ONE_MONTH, TimeFrames.FIFTEEN_MINUTES ]) == TimeFrames.FIFTEEN_MINUTES assert find_min_time_frame([TimeFrames.ONE_MONTH, TimeFrames.ONE_WEEK]) == TimeFrames.ONE_WEEK assert find_min_time_frame([TimeFrames.ONE_MINUTE ]) == TimeFrames.ONE_MINUTE
def _filter_exchange_time_frames(self): """ Populates self.filtered_timeframes from self.time_frames when time frame is supported by the cryptofeed exchange """ for time_frame in self.time_frames: self._add_time_frame(time_frame) self.min_timeframe = time_frame_manager.find_min_time_frame( self.filtered_timeframes)
def _get_exchanges_report(self, reference_market, trading_mode): SYMBOL_REPORT = "symbol_report" BOT_REPORT = "bot_report" CHART_IDENTIFIERS = "chart_identifiers" ERRORS_COUNT = "errors_count" report = { SYMBOL_REPORT: [], BOT_REPORT: {}, CHART_IDENTIFIERS: [], ERRORS_COUNT: logging.get_backtesting_errors_count() } profitabilities = {} market_average_profitabilities = {} starting_portfolios = {} end_portfolios = {} for exchange_id in self.octobot_backtesting.exchange_manager_ids: exchange_manager = trading_api.get_exchange_manager_from_exchange_id( exchange_id) _, profitability, _, market_average_profitability, _ = trading_api.get_profitability_stats( exchange_manager) min_timeframe = time_frame_manager.find_min_time_frame( trading_api.get_watched_timeframes(exchange_manager)) exchange_name = trading_api.get_exchange_name(exchange_manager) for symbol in self.symbols_to_create_exchange_classes[ exchange_name]: market_delta = self._get_market_delta(symbol, exchange_manager, min_timeframe) report[SYMBOL_REPORT].append({symbol: market_delta * 100}) report[CHART_IDENTIFIERS].append({ "symbol": symbol, "exchange_id": exchange_id, "exchange_name": exchange_name, "time_frame": min_timeframe.value }) profitabilities[exchange_name] = profitability market_average_profitabilities[ exchange_name] = market_average_profitability starting_portfolios[ exchange_name] = trading_api.get_origin_portfolio( exchange_manager) end_portfolios[exchange_name] = trading_api.get_portfolio( exchange_manager) report[BOT_REPORT] = { "profitability": profitabilities, "market_average_profitability": market_average_profitabilities, "reference_market": reference_market, "end_portfolio": end_portfolios, "starting_portfolio": starting_portfolios, "trading_mode": trading_mode } return report
def __init__(self, run_profitabilities, trades_counts, risk, time_frames, evaluators, strategy): self.run_profitabilities = run_profitabilities self.trades_counts = trades_counts self.risk = risk self.time_frames = time_frames self.min_time_frame = time_frame_manager.find_min_time_frame( self.time_frames) self.evaluators = evaluators self.strategy = strategy
def _set_config_time_frame(self): for time_frame in get_config_time_frame(self.config): if self.exchange_manager.time_frame_exists(time_frame.value): self.traded_time_frames.append(time_frame) # add shortest timeframe for realtime evaluators client_shortest_time_frame = find_min_time_frame( self.exchange_manager.client_time_frames[CONFIG_WILDCARD], MIN_EVAL_TIME_FRAME) if client_shortest_time_frame not in self.traded_time_frames: self.traded_time_frames.append(client_shortest_time_frame) self.traded_time_frames = sort_time_frames(self.traded_time_frames, reverse=True)
async def adapt_backtesting_channels(backtesting, config, importer_class, run_on_common_part_only=True, start_timestamp=None, end_timestamp=None): # set mininmum and maximum timestamp according to all importers data min_time_frame_to_consider = time_frame_manager.find_min_time_frame( time_frame_manager.get_config_time_frame(config)) importers = backtesting.get_importers(importer_class) if not importers: raise RuntimeError("No exchange importer has been found for this data file, backtesting can't start.") try: timestamps = [await api.get_data_timestamp_interval(importer, min_time_frame_to_consider) for importer in importers] # [(min, max) ... ] except errors.MissingTimeFrame as e: raise RuntimeError(f"Impossible to start backtesting on this configuration: {e}") min_timestamps = [timestamp[0] for timestamp in timestamps] max_timestamps = [timestamp[1] for timestamp in timestamps] min_timestamp = max(min_timestamps) if run_on_common_part_only else min(min_timestamps) max_timestamp = min(max_timestamps) if run_on_common_part_only else max(max_timestamps) if min_timestamp > max_timestamp: raise RuntimeError(f"No candle data to run backtesting on in this time window: starting at: {min_timestamp} " f"and ending at: {max_timestamp}") if start_timestamp is not None and end_timestamp is not None and \ start_timestamp > end_timestamp: raise RuntimeError(f"No candle data to run backtesting on in this time window: starting at: {start_timestamp} " f"and ending at: {end_timestamp}") if start_timestamp is not None: if min_timestamp <= start_timestamp < end_timestamp if end_timestamp else max_timestamp: min_timestamp = start_timestamp else: logging.get_logger("BacktestingAPI").warning(f"Can't set the minimum timestamp to {start_timestamp}. " f"The minimum available({min_timestamp}) will be used instead.") if end_timestamp is not None: if max_timestamp >= end_timestamp > start_timestamp if start_timestamp else min_timestamp: max_timestamp = end_timestamp else: logging.get_logger("BacktestingAPI").warning(f"Can't set the maximum timestamp to {end_timestamp}. " f"The maximum available({max_timestamp}) will be used instead.") await modify_backtesting_timestamps( backtesting, minimum_timestamp=min_timestamp, maximum_timestamp=max_timestamp) try: import octobot_trading.api as exchange_api if exchange_api.has_only_ohlcv(importers): set_time_updater_interval(backtesting, common_enums.TimeFramesMinutes[min_time_frame_to_consider] * common_constants.MINUTE_TO_SECONDS) except ImportError: logging.get_logger("BacktestingAPI").error("requires OctoBot-Trading package installed")
def _set_config_time_frame(self): for time_frame in get_config_time_frame(self.config): if self.exchange_manager.time_frame_exists(time_frame.value): self.traded_time_frames.append(time_frame) if not self.exchange_manager.is_backtesting or not self.traded_time_frames: # add shortest time frame for realtime evaluators or if no time frame at all has # been registered in backtesting client_shortest_time_frame = find_min_time_frame(self.exchange_manager.client_time_frames, MIN_EVAL_TIME_FRAME) self.real_time_time_frames.append(client_shortest_time_frame) self.traded_time_frames = list(set().union(self.traded_time_frames, self.real_time_time_frames)) self.traded_time_frames = sort_time_frames(self.traded_time_frames, reverse=True)
async def modify_channels(self): # set mininmum and maximum timestamp according to all importers data min_time_frame_to_consider = find_min_time_frame(get_config_time_frame(self.config)) timestamps = [await get_data_timestamp_interval(importer, min_time_frame_to_consider) for importer in self.exchange_importers] # [(min, max) ... ] await modify_backtesting_timestamps( self.backtesting, minimum_timestamp=min(timestamps)[0], maximum_timestamp=max(timestamps)[1]) if self._has_only_ohlcv(): set_time_updater_interval(self.backtesting, TimeFramesMinutes[min_time_frame_to_consider] * MINUTE_TO_SECONDS)
def log_report(self): self.logger.info(" **** Backtesting report ****") for exchange_id in self.octobot_backtesting.exchange_manager_ids: exchange_manager = trading_api.get_exchange_manager_from_exchange_id(exchange_id) exchange_name = trading_api.get_exchange_name(exchange_manager) self.logger.info(f" ========= Trades on {exchange_name} =========") self._log_trades_history(exchange_manager, exchange_name) self.logger.info(f" ========= Prices evolution on {exchange_name} =========") min_timeframe = time_frame_manager.find_min_time_frame(trading_api.get_watched_timeframes(exchange_manager)) for symbol in self.symbols_to_create_exchange_classes[exchange_name]: self._log_symbol_report(symbol, exchange_manager, min_timeframe) self.logger.info(" ========= Octobot end state =========") self._log_global_report(exchange_manager)
async def get_database_description(database): description = (await database.select(enums.DataTables.DESCRIPTION, size=1))[0] version = description[1] if version == "1.0": return { enums.DataFormatKeys.TIMESTAMP.value: description[0], enums.DataFormatKeys.VERSION.value: description[1], enums.DataFormatKeys.EXCHANGE.value: description[2], enums.DataFormatKeys.SYMBOLS.value: json.loads(description[3]), enums.DataFormatKeys.TIME_FRAMES.value: [common_enums.TimeFrames(tf) for tf in json.loads(description[4])], enums.DataFormatKeys.START_TIMESTAMP.value: 0, enums.DataFormatKeys.END_TIMESTAMP.value: 0, enums.DataFormatKeys.CANDLES_LENGTH.value: int((await database.select_count(enums.ExchangeDataTables.OHLCV, ["*"],\ time_frame=tmf_manager.find_min_time_frame([common_enums.TimeFrames(tf) for tf in json.loads(description[4])]).value))[0][0] / len(json.loads(description[3]))) } elif version == "1.1": return { enums.DataFormatKeys.TIMESTAMP.value: description[0], enums.DataFormatKeys.VERSION.value: description[1], enums.DataFormatKeys.EXCHANGE.value: description[2], enums.DataFormatKeys.SYMBOLS.value: json.loads(description[3]), enums.DataFormatKeys.TIME_FRAMES.value: [common_enums.TimeFrames(tf) for tf in json.loads(description[4])], enums.DataFormatKeys.START_TIMESTAMP.value: description[5], enums.DataFormatKeys.END_TIMESTAMP.value: description[6], enums.DataFormatKeys.CANDLES_LENGTH.value: int((await database.select_count(enums.ExchangeDataTables.OHLCV, ["*"],\ time_frame=tmf_manager.find_min_time_frame([common_enums.TimeFrames(tf) for tf in json.loads(description[4])]).value))[0][0] / len(json.loads(description[3]))) } else: raise RuntimeError(f"Unknown datafile version: {version}")
async def start(self): self.should_stop = False should_stop_database = True try: self.exchange_manager = await trading_api.create_exchange_builder(self.config, self.exchange_name) \ .is_simulated() \ .is_rest_only() \ .is_exchange_only() \ .is_collecting() \ .is_ignoring_config() \ .disable_trading_mode() \ .use_tentacles_setup_config(self.tentacles_setup_config) \ .build() self.exchange = self.exchange_manager.exchange self._load_timeframes_if_necessary() if self.start_timestamp is not None: lowest_timestamp = min([ await self.get_first_candle_timestamp( symbol, time_frame_manager.find_min_time_frame( self.time_frames)) for symbol in self.symbols ]) if lowest_timestamp > self.start_timestamp: self.start_timestamp = lowest_timestamp if self.start_timestamp > (self.end_timestamp if self.end_timestamp else (time.time() * 1000)): raise errors.DataCollectorError( "start_timestamp is higher than end_timestamp") # create description await self._create_description() self.total_steps = len(self.time_frames) * len(self.symbols) self.in_progress = True self.logger.info( f"Start collecting history on {self.exchange_name}") for symbol_index, symbol in enumerate(self.symbols): self.logger.info(f"Collecting history for {symbol}...") await self.get_ticker_history(self.exchange_name, symbol) await self.get_order_book_history(self.exchange_name, symbol) await self.get_recent_trades_history(self.exchange_name, symbol) for time_frame_index, time_frame in enumerate( self.time_frames): self.current_step_index = (symbol_index * len( self.time_frames)) + time_frame_index + 1 self.logger.info( f"[{time_frame_index}/{len(self.time_frames)}] Collecting {symbol} history on {time_frame}..." ) await self.get_ohlcv_history(self.exchange_name, symbol, time_frame) await self.get_kline_history(self.exchange_name, symbol, time_frame) except Exception as err: await self.database.stop() should_stop_database = False # Do not keep errored data file if os.path.isfile(self.temp_file_path): os.remove(self.temp_file_path) if not self.should_stop: self.logger.exception( err, True, f"Error when collecting {self.exchange_name} history for " f"{', '.join(self.symbols)}: {err}") raise errors.DataCollectorError(err) finally: await self.stop(should_stop_database=should_stop_database)