def test_find_min_time_frame():
assert find_min_time_frame([
TimeFrames.FOUR_HOURS, TimeFrames.ONE_DAY, TimeFrames.ONE_MONTH,
TimeFrames.FIFTEEN_MINUTES
]) == TimeFrames.FIFTEEN_MINUTES
assert find_min_time_frame([TimeFrames.ONE_MONTH,
TimeFrames.ONE_WEEK]) == TimeFrames.ONE_WEEK
assert find_min_time_frame([TimeFrames.ONE_MINUTE
]) == TimeFrames.ONE_MINUTE
def _filter_exchange_time_frames(self):
"""
Populates self.filtered_timeframes from self.time_frames when time frame is supported by the cryptofeed exchange
"""
for time_frame in self.time_frames:
self._add_time_frame(time_frame)
self.min_timeframe = time_frame_manager.find_min_time_frame(
self.filtered_timeframes)
def _get_exchanges_report(self, reference_market, trading_mode):
SYMBOL_REPORT = "symbol_report"
BOT_REPORT = "bot_report"
CHART_IDENTIFIERS = "chart_identifiers"
ERRORS_COUNT = "errors_count"
report = {
SYMBOL_REPORT: [],
BOT_REPORT: {},
CHART_IDENTIFIERS: [],
ERRORS_COUNT: logging.get_backtesting_errors_count()
}
profitabilities = {}
market_average_profitabilities = {}
starting_portfolios = {}
end_portfolios = {}
for exchange_id in self.octobot_backtesting.exchange_manager_ids:
exchange_manager = trading_api.get_exchange_manager_from_exchange_id(
exchange_id)
_, profitability, _, market_average_profitability, _ = trading_api.get_profitability_stats(
exchange_manager)
min_timeframe = time_frame_manager.find_min_time_frame(
trading_api.get_watched_timeframes(exchange_manager))
exchange_name = trading_api.get_exchange_name(exchange_manager)
for symbol in self.symbols_to_create_exchange_classes[
exchange_name]:
market_delta = self._get_market_delta(symbol, exchange_manager,
min_timeframe)
report[SYMBOL_REPORT].append({symbol: market_delta * 100})
report[CHART_IDENTIFIERS].append({
"symbol":
symbol,
"exchange_id":
exchange_id,
"exchange_name":
exchange_name,
"time_frame":
min_timeframe.value
})
profitabilities[exchange_name] = profitability
market_average_profitabilities[
exchange_name] = market_average_profitability
starting_portfolios[
exchange_name] = trading_api.get_origin_portfolio(
exchange_manager)
end_portfolios[exchange_name] = trading_api.get_portfolio(
exchange_manager)
report[BOT_REPORT] = {
"profitability": profitabilities,
"market_average_profitability": market_average_profitabilities,
"reference_market": reference_market,
"end_portfolio": end_portfolios,
"starting_portfolio": starting_portfolios,
"trading_mode": trading_mode
}
return report
def __init__(self, run_profitabilities, trades_counts, risk, time_frames,
evaluators, strategy):
self.run_profitabilities = run_profitabilities
self.trades_counts = trades_counts
self.risk = risk
self.time_frames = time_frames
self.min_time_frame = time_frame_manager.find_min_time_frame(
self.time_frames)
self.evaluators = evaluators
self.strategy = strategy
def _set_config_time_frame(self):
for time_frame in get_config_time_frame(self.config):
if self.exchange_manager.time_frame_exists(time_frame.value):
self.traded_time_frames.append(time_frame)
# add shortest timeframe for realtime evaluators
client_shortest_time_frame = find_min_time_frame(
self.exchange_manager.client_time_frames[CONFIG_WILDCARD], MIN_EVAL_TIME_FRAME)
if client_shortest_time_frame not in self.traded_time_frames:
self.traded_time_frames.append(client_shortest_time_frame)
self.traded_time_frames = sort_time_frames(self.traded_time_frames, reverse=True)
async def adapt_backtesting_channels(backtesting, config, importer_class, run_on_common_part_only=True,
start_timestamp=None, end_timestamp=None):
# set mininmum and maximum timestamp according to all importers data
min_time_frame_to_consider = time_frame_manager.find_min_time_frame(
time_frame_manager.get_config_time_frame(config))
importers = backtesting.get_importers(importer_class)
if not importers:
raise RuntimeError("No exchange importer has been found for this data file, backtesting can't start.")
try:
timestamps = [await api.get_data_timestamp_interval(importer, min_time_frame_to_consider)
for importer in importers] # [(min, max) ... ]
except errors.MissingTimeFrame as e:
raise RuntimeError(f"Impossible to start backtesting on this configuration: {e}")
min_timestamps = [timestamp[0] for timestamp in timestamps]
max_timestamps = [timestamp[1] for timestamp in timestamps]
min_timestamp = max(min_timestamps) if run_on_common_part_only else min(min_timestamps)
max_timestamp = min(max_timestamps) if run_on_common_part_only else max(max_timestamps)
if min_timestamp > max_timestamp:
raise RuntimeError(f"No candle data to run backtesting on in this time window: starting at: {min_timestamp} "
f"and ending at: {max_timestamp}")
if start_timestamp is not None and end_timestamp is not None and \
start_timestamp > end_timestamp:
raise RuntimeError(f"No candle data to run backtesting on in this time window: starting at: {start_timestamp} "
f"and ending at: {end_timestamp}")
if start_timestamp is not None:
if min_timestamp <= start_timestamp < end_timestamp if end_timestamp else max_timestamp:
min_timestamp = start_timestamp
else:
logging.get_logger("BacktestingAPI").warning(f"Can't set the minimum timestamp to {start_timestamp}. "
f"The minimum available({min_timestamp}) will be used instead.")
if end_timestamp is not None:
if max_timestamp >= end_timestamp > start_timestamp if start_timestamp else min_timestamp:
max_timestamp = end_timestamp
else:
logging.get_logger("BacktestingAPI").warning(f"Can't set the maximum timestamp to {end_timestamp}. "
f"The maximum available({max_timestamp}) will be used instead.")
await modify_backtesting_timestamps(
backtesting,
minimum_timestamp=min_timestamp,
maximum_timestamp=max_timestamp)
try:
import octobot_trading.api as exchange_api
if exchange_api.has_only_ohlcv(importers):
set_time_updater_interval(backtesting,
common_enums.TimeFramesMinutes[min_time_frame_to_consider] *
common_constants.MINUTE_TO_SECONDS)
except ImportError:
logging.get_logger("BacktestingAPI").error("requires OctoBot-Trading package installed")
def _set_config_time_frame(self):
for time_frame in get_config_time_frame(self.config):
if self.exchange_manager.time_frame_exists(time_frame.value):
self.traded_time_frames.append(time_frame)
if not self.exchange_manager.is_backtesting or not self.traded_time_frames:
# add shortest time frame for realtime evaluators or if no time frame at all has
# been registered in backtesting
client_shortest_time_frame = find_min_time_frame(self.exchange_manager.client_time_frames,
MIN_EVAL_TIME_FRAME)
self.real_time_time_frames.append(client_shortest_time_frame)
self.traded_time_frames = list(set().union(self.traded_time_frames, self.real_time_time_frames))
self.traded_time_frames = sort_time_frames(self.traded_time_frames, reverse=True)
async def modify_channels(self):
# set mininmum and maximum timestamp according to all importers data
min_time_frame_to_consider = find_min_time_frame(get_config_time_frame(self.config))
timestamps = [await get_data_timestamp_interval(importer, min_time_frame_to_consider)
for importer in self.exchange_importers] # [(min, max) ... ]
await modify_backtesting_timestamps(
self.backtesting,
minimum_timestamp=min(timestamps)[0],
maximum_timestamp=max(timestamps)[1])
if self._has_only_ohlcv():
set_time_updater_interval(self.backtesting,
TimeFramesMinutes[min_time_frame_to_consider] * MINUTE_TO_SECONDS)
def log_report(self):
self.logger.info(" **** Backtesting report ****")
for exchange_id in self.octobot_backtesting.exchange_manager_ids:
exchange_manager = trading_api.get_exchange_manager_from_exchange_id(exchange_id)
exchange_name = trading_api.get_exchange_name(exchange_manager)
self.logger.info(f" ========= Trades on {exchange_name} =========")
self._log_trades_history(exchange_manager, exchange_name)
self.logger.info(f" ========= Prices evolution on {exchange_name} =========")
min_timeframe = time_frame_manager.find_min_time_frame(trading_api.get_watched_timeframes(exchange_manager))
for symbol in self.symbols_to_create_exchange_classes[exchange_name]:
self._log_symbol_report(symbol, exchange_manager, min_timeframe)
self.logger.info(" ========= Octobot end state =========")
self._log_global_report(exchange_manager)
async def get_database_description(database):
description = (await database.select(enums.DataTables.DESCRIPTION,
size=1))[0]
version = description[1]
if version == "1.0":
return {
enums.DataFormatKeys.TIMESTAMP.value: description[0],
enums.DataFormatKeys.VERSION.value: description[1],
enums.DataFormatKeys.EXCHANGE.value: description[2],
enums.DataFormatKeys.SYMBOLS.value: json.loads(description[3]),
enums.DataFormatKeys.TIME_FRAMES.value: [common_enums.TimeFrames(tf) for tf in json.loads(description[4])],
enums.DataFormatKeys.START_TIMESTAMP.value: 0,
enums.DataFormatKeys.END_TIMESTAMP.value: 0,
enums.DataFormatKeys.CANDLES_LENGTH.value:
int((await database.select_count(enums.ExchangeDataTables.OHLCV, ["*"],\
time_frame=tmf_manager.find_min_time_frame([common_enums.TimeFrames(tf)
for tf in json.loads(description[4])]).value))[0][0]
/ len(json.loads(description[3])))
}
elif version == "1.1":
return {
enums.DataFormatKeys.TIMESTAMP.value: description[0],
enums.DataFormatKeys.VERSION.value: description[1],
enums.DataFormatKeys.EXCHANGE.value: description[2],
enums.DataFormatKeys.SYMBOLS.value: json.loads(description[3]),
enums.DataFormatKeys.TIME_FRAMES.value: [common_enums.TimeFrames(tf) for tf in json.loads(description[4])],
enums.DataFormatKeys.START_TIMESTAMP.value: description[5],
enums.DataFormatKeys.END_TIMESTAMP.value: description[6],
enums.DataFormatKeys.CANDLES_LENGTH.value:
int((await database.select_count(enums.ExchangeDataTables.OHLCV, ["*"],\
time_frame=tmf_manager.find_min_time_frame([common_enums.TimeFrames(tf)
for tf in json.loads(description[4])]).value))[0][0]
/ len(json.loads(description[3])))
}
else:
raise RuntimeError(f"Unknown datafile version: {version}")
async def start(self):
self.should_stop = False
should_stop_database = True
try:
self.exchange_manager = await trading_api.create_exchange_builder(self.config, self.exchange_name) \
.is_simulated() \
.is_rest_only() \
.is_exchange_only() \
.is_collecting() \
.is_ignoring_config() \
.disable_trading_mode() \
.use_tentacles_setup_config(self.tentacles_setup_config) \
.build()
self.exchange = self.exchange_manager.exchange
self._load_timeframes_if_necessary()
if self.start_timestamp is not None:
lowest_timestamp = min([
await self.get_first_candle_timestamp(
symbol,
time_frame_manager.find_min_time_frame(
self.time_frames)) for symbol in self.symbols
])
if lowest_timestamp > self.start_timestamp:
self.start_timestamp = lowest_timestamp
if self.start_timestamp > (self.end_timestamp
if self.end_timestamp else
(time.time() * 1000)):
raise errors.DataCollectorError(
"start_timestamp is higher than end_timestamp")
# create description
await self._create_description()
self.total_steps = len(self.time_frames) * len(self.symbols)
self.in_progress = True
self.logger.info(
f"Start collecting history on {self.exchange_name}")
for symbol_index, symbol in enumerate(self.symbols):
self.logger.info(f"Collecting history for {symbol}...")
await self.get_ticker_history(self.exchange_name, symbol)
await self.get_order_book_history(self.exchange_name, symbol)
await self.get_recent_trades_history(self.exchange_name,
symbol)
for time_frame_index, time_frame in enumerate(
self.time_frames):
self.current_step_index = (symbol_index * len(
self.time_frames)) + time_frame_index + 1
self.logger.info(
f"[{time_frame_index}/{len(self.time_frames)}] Collecting {symbol} history on {time_frame}..."
)
await self.get_ohlcv_history(self.exchange_name, symbol,
time_frame)
await self.get_kline_history(self.exchange_name, symbol,
time_frame)
except Exception as err:
await self.database.stop()
should_stop_database = False
# Do not keep errored data file
if os.path.isfile(self.temp_file_path):
os.remove(self.temp_file_path)
if not self.should_stop:
self.logger.exception(
err, True,
f"Error when collecting {self.exchange_name} history for "
f"{', '.join(self.symbols)}: {err}")
raise errors.DataCollectorError(err)
finally:
await self.stop(should_stop_database=should_stop_database)
