def _fetch_recent_trades(self, symbol, timeframe, index):
time_frame = self.get_ohlcv(symbol)[timeframe.value][index]
start_timestamp = time_frame[PriceIndexes.IND_PRICE_TIME.value] \
if backtesting_enabled(self.config) else time.time()
if not self.handles_trades_history(symbol) or not backtesting_enabled(self.config):
return self.generate_trades(time_frame, start_timestamp)
else:
end_timestamp = self.get_ohlcv(symbol)[timeframe.value][index+1][PriceIndexes.IND_PRICE_TIME.value] \
if len(self.get_ohlcv(symbol)[timeframe.value]) > index+1 else -1
return self.select_trades(start_timestamp, end_timestamp, symbol)
def _fetch_recent_trades(self, symbol, timeframe, index):
# Use index + 1 to use prices from the current candle. Currently evaluated candle (previous candle) is closed.
current_candle = self.get_ohlcv(symbol)[timeframe.value][index + 1]
start_timestamp = current_candle[PriceIndexes.IND_PRICE_TIME.value] \
if backtesting_enabled(self.config) else time.time()
if not self.handles_trades_history(symbol) or not backtesting_enabled(
self.config):
return self._generate_trades(current_candle, start_timestamp)
else:
end_timestamp = self.get_ohlcv(symbol)[timeframe.value][index+1][PriceIndexes.IND_PRICE_TIME.value] \
if len(self.get_ohlcv(symbol)[timeframe.value]) > index+1 else -1
return self.select_trades(start_timestamp, end_timestamp, symbol)
def _create_real_time_ta_list(self, exchange, symbol_evaluator):
real_time_ta_eval_list = []
if not backtesting_enabled(self.octobot.get_config()):
real_time_ta_eval_list = self._create_real_time_ta_evaluators(exchange, symbol_evaluator)
self.real_time_eval_tasks += real_time_ta_eval_list
return real_time_ta_eval_list
def run_in_main_asyncio_loop(self, coroutine):
# restart a new loop if necessary (for backtesting analysis)
if backtesting_enabled(self.octobot.get_config()) and self.async_loop.is_closed():
self.logger.debug("Main loop is closed, starting a new main loop.")
self._create_new_asyncio_main_loop()
return run_coroutine_in_asyncio_loop(coroutine, self.async_loop)
async def _update_last_symbol_prices(self,
symbol,
uniformize_timestamps=False):
"""
Ask to update a specific symbol with exchange data
"""
exchange = self.trader.get_exchange()
if backtesting_enabled(self.config):
last_symbol_price = await self.trader.get_exchange(
).get_recent_trades(symbol)
# Exchange call when not backtesting
else:
last_symbol_price = []
try:
last_symbol_price = await exchange.get_recent_trades(symbol)
except BaseError as e:
self.logger.warning(
f"Failed to get recent trade: {e}, skipping simulated {symbol} order(s) update for "
f"this time. Next update in {self.order_refresh_time} second(s)."
)
if uniformize_timestamps and last_symbol_price:
timestamp_sample = last_symbol_price[0][eC.TIMESTAMP.value]
if exchange.get_exchange_manager(
).need_to_uniformize_timestamp(timestamp_sample):
for order in last_symbol_price:
order[eC.TIMESTAMP.
value] = exchange.get_uniform_timestamp(
order[eC.TIMESTAMP.value])
# Check if exchange request failed
if last_symbol_price:
self.last_symbol_prices[symbol] = last_symbol_price
async def _init_origin_portfolio_and_currencies_value_from_scratch(
self, previous_state_manager):
self.origin_crypto_currencies_values = await self._evaluate_config_crypto_currencies_values(
)
async with self.portfolio.get_lock():
self.origin_portfolio = deepcopy(self.portfolio.get_portfolio())
self.portfolio_origin_value = \
await self.update_portfolio_current_value(self.origin_portfolio,
currencies_values=self.origin_crypto_currencies_values)
if not backtesting_enabled(
self.config) and previous_state_manager is not None:
if self.trader.simulate:
previous_state_manager.update_previous_states(
self.exchange,
simulated_initial_portfolio=deepcopy(
self.origin_portfolio),
simulated_initial_portfolio_value=self.
portfolio_origin_value,
watched_markets_initial_values=self.
origin_crypto_currencies_values,
reference_market=self.reference_market)
else:
previous_state_manager.update_previous_states(
self.exchange,
real_initial_portfolio=deepcopy(self.origin_portfolio),
real_initial_portfolio_value=self.portfolio_origin_value,
watched_markets_initial_values=self.
origin_crypto_currencies_values,
reference_market=self.reference_market)
def create_previous_state_manager(self):
if not backtesting_enabled(self.octobot.get_config()) and \
PreviousTradingStateManager.enabled(self.octobot.get_config()):
self.previous_trading_state_manager = PreviousTradingStateManager(
self.octobot.get_config()[CONFIG_EXCHANGES],
self.octobot.reset_trading_history, self.octobot.get_config())
async def _create_services(self):
# Add services to self.octobot.get_config()[CONFIG_CATEGORY_SERVICES]
await ServiceCreator.create_services(self.octobot.get_config(), backtesting_enabled(self.octobot.get_config()))
# Notify starting
if self.octobot.get_config()[CONFIG_NOTIFICATION_INSTANCE].enabled(CONFIG_NOTIFICATION_GLOBAL_INFO):
await self.octobot.get_config()[CONFIG_NOTIFICATION_INSTANCE].notify_with_all(NOTIFICATION_STARTING_MESSAGE,
False)
async def _init_origin_portfolio_and_currencies_value(
self, force_ignore_history=False):
previous_state_manager = self.trader.get_previous_state_manager()
if backtesting_enabled(self.config) or force_ignore_history or \
previous_state_manager is None or previous_state_manager.should_initialize_data():
await self._init_origin_portfolio_and_currencies_value_from_scratch(
previous_state_manager)
else:
await self._init_origin_portfolio_and_currencies_value_from_previous_executions(
previous_state_manager)
def _init_time_frames(self):
# Init time frames using enabled strategies
EvaluatorCreator.init_time_frames_from_strategies(self.octobot.get_config())
self.time_frames = copy.copy(TimeFrameManager.get_config_time_frame(self.octobot.get_config()))
# Init display time frame
config_time_frames = TimeFrameManager.get_config_time_frame(self.octobot.get_config())
if TimeFrames.ONE_HOUR not in config_time_frames and not backtesting_enabled(self.octobot.get_config()):
config_time_frames.append(TimeFrames.ONE_HOUR)
TimeFrameManager.sort_config_time_frames(self.octobot.get_config())
def _create_exchange_manager(self, exchange_type) -> ExchangeManager:
# Backtesting Exchange
if backtesting_enabled(self.octobot.get_config()):
return ExchangeManager(self.octobot.get_config(),
exchange_type,
is_simulated=True)
else:
# Real Exchange
return ExchangeManager(self.octobot.get_config(),
exchange_type,
ignore_config=self.ignore_config)
def _init_backtesting_if_necessary(self, time_frames):
# figure out from an evaluator if back testing is running for this symbol
evaluator_task_manager = \
self.evaluator_task_manager_by_time_frame_by_symbol[time_frames[0]][self.symbols[0]]
# test if we need to initialize backtesting features
is_backtesting_enabled = backtesting_enabled(evaluator_task_manager.get_evaluator().get_config())
if is_backtesting_enabled:
for symbol in self.symbols:
self.exchange.get_exchange().init_candles_offset(time_frames, symbol)
return is_backtesting_enabled
async def start_update_loop(self):
error = None
force_backtesting_exit = False
try:
time_frames = self.evaluator_task_manager_by_time_frame_by_symbol.keys()
# sort time frames to update them in order of accuracy
time_frames = TimeFrameManager.sort_time_frames(time_frames)
if time_frames and self.symbols:
self.in_backtesting = self._init_backtesting_if_necessary(time_frames)
# init refreshed_times at 0 for each time frame
self.refreshed_times = {key: {symbol: 0 for symbol in self.symbols} for key in time_frames}
# init last refresh times at 0 for each time frame
self.time_frame_last_update = {key: {symbol: 0 for symbol in self.symbols} for key in time_frames}
while self.keep_running:
try:
await self._trigger_update(time_frames)
except CancelledError:
self.logger.info("Update tasks cancelled.")
except Exception as e:
self.logger.error(f"exception when triggering update: {e}")
self.logger.exception(e)
else:
if backtesting_enabled(self.exchange.config):
force_backtesting_exit = True
else:
self.logger.warning("No time frames to monitor, going to sleep. "
"This is normal if you did not activate any technical analysis evaluator.")
except Exception as e:
self.logger.exception(e)
if self.watcher is not None:
error = e
finally:
if force_backtesting_exit \
or (self.in_backtesting
and self.symbols is not None
and not self.exchange.get_exchange().get_backtesting().get_is_finished(self.symbols)):
if error is None:
error = "backtesting did not finish properly."
if self.watcher is not None:
self.watcher.set_error(error)
self.logger.error(error)
async def initialize_impl(self):
if self.enable:
if self.previous_state_manager is not None:
self.load_previous_state_if_any()
try:
await self.portfolio.initialize()
await self.trades_manager.initialize()
except Exception as e:
self.enable = False
self.logger.error(
f"Error when initializing portfolio: {e}. "
f"{self.exchange.get_name()} trader disabled.")
self.logger.exception(e)
if backtesting_enabled(self.config):
raise e
def _find_min_time_frame_to_consider(self, time_frames, symbol):
time_frames_to_consider = copy.copy(time_frames)
self.min_time_frame_to_consider[symbol] = None
while not self.min_time_frame_to_consider[symbol] and time_frames_to_consider:
potential_min_time_frame_to_consider = TimeFrameManager.find_min_time_frame(time_frames_to_consider).value
if potential_min_time_frame_to_consider in self.get_ohlcv(symbol):
self.min_time_frame_to_consider[symbol] = potential_min_time_frame_to_consider
else:
time_frames_to_consider.remove(potential_min_time_frame_to_consider)
if self.min_time_frame_to_consider[symbol]:
return self.get_ohlcv(symbol)[self.min_time_frame_to_consider[symbol]][self.MIN_LIMIT] \
[PriceIndexes.IND_PRICE_TIME.value]
else:
self.logger.error(f"No data for the timeframes: {time_frames} in loaded backtesting file.")
if backtesting_enabled(self.config):
self.backtesting.end(symbol)
def get_currency_price_graph_update(exchange_name,
symbol,
time_frame,
list_arrays=True,
backtesting=False):
bot = get_bot()
if backtesting and bot.get_tools() and bot.get_tools(
)[BOT_TOOLS_BACKTESTING]:
bot = bot.get_tools()[BOT_TOOLS_BACKTESTING].get_bot()
symbol = parse_get_symbol(symbol)
symbol_evaluator_list = bot.get_symbol_evaluator_list()
in_backtesting = backtesting_enabled(get_global_config()) or backtesting
exchange = exchange_name
exchange_list = bot.get_exchanges_list()
if backtesting:
exchanges = [key for key in exchange_list if exchange_name in key]
if exchanges:
exchange = exchanges[0]
if time_frame is not None:
if symbol_evaluator_list:
evaluator_thread_managers = symbol_evaluator_list[
symbol].get_evaluator_task_managers(exchange_list[exchange])
data = None
if time_frame in evaluator_thread_managers:
if backtesting:
exchange_simulator = exchange_list[exchange].get_exchange()
data = exchange_simulator.get_full_candles_data(
symbol, time_frame)
else:
evaluator_thread_manager = evaluator_thread_managers[
time_frame]
data = evaluator_thread_manager.get_evaluator().get_data()
elif not backtesting and time_frame in get_exchange_time_frames(
exchange_name)[0]:
# might be the real-time evaluator time frame => check in symbol data
data = get_bot().run_in_main_asyncio_loop(
exchange_list[exchange].get_symbol_prices(
symbol, time_frame, return_list=False))
if data is not None:
return create_candles_data(symbol, time_frame, data, bot,
list_arrays, in_backtesting)
return None
async def start_task(self):
while self.keep_running:
now = time.time()
if self.is_active:
try:
await self._refresh_data()
except Exception as e:
self.logger.error(
f"error when refreshing data for {self.symbol}: {e}")
if self._should_eval():
await self.eval()
if not backtesting_enabled(self.config):
sleeping_time = self.specific_config[CONFIG_REFRESH_RATE] - (
time.time() - now)
if sleeping_time > 0:
await asyncio.sleep(sleeping_time)
def remove_loaded_only_element(config):
# remove service instances
for service in config[CONFIG_CATEGORY_SERVICES]:
config[CONFIG_CATEGORY_SERVICES][service].pop(CONFIG_SERVICE_INSTANCE, None)
# remove non config keys
config.pop(CONFIG_EVALUATOR, None)
config.pop(CONFIG_TRADING_TENTACLES, None)
config.pop(CONFIG_INTERFACES, None)
config.pop(CONFIG_ADVANCED_CLASSES, None)
config.pop(CONFIG_TIME_FRAME, None)
config.pop(CONFIG_NOTIFICATION_INSTANCE, None)
config.pop(CONFIG_ADVANCED_INSTANCES, None)
# remove backtesting specific differences
if backtesting_enabled(config):
if CONFIG_BACKTESTING in config:
config[CONFIG_BACKTESTING].pop(CONFIG_ENABLED_OPTION, None)
config[CONFIG_BACKTESTING].pop(CONFIG_ANALYSIS_ENABLED_OPTION, None)
def config():
if request.method == 'POST':
request_data = request.get_json()
success = True
response = ""
if request_data:
# update global config if required
if GLOBAL_CONFIG_KEY in request_data and request_data[
GLOBAL_CONFIG_KEY]:
success = update_global_config(request_data[GLOBAL_CONFIG_KEY])
else:
request_data[GLOBAL_CONFIG_KEY] = ""
# update trading config if required
if TRADING_CONFIG_KEY in request_data and request_data[
TRADING_CONFIG_KEY]:
success = success and update_trading_config(
request_data[TRADING_CONFIG_KEY])
else:
request_data[TRADING_CONFIG_KEY] = ""
# update evaluator config if required
if EVALUATOR_CONFIG_KEY in request_data and request_data[
EVALUATOR_CONFIG_KEY]:
success = success and update_evaluator_config(
request_data[EVALUATOR_CONFIG_KEY])
else:
request_data[EVALUATOR_CONFIG_KEY] = ""
# remove elements from global config if any to remove
removed_elements_key = "removed_elements"
if removed_elements_key in request_data and request_data[
removed_elements_key]:
success = success and update_global_config(
request_data[removed_elements_key], delete=True)
else:
request_data[removed_elements_key] = ""
response = {
"evaluator_updated_config": request_data[EVALUATOR_CONFIG_KEY],
"trading_updated_config": request_data[TRADING_CONFIG_KEY],
"global_updated_config": request_data[GLOBAL_CONFIG_KEY],
removed_elements_key: request_data[removed_elements_key]
}
if success:
return get_rest_reply(jsonify(response))
else:
return get_rest_reply('{"update": "ko"}', 500)
else:
display_config = get_edited_config()
# service lists
service_list, service_name_list = get_services_list()
return render_template(
'config.html',
config_exchanges=display_config[CONFIG_EXCHANGES],
config_trading=display_config[CONFIG_TRADING],
config_trader=display_config[CONFIG_TRADER],
config_trader_simulator=display_config[CONFIG_SIMULATOR],
config_notifications=display_config[CONFIG_CATEGORY_NOTIFICATION],
config_services=display_config[CONFIG_CATEGORY_SERVICES],
config_symbols=display_config[CONFIG_CRYPTO_CURRENCIES],
config_reference_market=display_config[CONFIG_TRADING]
[CONFIG_TRADER_REFERENCE_MARKET],
ccxt_tested_exchanges=get_tested_exchange_list(),
ccxt_simulated_tested_exchanges=get_simulated_exchange_list(),
ccxt_other_exchanges=sorted(get_other_exchange_list()),
services_list=service_list,
service_name_list=service_name_list,
symbol_list=sorted(
get_symbol_list([
exchange for exchange in display_config[CONFIG_EXCHANGES]
])),
full_symbol_list=get_all_symbol_list(),
strategy_config=get_strategy_config(),
evaluator_startup_config=get_evaluator_startup_config(),
trading_startup_config=get_trading_startup_config(),
is_trading_persistence_activated=is_trading_persistence_activated(
),
in_backtesting=backtesting_enabled(display_config))
async def start_order_manager(self):
if not backtesting_enabled(self.config):
await self.order_manager.poll_update()
def _init_metrics(self):
if not backtesting_enabled(self.octobot.get_config()):
self.octobot.metrics_handler = MetricsManager(self.octobot)
def should_be_ready(cls, config):
on_backtesting = backtesting_enabled(config)
return not on_backtesting or (on_backtesting
and cls.BACKTESTING_ENABLED)
def get_in_backtesting_mode():
return backtesting_enabled(get_bot().get_config())
def filter_to_update_data(to_update_data, current_config):
if backtesting_enabled(current_config):
for key in set(to_update_data.keys()):
# remove changes to currency config when in backtesting
if CONFIG_CRYPTO_CURRENCIES in key:
to_update_data.pop(key)
