async def start_backtesting_bot(bot, in_thread=False, watcher=None):
BotLogger.reset_backtesting_errors()
await initialize_bot(bot)
# fix backtesting exit
for exchange in bot.get_exchanges_list():
exchange_inst = bot.get_exchanges_list()[exchange].get_exchange()
try:
exchange_inst.backtesting.force_exit_at_end = False
except Exception:
get_logger(
f"fail to stop force exit for exchange {exchange_inst.get_name()}"
)
if not bot.get_symbols_tasks_manager():
raise RuntimeError(
f"No candles data for the current configuration. Please ensure the required data files for "
f"the activated symbol(s) are available. Symbol(s): {list(bot.get_symbols_list())}"
)
if watcher is not None:
bot.set_watcher(watcher)
if in_thread:
await start_bot(bot, True)
return True
else:
await start_bot(bot)
trader = next(iter(bot.get_exchange_trader_simulators().values()))
return await Backtesting.get_profitability(trader)
def get_class(config, class_type):
classes = AdvancedManager.get_classes(config, class_type)
if classes and len(classes) > 1:
get_logger(AdvancedManager.__name__).warning(
f"More than one instance of {class_type} available, "
f"using {classes[0]}.")
return classes[0]
def check_last_prices(self, price, inferior, simulated_time=False):
if self.last_prices is not None:
prices = [
p["price"]
for p in self.last_prices[-SIMULATOR_LAST_PRICES_TO_CHECK:]
if not math.isnan(p["price"]) and
(p[eC.TIMESTAMP.value] >= self.creation_time or simulated_time)
]
if prices:
if inferior:
if float(min(prices)) < price:
get_logger(self.get_name()).debug(
f"{self.symbol} last prices: {prices}, "
f"ask for {'inferior' if inferior else 'superior'} "
f"to {price}")
return True
else:
if float(max(prices)) > price:
get_logger(self.get_name()).debug(
f"{self.symbol} last prices: {prices}, "
f"ask for {'inferior' if inferior else 'superior'} "
f"to {price}")
return True
return False
def create_services(config, backtesting_enabled):
logger = get_logger(ServiceCreator.get_name())
for service_class in AbstractService.__subclasses__():
service_instance = service_class()
if service_instance.get_is_enabled() and (
not backtesting_enabled
or service_instance.BACKTESTING_ENABLED):
service_instance.set_logger(
get_logger(service_class.get_name()))
service_instance.set_config(config)
if service_instance.has_required_configuration():
try:
service_instance.prepare()
config[CONFIG_CATEGORY_SERVICES][service_instance.get_type()][CONFIG_SERVICE_INSTANCE] = \
service_instance
service_instance.say_hello()
except Exception as e:
logger.error(
f"{service_class.get_name()} preparation produced the following error: {e}"
)
logger.exception(e)
else:
if service_instance.get_should_warn():
logger.warning(
f"{service_class.get_name()} can't be initialized: configuration is missing, "
f"wrong or incomplete !")
def start_backtesting_bot(bot, in_thread=False, watcher=None):
bot.create_exchange_traders()
# fix backtesting exit
for exchange in bot.exchanges_list:
exchange_inst = bot.exchanges_list[exchange].get_exchange()
try:
exchange_inst.backtesting.force_exit_at_end = False
except Exception:
get_logger(
f"fail to stop force exit for exchange {exchange_inst.get_name()}"
)
bot.create_evaluation_threads()
if not bot.get_symbols_threads_manager():
raise RuntimeError(
f"No candles data for the current configuration. Please ensure the required data files for "
f"the activated symbol(s) are available. Symbol(s): {list(bot.get_symbols_list())}"
)
if watcher is not None:
bot.set_watcher(watcher)
bot.start_threads()
if not in_thread:
bot.join_threads()
trader = next(iter(bot.get_exchange_trader_simulators().values()))
return Backtesting.get_profitability(trader)
else:
return True
def create_social_eval(config, symbol, dispatchers_list, relevant_evaluators):
social_eval_list = []
for social_eval_class in AdvancedManager.create_advanced_evaluator_types_list(SocialEvaluator, config):
social_eval_class_instance = social_eval_class()
social_eval_class_instance.set_config(config)
if EvaluatorCreator.is_relevant_evaluator(social_eval_class_instance, relevant_evaluators):
is_evaluator_to_be_used = True
social_eval_class_instance.set_logger(get_logger(social_eval_class.get_name()))
social_eval_class_instance.set_symbol(symbol)
social_eval_class_instance.prepare()
# If evaluator is a dispatcher client --> check if dispatcher exists
# else warn and pass this evaluator
if social_eval_class.get_is_dispatcher_client():
client_found_dispatcher = EvaluatorCreator.set_social_eval_dispatcher(social_eval_class_instance,
dispatchers_list)
if not client_found_dispatcher:
is_evaluator_to_be_used = False
get_logger(EvaluatorCreator.get_name()).warning(
"No dispatcher found for evaluator: {0} for symbol: {1}, evaluator has been disabled."
.format(social_eval_class_instance.get_name(), symbol))
# start refreshing thread if the thread is not manage by dispatcher
elif is_evaluator_to_be_used and social_eval_class_instance.get_is_threaded():
social_eval_class_instance.start()
if is_evaluator_to_be_used:
social_eval_list.append(social_eval_class_instance)
return social_eval_list
def get_class(config, class_type):
classes = AdvancedManager.get_classes(config, class_type)
if classes and len(classes) > 1:
get_logger(AdvancedManager.__name__).warning(
"More than one instance of {0} available, using {1}.".format(
class_type, classes[0]))
return classes[0]
def _activate_deactivate_strategies(self, strategies, exchange, activate=True):
try:
for symbol_evaluator in self.symbol_evaluator_list.values():
symbol_evaluator.activate_deactivate_strategies(strategies, exchange, activate)
except Exception as e:
get_logger(self.__class__.__name__)\
.error(f"{self.crypto_currency} error in activate_deactivate_strategies(): {e}")
def _init_strategies_instances(self, symbol, all_strategy_instances):
all_strategy_classes = [s.__class__ for s in all_strategy_instances]
required_strategies, required_strategies_min_count = self.get_required_strategies()
missing_strategies = []
found_strategy_count = 0
for required_class in required_strategies:
if required_class in all_strategy_classes:
self.strategy_instances_by_classes[symbol][required_class] = \
all_strategy_instances[all_strategy_classes.index(required_class)]
found_strategy_count += 1
else:
subclass = AdvancedManager.get_class(self.config, required_class)
if subclass in all_strategy_classes:
self.strategy_instances_by_classes[symbol][required_class] = \
all_strategy_instances[all_strategy_classes.index(subclass)]
found_strategy_count += 1
if required_class not in self.strategy_instances_by_classes[symbol]:
missing_strategies.append(required_class)
if found_strategy_count < required_strategies_min_count:
for missing_strategy in missing_strategies:
get_logger(self.get_name()).error(f"No instance of {missing_strategy.__name__} "
f"or advanced equivalent found, {self.get_name()} trading "
"mode can't work properly ! Maybe this strategy is disabled in"
f" tentacles/Evaluator/evaluator_config.json (missing "
f"{required_strategies_min_count-found_strategy_count} out of "
f"{required_strategies_min_count} minimum required strategies).")
def get_activated_trading_mode(config):
if CONFIG_TRADING_TENTACLES in config:
try:
trading_modes = [
class_str for class_str, activated in
config[CONFIG_TRADING_TENTACLES].items() if activated
and get_class_from_string(class_str,
AbstractTradingMode,
modes,
error_when_not_found=True)
]
if len(trading_modes) > 1:
raise ConfigTradingError(
f"More than one activated trading mode found in your {CONFIG_TRADING_FILE_PATH} file, "
f"please activate only one")
elif trading_modes:
trading_mode_class = get_deep_class_from_string(
trading_modes[0], modes)
if trading_mode_class is not None:
return AdvancedManager.get_class(config,
trading_mode_class)
except ModuleNotFoundError as e:
get_logger("get_activated_trading_mode").error(
f"Error when loading a trading mode: {e} "
f"referenced in {CONFIG_TRADING_FILE_PATH} file")
raise ConfigTradingError(
f"Please ensure your {CONFIG_TRADING_FILE_PATH} file is valid and at least one trading "
f"mode is activated")
def get_all_symbol_list():
try:
currencies_list = json.loads(requests.get(COIN_MARKET_CAP_CURRENCIES_LIST_URL).text)
return {
currency_data["name"]: currency_data["symbol"]
for currency_data in currencies_list["data"]
}
except Exception as e:
get_logger("Configuration").error(f"Failed to get currencies list from coinmarketcap : {e}")
def run_coroutine_in_asyncio_loop(coroutine, async_loop):
future = asyncio.run_coroutine_threadsafe(coroutine, async_loop)
try:
return future.result(DEFAULT_FUTURE_TIMEOUT)
except asyncio.TimeoutError as e:
get_logger("run_coroutine_in_asyncio_loop")\
.error(f'{coroutine} coroutine coroutine too long, cancelling the task.')
future.cancel()
raise e
def parse_time_frames(time_frames_string_list):
result_list = []
for time_frame_string in time_frames_string_list:
try:
result_list.append(TimeFrames(time_frame_string))
except ValueError:
get_logger(TimeFrameManager.__name__).error("No time frame available for: '{0}'. Available time "
"frames are: {1}. '{0}' time frame requirement "
"ignored.".
format(time_frame_string,
[t.value for t in TimeFrames]))
return result_list
def create_advanced_evaluator_types_list(evaluator_class, config):
evaluator_advanced_eval_class_list = []
for evaluator_subclass in evaluator_class.__subclasses__():
for eval_class in evaluator_subclass.__subclasses__():
for eval_class_type in AdvancedManager.get_classes(
config, eval_class):
evaluator_advanced_eval_class_list.append(eval_class_type)
if not AdvancedManager._check_duplicate(
evaluator_advanced_eval_class_list):
get_logger(
AdvancedManager.__name__).warning("Duplicate evaluator name.")
return evaluator_advanced_eval_class_list
def exchange_keys_encrypter(catch=False):
try:
api_key_crypted = encrypt(input("ENTER YOUR API-KEY : ")).decode()
api_secret_crypted = encrypt(input("ENTER YOUR API-SECRET : ")).decode()
print(f"Here are your encrypted exchanges keys : \n "
f"\t- API-KEY : {api_key_crypted}\n"
f"\t- API-SECRET : {api_secret_crypted}\n\n"
f"Your new exchange key configuration is : \n"
f'\t"api-key": "{api_key_crypted}",\n'
f'\t"api-secret": "{api_secret_crypted}"\n')
except Exception as e:
if not catch:
get_logger(Commands.__name__).error(f"Fail to encrypt your exchange keys, please try again ({e}).")
raise e
def get_external_resource(resource_key,
catch_exception=False,
default_response=""):
try:
external_resource_url = f"{GITHUB_RAW_CONTENT_URL}/{GITHUB_REPOSITORY}/{ASSETS_BRANCH}/{EXTERNAL_RESOURCES_FILE}"
external_resources = json.loads(
requests.get(external_resource_url).text)
return external_resources[resource_key]
except Exception as e:
if catch_exception:
get_logger("ExternalResourcesManager")\
.error(f"Exception when calling get_external_resource for {resource_key} key: {e}")
return default_response
else:
raise e
def get_symbol_list(exchanges):
result = []
for exchange in exchanges:
try:
inst = getattr(ccxt, exchange)({'verbose': False})
inst.load_markets()
result += inst.symbols
except Exception as e:
get_logger("Configuration").error(f"error when loading symbol list for {exchange}: {e}")
# filter symbols with a "." or no "/" because bot can't handle them for now
symbols = [res for res in result if "/" in res]
return list(set(symbols))
def __init__(self,
config,
exchange,
order_refresh_time=None,
previous_state_manager=None):
super().__init__()
self.exchange = exchange
self.config = config
self.risk = None
self.order_refresh_time = order_refresh_time
self.set_risk(self.config[CONFIG_TRADING][CONFIG_TRADER_RISK])
# logging
self.trader_type_str = REAL_TRADER_STR
self.logger = get_logger(
f"{self.__class__.__name__}[{self.exchange.get_name()}]")
self.previous_state_manager = previous_state_manager
self.loaded_previous_state = False
if not hasattr(self, 'simulate'):
self.simulate = False
self.enable = self.enabled(self.config)
self.order_manager = None
self.portfolio = None
self.trades_manager = None
self.notifier = None
self.trading_modes = []
if self.enable:
self.initialize_trader()
def __init__(self):
super().__init__()
self.keep_running = True
self.interval = CONFIG_DEBUG_OPTION_PERF_REFRESH_TIME_MIN * MINUTE_TO_SECONDS
self.logger = get_logger(self.__class__.__name__)
self.pid = os.getpid()
self.py = psutil.Process(self.pid)
def __init__(self, config):
self.config = config
self.tentacle_package_manager = TentaclePackageManager(config, self)
self.default_package = None
self.advanced_package_list = []
self.logger = get_logger(self.__class__.__name__)
self.force_actions = False
def __init__(self, config, trader):
super().__init__()
self.config = config
self.trader = trader
self.portfolio = trader.get_portfolio()
self.exchange = trader.get_exchange()
self.logger = get_logger(self.__class__.__name__)
self.trade_history = []
self.profitability = 0
self.profitability_percent = 0
self.profitability_diff = 0
self.currencies_last_prices = {}
self.origin_crypto_currencies_values = {}
self.current_crypto_currencies_values = {}
self.origin_portfolio = None
# buffer of currencies excluding market only used currencies ex: conf = btc/usd, eth/btc, ltc/btc, here usd
# is market only => not used to compute market average profitability
self.traded_currencies_without_market_specific = set()
# buffer of currencies containing currencies that have already been logged as without matching symbol
# (used not to spam logs)
self.already_informed_no_matching_symbol_currency = set()
self.portfolio_origin_value = 0
self.portfolio_current_value = 0
self.trades_value = 0
self.reference_market = TradesManager.get_reference_market(self.config)
def __init__(self, config):
self.start_time = time.time()
self.config = config
self.startup_config = copy.deepcopy(config)
self.edited_config = copy.deepcopy(config)
self.ready = False
self.watcher = None
# tools: used for alternative operations on a bot on the fly (ex: backtesting started from web interface)
self.tools = {
BOT_TOOLS_BACKTESTING: None,
BOT_TOOLS_STRATEGY_OPTIMIZER: None,
BOT_TOOLS_RECORDER: None,
}
# Logger
self.logger = get_logger(self.__class__.__name__)
# Advanced
AdvancedManager.init_advanced_classes_if_necessary(self.config)
# Debug tools
self.performance_analyser = None
if CONFIG_DEBUG_OPTION_PERF in self.config and self.config[
CONFIG_DEBUG_OPTION_PERF]:
self.performance_analyser = PerformanceAnalyser()
# Init time frames using enabled strategies
EvaluatorCreator.init_time_frames_from_strategies(self.config)
self.time_frames = TimeFrameManager.get_config_time_frame(self.config)
# Init relevant evaluator names list using enabled strategies
self.relevant_evaluators = EvaluatorCreator.get_relevant_evaluators_from_strategies(
self.config)
# Backtesting
self.backtesting_enabled = Backtesting.enabled(self.config)
# Add services to self.config[CONFIG_CATEGORY_SERVICES]
ServiceCreator.create_services(self.config, self.backtesting_enabled)
# Notifier
self.config[CONFIG_NOTIFICATION_INSTANCE] = Notification(self.config)
# Notify starting
if self.config[CONFIG_NOTIFICATION_INSTANCE].enabled(
CONFIG_NOTIFICATION_GLOBAL_INFO):
self.config[CONFIG_NOTIFICATION_INSTANCE].notify_with_all(
NOTIFICATION_STARTING_MESSAGE, False)
self.symbol_threads_manager = {}
self.exchange_traders = {}
self.exchange_trader_simulators = {}
self.trading_mode = None
self.exchange_trading_modes = {}
self.exchanges_list = {}
self.symbol_evaluator_list = {}
self.crypto_currency_evaluator_list = {}
self.dispatchers_list = []
self.symbol_time_frame_updater_threads = []
def __init__(self,
target_exchanges,
reset_simulator,
config,
save_file=SIMULATOR_STATE_SAVE_FILE,
log_file=LOG_FILE):
self.logger = get_logger(self.__class__.__name__)
self.save_file = save_file
self.log_file = log_file
if reset_simulator:
self.reset_trading_history()
self.reset_state_history = False
self._previous_state = {}
try:
if not self._load_previous_state(target_exchanges, config):
self._previous_state = self._initialize_new_previous_state(
target_exchanges)
self.first_data = True
else:
self.first_data = False
except Exception as e:
self.logger.error(f"{self.ERROR_MESSAGE}{e}")
self.logger.exception(e)
self._previous_state = self._initialize_new_previous_state(
target_exchanges)
self.first_data = True
def __init__(self, social_evaluator_list):
super().__init__()
self.social_evaluator_list = social_evaluator_list
self.social_evaluator_list_timers = []
self.logger = get_logger(self.__class__.__name__)
self._create_eval_timers()
self.keep_running = True
def __init__(self):
super().__init__()
self._profitability_results = []
self._trades_counts = []
self.logger = get_logger(self.__class__.__name__)
self.current_progress = 0
self.exceptions = []
def _init_strategies_instances(self, symbol, all_strategy_instances):
all_strategy_classes = [s.__class__ for s in all_strategy_instances]
for required_class in self.get_required_strategies():
if required_class in all_strategy_classes:
self.strategy_instances_by_classes[symbol][required_class] = \
all_strategy_instances[all_strategy_classes.index(required_class)]
else:
subclass = AdvancedManager.get_class(self.config, required_class)
if subclass in all_strategy_classes:
self.strategy_instances_by_classes[symbol][required_class] = \
all_strategy_instances[all_strategy_classes.index(subclass)]
if required_class not in self.strategy_instances_by_classes[symbol]:
get_logger(self.get_name()).error(f"No instance of {required_class.__name__} "
f"or advanced equivalent found, {self.get_name()} trading "
"mode can't work properly ! Maybe this strategy is disabled in"
" tentacles/Evaluator/evaluator_config.json.")
def __init__(self, config, trader):
self.config = config
self.trader = trader
self.portfolio = trader.get_portfolio()
self.exchange = trader.get_exchange()
self.logger = get_logger(self.__class__.__name__)
self.trade_history = []
self.profitability = 0
self.profitability_percent = 0
self.profitability_diff = 0
self.currencies_last_prices = {}
self.origin_crypto_currencies_values = {}
self.current_crypto_currencies_values = {}
self.origin_portfolio = None
# buffer of currencies excluding market only used currencies ex: conf = btc/usd, eth/btc, ltc/btc, here usd
# is market only => not used to compute market average profitability
self.traded_currencies_without_market_specific = set()
self.portfolio_origin_value = 0
self.portfolio_current_value = 0
self.trades_value = 0
self.reference_market = TradesManager.get_reference_market(self.config)
self._init_origin_portfolio_and_currencies_value()
def __init__(self,
config,
ignore_config=False,
reset_trading_history=False):
self.start_time = time.time()
self.config = config
self.reset_trading_history = reset_trading_history
self.startup_config = copy.deepcopy(config)
self.edited_config = copy.deepcopy(config)
# tools: used for alternative operations on a bot on the fly (ex: backtesting started from web interface)
self.tools = {
BOT_TOOLS_BACKTESTING: None,
BOT_TOOLS_STRATEGY_OPTIMIZER: None,
BOT_TOOLS_RECORDER: None,
}
# unique aiohttp session: to be initialized from getter in a task
self._aiohttp_session = None
# metrics if enabled
self.metrics_handler = None
# Logger
self.logger = get_logger(self.__class__.__name__)
self.initializer = Initializer(self)
self.task_manager = TaskManager(self)
self.exchange_factory = ExchangeFactory(self,
ignore_config=ignore_config)
self.evaluator_factory = EvaluatorFactory(self)
def __init__(self, config, strategy_name):
self.is_properly_initialized = False
self.logger = get_logger(self.get_name())
AdvancedManager.init_advanced_classes_if_necessary(config)
self.trading_mode = get_activated_trading_mode(config)
self.config = create_blank_config_using_loaded_one(config)
self.strategy_class = get_class_from_string(
strategy_name, StrategiesEvaluator, Strategies,
evaluator_parent_inspection)
self.run_results = []
self.results_report = []
self.sorted_results_by_time_frame = {}
self.sorted_results_through_all_time_frame = {}
self.all_time_frames = []
self.all_TAs = []
self.risks = []
self.current_test_suite = None
self.errors = set()
self.is_computing = False
self.run_id = 0
self.total_nb_runs = 0
if not self.strategy_class:
self.logger.error(
f"Impossible to find a strategy matching class name: {strategy_name} in installed "
f"strategies. Please make sure to enter the name of the class, "
f"ex: FullMixedStrategiesEvaluator")
else:
self.is_properly_initialized = True
async def create_service(logger, service_class, config,
backtesting_enabled):
service_instance = service_class()
if service_instance.get_is_enabled(config) and \
(not backtesting_enabled or service_instance.BACKTESTING_ENABLED):
service_instance.set_logger(get_logger(service_class.get_name()))
service_instance.set_config(config)
if service_instance.has_required_configuration():
try:
await service_instance.prepare()
# notifier
if isinstance(service_instance, NotifierService):
config[CONFIG_CATEGORY_SERVICES][CONFIG_NOTIFIER][CONFIG_SERVICE_INSTANCE] \
.append(service_instance)
else:
config[CONFIG_CATEGORY_SERVICES][service_instance.get_type()][CONFIG_SERVICE_INSTANCE] = \
service_instance
if not await service_instance.say_hello():
logger.warning(
f"{service_class.get_name()} initial checkup failed."
)
except Exception as e:
logger.error(
f"{service_class.get_name()} preparation produced the following error: {e}"
)
logger.exception(e)
else:
if service_instance.get_should_warn():
logger.warning(
f"{service_class.get_name()} can't be initialized: configuration is missing, "
f"wrong or incomplete !")