def start(self) -> None:
self.random_state = self._set_random_state(
self.config.get('hyperopt_random_state', None))
logger.info(f"Using optimizer random state: {self.random_state}")
self.hyperopt_table_header = -1
# Initialize spaces ...
self.init_spaces()
self.prepare_hyperopt_data()
# We don't need exchange instance anymore while running hyperopt
self.backtesting.exchange.close()
self.backtesting.exchange._api = None # type: ignore
self.backtesting.exchange._api_async = None # type: ignore
# self.backtesting.exchange = None # type: ignore
self.backtesting.pairlists = None # type: ignore
cpus = cpu_count()
logger.info(f"Found {cpus} CPU cores. Let's make them scream!")
config_jobs = self.config.get('hyperopt_jobs', -1)
logger.info(f'Number of parallel jobs set as: {config_jobs}')
self.opt = self.get_optimizer(self.dimensions, config_jobs)
if self.print_colorized:
colorama_init(autoreset=True)
try:
with Parallel(n_jobs=config_jobs) as parallel:
jobs = parallel._effective_n_jobs()
logger.info(
f'Effective number of parallel workers used: {jobs}')
# Define progressbar
if self.print_colorized:
widgets = [
' [Epoch ',
progressbar.Counter(),
' of ',
str(self.total_epochs),
' (',
progressbar.Percentage(),
')] ',
progressbar.Bar(marker=progressbar.AnimatedMarker(
fill='\N{FULL BLOCK}',
fill_wrap=Fore.GREEN + '{}' + Fore.RESET,
marker_wrap=Style.BRIGHT + '{}' + Style.RESET_ALL,
)),
' [',
progressbar.ETA(),
', ',
progressbar.Timer(),
']',
]
else:
widgets = [
' [Epoch ',
progressbar.Counter(),
' of ',
str(self.total_epochs),
' (',
progressbar.Percentage(),
')] ',
progressbar.Bar(marker=progressbar.AnimatedMarker(
fill='\N{FULL BLOCK}', )),
' [',
progressbar.ETA(),
', ',
progressbar.Timer(),
']',
]
with progressbar.ProgressBar(max_value=self.total_epochs,
redirect_stdout=False,
redirect_stderr=False,
widgets=widgets) as pbar:
EVALS = ceil(self.total_epochs / jobs)
for i in range(EVALS):
# Correct the number of epochs to be processed for the last
# iteration (should not exceed self.total_epochs in total)
n_rest = (i + 1) * jobs - self.total_epochs
current_jobs = jobs - n_rest if n_rest > 0 else jobs
asked = self.opt.ask(n_points=current_jobs)
f_val = self.run_optimizer_parallel(parallel, asked, i)
self.opt.tell(asked, [v['loss'] for v in f_val])
# Calculate progressbar outputs
for j, val in enumerate(f_val):
# Use human-friendly indexes here (starting from 1)
current = i * jobs + j + 1
val['current_epoch'] = current
val['is_initial_point'] = current <= INITIAL_POINTS
logger.debug(f"Optimizer epoch evaluated: {val}")
is_best = HyperoptTools.is_best_loss(
val, self.current_best_loss)
# This value is assigned here and not in the optimization method
# to keep proper order in the list of results. That's because
# evaluations can take different time. Here they are aligned in the
# order they will be shown to the user.
val['is_best'] = is_best
self.print_results(val)
if is_best:
self.current_best_loss = val['loss']
self.current_best_epoch = val
self._save_result(val)
pbar.update(current)
except KeyboardInterrupt:
print('User interrupted..')
logger.info(
f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} "
f"saved to '{self.results_file}'.")
if self.current_best_epoch:
HyperoptTools.try_export_params(
self.config, self.backtesting.strategy.get_strategy_name(),
self.current_best_epoch)
HyperoptTools.show_epoch_details(self.current_best_epoch,
self.total_epochs,
self.print_json)
else:
# This is printed when Ctrl+C is pressed quickly, before first epochs have
# a chance to be evaluated.
print("No epochs evaluated yet, no best result.")
def start(self) -> None:
self.random_state = self._set_random_state(
self.config.get('hyperopt_random_state', None))
logger.info(f"Using optimizer random state: {self.random_state}")
self.hyperopt_table_header = -1
data, timerange = self.backtesting.load_bt_data()
logger.info("Dataload complete. Calculating indicators")
preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data)
# Trim startup period from analyzed dataframe
for pair, df in preprocessed.items():
preprocessed[pair] = trim_dataframe(
df,
timerange,
startup_candles=self.backtesting.required_startup)
min_date, max_date = get_timerange(preprocessed)
logger.info(
f'Hyperopting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(max_date - min_date).days} days)..')
dump(preprocessed, self.data_pickle_file)
# We don't need exchange instance anymore while running hyperopt
self.backtesting.exchange.close()
self.backtesting.exchange._api = None # type: ignore
self.backtesting.exchange._api_async = None # type: ignore
# self.backtesting.exchange = None # type: ignore
self.backtesting.pairlists = None # type: ignore
self.backtesting.strategy.dp = None # type: ignore
IStrategy.dp = None # type: ignore
cpus = cpu_count()
logger.info(f"Found {cpus} CPU cores. Let's make them scream!")
config_jobs = self.config.get('hyperopt_jobs', -1)
logger.info(f'Number of parallel jobs set as: {config_jobs}')
self.dimensions: List[Dimension] = self.hyperopt_space()
self.opt = self.get_optimizer(self.dimensions, config_jobs)
if self.print_colorized:
colorama_init(autoreset=True)
try:
with Parallel(n_jobs=config_jobs) as parallel:
jobs = parallel._effective_n_jobs()
logger.info(
f'Effective number of parallel workers used: {jobs}')
# Define progressbar
if self.print_colorized:
widgets = [
' [Epoch ',
progressbar.Counter(),
' of ',
str(self.total_epochs),
' (',
progressbar.Percentage(),
')] ',
progressbar.Bar(marker=progressbar.AnimatedMarker(
fill='\N{FULL BLOCK}',
fill_wrap=Fore.GREEN + '{}' + Fore.RESET,
marker_wrap=Style.BRIGHT + '{}' + Style.RESET_ALL,
)),
' [',
progressbar.ETA(),
', ',
progressbar.Timer(),
']',
]
else:
widgets = [
' [Epoch ',
progressbar.Counter(),
' of ',
str(self.total_epochs),
' (',
progressbar.Percentage(),
')] ',
progressbar.Bar(marker=progressbar.AnimatedMarker(
fill='\N{FULL BLOCK}', )),
' [',
progressbar.ETA(),
', ',
progressbar.Timer(),
']',
]
with progressbar.ProgressBar(max_value=self.total_epochs,
redirect_stdout=False,
redirect_stderr=False,
widgets=widgets) as pbar:
EVALS = ceil(self.total_epochs / jobs)
for i in range(EVALS):
# Correct the number of epochs to be processed for the last
# iteration (should not exceed self.total_epochs in total)
n_rest = (i + 1) * jobs - self.total_epochs
current_jobs = jobs - n_rest if n_rest > 0 else jobs
asked = self.opt.ask(n_points=current_jobs)
f_val = self.run_optimizer_parallel(parallel, asked, i)
self.opt.tell(asked, [v['loss'] for v in f_val])
# Calculate progressbar outputs
for j, val in enumerate(f_val):
# Use human-friendly indexes here (starting from 1)
current = i * jobs + j + 1
val['current_epoch'] = current
val['is_initial_point'] = current <= INITIAL_POINTS
logger.debug(f"Optimizer epoch evaluated: {val}")
is_best = HyperoptTools.is_best_loss(
val, self.current_best_loss)
# This value is assigned here and not in the optimization method
# to keep proper order in the list of results. That's because
# evaluations can take different time. Here they are aligned in the
# order they will be shown to the user.
val['is_best'] = is_best
self.print_results(val)
if is_best:
self.current_best_loss = val['loss']
self.epochs.append(val)
# Save results after each best epoch and every 100 epochs
if is_best or current % 100 == 0:
self._save_results()
pbar.update(current)
except KeyboardInterrupt:
print('User interrupted..')
self._save_results()
logger.info(
f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} "
f"saved to '{self.results_file}'.")
if self.epochs:
sorted_epochs = sorted(self.epochs, key=itemgetter('loss'))
best_epoch = sorted_epochs[0]
HyperoptTools.print_epoch_details(best_epoch, self.total_epochs,
self.print_json)
else:
# This is printed when Ctrl+C is pressed quickly, before first epochs have
# a chance to be evaluated.
print("No epochs evaluated yet, no best result.")