def _final_iteration_check(env, eval_name_splitted, i):
if env.iteration == env.end_iteration - 1:
if verbose:
print(
'Did not meet early stopping. Best iteration is:\n[%d]\t%s'
% (best_iter[i] + 1, '\t'.join(
[_format_eval_result(x) for x in best_score_list[i]])))
if first_metric_only:
print("Evaluated only: {}".format(eval_name_splitted[-1]))
raise EarlyStopException(best_iter[i], best_score_list[i])
def _callback(env):
if len(start_time) == 0:
start_time.append(time.time())
elif len(time_for_each_iteration) == 0:
time_for_each_iteration.append(time.time() - start_time[0])
start_time[0] = time.time()
print("time for each iteration is %f"%time_for_each_iteration[0])
#elif len(validation_time_for_each_iteration) == 0 and len(env.evaluation_result_list)>1:
# validation_time_for_each_iteration.append(time.time() - start_time[0])
# print("all time is %f"%validation_time_for_each_iteration[0])
# print("time for each evaluation is %f"%(validation_time_for_each_iteration[0]-100*time_for_each_iteration[0]))
else:
time_for_each_iteration.append(time.time() - start_time[0])
start_time[0] = time.time()
if env.iteration%100 == 0:
print("time for the current iteration is %f"%time_for_each_iteration[-1])
if reserved_time is None:
if config.time_left() < (env.iteration+1)*time_for_each_iteration[0]/3.0:
raise EarlyStopException(env.iteration, env.evaluation_result_list)
else:
if config.time_left() < reserved_time:
print("Time is not enough. The reserved time for post process is %d"%reserved_time)
raise EarlyStopException(env.iteration, env.evaluation_result_list)
def _callback(env):
if not dict_eval:
_init(env)
for data_name, eval_name, result, _ in env.evaluation_result_list:
if data_name == "valid_1":
if dict_eval["best_score"] > result:
dict_eval["best_score"] = result
dict_eval["eval_name"] = eval_name
dict_eval["best_iter"] = env.iteration
dict_eval[
"best_result_list"] = env.evaluation_result_list
break
if isinstance(
stopping_rounds, int
) and env.iteration - dict_eval["best_iter"] >= stopping_rounds:
logger.info(
f'early stopping. iteration: {dict_eval["best_iter"]}, score: {dict_eval["best_score"]}'
)
raise EarlyStopException(dict_eval["best_iter"],
dict_eval["best_result_list"])
def _callback(env):
if not cmp_op:
_init(env)
if not enabled[0]:
return
# print(f'iteration:{env.iteration}')
for i in range(len(env.evaluation_result_list)):
score = env.evaluation_result_list[i][2]
# print(f'score:{score}')
if best_score_list[i] is None or cmp_op[i](score, best_score[i]):
best_score[i] = score
best_iter[i] = env.iteration
best_score_list[i] = env.evaluation_result_list
best_model[i] = env.model.model_to_string()
# split is needed for "<dataset type> <metric>" case (e.g. "train l1")
eval_name_splitted = env.evaluation_result_list[i][1].split(" ")
if first_metric_only and first_metric[0] != eval_name_splitted[-1]:
continue # use only the first metric for early stopping
if ((env.evaluation_result_list[i][0] == "cv_agg"
and eval_name_splitted[0] == "train"
or env.evaluation_result_list[i][0]
== env.model._train_data_name)):
_final_iteration_check(env, eval_name_splitted, i)
continue # train datasets for lgb.cv or sklearn wrapper (underlying lgb.train)
elif env.iteration - best_iter[i] >= stopping_rounds:
if verbose:
print('Early stopping, best iteration is:\n[%d]\t%s' %
(best_iter[i] + 1, '\t'.join([
_format_eval_result(x)
for x in best_score_list[i]
])))
if first_metric_only:
print("Evaluated only: {}".format(
eval_name_splitted[-1]))
# recover best model from string
env.model.model_from_string(best_model[i])
raise EarlyStopException(best_iter[i], best_score_list[i])
_final_iteration_check(env, eval_name_splitted, i)
def _callback(env):
try_import_lightgbm()
from lightgbm.callback import _format_eval_result, EarlyStopException
cur_time = time.time()
# if verbose:
# print(cur_time - timex[0])
timex[0] = cur_time
if not cmp_op:
_init(env)
if not enabled[0]:
return
if train_loss_name is not None:
train_loss_evals = [
eval for i, eval in enumerate(env.evaluation_result_list)
if eval[0] == 'train_set' and eval[1] == train_loss_name
]
train_loss_val = train_loss_evals[0][2]
else:
train_loss_val = 0.0
for i in indices_to_check:
score = env.evaluation_result_list[i][2]
if i == indices_to_check[
0]: # TODO: documentation needs to note that we assume 0th index is the 'official' validation performance metric.
if cmp_op[i] == gt:
validation_perf = score
else:
validation_perf = -score
if best_score_list[i] is None or cmp_op[i](score, best_score[i]):
best_score[i] = score
best_iter[i] = env.iteration
best_score_list[i] = env.evaluation_result_list
if train_loss_name is not None:
best_trainloss[i] = train_loss_val # same for all i
elif env.iteration - best_iter[i] >= stopping_rounds:
if verbose:
logger.log(
15, 'Early stopping, best iteration is:\n[%d]\t%s' %
(best_iter[i] + 1, '\t'.join([
_format_eval_result(x) for x in best_score_list[i]
])))
raise EarlyStopException(best_iter[i], best_score_list[i])
elif (max_diff
is not None) and (abs(score - best_score[i]) > max_diff):
if verbose:
logger.debug('max_diff breached!')
logger.debug(str(abs(score - best_score[i])))
logger.log(
15, 'Early stopping, best iteration is:\n[%d]\t%s' %
(best_iter[i] + 1, '\t'.join([
_format_eval_result(x) for x in best_score_list[i]
])))
raise EarlyStopException(best_iter[i], best_score_list[i])
if env.iteration == env.end_iteration - 1:
if verbose:
logger.log(
15,
'Did not meet early stopping criterion. Best iteration is:\n[%d]\t%s'
% (best_iter[i] + 1, '\t'.join([
_format_eval_result(x) for x in best_score_list[i]
])))
raise EarlyStopException(best_iter[i], best_score_list[i])
if verbose:
logger.debug(
str(env.iteration - best_iter[i],
env.evaluation_result_list[i]))
if manual_stop_file:
if os.path.exists(manual_stop_file):
logger.log(
20,
'Found manual stop file, early stopping. Best iteration is:\n[%d]\t%s'
% (best_iter[i] + 1, '\t'.join([
_format_eval_result(x) for x in best_score_list[i]
])))
raise EarlyStopException(best_iter[i], best_score_list[i])
# TODO: This should be moved inside for loop at the start, otherwise it won't record the final iteration
idx = indices_to_check[0]
eval_results['best_iter'] = best_iter[
idx] + 1 # add one to index to align with lightgbm best_iteration instance variable.
eval_results['best_valperf'] = best_score[
idx] # validation performance at round = best_iter
eval_results['best_trainloss'] = best_trainloss[
idx] # training loss at round = best_iter
reporter(epoch=env.iteration,
validation_performance=validation_perf,
train_loss=train_loss_val,
best_iter_sofar=eval_results['best_iter'],
best_valperf_sofar=eval_results['best_valperf'])
def _callback(env):
try_import_lightgbm()
from lightgbm.callback import _format_eval_result, EarlyStopException
if not cmp_op:
_init(env)
if not enabled[0]:
return
for i in indices_to_check:
score = env.evaluation_result_list[i][2]
if best_score_list[i] is None or cmp_op[i](score, best_score[i]):
best_score[i] = score
best_iter[i] = env.iteration
best_score_list[i] = env.evaluation_result_list
elif env.iteration - best_iter[i] >= stopping_rounds:
if verbose:
logger.log(
15, 'Early stopping, best iteration is:\n[%d]\t%s' %
(best_iter[i] + 1, '\t'.join([
_format_eval_result(x) for x in best_score_list[i]
])))
raise EarlyStopException(best_iter[i], best_score_list[i])
elif (max_diff
is not None) and (abs(score - best_score[i]) > max_diff):
if verbose:
logger.debug('max_diff breached!')
logger.debug(abs(score - best_score[i]))
logger.log(
15, 'Early stopping, best iteration is:\n[%d]\t%s' %
(best_iter[i] + 1, '\t'.join([
_format_eval_result(x) for x in best_score_list[i]
])))
raise EarlyStopException(best_iter[i], best_score_list[i])
if env.iteration == env.end_iteration - 1:
if verbose:
logger.log(
15,
'Did not meet early stopping criterion. Best iteration is:\n[%d]\t%s'
% (best_iter[i] + 1, '\t'.join([
_format_eval_result(x) for x in best_score_list[i]
])))
raise EarlyStopException(best_iter[i], best_score_list[i])
if verbose:
logger.debug((env.iteration - best_iter[i],
env.evaluation_result_list[i]))
if manual_stop_file:
if os.path.exists(manual_stop_file):
i = indices_to_check[0]
logger.log(
20,
'Found manual stop file, early stopping. Best iteration is:\n[%d]\t%s'
% (best_iter[i] + 1, '\t'.join(
[_format_eval_result(x) for x in best_score_list[i]])))
raise EarlyStopException(best_iter[i], best_score_list[i])
# TODO: Add toggle parameter to early_stopping to disable this
# TODO: Identify optimal threshold values for early_stopping based on lack of memory
if env.iteration % 10 == 0:
available = psutil.virtual_memory().available
cur_rss = mem_status.memory_info().rss
if cur_rss < init_mem_rss[0]:
init_mem_rss[0] = cur_rss
estimated_model_size_mb = (cur_rss - init_mem_rss[0]) >> 20
available_mb = available >> 20
model_size_memory_ratio = estimated_model_size_mb / available_mb
if verbose or (model_size_memory_ratio > 0.25):
logging.debug('Available Memory: ' + str(available_mb) + ' MB')
logging.debug('Estimated Model Size: ' +
str(estimated_model_size_mb) + ' MB')
early_stop = False
if model_size_memory_ratio > 1.0:
logger.warning(
'Warning: Large GBM model size may cause OOM error if training continues'
)
logger.warning('Available Memory: ' + str(available_mb) +
' MB')
logger.warning('Estimated GBM model size: ' +
str(estimated_model_size_mb) + ' MB')
early_stop = True
# TODO: We will want to track size of model as well, even if we early stop before OOM, we will still crash when saving if the model is large enough
if available_mb < 512: # Less than 500 MB
logger.warning(
'Warning: Low available memory may cause OOM error if training continues'
)
logger.warning('Available Memory: ' + str(available_mb) +
' MB')
logger.warning('Estimated GBM model size: ' +
str(estimated_model_size_mb) + ' MB')
early_stop = True
if early_stop:
logger.warning(
'Warning: Early stopped GBM model prior to optimal result to avoid OOM error. Please increase available memory to avoid subpar model quality.'
)
logger.log(
15, 'Early stopping, best iteration is:\n[%d]\t%s' %
(best_iter[0] + 1, '\t'.join(
[_format_eval_result(x) for x in best_score_list[0]])))
raise EarlyStopException(best_iter[0], best_score_list[0])
def _callback(env):
try_import_lightgbm()
from lightgbm.callback import _format_eval_result, EarlyStopException
if not cmp_op:
_init(env)
if not enabled[0]:
return
if train_loss_name is not None:
train_loss_evals = [
eval for eval in env.evaluation_result_list
if eval[0] == 'train_set' and eval[1] == train_loss_name
]
train_loss_val = train_loss_evals[0][2]
else:
train_loss_val = 0.0
for i in indices_to_check:
score = env.evaluation_result_list[i][2]
if best_score_list[i] is None or cmp_op[i](score, best_score[i]):
best_score[i] = score
best_iter[i] = env.iteration
best_score_list[i] = env.evaluation_result_list
best_trainloss[i] = train_loss_val
if reporter is not None: # Report current best scores for iteration, used in HPO
if i == indices_to_check[
0]: # TODO: documentation needs to note that we assume 0th index is the 'official' validation performance metric.
if cmp_op[i] == gt:
validation_perf = score
else:
validation_perf = -score
reporter(epoch=env.iteration + 1,
validation_performance=validation_perf,
train_loss=best_trainloss[i],
best_iter_sofar=best_iter[i] + 1,
best_valperf_sofar=best_score[i])
if env.iteration - best_iter[i] >= stopping_rounds:
if verbose:
logger.log(
15, 'Early stopping, best iteration is:\n[%d]\t%s' %
(best_iter[i] + 1, '\t'.join([
_format_eval_result(x) for x in best_score_list[i]
])))
raise EarlyStopException(best_iter[i], best_score_list[i])
elif (max_diff
is not None) and (abs(score - best_score[i]) > max_diff):
if verbose:
logger.debug('max_diff breached!')
logger.debug(abs(score - best_score[i]))
logger.log(
15, 'Early stopping, best iteration is:\n[%d]\t%s' %
(best_iter[i] + 1, '\t'.join([
_format_eval_result(x) for x in best_score_list[i]
])))
raise EarlyStopException(best_iter[i], best_score_list[i])
if env.iteration == env.end_iteration - 1:
if verbose:
logger.log(
15,
'Did not meet early stopping criterion. Best iteration is:\n[%d]\t%s'
% (best_iter[i] + 1, '\t'.join([
_format_eval_result(x) for x in best_score_list[i]
])))
raise EarlyStopException(best_iter[i], best_score_list[i])
if verbose:
logger.debug((env.iteration - best_iter[i],
env.evaluation_result_list[i]))
if manual_stop_file:
if os.path.exists(manual_stop_file):
i = indices_to_check[0]
logger.log(
20,
'Found manual stop file, early stopping. Best iteration is:\n[%d]\t%s'
% (best_iter[i] + 1, '\t'.join(
[_format_eval_result(x) for x in best_score_list[i]])))
raise EarlyStopException(best_iter[i], best_score_list[i])
if time_limit:
time_elapsed = time.time() - start_time
time_left = time_limit - time_elapsed
if time_left <= 0:
i = indices_to_check[0]
logger.log(
20, '\tRan out of time, early stopping on iteration ' +
str(env.iteration + 1) +
'. Best iteration is:\n\t[%d]\t%s' %
(best_iter[i] + 1, '\t'.join(
[_format_eval_result(x) for x in best_score_list[i]])))
raise EarlyStopException(best_iter[i], best_score_list[i])
# TODO: Add toggle parameter to early_stopping to disable this
# TODO: Identify optimal threshold values for early_stopping based on lack of memory
if env.iteration % 10 == 0:
available = psutil.virtual_memory().available
cur_rss = mem_status.memory_info().rss
if cur_rss < init_mem_rss[0]:
init_mem_rss[0] = cur_rss
estimated_model_size_mb = (cur_rss - init_mem_rss[0]) >> 20
available_mb = available >> 20
model_size_memory_ratio = estimated_model_size_mb / available_mb
if verbose or (model_size_memory_ratio > 0.25):
logging.debug('Available Memory: ' + str(available_mb) + ' MB')
logging.debug('Estimated Model Size: ' +
str(estimated_model_size_mb) + ' MB')
early_stop = False
if model_size_memory_ratio > 1.0:
logger.warning(
'Warning: Large GBM model size may cause OOM error if training continues'
)
logger.warning('Available Memory: ' + str(available_mb) +
' MB')
logger.warning('Estimated GBM model size: ' +
str(estimated_model_size_mb) + ' MB')
early_stop = True
# TODO: We will want to track size of model as well, even if we early stop before OOM, we will still crash when saving if the model is large enough
if available_mb < 512: # Less than 500 MB
logger.warning(
'Warning: Low available memory may cause OOM error if training continues'
)
logger.warning('Available Memory: ' + str(available_mb) +
' MB')
logger.warning('Estimated GBM model size: ' +
str(estimated_model_size_mb) + ' MB')
early_stop = True
if early_stop:
logger.warning(
'Warning: Early stopped GBM model prior to optimal result to avoid OOM error. Please increase available memory to avoid subpar model quality.'
)
logger.log(
15, 'Early stopping, best iteration is:\n[%d]\t%s' %
(best_iter[0] + 1, '\t'.join(
[_format_eval_result(x) for x in best_score_list[0]])))
raise EarlyStopException(best_iter[0], best_score_list[0])
def __call__(self, env: "CallbackEnv") -> None:
# If this callback has been passed to `lightgbm.cv` function,
# the value of `is_cv` becomes `True`. See also:
# https://github.com/Microsoft/LightGBM/blob/v2.2.2/python-package/lightgbm/engine.py#L329
# Note that `5` is not the number of folds but the length of sequence.
is_cv = len(env.evaluation_result_list) > 0 and len(
env.evaluation_result_list[0]) == 5
if is_cv:
target_valid_name = "cv_agg"
else:
target_valid_name = self._valid_name
if target_valid_name is None:
# just use first key
target_valid_name = env.evaluation_result_list[0][0]
if self._metric is None:
# just use first key
self._metric = env.evaluation_result_list[0][1]
for evaluation_result in env.evaluation_result_list:
valid_name, metric, current_score, is_higher_better = evaluation_result[:
4]
if valid_name != target_valid_name or metric != self._metric:
continue
sig = 1.0
if is_higher_better:
if self._trial.study.direction != optuna.study.StudyDirection.MAXIMIZE:
sig = -1.0
#raise ValueError(
# "The intermediate values are inconsistent with the objective values in "
# "terms of study directions. Please specify a metric to be minimized for "
# "LightGBMPruningCallback."
#)
else:
if self._trial.study.direction != optuna.study.StudyDirection.MINIMIZE:
sig = -1.0
#raise ValueError(
# "The intermediate values are inconsistent with the objective values in "
# "terms of study directions. Please specify a metric to be maximized for "
# "LightGBMPruningCallback."
#)
self._trial.report(sig * current_score, step=env.iteration)
self._trial.was_pruned = False
if self._trial.should_prune():
message = "Trial was pruned at iteration {}.".format(
env.iteration)
self._trial.was_pruned = True
if self._raise_type is None or self._raise_type == "prune":
raise optuna.TrialPruned(message)
else:
raise EarlyStopException(env.iteration,
env.evaluation_result_list)
return None
raise ValueError(
'The entry associated with the validation name "{}" and the metric name "{}" '
"is not found in the evaluation result list {}.".format(
target_valid_name, self._metric,
str(env.evaluation_result_list)))