def callback(env):
"""internal function"""
if not state:
init(env)
bst = env.model
i = env.iteration
cvfolds = env.cvfolds
res_dict = {}
##### evaluation #####
if cvfolds is not None:
for feval in fevals:
tmp = aggcv([f.eval(i, feval) for f in cvfolds])
for k, mean, std in tmp:
res_dict[k] = [mean, std]
else:
for feval in fevals:
bst_eval = bst.eval_set(evals, i, feval)
res = [x.split(":") for x in bst_eval.split()]
for kv in res[1:]:
res_dict[kv[0]] = [float(kv[1])]
eval_res = []
keys = list(res_dict.keys())
keys.sort(key=lambda x: x if metric_shortname not in x else "a" + x)
for key in keys:
v = res_dict[key]
eval_res.append([key] + v)
##### print eval result #####
infos = ["XGB iter: %3d" % i]
for item in eval_res:
if "null" in item[0]:
continue
infos.append("%s: %.6f" % (item[0], item[1]))
if not isinstance(verbose_eval,
bool) and verbose_eval and i % verbose_eval == 0:
logger.debug("\t".join(infos))
if log_file:
with open(log_file, "a") as fout:
fout.write("\t".join(infos) + "\n")
##### choose score and do early stopping #####
score = None
for item in eval_res:
if item[0] == metric:
score = item[1]
break
assert score is not None
best_score = state["best_score"]
best_iteration = state["best_iteration"]
maximize_score = state["maximize_score"]
if (maximize_score and score > best_score) or (not maximize_score
and score < best_score):
msg = "[%d] %s" % (env.iteration, "\t".join(
[_fmt_metric(x) for x in eval_res]))
state["best_msg"] = msg
state["best_score"] = score
state["best_iteration"] = env.iteration
# save the property to attributes, so they will occur in checkpoint.
if env.model is not None:
env.model.set_attr(
best_score=str(state["best_score"]),
best_iteration=str(state["best_iteration"]),
best_msg=state["best_msg"],
)
elif env.iteration - best_iteration >= stopping_rounds:
best_msg = state["best_msg"]
if verbose_eval and env.rank == 0:
logger.debug("XGB stopped. Best iteration: %s ", best_msg)
raise EarlyStopException(best_iteration)
def callback(env: "xgb.core.CallbackEnv"):
# pylint:disable = import-outside-toplevel
import xgboost as xgb
from xgboost.callback import _fmt_metric # type: ignore
from xgboost.core import EarlyStopException # type: ignore
try:
from xgboost.training import aggcv # type: ignore
except ImportError:
from xgboost.callback import _aggcv as aggcv # type: ignore
# pylint:enable = import-outside-toplevel
if not state:
init(env)
booster: xgb.Booster = env.model
iteration: int = env.iteration
cvfolds: List[xgb.training.CVPack] = env.cvfolds
##### Evaluation #####
# `eval_result` is a list of (key, score)
eval_result: List[Tuple[str, float]] = []
if cvfolds is None:
eval_result = list(
itertools_chain.from_iterable([(key, float(value))
for key, value in map(
lambda x: x.split(":"),
booster.eval_set(
evals=evals,
iteration=iteration,
feval=feval,
).split()[1:],
)] for feval in fevals))
else:
eval_result = list(
itertools_chain.from_iterable([(key, score)
for key, score, _std in aggcv(
fold.eval(
iteration=iteration,
feval=feval,
) for fold in cvfolds)]
for feval in fevals))
eval_result = list(eval_result)
eval_result.sort(key=sort_key)
##### Print eval result #####
if verbose_eval and iteration % verbose_eval == 0:
info = []
for key, score in eval_result:
if "null" not in key:
info.append(f"{key}: {score:.6f}")
logger.debug("XGB iter %3d: %s", iteration, "\t".join(info))
##### Choose score and do early stopping #####
score = None
for key, _score in eval_result:
if key == focused_metric:
score = _score
break
assert score is not None
best_score = state["best_score"]
best_iteration = state["best_iteration"]
if score < best_score:
tab = "\t" # to work with f-string
msg = f"[{env.iteration}] {tab.join([_fmt_metric(x) for x in eval_result])}"
state["best_msg"] = msg
state["best_score"] = score
state["best_iteration"] = env.iteration
# save the property to attributes, so they will occur in checkpoint.
if env.model is not None:
env.model.set_attr(
best_score=str(state["best_score"]),
best_iteration=str(state["best_iteration"]),
best_msg=state["best_msg"],
)
elif env.iteration - best_iteration >= early_stopping_rounds:
best_msg = state["best_msg"]
if verbose_eval and env.rank == 0:
logger.debug("XGB stopped. Best iteration: %s ", best_msg)
raise EarlyStopException(best_iteration)
def callback(env):
"""internal function"""
if not best_state:
init(env)
bst = env.model
i = env.iteration
cvfolds = env.cvfolds
res_dict = {}
##### evaluation #####
if cvfolds is not None:
for feval in fevals:
tmp = aggcv([f.eval(i, feval) for f in cvfolds])
for k, mean, std in tmp:
res_dict[k] = [mean, std]
else:
for feval in fevals:
bst_eval = bst.eval_set(evals, i, feval)
res = [x.split(':') for x in bst_eval.split()]
for kv in res[1:]:
res_dict[kv[0]] = [float(kv[1])]
eval_res = []
keys = list(res_dict.keys())
keys.sort(key=lambda x: x if metric_shortname not in x else "a" + x)
for key in keys:
v = res_dict[key]
eval_res.append([key] + v)
##### print eval result #####
infos = ["XGB iter: %3d" % i]
for item in eval_res:
if 'null' in item[0]:
continue
infos.append("%s: %.6f" % (item[0], item[1]))
if not isinstance(verbose_eval, bool) and verbose_eval and i % verbose_eval == 0:
logger.debug("\t".join(infos))
if log_file:
with open(log_file, "a") as fout:
fout.write("\t".join(infos) + '\n')
##### choose score and do early stopping #####
score = None
for item in eval_res:
if item[0] == metric:
score = item[1]
break
best_score = best_state['score']
best_iteration = best_state['iteration']
if score and score > best_score:
msg = '[%d] %s' % (env.iteration, '\t'.join([_fmt_metric(x) for x in eval_res]))
if bst is not None:
bst.set_attr(best_score=str(score), best_iteration=str(env.iteration), best_msg=msg)
best_state['msg'] = msg
best_state['score'] = score
best_state['iteration'] = env.iteration
elif env.iteration - best_iteration >= stopping_rounds:
best_msg = best_state['msg']
if verbose_eval and env.rank == 0:
logger.debug("XGB stopped. Best iteration: %s ", best_msg)
raise EarlyStopException(best_iteration)
def callback(env):
"""internal function"""
if not state:
init(env)
bst = env.model
i = env.iteration
cvfolds = env.cvfolds
res_dict = {}
##### evaluation #####
if cvfolds is not None:
for feval in fevals:
tmp = aggcv([f.eval(i, feval) for f in cvfolds])
for k, mean, std in tmp:
res_dict[k] = [mean, std]
else:
for feval in fevals:
bst_eval = bst.eval_set(evals, i, feval)
res = [x.split(':') for x in bst_eval.split()]
for kv in res[1:]:
res_dict[kv[0]] = [float(kv[1])]
eval_res = []
keys = list(res_dict.keys())
keys.sort(key=lambda x: x if metric_shortname not in x else "a" + x)
for key in keys:
v = res_dict[key]
eval_res.append([key] + v)
##### print eval result #####
infos = ["XGB iter: %3d" % i]
for item in eval_res:
if 'null' in item[0]:
continue
infos.append("%s: %.6f" % (item[0], item[1]))
if not isinstance(verbose_eval, bool) and verbose_eval and i % verbose_eval == 0:
logger.debug("\t".join(infos))
if log_file:
with open(log_file, "a") as fout:
fout.write("\t".join(infos) + '\n')
##### choose score and do early stopping #####
score = None
for item in eval_res:
if item[0] == metric:
score = item[1]
break
assert score is not None
best_score = state['best_score']
best_iteration = state['best_iteration']
maximize_score = state['maximize_score']
if (maximize_score and score > best_score) or \
(not maximize_score and score < best_score):
msg = '[%d] %s' % (
env.iteration,
'\t'.join([_fmt_metric(x) for x in eval_res]))
state['best_msg'] = msg
state['best_score'] = score
state['best_iteration'] = env.iteration
# save the property to attributes, so they will occur in checkpoint.
if env.model is not None:
env.model.set_attr(best_score=str(state['best_score']),
best_iteration=str(state['best_iteration']),
best_msg=state['best_msg'])
elif env.iteration - best_iteration >= stopping_rounds:
best_msg = state['best_msg']
if verbose_eval and env.rank == 0:
logger.debug("XGB stopped. Best iteration: %s ", best_msg)
raise EarlyStopException(best_iteration)
