def callback(env): """internal function""" score_train = env.evaluation_result_list[0][1] score = env.evaluation_result_list[1][1] if not state: init(env) best_score = state['best_score'] best_iteration = state['best_iteration'] maximize_score = state['maximize_score'] if (maximize_score and score > best_score and ((not symetric_overfit and score_train - score <= max_overfit) or (symetric_overfit and abs(score_train - score) <= max_overfit))) or \ (not maximize_score and score < best_score and ((not symetric_overfit and score - score_train <= max_overfit) or (symetric_overfit and abs(score - score_train) <= max_overfit))): msg = '[%d]\t%s' % (env.iteration, '\t'.join( [_fmt_metric(x) for x in env.evaluation_result_list])) state['best_msg'] = msg state['best_score'] = score state['best_score_train'] = score_train state['best_iteration'] = env.iteration # save the property to attributes, so they will occur in checkpoint. if env.model is not None: env.model.set_attr(best_score=str(state['best_score']), best_score_train=str( state['best_score_train']), best_iteration=str(state['best_iteration']), best_msg=state['best_msg']) elif env.iteration - best_iteration >= stopping_rounds: best_msg = state['best_msg'] if verbose and env.rank == 0: msg = "Stopping. Best iteration:\n{}\n\n" rabit.tracker_print(msg.format(best_msg)) raise EarlyStopException(best_iteration)
def callback(env): """internal function""" if not state: init(env) bst = env.model i = env.iteration cvfolds = env.cvfolds res_dict = {} ##### evaluation ##### if cvfolds is not None: for feval in fevals: tmp = aggcv([f.eval(i, feval) for f in cvfolds]) for k, mean, std in tmp: res_dict[k] = [mean, std] else: for feval in fevals: bst_eval = bst.eval_set(evals, i, feval) res = [x.split(":") for x in bst_eval.split()] for kv in res[1:]: res_dict[kv[0]] = [float(kv[1])] eval_res = [] keys = list(res_dict.keys()) keys.sort(key=lambda x: x if metric_shortname not in x else "a" + x) for key in keys: v = res_dict[key] eval_res.append([key] + v) ##### print eval result ##### infos = ["XGB iter: %3d" % i] for item in eval_res: if "null" in item[0]: continue infos.append("%s: %.6f" % (item[0], item[1])) if not isinstance(verbose_eval, bool) and verbose_eval and i % verbose_eval == 0: logger.debug("\t".join(infos)) if log_file: with open(log_file, "a") as fout: fout.write("\t".join(infos) + "\n") ##### choose score and do early stopping ##### score = None for item in eval_res: if item[0] == metric: score = item[1] break assert score is not None best_score = state["best_score"] best_iteration = state["best_iteration"] maximize_score = state["maximize_score"] if (maximize_score and score > best_score) or (not maximize_score and score < best_score): msg = "[%d] %s" % (env.iteration, "\t".join( [_fmt_metric(x) for x in eval_res])) state["best_msg"] = msg state["best_score"] = score state["best_iteration"] = env.iteration # save the property to attributes, so they will occur in checkpoint. if env.model is not None: env.model.set_attr( best_score=str(state["best_score"]), best_iteration=str(state["best_iteration"]), best_msg=state["best_msg"], ) elif env.iteration - best_iteration >= stopping_rounds: best_msg = state["best_msg"] if verbose_eval and env.rank == 0: logger.debug("XGB stopped. Best iteration: %s ", best_msg) raise EarlyStopException(best_iteration)
def callback(env): """internal function""" if not best_state: init(env) bst = env.model i = env.iteration cvfolds = env.cvfolds res_dict = {} ##### evaluation ##### if cvfolds is not None: for feval in fevals: tmp = aggcv([f.eval(i, feval) for f in cvfolds]) for k, mean, std in tmp: res_dict[k] = [mean, std] else: for feval in fevals: bst_eval = bst.eval_set(evals, i, feval) res = [x.split(':') for x in bst_eval.split()] for kv in res[1:]: res_dict[kv[0]] = [float(kv[1])] eval_res = [] keys = list(res_dict.keys()) keys.sort(key=lambda x: x if metric_shortname not in x else "a" + x) for key in keys: v = res_dict[key] eval_res.append([key] + v) ##### print eval result ##### infos = ["XGB iter: %3d" % i] for item in eval_res: if 'null' in item[0]: continue infos.append("%s: %.6f" % (item[0], item[1])) if not isinstance(verbose_eval, bool) and verbose_eval and i % verbose_eval == 0: logger.debug("\t".join(infos)) if log_file: with open(log_file, "a") as fout: fout.write("\t".join(infos) + '\n') ##### choose score and do early stopping ##### score = None for item in eval_res: if item[0] == metric: score = item[1] break best_score = best_state['score'] best_iteration = best_state['iteration'] if score and score > best_score: msg = '[%d] %s' % (env.iteration, '\t'.join([_fmt_metric(x) for x in eval_res])) if bst is not None: bst.set_attr(best_score=str(score), best_iteration=str(env.iteration), best_msg=msg) best_state['msg'] = msg best_state['score'] = score best_state['iteration'] = env.iteration elif env.iteration - best_iteration >= stopping_rounds: best_msg = best_state['msg'] if verbose_eval and env.rank == 0: logger.debug("XGB stopped. Best iteration: %s ", best_msg) raise EarlyStopException(best_iteration)
def callback(env): """internal function""" if not state: init(env) bst = env.model i = env.iteration cvfolds = env.cvfolds res_dict = {} ##### evaluation ##### if cvfolds is not None: for feval in fevals: tmp = aggcv([f.eval(i, feval) for f in cvfolds]) for k, mean, std in tmp: res_dict[k] = [mean, std] else: for feval in fevals: bst_eval = bst.eval_set(evals, i, feval) res = [x.split(':') for x in bst_eval.split()] for kv in res[1:]: res_dict[kv[0]] = [float(kv[1])] eval_res = [] keys = list(res_dict.keys()) keys.sort(key=lambda x: x if metric_shortname not in x else "a" + x) for key in keys: v = res_dict[key] eval_res.append([key] + v) ##### print eval result ##### infos = ["XGB iter: %3d" % i] for item in eval_res: if 'null' in item[0]: continue infos.append("%s: %.6f" % (item[0], item[1])) if not isinstance(verbose_eval, bool) and verbose_eval and i % verbose_eval == 0: logger.debug("\t".join(infos)) if log_file: with open(log_file, "a") as fout: fout.write("\t".join(infos) + '\n') ##### choose score and do early stopping ##### score = None for item in eval_res: if item[0] == metric: score = item[1] break assert score is not None best_score = state['best_score'] best_iteration = state['best_iteration'] maximize_score = state['maximize_score'] if (maximize_score and score > best_score) or \ (not maximize_score and score < best_score): msg = '[%d] %s' % ( env.iteration, '\t'.join([_fmt_metric(x) for x in eval_res])) state['best_msg'] = msg state['best_score'] = score state['best_iteration'] = env.iteration # save the property to attributes, so they will occur in checkpoint. if env.model is not None: env.model.set_attr(best_score=str(state['best_score']), best_iteration=str(state['best_iteration']), best_msg=state['best_msg']) elif env.iteration - best_iteration >= stopping_rounds: best_msg = state['best_msg'] if verbose_eval and env.rank == 0: logger.debug("XGB stopped. Best iteration: %s ", best_msg) raise EarlyStopException(best_iteration)