def cross_validate_sklearn(args: SklearnTrainArgs,
logger: Logger = None) -> Tuple[float, float]:
info = logger.info if logger is not None else print
init_seed = args.seed
save_dir = args.save_dir
# Run training on different random seeds for each fold
all_scores = []
for fold_num in range(args.num_folds):
info(f'Fold {fold_num}')
args.seed = init_seed + fold_num
args.save_dir = os.path.join(save_dir, f'fold_{fold_num}')
makedirs(args.save_dir)
model_scores = run_sklearn(args, logger)
all_scores.append(model_scores)
all_scores = np.array(all_scores)
# Report scores for each fold
for fold_num, scores in enumerate(all_scores):
info(
f'Seed {init_seed + fold_num} ==> test {args.metric} = {np.nanmean(scores):.6f}'
)
# Report scores across folds
avg_scores = np.nanmean(
all_scores, axis=1) # average score for each model across tasks
mean_score, std_score = np.nanmean(avg_scores), np.nanstd(avg_scores)
info(f'Overall test {args.metric} = {mean_score:.6f} +/- {std_score:.6f}')
return mean_score, std_score
def cross_validate_sklearn(args: SklearnTrainArgs) -> Tuple[float, float]:
"""
Runs k-fold cross-validation for a scikit-learn model.
For each of k splits (folds) of the data, trains and tests a model on that split
and aggregates the performance across folds.
:param args: A :class:`~chemprop.args.SklearnTrainArgs` object containing arguments for
loading data and training the scikit-learn model.
:return: A tuple containing the mean and standard deviation performance across folds.
"""
logger = create_logger(name=SKLEARN_TRAIN_LOGGER_NAME,
save_dir=args.save_dir,
quiet=args.quiet)
info = logger.info if logger is not None else print
init_seed = args.seed
save_dir = args.save_dir
# Run training on different random seeds for each fold
all_scores = []
for fold_num in range(args.num_folds):
info(f'Fold {fold_num}')
args.seed = init_seed + fold_num
args.save_dir = os.path.join(save_dir, f'fold_{fold_num}')
makedirs(args.save_dir)
model_scores = run_sklearn(args, logger)
all_scores.append(model_scores)
all_scores = np.array(all_scores)
# Report scores for each fold
for fold_num, scores in enumerate(all_scores):
info(
f'Seed {init_seed + fold_num} ==> test {args.metric} = {np.nanmean(scores):.6f}'
)
# Report scores across folds
avg_scores = np.nanmean(
all_scores, axis=1) # average score for each model across tasks
mean_score, std_score = np.nanmean(avg_scores), np.nanstd(avg_scores)
info(f'Overall test {args.metric} = {mean_score:.6f} +/- {std_score:.6f}')
return mean_score, std_score