def make_mode_nested_cv(data, seed, configuration, num_run, inner_folds, outer_folds): global evaluator evaluator = NestedCVEvaluator(data, configuration, inner_cv_folds=inner_folds, outer_cv_folds=outer_folds, seed=seed, num_run=num_run, **_get_base_dict()) evaluator.fit() signal.signal(15, empty_signal_handler) evaluator.finish_up()
def make_mode_nested_cv(data, seed, configuration, num_run, inner_folds, outer_folds, output_dir): global evaluator evaluator = NestedCVEvaluator(data, output_dir, configuration, inner_cv_folds=inner_folds, outer_cv_folds=outer_folds, seed=seed, all_scoring_functions=False, num_run=num_run, **_get_base_dict()) loss, opt_pred, valid_pred, test_pred = evaluator.fit_predict_and_loss() evaluator.finish_up(loss, opt_pred, valid_pred, test_pred)
def test_datasets(self): for getter in get_dataset_getters(): testname = "%s_%s" % (os.path.basename(__file__).replace(".pyc", "").replace(".py", ""), getter.__name__) with self.subTest(testname): D = getter() output_directory = os.path.join(os.getcwd(), ".%s" % testname) err = np.zeros([N_TEST_RUNS]) for i in range(N_TEST_RUNS): D_ = copy.deepcopy(D) evaluator = NestedCVEvaluator(D_, output_directory, None) err[i] = evaluator.fit_predict_and_loss()[0] self.assertTrue(np.isfinite(err[i])) for model_idx in range(5): model = evaluator.outer_models[model_idx] self.assertIsNotNone(model) model = evaluator.inner_models[model_idx] self.assertIsNotNone(model)
def test_datasets(self): for getter in get_dataset_getters(): testname = '%s_%s' % (os.path.basename(__file__).replace( '.pyc', '').replace('.py', ''), getter.__name__) with self.subTest(testname): D = getter() output_directory = os.path.join(os.getcwd(), '.%s' % testname) err = np.zeros([N_TEST_RUNS]) for i in range(N_TEST_RUNS): D_ = copy.deepcopy(D) evaluator = NestedCVEvaluator(D_, output_directory, None) err[i] = evaluator.fit_predict_and_loss()[0] self.assertTrue(np.isfinite(err[i])) for model_idx in range(5): model = evaluator.outer_models[model_idx] self.assertIsNotNone(model) model = evaluator.inner_models[model_idx] self.assertIsNotNone(model)