def make_mode_nested_cv(data, seed, configuration, num_run, inner_folds,
outer_folds):
global evaluator
evaluator = NestedCVEvaluator(data, configuration,
inner_cv_folds=inner_folds,
outer_cv_folds=outer_folds,
seed=seed,
num_run=num_run,
**_get_base_dict())
evaluator.fit()
signal.signal(15, empty_signal_handler)
evaluator.finish_up()
def make_mode_nested_cv(data, seed, configuration, num_run, inner_folds,
outer_folds, output_dir):
global evaluator
evaluator = NestedCVEvaluator(data, output_dir, configuration,
inner_cv_folds=inner_folds,
outer_cv_folds=outer_folds,
seed=seed,
all_scoring_functions=False,
num_run=num_run,
**_get_base_dict())
loss, opt_pred, valid_pred, test_pred = evaluator.fit_predict_and_loss()
evaluator.finish_up(loss, opt_pred, valid_pred, test_pred)
def make_mode_nested_cv(data, seed, configuration, num_run, inner_folds,
outer_folds, output_dir):
global evaluator
evaluator = NestedCVEvaluator(data,
output_dir,
configuration,
inner_cv_folds=inner_folds,
outer_cv_folds=outer_folds,
seed=seed,
all_scoring_functions=False,
num_run=num_run,
**_get_base_dict())
loss, opt_pred, valid_pred, test_pred = evaluator.fit_predict_and_loss()
evaluator.finish_up(loss, opt_pred, valid_pred, test_pred)
def test_datasets(self):
for getter in get_dataset_getters():
testname = "%s_%s" % (os.path.basename(__file__).replace(".pyc", "").replace(".py", ""), getter.__name__)
with self.subTest(testname):
D = getter()
output_directory = os.path.join(os.getcwd(), ".%s" % testname)
err = np.zeros([N_TEST_RUNS])
for i in range(N_TEST_RUNS):
D_ = copy.deepcopy(D)
evaluator = NestedCVEvaluator(D_, output_directory, None)
err[i] = evaluator.fit_predict_and_loss()[0]
self.assertTrue(np.isfinite(err[i]))
for model_idx in range(5):
model = evaluator.outer_models[model_idx]
self.assertIsNotNone(model)
model = evaluator.inner_models[model_idx]
self.assertIsNotNone(model)
def test_datasets(self):
for getter in get_dataset_getters():
testname = '%s_%s' % (os.path.basename(__file__).replace(
'.pyc', '').replace('.py', ''), getter.__name__)
with self.subTest(testname):
D = getter()
output_directory = os.path.join(os.getcwd(), '.%s' % testname)
err = np.zeros([N_TEST_RUNS])
for i in range(N_TEST_RUNS):
D_ = copy.deepcopy(D)
evaluator = NestedCVEvaluator(D_, output_directory, None)
err[i] = evaluator.fit_predict_and_loss()[0]
self.assertTrue(np.isfinite(err[i]))
for model_idx in range(5):
model = evaluator.outer_models[model_idx]
self.assertIsNotNone(model)
model = evaluator.inner_models[model_idx]
self.assertIsNotNone(model)
def make_mode_nested_cv(data, seed, configuration, num_run, inner_folds,
outer_folds):
global evaluator
evaluator = NestedCVEvaluator(data,
configuration,
inner_cv_folds=inner_folds,
outer_cv_folds=outer_folds,
seed=seed,
num_run=num_run,
**_get_base_dict())
evaluator.fit()
signal.signal(15, empty_signal_handler)
evaluator.finish_up()
