def test_datasets(self): for getter in get_dataset_getters(): testname = '%s_%s' % (os.path.basename(__file__). replace('.pyc', '').replace('.py', ''), getter.__name__) with self.subTest(testname): D = getter() output_directory = os.path.join(os.path.dirname(__file__), '.%s' % testname) self.output_directories.append(output_directory) err = np.zeros([N_TEST_RUNS]) for i in range(N_TEST_RUNS): D_ = copy.deepcopy(D) evaluator = CVEvaluator(D_, output_directory, None) err[i] = evaluator.fit_predict_and_loss()[0] self.assertTrue(np.isfinite(err[i])) self.assertEqual(err[i], 1.0) for model_idx in range(10): indices = evaluator.indices[model_idx] self.assertIsNotNone(indices) D_ = copy.deepcopy(D) evaluator = CVEvaluator(D_, output_directory, None) for j in range(5): evaluator.partial_fit_predict_and_loss(j) indices = evaluator.indices[j] self.assertIsNotNone(indices) for j in range(5, 10): indices = evaluator.indices[j] self.assertIsNone(indices)
def test_datasets(self): for getter in get_dataset_getters(): testname = '%s_%s' % (os.path.basename(__file__).replace( '.pyc', '').replace('.py', ''), getter.__name__) with self.subTest(testname): D = getter() output_directory = os.path.join(os.path.dirname(__file__), '.%s' % testname) self.output_directories.append(output_directory) err = np.zeros([N_TEST_RUNS]) for i in range(N_TEST_RUNS): D_ = copy.deepcopy(D) evaluator = CVEvaluator(D_, output_directory, None) err[i] = evaluator.fit_predict_and_loss()[0] self.assertTrue(np.isfinite(err[i])) self.assertEqual(err[i], 1.0) for model_idx in range(10): indices = evaluator.indices[model_idx] self.assertIsNotNone(indices) D_ = copy.deepcopy(D) evaluator = CVEvaluator(D_, output_directory, None) for j in range(5): evaluator.partial_fit_predict_and_loss(j) indices = evaluator.indices[j] self.assertIsNotNone(indices) for j in range(5, 10): indices = evaluator.indices[j] self.assertIsNone(indices)
def make_mode_partial_cv(data, seed, configuration, num_run, metric, fold, folds, output_dir): global evaluator evaluator = CVEvaluator(data, output_dir, configuration, cv_folds=folds, seed=seed, num_run=num_run, all_scoring_functions=False, **_get_base_dict()) loss, opt_pred, valid_pred, test_pred = \ evaluator.partial_fit_predict_and_loss(fold) duration = time.time() - evaluator.starttime additional_run_info = 'duration: ' + str(duration) print(metric, loss, additional_run_info) print('Result for ParamILS: %s, %f, 1, %f, %d, %s' % ('SAT', abs(duration), loss, evaluator.seed, additional_run_info))
def make_mode_partial_cv(data, seed, configuration, num_run, metric, fold, folds, output_dir): global evaluator evaluator = CVEvaluator(data, output_dir, configuration, cv_folds=folds, seed=seed, num_run=num_run, all_scoring_functions=False, **_get_base_dict()) loss, opt_pred, valid_pred, test_pred = \ evaluator.partial_fit_predict_and_loss(fold) duration = time.time() - evaluator.starttime additional_run_info = 'duration: ' + str(duration) print(metric, loss, additional_run_info) print('Result for ParamILS: %s, %f, 1, %f, %d, %s' % ('SAT', abs(duration), loss, evaluator.seed, additional_run_info))