def test_predict_proba_binary_classification(self):
self.output_dir = os.path.join(
os.getcwd(), '.test_predict_proba_binary_classification')
D = get_binary_classification_datamanager()
class Dummy2(object):
def predict_proba(self, y, batch_size=200):
return np.array([[0.1, 0.9]] * 23)
def fit(self, X, y):
return self
model = Dummy2()
configuration_space = get_configuration_space(
D.info,
include_estimators=['extra_trees'],
include_preprocessors=['select_rates'])
configuration = configuration_space.sample_configuration()
evaluator = HoldoutEvaluator(D, self.output_dir, configuration)
evaluator.model = model
loss, Y_optimization_pred, Y_valid_pred, Y_test_pred = \
evaluator.fit_predict_and_loss()
for i in range(23):
self.assertEqual(0.9, Y_optimization_pred[i][1])
def test_file_output(self):
self.output_dir = os.path.join(os.getcwd(), '.test_file_output')
D = get_regression_datamanager()
D.name = 'test'
configuration_space = get_configuration_space(D.info)
configuration = configuration_space.sample_configuration()
backend_api = backend.create(self.output_dir, self.output_dir)
evaluator = HoldoutEvaluator(D,
backend_api,
configuration,
with_predictions=True,
all_scoring_functions=True,
output_y_test=True)
loss, Y_optimization_pred, Y_valid_pred, Y_test_pred = \
evaluator.fit_predict_and_loss()
evaluator.file_output(loss, Y_optimization_pred, Y_valid_pred,
Y_test_pred)
self.assertTrue(
os.path.exists(
os.path.join(self.output_dir, '.auto-sklearn',
'true_targets_ensemble.npy')))
def test_predict_proba_binary_classification(self):
self.output_dir = os.path.join(os.getcwd(),
'.test_predict_proba_binary_classification')
D = get_binary_classification_datamanager()
class Dummy2(object):
def predict_proba(self, y, batch_size=200):
return np.array([[0.1, 0.9]] * 23)
def fit(self, X, y):
return self
model = Dummy2()
configuration_space = get_configuration_space(
D.info,
include_estimators=['extra_trees'],
include_preprocessors=['select_rates'])
configuration = configuration_space.sample_configuration()
evaluator = HoldoutEvaluator(D, self.output_dir, configuration)
evaluator.model = model
loss, Y_optimization_pred, Y_valid_pred, Y_test_pred = \
evaluator.fit_predict_and_loss()
for i in range(23):
self.assertEqual(0.9, Y_optimization_pred[i][1])
def test_datasets(self):
for getter in get_dataset_getters():
testname = '%s_%s' % (os.path.basename(__file__).
replace('.pyc', '').replace('.py', ''),
getter.__name__)
with self.subTest(testname):
D = getter()
output_directory = os.path.join(os.getcwd(), '.%s' % testname)
self.output_directory = output_directory
err = np.zeros([N_TEST_RUNS])
for i in range(N_TEST_RUNS):
D_ = copy.deepcopy(D)
evaluator = HoldoutEvaluator(D_, self.output_directory, None)
err[i] = evaluator.fit_predict_and_loss()[0]
self.assertTrue(np.isfinite(err[i]))
def test_datasets(self):
for getter in get_dataset_getters():
testname = '%s_%s' % (os.path.basename(__file__).replace(
'.pyc', '').replace('.py', ''), getter.__name__)
with self.subTest(testname):
D = getter()
output_directory = os.path.join(os.getcwd(), '.%s' % testname)
self.output_directory = output_directory
err = np.zeros([N_TEST_RUNS])
for i in range(N_TEST_RUNS):
D_ = copy.deepcopy(D)
evaluator = HoldoutEvaluator(D_, self.output_directory,
None)
err[i] = evaluator.fit_predict_and_loss()[0]
self.assertTrue(np.isfinite(err[i]))
def test_file_output(self):
self.output_dir = os.path.join(os.getcwd(), '.test')
D = get_regression_datamanager()
D.name = 'test'
configuration_space = get_configuration_space(D.info)
configuration = configuration_space.sample_configuration()
evaluator = HoldoutEvaluator(D, self.output_dir, configuration,
with_predictions=True,
all_scoring_functions=True,
output_y_test=True)
loss, Y_optimization_pred, Y_valid_pred, Y_test_pred = \
evaluator.fit_predict_and_loss()
evaluator.file_output(loss, Y_optimization_pred, Y_valid_pred,
Y_test_pred)
self.assertTrue(os.path.exists(os.path.join(
self.output_dir, '.auto-sklearn', 'true_targets_ensemble.npy')))
