def test_datasets(self):
for getter in get_dataset_getters():
testname = '%s_%s' % (os.path.basename(__file__).replace(
'.pyc', '').replace('.py', ''), getter.__name__)
with self.subTest(testname):
backend_mock = unittest.mock.Mock(spec=Backend)
backend_mock.get_model_dir.return_value = 'dutirapbdxvltcrpbdlcatepdeau'
D = getter()
D_ = copy.deepcopy(D)
y = D.data['Y_train']
if len(y.shape) == 2 and y.shape[1] == 1:
D_.data['Y_train'] = y.flatten()
backend_mock.load_datamanager.return_value = D_
metric_lookup = {
MULTILABEL_CLASSIFICATION: f1_macro,
BINARY_CLASSIFICATION: accuracy,
MULTICLASS_CLASSIFICATION: accuracy,
REGRESSION: r2
}
queue_ = multiprocessing.Queue()
evaluator = TestEvaluator(backend_mock,
queue_,
metric=metric_lookup[D.info['task']])
evaluator.fit_predict_and_loss()
rval = read_queue(evaluator.queue)
self.assertEqual(len(rval), 1)
self.assertEqual(len(rval[0]), 3)
self.assertTrue(np.isfinite(rval[0]['loss']))
def test_datasets(self):
for getter in get_dataset_getters():
testname = '%s_%s' % (os.path.basename(__file__).
replace('.pyc', '').replace('.py', ''),
getter.__name__)
with self.subTest(testname):
backend_mock = get_evaluation_backend()
D = getter()
D_ = copy.deepcopy(D)
y = D.data['Y_train']
if len(y.shape) == 2 and y.shape[1] == 1:
D_.data['Y_train'] = y.flatten()
backend_mock.load_datamanager.return_value = D_
metric_lookup = {MULTILABEL_CLASSIFICATION: f1_macro,
BINARY_CLASSIFICATION: accuracy,
MULTICLASS_CLASSIFICATION: accuracy,
REGRESSION: r2}
queue_ = multiprocessing.Queue()
evaluator = TestEvaluator(
backend_mock,
queue_,
metric=metric_lookup[D.info['task']],
port=logging.handlers.DEFAULT_TCP_LOGGING_PORT,
additional_components=dict(),
)
evaluator.fit_predict_and_loss()
rval = read_queue(evaluator.queue)
self.assertEqual(len(rval), 1)
self.assertEqual(len(rval[0]), 3)
self.assertTrue(np.isfinite(rval[0]['loss']))
def test_eval_test_all_loss_functions(self):
eval_t(
queue=self.queue,
backend=self.backend,
config=self.configuration,
metric=accuracy,
seed=1,
num_run=1,
scoring_functions=SCORER_LIST,
output_y_hat_optimization=False,
include=None,
exclude=None,
disable_file_output=False,
instance=self.dataset_name,
port=self.port,
)
rval = read_queue(self.queue)
self.assertEqual(len(rval), 1)
# Note: All metric here should be minimized
fixture = {
'accuracy': 0.040000000000000036,
'balanced_accuracy': 0.02777777777777779,
'f1_macro': 0.0341005967604433,
'f1_micro': 0.040000000000000036,
'f1_weighted': 0.039693094629155934,
'log_loss': 0.13966929787769913,
'precision_macro': 0.03703703703703709,
'precision_micro': 0.040000000000000036,
'precision_weighted': 0.03555555555555556,
'recall_macro': 0.02777777777777779,
'recall_micro': 0.040000000000000036,
'recall_weighted': 0.040000000000000036,
'num_run': -1
}
additional_run_info = rval[0]['additional_run_info']
for key, value in fixture.items():
self.assertAlmostEqual(additional_run_info[key],
fixture[key],
msg=key)
self.assertEqual(len(additional_run_info),
len(fixture) + 1,
msg=sorted(additional_run_info.items()))
self.assertIn('duration', additional_run_info)
self.assertAlmostEqual(rval[0]['loss'], 0.040000000000000036)
self.assertEqual(rval[0]['status'], StatusType.SUCCESS)
def test_eval_test_all_loss_functions(self):
eval_t(
queue=self.queue,
backend=self.backend,
config=self.configuration,
metric=accuracy,
seed=1,
num_run=1,
all_scoring_functions=True,
output_y_hat_optimization=False,
include=None,
exclude=None,
disable_file_output=False,
instance=self.dataset_name,
)
rval = read_queue(self.queue)
self.assertEqual(len(rval), 1)
fixture = {
'accuracy': 0.08,
'balanced_accuracy': 0.05555555555555547,
'f1_macro': 0.06734006734006737,
'f1_micro': 0.08,
'f1_weighted': 0.07919191919191915,
'log_loss': 1.128776115477085,
'pac_score': 0.187005982641133,
'precision_macro': 0.06666666666666676,
'precision_micro': 0.08,
'precision_weighted': 0.064,
'recall_macro': 0.05555555555555547,
'recall_micro': 0.08,
'recall_weighted': 0.08,
'num_run': -1
}
additional_run_info = rval[0]['additional_run_info']
for key, value in fixture.items():
self.assertAlmostEqual(additional_run_info[key],
fixture[key],
msg=key)
self.assertEqual(len(additional_run_info),
len(fixture) + 1,
msg=sorted(additional_run_info.items()))
self.assertIn('duration', additional_run_info)
self.assertAlmostEqual(rval[0]['loss'], 0.08)
self.assertEqual(rval[0]['status'], StatusType.SUCCESS)
def test_eval_test(self):
eval_t(queue=self.queue,
backend=self.backend,
config=self.configuration,
metric=accuracy,
seed=1,
num_run=1,
all_scoring_functions=False,
output_y_hat_optimization=False,
include=None,
exclude=None,
disable_file_output=False,
instance=self.dataset_name)
rval = read_queue(self.queue)
self.assertEqual(len(rval), 1)
self.assertAlmostEqual(rval[0]['loss'], 0.08)
self.assertEqual(rval[0]['status'], StatusType.SUCCESS)
self.assertNotIn('bac_metric', rval[0]['additional_run_info'])
