def test_datasets(self):
for getter in get_dataset_getters():
testname = '%s_%s' % (os.path.basename(__file__).replace(
'.pyc', '').replace('.py', ''), getter.__name__)
with self.subTest(testname):
D = getter()
output_directory = os.path.join(os.path.dirname(__file__),
'.%s' % testname)
self.output_directories.append(output_directory)
err = np.zeros([N_TEST_RUNS])
for i in range(N_TEST_RUNS):
D_ = copy.deepcopy(D)
evaluator = CVEvaluator(D_, output_directory, None)
err[i] = evaluator.fit_predict_and_loss()[0]
self.assertTrue(np.isfinite(err[i]))
self.assertEqual(err[i], 1.0)
for model_idx in range(10):
indices = evaluator.indices[model_idx]
self.assertIsNotNone(indices)
D_ = copy.deepcopy(D)
evaluator = CVEvaluator(D_, output_directory, None)
for j in range(5):
evaluator.partial_fit_predict_and_loss(j)
indices = evaluator.indices[j]
self.assertIsNotNone(indices)
for j in range(5, 10):
indices = evaluator.indices[j]
self.assertIsNone(indices)
def test_datasets(self):
for getter in get_dataset_getters():
testname = '%s_%s' % (os.path.basename(__file__).replace(
'.pyc', '').replace('.py', ''), getter.__name__)
with self.subTest(testname):
backend_mock = unittest.mock.Mock(spec=Backend)
backend_mock.get_model_dir.return_value = 'dutirapbdxvltcrpbdlcatepdeau'
D = getter()
D_ = copy.deepcopy(D)
y = D.data['Y_train']
if len(y.shape) == 2 and y.shape[1] == 1:
D_.data['Y_train'] = y.flatten()
backend_mock.load_datamanager.return_value = D_
metric_lookup = {
MULTILABEL_CLASSIFICATION: f1_macro,
BINARY_CLASSIFICATION: accuracy,
MULTICLASS_CLASSIFICATION: accuracy,
REGRESSION: r2
}
queue_ = multiprocessing.Queue()
evaluator = TestEvaluator(backend_mock,
queue_,
metric=metric_lookup[D.info['task']])
evaluator.fit_predict_and_loss()
rval = read_queue(evaluator.queue)
self.assertEqual(len(rval), 1)
self.assertEqual(len(rval[0]), 3)
self.assertTrue(np.isfinite(rval[0]['loss']))
def test_datasets(self):
for getter in get_dataset_getters():
testname = '%s_%s' % (os.path.basename(__file__).
replace('.pyc', '').replace('.py', ''),
getter.__name__)
with self.subTest(testname):
backend_mock = get_evaluation_backend()
D = getter()
D_ = copy.deepcopy(D)
y = D.data['Y_train']
if len(y.shape) == 2 and y.shape[1] == 1:
D_.data['Y_train'] = y.flatten()
backend_mock.load_datamanager.return_value = D_
metric_lookup = {MULTILABEL_CLASSIFICATION: f1_macro,
BINARY_CLASSIFICATION: accuracy,
MULTICLASS_CLASSIFICATION: accuracy,
REGRESSION: r2}
queue_ = multiprocessing.Queue()
evaluator = TestEvaluator(
backend_mock,
queue_,
metric=metric_lookup[D.info['task']],
port=logging.handlers.DEFAULT_TCP_LOGGING_PORT,
additional_components=dict(),
)
evaluator.fit_predict_and_loss()
rval = read_queue(evaluator.queue)
self.assertEqual(len(rval), 1)
self.assertEqual(len(rval[0]), 3)
self.assertTrue(np.isfinite(rval[0]['loss']))
def test_datasets(self):
for getter in get_dataset_getters():
testname = '%s_%s' % (os.path.basename(__file__).
replace('.pyc', '').replace('.py', ''),
getter.__name__)
with self.subTest(testname):
D = getter()
output_directory = os.path.join(os.path.dirname(__file__),
'.%s' % testname)
self.output_directories.append(output_directory)
err = np.zeros([N_TEST_RUNS])
for i in range(N_TEST_RUNS):
D_ = copy.deepcopy(D)
evaluator = CVEvaluator(D_, output_directory, None)
err[i] = evaluator.fit_predict_and_loss()[0]
self.assertTrue(np.isfinite(err[i]))
self.assertEqual(err[i], 1.0)
for model_idx in range(10):
indices = evaluator.indices[model_idx]
self.assertIsNotNone(indices)
D_ = copy.deepcopy(D)
evaluator = CVEvaluator(D_, output_directory, None)
for j in range(5):
evaluator.partial_fit_predict_and_loss(j)
indices = evaluator.indices[j]
self.assertIsNotNone(indices)
for j in range(5, 10):
indices = evaluator.indices[j]
self.assertIsNone(indices)
def test_datasets(self):
for getter in get_dataset_getters():
testname = '%s_%s' % (os.path.basename(__file__).replace(
'.pyc', '').replace('.py', ''), getter.__name__)
with self.subTest(testname):
backend_mock = unittest.mock.Mock(spec=backend.Backend)
backend_mock.get_model_dir.return_value = 'dutirapbdxvltcrpbdlcatepdeau'
D = getter()
D_ = copy.deepcopy(D)
y = D.data['Y_train']
if len(y.shape) == 2 and y.shape[1] == 1:
D_.data['Y_train'] = y.flatten()
backend_mock.load_datamanager.return_value = D_
queue_ = multiprocessing.Queue()
metric_lookup = {
MULTILABEL_CLASSIFICATION: f1_macro,
BINARY_CLASSIFICATION: accuracy,
MULTICLASS_CLASSIFICATION: accuracy,
REGRESSION: r2
}
evaluator = TrainEvaluator(
backend_mock,
queue_,
resampling_strategy='cv',
resampling_strategy_args={'folds': 2},
output_y_hat_optimization=False,
metric=metric_lookup[D.info['task']])
evaluator.fit_predict_and_loss()
rval = evaluator.queue.get(timeout=1)
self.assertTrue(np.isfinite(rval['loss']))
def test_datasets(self):
for getter in get_dataset_getters():
testname = '%s_%s' % (os.path.basename(__file__).
replace('.pyc', '').replace('.py', ''),
getter.__name__)
with self.subTest(testname):
D = getter()
output_directory = os.path.join(os.getcwd(), '.%s' % testname)
self.output_directory = output_directory
err = np.zeros([N_TEST_RUNS])
for i in range(N_TEST_RUNS):
D_ = copy.deepcopy(D)
evaluator = HoldoutEvaluator(D_, self.output_directory, None)
err[i] = evaluator.fit_predict_and_loss()[0]
self.assertTrue(np.isfinite(err[i]))
def test_datasets(self):
for getter in get_dataset_getters():
testname = '%s_%s' % (os.path.basename(__file__).replace(
'.pyc', '').replace('.py', ''), getter.__name__)
with self.subTest(testname):
D = getter()
output_directory = os.path.join(os.getcwd(), '.%s' % testname)
self.output_directory = output_directory
err = np.zeros([N_TEST_RUNS])
for i in range(N_TEST_RUNS):
D_ = copy.deepcopy(D)
evaluator = HoldoutEvaluator(D_, self.output_directory,
None)
err[i] = evaluator.fit_predict_and_loss()[0]
self.assertTrue(np.isfinite(err[i]))
def test_datasets(self):
for getter in get_dataset_getters():
testname = "%s_%s" % (os.path.basename(__file__).replace(".pyc", "").replace(".py", ""), getter.__name__)
with self.subTest(testname):
D = getter()
output_directory = os.path.join(os.getcwd(), ".%s" % testname)
err = np.zeros([N_TEST_RUNS])
for i in range(N_TEST_RUNS):
D_ = copy.deepcopy(D)
evaluator = NestedCVEvaluator(D_, output_directory, None)
err[i] = evaluator.fit_predict_and_loss()[0]
self.assertTrue(np.isfinite(err[i]))
for model_idx in range(5):
model = evaluator.outer_models[model_idx]
self.assertIsNotNone(model)
model = evaluator.inner_models[model_idx]
self.assertIsNotNone(model)
def test_datasets(self):
for getter in get_dataset_getters():
testname = '%s_%s' % (os.path.basename(__file__).replace(
'.pyc', '').replace('.py', ''), getter.__name__)
with self.subTest(testname):
backend_mock = unittest.mock.Mock(spec=Backend)
backend_mock.get_model_dir.return_value = 'dutirapbdxvltcrpbdlcatepdeau'
D = getter()
D_ = copy.deepcopy(D)
y = D.data['Y_train']
if len(y.shape) == 2 and y.shape[1] == 1:
D_.data['Y_train'] = y.flatten()
queue_ = multiprocessing.Queue()
evaluator = TestEvaluator(D_, backend_mock, queue_)
evaluator.fit_predict_and_loss()
duration, result, seed, run_info, status = evaluator.queue.get(
timeout=1)
self.assertTrue(np.isfinite(result))