def testNBest(self):
for ensemble_nbest, models_on_disc, exp in (
(1, None, 1),
(1.0, None, 2),
(0.1, None, 1),
(0.9, None, 1),
(1, 2, 1),
(2, 1, 1),
):
ensbuilder = EnsembleBuilder(
backend=self.backend,
dataset_name="TEST",
task_type=1, # Binary Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
ensemble_nbest=ensemble_nbest,
max_models_on_disc=models_on_disc,
)
ensbuilder.score_ensemble_preds()
sel_keys = ensbuilder.get_n_best_preds()
self.assertEqual(len(sel_keys), exp)
fixture = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2_0.0.npy"
)
self.assertEqual(sel_keys[0], fixture)
def testRead(self):
ensbuilder = EnsembleBuilder(
backend=self.backend,
dataset_name="TEST",
task_type=1, #Binary Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
)
success = ensbuilder.read_ensemble_preds()
self.assertTrue(success, str(ensbuilder.read_preds))
self.assertEqual(len(ensbuilder.read_preds), 2)
filename = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1.npy"
)
self.assertEqual(ensbuilder.read_preds[filename]["ens_score"], 0.5)
filename = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy"
)
self.assertEqual(ensbuilder.read_preds[filename]["ens_score"], 1.0)
def testPerformanceRangeThresholdMaxBest(self):
to_test = ((0.0, 1, 1), (0.0, 1.0, 4), (0.1, 2, 2), (0.3, 4, 3),
(0.5, 1, 1), (0.6, 10, 2), (0.8, 0.5, 1), (1, 1.0, 1))
for performance_range_threshold, ensemble_nbest, exp in to_test:
ensbuilder = EnsembleBuilder(
backend=self.backend,
dataset_name="TEST",
task_type=1, # Binary Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
ensemble_nbest=ensemble_nbest,
performance_range_threshold=performance_range_threshold,
max_models_on_disc=None,
)
ensbuilder.read_preds = {
'A': {'ens_score': 1, 'num_run': 1, 0: True, 'loaded': -1, "seed": 1},
'B': {'ens_score': 2, 'num_run': 2, 0: True, 'loaded': -1, "seed": 1},
'C': {'ens_score': 3, 'num_run': 3, 0: True, 'loaded': -1, "seed": 1},
'D': {'ens_score': 4, 'num_run': 4, 0: True, 'loaded': -1, "seed": 1},
'E': {'ens_score': 5, 'num_run': 5, 0: True, 'loaded': -1, "seed": 1},
}
sel_keys = ensbuilder.get_n_best_preds()
self.assertEqual(len(sel_keys), exp)
def testEntireEnsembleBuilder(self):
ensbuilder = EnsembleBuilder(
backend=self.backend,
dataset_name="TEST",
task_type=1, # Binary Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
ensemble_nbest=2,
)
ensbuilder.SAVE2DISC = False
ensbuilder.score_ensemble_preds()
d2 = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2_0.0.npy"
)
sel_keys = ensbuilder.get_n_best_preds()
self.assertGreater(len(sel_keys), 0)
ensemble = ensbuilder.fit_ensemble(selected_keys=sel_keys)
print(ensemble, sel_keys)
n_sel_valid, n_sel_test = ensbuilder.get_valid_test_preds(
selected_keys=sel_keys)
# both valid and test prediction files are available
self.assertGreater(len(n_sel_valid), 0)
self.assertEqual(n_sel_valid, n_sel_test)
y_valid = ensbuilder.predict(
set_="valid",
ensemble=ensemble,
selected_keys=n_sel_valid,
n_preds=len(sel_keys),
index_run=1,
)
y_test = ensbuilder.predict(
set_="test",
ensemble=ensemble,
selected_keys=n_sel_test,
n_preds=len(sel_keys),
index_run=1,
)
# predictions for valid and test are the same
# --> should results in the same predictions
np.testing.assert_array_almost_equal(y_valid, y_test)
# since d2 provides perfect predictions
# it should get a higher weight
# so that y_valid should be exactly y_valid_d2
y_valid_d2 = ensbuilder.read_preds[d2][Y_VALID][:, 1]
np.testing.assert_array_almost_equal(y_valid, y_valid_d2)
def testEntireEnsembleBuilder(ensemble_backend):
ensbuilder = EnsembleBuilder(
backend=ensemble_backend,
dataset_name="TEST",
task_type=BINARY_CLASSIFICATION,
metric=roc_auc,
seed=0, # important to find the test files
ensemble_nbest=2,
)
ensbuilder.SAVE2DISC = False
ensbuilder.compute_loss_per_model()
d2 = os.path.join(
ensemble_backend.temporary_directory,
".auto-sklearn/runs/0_2_0.0/predictions_ensemble_0_2_0.0.npy")
sel_keys = ensbuilder.get_n_best_preds()
assert len(sel_keys) > 0
ensemble = ensbuilder.fit_ensemble(selected_keys=sel_keys)
print(ensemble, sel_keys)
n_sel_valid, n_sel_test = ensbuilder.get_valid_test_preds(
selected_keys=sel_keys)
# both valid and test prediction files are available
assert len(n_sel_valid) > 0
assert n_sel_valid == n_sel_test
y_valid = ensbuilder.predict(
set_="valid",
ensemble=ensemble,
selected_keys=n_sel_valid,
n_preds=len(sel_keys),
index_run=1,
)
y_test = ensbuilder.predict(
set_="test",
ensemble=ensemble,
selected_keys=n_sel_test,
n_preds=len(sel_keys),
index_run=1,
)
# predictions for valid and test are the same
# --> should results in the same predictions
np.testing.assert_array_almost_equal(y_valid, y_test)
# since d2 provides perfect predictions
# it should get a higher weight
# so that y_valid should be exactly y_valid_d2
y_valid_d2 = ensbuilder.read_preds[d2][Y_VALID][:, 1]
np.testing.assert_array_almost_equal(y_valid, y_valid_d2)
def testPerformanceRangeThresholdMaxBest(ensemble_backend,
performance_range_threshold,
ensemble_nbest, exp):
ensbuilder = EnsembleBuilder(
backend=ensemble_backend,
dataset_name="TEST",
task_type=BINARY_CLASSIFICATION,
metric=roc_auc,
seed=0, # important to find the test files
ensemble_nbest=ensemble_nbest,
performance_range_threshold=performance_range_threshold,
max_models_on_disc=None,
)
ensbuilder.read_scores = {
'A': {
'ens_score': 1,
'num_run': 1,
'loaded': -1,
"seed": 1
},
'B': {
'ens_score': 2,
'num_run': 2,
'loaded': -1,
"seed": 1
},
'C': {
'ens_score': 3,
'num_run': 3,
'loaded': -1,
"seed": 1
},
'D': {
'ens_score': 4,
'num_run': 4,
'loaded': -1,
"seed": 1
},
'E': {
'ens_score': 5,
'num_run': 5,
'loaded': -1,
"seed": 1
},
}
ensbuilder.read_preds = {
key: {key_2: True
for key_2 in (Y_ENSEMBLE, Y_VALID, Y_TEST)}
for key in ensbuilder.read_scores
}
sel_keys = ensbuilder.get_n_best_preds()
assert len(sel_keys) == exp
def testEntireEnsembleBuilder(self):
ensbuilder = EnsembleBuilder(
backend=self.backend,
dataset_name="TEST",
task_type=1, #Binary Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
ensemble_nbest=2,
)
ensbuilder.SAVE2DISC = False
ensbuilder.read_ensemble_preds()
d2 = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy"
)
sel_keys = ensbuilder.get_n_best_preds()
self.assertGreater(len(sel_keys), 0)
ensemble = ensbuilder.fit_ensemble(selected_keys=sel_keys)
print(ensemble, sel_keys)
n_sel_valid, n_sel_test = ensbuilder.get_valid_test_preds(selected_keys=sel_keys)
# both valid and test prediction files are available
self.assertGreater(len(n_sel_valid), 0)
self.assertEqual(n_sel_valid, n_sel_test)
y_valid = ensbuilder.predict(
set_="valid",
ensemble=ensemble,
selected_keys=n_sel_valid,
n_preds=len(sel_keys),
index_run=1,
)
y_test = ensbuilder.predict(
set_="test",
ensemble=ensemble,
selected_keys=n_sel_test,
n_preds=len(sel_keys),
index_run=1,
)
# predictions for valid and test are the same
# --> should results in the same predictions
np.testing.assert_array_almost_equal(y_valid, y_test)
# since d2 provides perfect predictions
# it should get a higher weight
# so that y_valid should be exactly y_valid_d2
y_valid_d2 = ensbuilder.read_preds[d2][Y_VALID][:, 1]
np.testing.assert_array_almost_equal(y_valid, y_valid_d2)
def testMaxModelsOnDisc2(ensemble_backend):
# Test for Extreme scenarios
# Make sure that the best predictions are kept
ensbuilder = EnsembleBuilder(
backend=ensemble_backend,
dataset_name="TEST",
task_type=BINARY_CLASSIFICATION,
metric=roc_auc,
seed=0, # important to find the test files
ensemble_nbest=50,
max_models_on_disc=10000.0,
)
ensbuilder.read_preds = {}
for i in range(50):
ensbuilder.read_scores['pred' + str(i)] = {
'ens_score': i * 10,
'num_run': i,
'loaded': 1,
"seed": 1,
"disc_space_cost_mb": 50 * i,
}
ensbuilder.read_preds['pred' + str(i)] = {Y_ENSEMBLE: True}
sel_keys = ensbuilder.get_n_best_preds()
assert ['pred49', 'pred48', 'pred47'] == sel_keys
# Make sure at least one model is kept alive
ensbuilder.max_models_on_disc = 0.0
sel_keys = ensbuilder.get_n_best_preds()
assert ['pred49'] == sel_keys
def testMaxModelsOnDisc(ensemble_backend, test_case, exp):
ensemble_nbest = 4
ensbuilder = EnsembleBuilder(
backend=ensemble_backend,
dataset_name="TEST",
task_type=BINARY_CLASSIFICATION,
metric=roc_auc,
seed=0, # important to find the test files
ensemble_nbest=ensemble_nbest,
max_models_on_disc=test_case,
)
with unittest.mock.patch('os.path.getsize') as mock:
mock.return_value = 100 * 1024 * 1024
ensbuilder.score_ensemble_preds()
sel_keys = ensbuilder.get_n_best_preds()
assert len(sel_keys) == exp, test_case
def _get_ensemble_process(self,
time_left_for_ensembles,
task=None,
metric=None,
precision=None,
dataset_name=None,
max_iterations=None,
ensemble_nbest=None,
ensemble_size=None):
if task is None:
task = self._task
else:
self._task = task
if metric is None:
metric = self._metric
else:
self._metric = metric
if precision is None:
precision = self.precision
else:
self.precision = precision
if dataset_name is None:
dataset_name = self._dataset_name
else:
self._dataset_name = dataset_name
if ensemble_nbest is None:
ensemble_nbest = self._ensemble_nbest
else:
self._ensemble_nbest = ensemble_nbest
if ensemble_size is None:
ensemble_size = self._ensemble_size
else:
self._ensemble_size = ensemble_size
return EnsembleBuilder(
backend=self._backend,
dataset_name=dataset_name,
task_type=task,
metric=metric,
limit=time_left_for_ensembles,
ensemble_size=ensemble_size,
ensemble_nbest=ensemble_nbest,
max_models_on_disc=self._max_models_on_disc,
seed=self._seed,
shared_mode=self._shared_mode,
precision=precision,
max_iterations=max_iterations,
read_at_most=np.inf,
memory_limit=self._ensemble_memory_limit,
random_state=self._seed,
)
def testMain(self):
ensbuilder = EnsembleBuilder(backend=self.backend,
dataset_name="TEST",
task_type=1, #Binary Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
ensemble_nbest=2,
max_iterations=1 # prevents infinite loop
)
ensbuilder.SAVE2DISC = False
ensbuilder.main()
self.assertEqual(len(ensbuilder.read_preds), 2)
self.assertIsNotNone(ensbuilder.last_hash)
self.assertIsNotNone(ensbuilder.y_true_ensemble)
def test_run_end_at(ensemble_backend):
with unittest.mock.patch('pynisher.enforce_limits') as pynisher_mock:
ensbuilder = EnsembleBuilder(
backend=ensemble_backend,
dataset_name="TEST",
task_type=MULTILABEL_CLASSIFICATION, # Multilabel Classification
metric=roc_auc,
seed=0, # important to find the test files
ensemble_nbest=2,
max_models_on_disc=None,
)
ensbuilder.SAVE2DISC = False
current_time = time.time()
ensbuilder.run(end_at=current_time + 10, iteration=1)
# 4 seconds left because: 10 seconds - 5 seconds overhead - very little overhead,
# but then rounded to an integer
assert pynisher_mock.call_args_list[0][1]["wall_time_in_s"], 4
def testNBest(ensemble_backend, ensemble_nbest, max_models_on_disc, exp):
ensbuilder = EnsembleBuilder(
backend=ensemble_backend,
dataset_name="TEST",
task_type=BINARY_CLASSIFICATION,
metric=roc_auc,
seed=0, # important to find the test files
ensemble_nbest=ensemble_nbest,
max_models_on_disc=max_models_on_disc,
)
ensbuilder.score_ensemble_preds()
sel_keys = ensbuilder.get_n_best_preds()
assert len(sel_keys) == exp
fixture = os.path.join(
ensemble_backend.temporary_directory,
".auto-sklearn/runs/0_2_0.0/predictions_ensemble_0_2_0.0.npy")
assert sel_keys[0] == fixture
def testGetValidTestPreds(self):
ensbuilder = EnsembleBuilder(backend=self.backend,
dataset_name="TEST",
task_type=1, #Binary Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
ensemble_nbest=1
)
ensbuilder.read_ensemble_preds()
d2 = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy"
)
d1 = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1.npy"
)
sel_keys = ensbuilder.get_n_best_preds()
ensbuilder.get_valid_test_preds(selected_keys=sel_keys)
# selected --> read valid and test predictions
self.assertIsNotNone(ensbuilder.read_preds[d2][Y_VALID])
self.assertIsNotNone(ensbuilder.read_preds[d2][Y_TEST])
# not selected --> should still be None
self.assertIsNone(ensbuilder.read_preds[d1][Y_VALID])
self.assertIsNone(ensbuilder.read_preds[d1][Y_TEST])
def testFallBackNBest(self):
ensbuilder = EnsembleBuilder(backend=self.backend,
dataset_name="TEST",
task_type=1, #Binary Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
ensemble_nbest=1
)
ensbuilder.read_ensemble_preds()
filename = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy"
)
ensbuilder.read_preds[filename]["ens_score"] = -1
filename = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1.npy"
)
ensbuilder.read_preds[filename]["ens_score"] = -1
sel_keys = ensbuilder.get_n_best_preds()
fixture = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1.npy"
)
self.assertEquals(sel_keys[0], fixture)
def testGetValidTestPreds(self):
ensbuilder = EnsembleBuilder(
backend=self.backend,
dataset_name="TEST",
task_type=1, # Binary Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
ensemble_nbest=1)
ensbuilder.score_ensemble_preds()
d1 = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1_0.0.npy"
)
d2 = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2_0.0.npy"
)
d3 = os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_3_100.0.npy"
)
sel_keys = ensbuilder.get_n_best_preds()
self.assertEqual(len(sel_keys), 1)
ensbuilder.get_valid_test_preds(selected_keys=sel_keys)
# Number of read files should be three and
# predictions_ensemble_0_4_0.0.npy must not be in there
self.assertEqual(len(ensbuilder.read_preds), 3)
self.assertNotIn(
os.path.join(
self.backend.temporary_directory,
".auto-sklearn/predictions_ensemble/predictions_ensemble_0_4_0.0.npy"
), ensbuilder.read_preds)
# not selected --> should still be None
self.assertIsNone(ensbuilder.read_preds[d1][Y_VALID])
self.assertIsNone(ensbuilder.read_preds[d1][Y_TEST])
self.assertIsNone(ensbuilder.read_preds[d3][Y_VALID])
self.assertIsNone(ensbuilder.read_preds[d3][Y_TEST])
# selected --> read valid and test predictions
self.assertIsNotNone(ensbuilder.read_preds[d2][Y_VALID])
self.assertIsNotNone(ensbuilder.read_preds[d2][Y_TEST])
def testGetValidTestPreds(ensemble_backend):
ensbuilder = EnsembleBuilder(
backend=ensemble_backend,
dataset_name="TEST",
task_type=BINARY_CLASSIFICATION,
metric=roc_auc,
seed=0, # important to find the test files
ensemble_nbest=1)
ensbuilder.compute_loss_per_model()
# d1 is a dummt prediction. d2 and d3 have the same prediction with
# different name. num_run=2 is selected when doing sorted()
d1 = os.path.join(
ensemble_backend.temporary_directory,
".auto-sklearn/runs/0_1_0.0/predictions_ensemble_0_1_0.0.npy")
d2 = os.path.join(
ensemble_backend.temporary_directory,
".auto-sklearn/runs/0_2_0.0/predictions_ensemble_0_2_0.0.npy")
d3 = os.path.join(
ensemble_backend.temporary_directory,
".auto-sklearn/runs/0_3_100.0/predictions_ensemble_0_3_100.0.npy")
sel_keys = ensbuilder.get_n_best_preds()
assert len(sel_keys) == 1
ensbuilder.get_valid_test_preds(selected_keys=sel_keys)
# Number of read files should be three and
# predictions_ensemble_0_4_0.0.npy must not be in there
assert len(ensbuilder.read_preds) == 3
assert os.path.join(
ensemble_backend.temporary_directory,
".auto-sklearn/runs/0_4_0.0/predictions_ensemble_0_4_0.0.npy"
) not in ensbuilder.read_preds
# not selected --> should still be None
assert ensbuilder.read_preds[d1][Y_VALID] is None
assert ensbuilder.read_preds[d1][Y_TEST] is None
assert ensbuilder.read_preds[d3][Y_VALID] is None
assert ensbuilder.read_preds[d3][Y_TEST] is None
# selected --> read valid and test predictions
assert ensbuilder.read_preds[d2][Y_VALID] is not None
assert ensbuilder.read_preds[d2][Y_TEST] is not None
def testMain(self):
ensbuilder = EnsembleBuilder(
backend=self.backend,
dataset_name="TEST",
task_type=3, # Multilabel Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
ensemble_nbest=2,
max_iterations=1, # prevents infinite loop
max_models_on_disc=None,
)
ensbuilder.SAVE2DISC = False
ensbuilder.main()
self.assertEqual(len(ensbuilder.read_preds), 3)
self.assertIsNotNone(ensbuilder.last_hash)
self.assertIsNotNone(ensbuilder.y_true_ensemble)
# Make sure the run history is ok
run_history = ensbuilder.get_ensemble_history()
# We expect 1 element to be the ensemble
self.assertEqual(len(run_history), 1)
# As the data loader loads the same val/train/test
# we expect 1.0 as score and all keys available
expected_performance = {
'ensemble_val_score': 1.0,
'ensemble_test_score': 1.0,
'ensemble_optimization_score': 1.0,
}
self.assertDictContainsSubset(expected_performance, run_history[0])
self.assertIn('Timestamp', run_history[0])
self.assertIsInstance(run_history[0]['Timestamp'], pd.Timestamp)
def main(task_id, ensemble_dir, performance_range_threshold, ensemble_size,
max_keep_best, seed, only_portfolio_runs, call_from_cmd):
if max_keep_best > 1:
assert max_keep_best == int(max_keep_best)
max_keep_best = int(max_keep_best)
memory_limit = 4000
precision = 32
metric = make_scorer('balanced_accuracy_fast', BalancedAccuracy())
if not os.path.exists(ensemble_dir):
raise NotADirectoryError("%s does not exist")
if call_from_cmd:
assert str(task_id) in ensemble_dir
fl_name = "ensemble_results_%fthresh_%dsize_%fbest" % \
(performance_range_threshold, ensemble_size, max_keep_best)
if only_portfolio_runs:
fl_name += "_only_portfolio"
fl_name = os.path.join(ensemble_dir, fl_name)
if os.path.isfile(fl_name):
raise ValueError("Nothing left to do, %s already exists" % fl_name)
# figure out how many prediction files are in dir
if call_from_cmd:
pred_dir = os.path.join(ensemble_dir, "auto-sklearn-output",
".auto-sklearn", "predictions_ensemble")
n_models = glob.glob(pred_dir +
"/predictions_ensemble_%d_*.npy.gz" % seed)
else:
pred_dir = os.path.join(ensemble_dir, ".auto-sklearn",
"predictions_ensemble")
n_models = glob.glob(pred_dir +
"/predictions_ensemble_%d_*.npy" % seed)
n_models.sort(key=lambda x: int(float(x.split("_")[-2])))
print("\n".join(n_models))
print("Found %d ensemble predictions" % len(n_models))
if len(n_models) == 0:
raise ValueError("%s has no ensemble predictions" % pred_dir)
# Get start time of ensemble building: 1) load json 2) find key 3) get creation times
if call_from_cmd:
timestamps_fl = os.path.join(ensemble_dir, "auto-sklearn-output",
"timestamps.json")
else:
timestamps_fl = os.path.join(ensemble_dir, "timestamps.json")
with open(timestamps_fl, "r") as fh:
timestamps = json.load(fh)
model_timestamps = None
overall_start_time = None
for k in timestamps:
if "predictions_ensemble" in k:
model_timestamps = timestamps[k]
if "start_time_%d" % seed in timestamps[k]:
overall_start_time = timestamps[k]["start_time_%d" % seed]
timestamp_keys = list(model_timestamps.keys())
for timestamp_key in timestamp_keys:
if timestamp_key.endswith(
'lock') or 'predictions_ensemble' not in timestamp_key:
del model_timestamps[timestamp_key]
assert model_timestamps is not None and overall_start_time is not None
assert len(model_timestamps) == len(n_models), (len(model_timestamps),
len(n_models))
# Get overall timelimit
vanilla_results_fl = os.path.join(ensemble_dir, "result.json")
with open(vanilla_results_fl, "r") as fh:
vanilla_results = json.load(fh)
# If only portfolio configurations, read runhistory
if only_portfolio_runs:
if call_from_cmd:
runhistory_fl = os.path.join(ensemble_dir, "auto-sklearn-output",
"smac3-output", "run*",
"runhistory.json")
else:
runhistory_fl = os.path.join(ensemble_dir, "smac3-output", "run*",
"runhistory.json")
runhistory_fl = glob.glob(runhistory_fl)
assert len(runhistory_fl) == 1
with open(runhistory_fl[0], "r") as fh:
runhistory = json.load(fh)
init_design_num_runs = []
for i in runhistory["data"]:
if i[1][3]["configuration_origin"] == "Initial design":
if "error" in i[1][3]:
continue
init_design_num_runs.append(i[1][3]["num_run"])
print("Portfolio stopped after %s runs" % str(init_design_num_runs))
last_run = max(init_design_num_runs)
print("Cut down to only portfolio runs fom %d" % len(n_models))
for i, n in enumerate(n_models):
if int(float(n.split("_")[-2])) > last_run:
n_models = n_models[:i]
break
print("... to %d" % len(n_models))
# load data
X_train, y_train, X_test, y_test, cat = load_task(task_id)
if len(np.unique(y_test)) == 2:
task_type = BINARY_CLASSIFICATION
elif len(np.unique(y_test)) > 2:
task_type = MULTICLASS_CLASSIFICATION
else:
raise ValueError("Unknown task type for task %d" % task_id)
tmp_dir = tempfile.TemporaryDirectory()
loss_trajectory = []
# Construct ensemble builder
context = BackendContextMock(
temporary_directory=(ensemble_dir + "/auto-sklearn-output/"
if call_from_cmd else ensemble_dir),
output_directory=tmp_dir.name,
delete_tmp_folder_after_terminate=False,
delete_output_folder_after_terminate=False,
shared_mode=False)
backend = Backend(context)
ens_builder = EnsembleBuilder(
backend=backend,
dataset_name=str(task_id),
task_type=task_type,
metric=metric,
limit=np.inf,
ensemble_size=ensemble_size,
ensemble_nbest=max_keep_best,
performance_range_threshold=performance_range_threshold,
max_models_on_disc=None,
seed=seed,
shared_mode=False,
precision=precision,
max_iterations=1,
read_at_most=1,
memory_limit=memory_limit,
random_state=1,
sleep_duration=0)
try:
# iterate over all models, take construction time into account when creating new trajectory
current_ensemble_timestamp = 0
skipped = 1
for midx, model_path in enumerate(n_models):
tstamp = model_timestamps[model_path.split("/")[-1].replace(
'.gz', '')] - overall_start_time
if current_ensemble_timestamp > tstamp:
# while this model was built, the ensemble script was not yet done
skipped += 1
continue
# Do one ensemble building step
start = time.time()
ens_builder.random_state = check_random_state(1)
print("############## %d: Working on %s (skipped %d)" %
(midx + 1, model_path, skipped - 1))
logging.basicConfig(level=logging.DEBUG)
ens_builder.read_at_most = skipped
valid_pred, test_pred = ens_builder.main(return_pred=True)
last_dur = time.time() - start
current_ensemble_timestamp = tstamp + last_dur
if current_ensemble_timestamp >= vanilla_results["0"]["time_limit"]:
print("############## Went over time %f > %f; Stop here" %
(current_ensemble_timestamp,
vanilla_results["0"]["time_limit"]))
break
# Reset, since we have just read model files
skipped = 1
if test_pred is None:
# Adding this model did not change the ensemble, no new prediction
continue
if task_type == BINARY_CLASSIFICATION:
# Recreate nx2 array
test_pred = np.concatenate([
1 - test_pred.reshape([-1, 1]),
test_pred.reshape([-1, 1])
],
axis=1)
# Build trajectory entry
score = 1 - balanced_accuracy(y_true=y_test, y_pred=test_pred)
loss_trajectory.append((current_ensemble_timestamp, score))
print("############## Round %d took %g sec" %
(midx, time.time() - start))
except:
raise
finally:
tmp_dir.cleanup()
# Store results
result = dict()
result[ensemble_size] = {
'task_id': task_id,
'time_limit': vanilla_results["0"]["time_limit"],
'loss': loss_trajectory[-1][1],
'configuration': {
"n_models": n_models,
"performance_range_threshold": performance_range_threshold,
"ensemble_size": ensemble_size,
"max_keep_best": max_keep_best,
"seed": seed,
"memory_limit": memory_limit,
"precision": precision,
},
'n_models': len(n_models),
'trajectory': loss_trajectory,
}
with open(fl_name, 'wt') as fh:
json.dump(result, fh, indent=4)
print("Dumped to %s" % fl_name)
def testMaxModelsOnDisc(self):
ensemble_nbest = 4
for (test_case, exp) in [
# If None, no reduction
(None, 2),
# If Int, limit only on exceed
(4, 2),
(1, 1),
# If Float, translate float to # models.
# below, mock of each file is 100 Mb and
# 4 files .model and .npy (test/val/pred) exist
(700.0, 1),
(800.0, 2),
(9999.0, 2),
]:
ensbuilder = EnsembleBuilder(
backend=self.backend,
dataset_name="TEST",
task_type=1, # Binary Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
ensemble_nbest=ensemble_nbest,
max_models_on_disc=test_case,
)
with unittest.mock.patch('os.path.getsize') as mock:
mock.return_value = 100 * 1024 * 1024
ensbuilder.score_ensemble_preds()
sel_keys = ensbuilder.get_n_best_preds()
self.assertEqual(len(sel_keys), exp)
# Test for Extreme scenarios
# Make sure that the best predictions are kept
ensbuilder = EnsembleBuilder(
backend=self.backend,
dataset_name="TEST",
task_type=1, # Binary Classification
metric=roc_auc,
limit=-1, # not used,
seed=0, # important to find the test files
ensemble_nbest=50,
max_models_on_disc=10000.0,
)
ensbuilder.read_preds = {}
for i in range(50):
ensbuilder.read_preds['pred' + str(i)] = {
'ens_score': i * 10,
'num_run': i,
0: True,
'loaded': 1,
"seed": 1,
"disc_space_cost_mb": 50 * i,
}
sel_keys = ensbuilder.get_n_best_preds()
self.assertListEqual(['pred49', 'pred48', 'pred47', 'pred46'],
sel_keys)
# Make sure at least one model is kept alive
ensbuilder.max_models_on_disc = 0.0
sel_keys = ensbuilder.get_n_best_preds()
self.assertListEqual(['pred49'], sel_keys)
