def test_fit_pSMAC(self):
output = os.path.join(self.test_dir, '..', '.tmp_estimator_fit_pSMAC')
self._setUp(output)
X_train, Y_train, X_test, Y_test = putil.get_dataset('iris')
automl = AutoSklearnClassifier(time_left_for_this_task=15,
per_run_time_limit=15,
output_folder=output,
tmp_folder=output,
shared_mode=True,
seed=1,
initial_configurations_via_metalearning=0,
ensemble_size=0)
automl.fit(X_train, Y_train)
# Create a 'dummy model' for the first run, which has an accuracy of
# more than 99%; it should be in the final ensemble if the ensemble
# building of the second AutoSklearn classifier works correct
true_targets_ensemble_path = os.path.join(output, '.auto-sklearn',
'true_targets_ensemble.npy')
true_targets_ensemble = np.load(true_targets_ensemble_path)
true_targets_ensemble[-1] = 1 if true_targets_ensemble[-1] != 1 else 0
probas = np.zeros((len(true_targets_ensemble), 3), dtype=float)
for i, value in enumerate(true_targets_ensemble):
probas[i, value] = 1.0
dummy_predictions_path = os.path.join(output, '.auto-sklearn',
'predictions_ensemble',
'predictions_ensemble_1_00030.npy')
with open(dummy_predictions_path, 'wb') as fh:
np.save(fh, probas)
probas_test = np.zeros((len(Y_test), 3), dtype=float)
for i, value in enumerate(Y_test):
probas_test[i, value] = 1.0
dummy = ArrayReturningDummyPredictor(probas_test)
backend = Backend(output, output)
backend.save_model(dummy, 30, 1)
automl = AutoSklearnClassifier(time_left_for_this_task=15,
per_run_time_limit=15,
output_folder=output,
tmp_folder=output,
shared_mode=True,
seed=2,
initial_configurations_via_metalearning=0,
ensemble_size=0)
automl.fit(X_train, Y_train)
automl.run_ensemble_builder(0, 1, 50).wait()
score = automl.score(X_test, Y_test)
self.assertEqual(len(os.listdir(os.path.join(output, '.auto-sklearn',
'ensembles'))), 1)
self.assertGreaterEqual(score, 0.90)
self.assertEqual(automl._task, MULTICLASS_CLASSIFICATION)
del automl
self._tearDown(output)
def test_fit_pSMAC(self):
output = os.path.join(self.test_dir, '..', '.tmp_estimator_fit_pSMAC')
self._setUp(output)
X_train, Y_train, X_test, Y_test = putil.get_dataset('iris')
automl = AutoSklearnClassifier(time_left_for_this_task=15,
per_run_time_limit=15,
output_folder=output,
tmp_folder=output,
shared_mode=True,
seed=1,
initial_configurations_via_metalearning=0,
ensemble_size=0)
automl.fit(X_train, Y_train)
# Create a 'dummy model' for the first run, which has an accuracy of
# more than 99%; it should be in the final ensemble if the ensemble
# building of the second AutoSklearn classifier works correct
true_targets_ensemble_path = os.path.join(output, '.auto-sklearn',
'true_targets_ensemble.npy')
true_targets_ensemble = np.load(true_targets_ensemble_path)
true_targets_ensemble[-1] = 1 if true_targets_ensemble[-1] != 1 else 0
probas = np.zeros((len(true_targets_ensemble), 3), dtype=float)
for i, value in enumerate(true_targets_ensemble):
probas[i, value] = 1.0
dummy_predictions_path = os.path.join(output, '.auto-sklearn',
'predictions_ensemble',
'predictions_ensemble_1_00030.npy')
with open(dummy_predictions_path, 'wb') as fh:
np.save(fh, probas)
probas_test = np.zeros((len(Y_test), 3), dtype=float)
for i, value in enumerate(Y_test):
probas_test[i, value] = 1.0
dummy = ArrayReturningDummyPredictor(probas_test)
backend = Backend(output, output)
backend.save_model(dummy, 30, 1)
automl = AutoSklearnClassifier(time_left_for_this_task=15,
per_run_time_limit=15,
output_folder=output,
tmp_folder=output,
shared_mode=True,
seed=2,
initial_configurations_via_metalearning=0,
ensemble_size=0)
automl.fit(X_train, Y_train)
automl.run_ensemble_builder(0, 1, 50).wait()
score = automl.score(X_test, Y_test)
self.assertEqual(len(os.listdir(os.path.join(output, '.auto-sklearn',
'ensemble_indices'))), 1)
self.assertGreaterEqual(score, 0.90)
self.assertEqual(automl._task, MULTICLASS_CLASSIFICATION)
del automl
self._tearDown(output)
def test_file_output(self):
shutil.rmtree(self.working_directory, ignore_errors=True)
os.mkdir(self.working_directory)
queue_mock = unittest.mock.Mock()
context = BackendContext(
temporary_directory=os.path.join(self.working_directory, 'tmp'),
output_directory=os.path.join(self.working_directory, 'out'),
delete_tmp_folder_after_terminate=True,
delete_output_folder_after_terminate=True,
)
with unittest.mock.patch.object(
Backend, 'load_datamanager') as load_datamanager_mock:
load_datamanager_mock.return_value = get_multiclass_classification_datamanager(
)
backend = Backend(context)
ae = AbstractEvaluator(
backend=backend,
output_y_hat_optimization=False,
queue=queue_mock,
metric=accuracy,
port=self.port,
)
ae.model = sklearn.dummy.DummyClassifier()
rs = np.random.RandomState()
ae.Y_optimization = rs.rand(33, 3)
predictions_ensemble = rs.rand(33, 3)
predictions_test = rs.rand(25, 3)
predictions_valid = rs.rand(25, 3)
ae.file_output(
Y_optimization_pred=predictions_ensemble,
Y_valid_pred=predictions_valid,
Y_test_pred=predictions_test,
)
self.assertTrue(
os.path.exists(
os.path.join(self.working_directory, 'tmp',
'.auto-sklearn', 'runs', '1_0_None')))
shutil.rmtree(self.working_directory, ignore_errors=True)
def test_refit_shuffle_on_fail(self):
output = os.path.join(self.test_dir, '..', '.tmp_refit_shuffle_on_fail')
context = BackendContext(output, output, False, False)
backend = Backend(context)
failing_model = unittest.mock.Mock()
failing_model.fit.side_effect = [ValueError(), ValueError(), None]
auto = AutoML(backend, 20, 5)
ensemble_mock = unittest.mock.Mock()
auto.ensemble_ = ensemble_mock
ensemble_mock.get_model_identifiers.return_value = [1]
auto.models_ = {1: failing_model}
X = np.array([1, 2, 3])
y = np.array([1, 2, 3])
auto.refit(X, y)
self.assertEqual(failing_model.fit.call_count, 3)
def test_add_additional_components(self):
shutil.rmtree(self.working_directory, ignore_errors=True)
os.mkdir(self.working_directory)
queue_mock = unittest.mock.Mock()
context = BackendContext(
temporary_directory=os.path.join(self.working_directory, 'tmp'),
delete_tmp_folder_after_terminate=True,
)
with unittest.mock.patch.object(
Backend, 'load_datamanager') as load_datamanager_mock:
load_datamanager_mock.return_value = get_multiclass_classification_datamanager(
)
backend = Backend(context)
with unittest.mock.patch.object(_addons['classification'],
'add_component') as _:
# If the components in the argument `additional_components` are an empty dict
# there is no call to `add_component`, if there's something in it, `add_component
# is called (2nd case)
for fixture, case in ((0, dict()), (1, dict(abc='def'))):
thirdparty_components_patch = unittest.mock.Mock()
thirdparty_components_patch.components = case
additional_components = dict(
classification=thirdparty_components_patch)
AbstractEvaluator(
backend=backend,
output_y_hat_optimization=False,
queue=queue_mock,
metric=accuracy,
port=self.port,
additional_components=additional_components,
)
self.assertEqual(
_addons['classification'].add_component.call_count,
fixture)
def test_fit_pSMAC(self):
tmp = os.path.join(self.test_dir, '..', '.tmp_estimator_fit_pSMAC')
output = os.path.join(self.test_dir, '..', '.out_estimator_fit_pSMAC')
self._setUp(tmp)
self._setUp(output)
X_train, Y_train, X_test, Y_test = putil.get_dataset('digits')
# test parallel Classifier to predict classes, not only indexes
Y_train += 1
Y_test += 1
automl = AutoSklearnClassifier(
time_left_for_this_task=20,
per_run_time_limit=5,
output_folder=output,
tmp_folder=tmp,
shared_mode=True,
seed=1,
initial_configurations_via_metalearning=0,
ensemble_size=0,
)
automl.fit(X_train, Y_train)
# Create a 'dummy model' for the first run, which has an accuracy of
# more than 99%; it should be in the final ensemble if the ensemble
# building of the second AutoSklearn classifier works correct
true_targets_ensemble_path = os.path.join(tmp, '.auto-sklearn',
'true_targets_ensemble.npy')
with open(true_targets_ensemble_path, 'rb') as fh:
true_targets_ensemble = np.load(fh)
true_targets_ensemble[-1] = 1 if true_targets_ensemble[-1] != 1 else 0
true_targets_ensemble = true_targets_ensemble.astype(int)
probas = np.zeros((len(true_targets_ensemble), 10), dtype=float)
for i, value in enumerate(true_targets_ensemble):
probas[i, value] = 1.0
dummy_predictions_path = os.path.join(
tmp,
'.auto-sklearn',
'predictions_ensemble',
'predictions_ensemble_1_00030.npy',
)
with open(dummy_predictions_path, 'wb') as fh:
np.save(fh, probas)
probas_test = np.zeros((len(Y_test), 10), dtype=float)
for i, value in enumerate(Y_test):
probas_test[i, value - 1] = 1.0
dummy = ArrayReturningDummyPredictor(probas_test)
context = BackendContext(tmp, output, False, False, True)
backend = Backend(context)
backend.save_model(dummy, 30, 1)
automl = AutoSklearnClassifier(
time_left_for_this_task=20,
per_run_time_limit=5,
output_folder=output,
tmp_folder=tmp,
shared_mode=True,
seed=2,
initial_configurations_via_metalearning=0,
ensemble_size=0,
)
automl.fit_ensemble(
Y_train,
task=MULTICLASS_CLASSIFICATION,
metric=accuracy,
precision='32',
dataset_name='iris',
ensemble_size=20,
ensemble_nbest=50,
)
predictions = automl.predict(X_test)
score = sklearn.metrics.accuracy_score(Y_test, predictions)
self.assertEqual(
len(os.listdir(os.path.join(tmp, '.auto-sklearn', 'ensembles'))),
1)
self.assertGreaterEqual(score, 0.90)
self.assertEqual(automl._automl._task, MULTICLASS_CLASSIFICATION)
models = automl._automl.models_
classifier_types = [type(c) for c in models.values()]
self.assertIn(ArrayReturningDummyPredictor, classifier_types)
del automl
self._tearDown(tmp)
self._tearDown(output)
def main(task_id, ensemble_dir, performance_range_threshold, ensemble_size,
max_keep_best, seed, only_portfolio_runs, call_from_cmd):
if max_keep_best > 1:
assert max_keep_best == int(max_keep_best)
max_keep_best = int(max_keep_best)
memory_limit = 4000
precision = 32
metric = make_scorer('balanced_accuracy_fast', BalancedAccuracy())
if not os.path.exists(ensemble_dir):
raise NotADirectoryError("%s does not exist")
if call_from_cmd:
assert str(task_id) in ensemble_dir
fl_name = "ensemble_results_%fthresh_%dsize_%fbest" % \
(performance_range_threshold, ensemble_size, max_keep_best)
if only_portfolio_runs:
fl_name += "_only_portfolio"
fl_name = os.path.join(ensemble_dir, fl_name)
if os.path.isfile(fl_name):
raise ValueError("Nothing left to do, %s already exists" % fl_name)
# figure out how many prediction files are in dir
if call_from_cmd:
pred_dir = os.path.join(ensemble_dir, "auto-sklearn-output",
".auto-sklearn", "predictions_ensemble")
n_models = glob.glob(pred_dir +
"/predictions_ensemble_%d_*.npy.gz" % seed)
else:
pred_dir = os.path.join(ensemble_dir, ".auto-sklearn",
"predictions_ensemble")
n_models = glob.glob(pred_dir +
"/predictions_ensemble_%d_*.npy" % seed)
n_models.sort(key=lambda x: int(float(x.split("_")[-2])))
print("\n".join(n_models))
print("Found %d ensemble predictions" % len(n_models))
if len(n_models) == 0:
raise ValueError("%s has no ensemble predictions" % pred_dir)
# Get start time of ensemble building: 1) load json 2) find key 3) get creation times
if call_from_cmd:
timestamps_fl = os.path.join(ensemble_dir, "auto-sklearn-output",
"timestamps.json")
else:
timestamps_fl = os.path.join(ensemble_dir, "timestamps.json")
with open(timestamps_fl, "r") as fh:
timestamps = json.load(fh)
model_timestamps = None
overall_start_time = None
for k in timestamps:
if "predictions_ensemble" in k:
model_timestamps = timestamps[k]
if "start_time_%d" % seed in timestamps[k]:
overall_start_time = timestamps[k]["start_time_%d" % seed]
timestamp_keys = list(model_timestamps.keys())
for timestamp_key in timestamp_keys:
if timestamp_key.endswith(
'lock') or 'predictions_ensemble' not in timestamp_key:
del model_timestamps[timestamp_key]
assert model_timestamps is not None and overall_start_time is not None
assert len(model_timestamps) == len(n_models), (len(model_timestamps),
len(n_models))
# Get overall timelimit
vanilla_results_fl = os.path.join(ensemble_dir, "result.json")
with open(vanilla_results_fl, "r") as fh:
vanilla_results = json.load(fh)
# If only portfolio configurations, read runhistory
if only_portfolio_runs:
if call_from_cmd:
runhistory_fl = os.path.join(ensemble_dir, "auto-sklearn-output",
"smac3-output", "run*",
"runhistory.json")
else:
runhistory_fl = os.path.join(ensemble_dir, "smac3-output", "run*",
"runhistory.json")
runhistory_fl = glob.glob(runhistory_fl)
assert len(runhistory_fl) == 1
with open(runhistory_fl[0], "r") as fh:
runhistory = json.load(fh)
init_design_num_runs = []
for i in runhistory["data"]:
if i[1][3]["configuration_origin"] == "Initial design":
if "error" in i[1][3]:
continue
init_design_num_runs.append(i[1][3]["num_run"])
print("Portfolio stopped after %s runs" % str(init_design_num_runs))
last_run = max(init_design_num_runs)
print("Cut down to only portfolio runs fom %d" % len(n_models))
for i, n in enumerate(n_models):
if int(float(n.split("_")[-2])) > last_run:
n_models = n_models[:i]
break
print("... to %d" % len(n_models))
# load data
X_train, y_train, X_test, y_test, cat = load_task(task_id)
if len(np.unique(y_test)) == 2:
task_type = BINARY_CLASSIFICATION
elif len(np.unique(y_test)) > 2:
task_type = MULTICLASS_CLASSIFICATION
else:
raise ValueError("Unknown task type for task %d" % task_id)
tmp_dir = tempfile.TemporaryDirectory()
loss_trajectory = []
# Construct ensemble builder
context = BackendContextMock(
temporary_directory=(ensemble_dir + "/auto-sklearn-output/"
if call_from_cmd else ensemble_dir),
output_directory=tmp_dir.name,
delete_tmp_folder_after_terminate=False,
delete_output_folder_after_terminate=False,
shared_mode=False)
backend = Backend(context)
ens_builder = EnsembleBuilder(
backend=backend,
dataset_name=str(task_id),
task_type=task_type,
metric=metric,
limit=np.inf,
ensemble_size=ensemble_size,
ensemble_nbest=max_keep_best,
performance_range_threshold=performance_range_threshold,
max_models_on_disc=None,
seed=seed,
shared_mode=False,
precision=precision,
max_iterations=1,
read_at_most=1,
memory_limit=memory_limit,
random_state=1,
sleep_duration=0)
try:
# iterate over all models, take construction time into account when creating new trajectory
current_ensemble_timestamp = 0
skipped = 1
for midx, model_path in enumerate(n_models):
tstamp = model_timestamps[model_path.split("/")[-1].replace(
'.gz', '')] - overall_start_time
if current_ensemble_timestamp > tstamp:
# while this model was built, the ensemble script was not yet done
skipped += 1
continue
# Do one ensemble building step
start = time.time()
ens_builder.random_state = check_random_state(1)
print("############## %d: Working on %s (skipped %d)" %
(midx + 1, model_path, skipped - 1))
logging.basicConfig(level=logging.DEBUG)
ens_builder.read_at_most = skipped
valid_pred, test_pred = ens_builder.main(return_pred=True)
last_dur = time.time() - start
current_ensemble_timestamp = tstamp + last_dur
if current_ensemble_timestamp >= vanilla_results["0"]["time_limit"]:
print("############## Went over time %f > %f; Stop here" %
(current_ensemble_timestamp,
vanilla_results["0"]["time_limit"]))
break
# Reset, since we have just read model files
skipped = 1
if test_pred is None:
# Adding this model did not change the ensemble, no new prediction
continue
if task_type == BINARY_CLASSIFICATION:
# Recreate nx2 array
test_pred = np.concatenate([
1 - test_pred.reshape([-1, 1]),
test_pred.reshape([-1, 1])
],
axis=1)
# Build trajectory entry
score = 1 - balanced_accuracy(y_true=y_test, y_pred=test_pred)
loss_trajectory.append((current_ensemble_timestamp, score))
print("############## Round %d took %g sec" %
(midx, time.time() - start))
except:
raise
finally:
tmp_dir.cleanup()
# Store results
result = dict()
result[ensemble_size] = {
'task_id': task_id,
'time_limit': vanilla_results["0"]["time_limit"],
'loss': loss_trajectory[-1][1],
'configuration': {
"n_models": n_models,
"performance_range_threshold": performance_range_threshold,
"ensemble_size": ensemble_size,
"max_keep_best": max_keep_best,
"seed": seed,
"memory_limit": memory_limit,
"precision": precision,
},
'n_models': len(n_models),
'trajectory': loss_trajectory,
}
with open(fl_name, 'wt') as fh:
json.dump(result, fh, indent=4)
print("Dumped to %s" % fl_name)
def test_fit_pSMAC(self):
tmp = os.path.join(self.test_dir, '..', '.tmp_estimator_fit_pSMAC')
output = os.path.join(self.test_dir, '..', '.out_estimator_fit_pSMAC')
self._setUp(tmp)
self._setUp(output)
X_train, Y_train, X_test, Y_test = putil.get_dataset('breast_cancer')
# test parallel Classifier to predict classes, not only indices
Y_train += 1
Y_test += 1
automl = AutoSklearnClassifier(
time_left_for_this_task=30,
per_run_time_limit=5,
output_folder=output,
tmp_folder=tmp,
shared_mode=True,
seed=1,
initial_configurations_via_metalearning=0,
ensemble_size=0,
)
automl.fit(X_train, Y_train)
n_models_fit = len(automl.cv_results_['mean_test_score'])
cv_results = automl.cv_results_['mean_test_score']
automl = AutoSklearnClassifier(
time_left_for_this_task=30,
per_run_time_limit=5,
output_folder=output,
tmp_folder=tmp,
shared_mode=True,
seed=2,
initial_configurations_via_metalearning=0,
ensemble_size=0,
)
automl.fit(X_train, Y_train)
n_models_fit_2 = len(automl.cv_results_['mean_test_score'])
# Check that the results from the first run were actually read by the
# second run
self.assertGreater(n_models_fit_2, n_models_fit)
for score in cv_results:
self.assertIn(
score,
automl.cv_results_['mean_test_score'],
msg=str((automl.cv_results_['mean_test_score'], cv_results)),
)
# Create a 'dummy model' for the first run, which has an accuracy of
# more than 99%; it should be in the final ensemble if the ensemble
# building of the second AutoSklearn classifier works correct
true_targets_ensemble_path = os.path.join(tmp, '.auto-sklearn',
'true_targets_ensemble.npy')
with open(true_targets_ensemble_path, 'rb') as fh:
true_targets_ensemble = np.load(fh, allow_pickle=True)
true_targets_ensemble[-1] = 1 if true_targets_ensemble[-1] != 1 else 0
true_targets_ensemble = true_targets_ensemble.astype(int)
probas = np.zeros((len(true_targets_ensemble), 2), dtype=float)
for i, value in enumerate(true_targets_ensemble):
probas[i, value] = 1.0
dummy_predictions_path = os.path.join(
tmp,
'.auto-sklearn',
'predictions_ensemble',
'predictions_ensemble_0_999_0.0.npy',
)
with open(dummy_predictions_path, 'wb') as fh:
np.save(fh, probas)
probas_test = np.zeros((len(Y_test), 2), dtype=float)
for i, value in enumerate(Y_test):
probas_test[i, value - 1] = 1.0
dummy = ArrayReturningDummyPredictor(probas_test)
context = BackendContext(tmp, output, False, False, True)
backend = Backend(context)
model_path = backend.get_model_path(seed=0, idx=999, budget=0.0)
backend.save_model(model=dummy, filepath=model_path)
automl = AutoSklearnClassifier(
time_left_for_this_task=30,
per_run_time_limit=5,
output_folder=output,
tmp_folder=tmp,
shared_mode=True,
seed=3,
initial_configurations_via_metalearning=0,
ensemble_size=0,
metric=accuracy,
)
automl.fit_ensemble(Y_train, task=BINARY_CLASSIFICATION,
precision='32',
dataset_name='breast_cancer',
ensemble_size=20,
ensemble_nbest=50,
)
predictions = automl.predict(X_test)
score = sklearn.metrics.accuracy_score(Y_test, predictions)
self.assertEqual(len(os.listdir(os.path.join(tmp, '.auto-sklearn',
'ensembles'))), 1)
self.assertGreaterEqual(score, 0.90)
self.assertEqual(automl._automl[0]._task, BINARY_CLASSIFICATION)
models = automl._automl[0].models_
classifier_types = [type(c) for c in models.values()]
self.assertIn(ArrayReturningDummyPredictor, classifier_types)
del automl
self._tearDown(tmp)
self._tearDown(output)
def test_fit_pSMAC(self):
tmp = os.path.join(self.test_dir, '..', '.tmp_estimator_fit_pSMAC')
output = os.path.join(self.test_dir, '..', '.out_estimator_fit_pSMAC')
self._setUp(tmp)
self._setUp(output)
X_train, Y_train, X_test, Y_test = putil.get_dataset('digits')
# test parallel Classifier to predict classes, not only indexes
Y_train += 1
Y_test += 1
automl = AutoSklearnClassifier(
time_left_for_this_task=20,
per_run_time_limit=5,
output_folder=output,
tmp_folder=tmp,
shared_mode=True,
seed=1,
initial_configurations_via_metalearning=0,
ensemble_size=0,
)
automl.fit(X_train, Y_train)
# Create a 'dummy model' for the first run, which has an accuracy of
# more than 99%; it should be in the final ensemble if the ensemble
# building of the second AutoSklearn classifier works correct
true_targets_ensemble_path = os.path.join(tmp, '.auto-sklearn',
'true_targets_ensemble.npy')
with open(true_targets_ensemble_path, 'rb') as fh:
true_targets_ensemble = np.load(fh)
true_targets_ensemble[-1] = 1 if true_targets_ensemble[-1] != 1 else 0
true_targets_ensemble = true_targets_ensemble.astype(int)
probas = np.zeros((len(true_targets_ensemble), 10), dtype=float)
for i, value in enumerate(true_targets_ensemble):
probas[i, value] = 1.0
dummy_predictions_path = os.path.join(
tmp,
'.auto-sklearn',
'predictions_ensemble',
'predictions_ensemble_1_00030.npy',
)
with open(dummy_predictions_path, 'wb') as fh:
np.save(fh, probas)
probas_test = np.zeros((len(Y_test), 10), dtype=float)
for i, value in enumerate(Y_test):
probas_test[i, value - 1] = 1.0
dummy = ArrayReturningDummyPredictor(probas_test)
context = BackendContext(tmp, output, False, False, True)
backend = Backend(context)
backend.save_model(dummy, 30, 1)
automl = AutoSklearnClassifier(
time_left_for_this_task=20,
per_run_time_limit=5,
output_folder=output,
tmp_folder=tmp,
shared_mode=True,
seed=2,
initial_configurations_via_metalearning=0,
ensemble_size=0,
)
automl.fit_ensemble(Y_train, task=MULTICLASS_CLASSIFICATION,
metric=accuracy,
precision='32',
dataset_name='iris',
ensemble_size=20,
ensemble_nbest=50,
)
predictions = automl.predict(X_test)
score = sklearn.metrics.accuracy_score(Y_test, predictions)
self.assertEqual(len(os.listdir(os.path.join(tmp, '.auto-sklearn',
'ensembles'))), 1)
self.assertGreaterEqual(score, 0.90)
self.assertEqual(automl._automl._task, MULTICLASS_CLASSIFICATION)
models = automl._automl.models_
classifier_types = [type(c) for c in models.values()]
self.assertIn(ArrayReturningDummyPredictor, classifier_types)
del automl
self._tearDown(tmp)
self._tearDown(output)
