def get_classification_metric(metric):
standard_classification_metrics = \
{"accuracy":
make_scorer('accuracy', sklearn.metrics.accuracy_score),
"balanced_accuracy":
make_scorer('balanced_accuracy',
classification_metrics.balanced_accuracy),
"f1":
make_scorer('f1', sklearn.metrics.f1_score),
"roc_auc":
make_scorer('roc_auc', sklearn.metrics.roc_auc_score,
greater_is_better=True, needs_threshold=True),
"average_precision":
make_scorer('average_precision',
sklearn.metrics.average_precision_score,
needs_threshold=True),
"precision":
make_scorer('precision', sklearn.metrics.precision_score),
"recall":
make_scorer('recall', sklearn.metrics.recall_score)
}
classification_metric = standard_classification_metrics.get(metric)
if metric is not None and classification_metric is None:
warnings.warn(
"This metric \"{}\" is not a supported metric for "
"classification. The metric will be reset as default "
"\"accuracy\".".format(metric))
classification_metric = standard_classification_metrics.get(
"accuracy")
return classification_metric
def get_regression_metric(metric):
standard_regression_metrics = \
{"r2":
make_scorer('r2', sklearn.metrics.r2_score),
"mean_squared_error":
make_scorer('mean_squared_error',
sklearn.metrics.mean_squared_error,
greater_is_better=False),
"mean_absolute_error":
make_scorer('mean_absolute_error',
sklearn.metrics.mean_absolute_error,
greater_is_better=False),
"median_absolute_error":
make_scorer('median_absolute_error',
sklearn.metrics.median_absolute_error,
greater_is_better=False)
}
regression_metric = standard_regression_metrics.get(metric)
if metric is not None and regression_metric is None:
warnings.warn("This metric \"{}\" is not a supported metric for "
"regression. The metric will be reset as default "
"\"r2\".".format(metric))
regression_metric = standard_regression_metrics.get("r2")
return regression_metric
def evaluate_ausk(run_id,
task_type,
mth,
dataset,
ens_method,
enable_meta,
eval_type='holdout',
time_limit=1200,
seed=1):
tmp_dir = 'data/exp_sys/ausk_tmp_%s_%s_%s_%d_%d' % (
task_type, mth, dataset, time_limit, run_id)
output_dir = 'data/exp_sys/ausk_output_%s_%s_%s_%d_%d' % (
task_type, mth, dataset, time_limit, run_id)
initial_configs = 25 if enable_meta == 'true' else 0
if os.path.exists(tmp_dir):
try:
shutil.rmtree(tmp_dir)
shutil.rmtree(output_dir)
except:
pass
if task_type == 'cls':
automl = autosklearn.classification.AutoSklearnClassifier(
time_left_for_this_task=int(time_limit),
per_run_time_limit=300,
n_jobs=1,
include_estimators=['random_forest'],
include_preprocessors=[
'extra_trees_preproc_for_classification',
'liblinear_svc_preprocessor',
'select_percentile_classification', 'select_rates'
],
ensemble_memory_limit=16384,
ml_memory_limit=16384,
ensemble_size=1 if ens_method is None else 50,
initial_configurations_via_metalearning=initial_configs,
tmp_folder=tmp_dir,
output_folder=output_dir,
delete_tmp_folder_after_terminate=False,
delete_output_folder_after_terminate=False,
seed=int(seed),
resampling_strategy='holdout',
resampling_strategy_arguments={'train_size': 0.67})
else:
automl = autosklearn.regression.AutoSklearnRegressor(
time_left_for_this_task=int(time_limit),
per_run_time_limit=300,
n_jobs=1,
include_estimators=['random_forest'],
include_preprocessors=[
'extra_trees_preproc_for_regression',
'select_percentile_regression', 'select_rates'
],
ensemble_memory_limit=16384,
ml_memory_limit=16384,
ensemble_size=1 if ens_method is None else 50,
initial_configurations_via_metalearning=initial_configs,
tmp_folder=tmp_dir,
output_folder=output_dir,
delete_tmp_folder_after_terminate=False,
delete_output_folder_after_terminate=False,
seed=int(seed),
resampling_strategy='holdout',
resampling_strategy_arguments={'train_size': 0.67})
print(automl)
_task_type = MULTICLASS_CLS if task_type == 'cls' else REGRESSION
train_data, test_data = load_train_test_data(dataset, task_type=_task_type)
X, y = train_data.data
X_test, y_test = test_data.data
feat_type = [
'Categorical' if _type == CATEGORICAL else 'Numerical'
for _type in train_data.feature_types
]
from autosklearn.metrics import make_scorer
if task_type == 'cls':
scorer = make_scorer(name='balanced_accuracy',
score_func=balanced_accuracy_score)
score_func = balanced_accuracy_score
else:
scorer = make_scorer(name='mean_squared_error',
score_func=mean_squared_error,
greater_is_better=False)
score_func = mean_squared_error
start_time = time.time()
automl.fit(X.copy(), y.copy(), feat_type=feat_type, metric=scorer)
valid_results = automl.cv_results_['mean_test_score']
if task_type == 'cls':
validation_score = np.max(valid_results)
else:
valid_results = [ele - valid_results[-1] for ele in valid_results[:-1]]
validation_score = np.min(valid_results)
# automl.refit(X.copy(), y.copy())
predictions = automl.predict(X_test)
test_score = score_func(y_test, predictions)
model_desc = automl.show_models()
str_stats = automl.sprint_statistics()
result_score = automl.cv_results_['mean_test_score']
result_time = automl.cv_results_['mean_fit_time']
print('=' * 10)
# print(model_desc)
print(str_stats)
print('=' * 10)
print('Validation score', validation_score)
print('Test score', test_score)
# print(automl.show_models())
save_path = save_folder + 'extremely_small_%s_%s_%s_%s_%d_%d_%d.pkl' % (
task_type, mth, dataset, enable_meta, time_limit,
(ens_method is None), run_id)
with open(save_path, 'wb') as f:
pickle.dump([
dataset, validation_score, test_score, start_time, result_score,
result_time
], f)
shutil.rmtree(output_dir)
shutil.rmtree(os.path.join(tmp_dir, '.auto-sklearn'))
def main(task_id, ensemble_dir, performance_range_threshold, ensemble_size,
max_keep_best, seed, only_portfolio_runs, call_from_cmd):
if max_keep_best > 1:
assert max_keep_best == int(max_keep_best)
max_keep_best = int(max_keep_best)
memory_limit = 4000
precision = 32
metric = make_scorer('balanced_accuracy_fast', BalancedAccuracy())
if not os.path.exists(ensemble_dir):
raise NotADirectoryError("%s does not exist")
if call_from_cmd:
assert str(task_id) in ensemble_dir
fl_name = "ensemble_results_%fthresh_%dsize_%fbest" % \
(performance_range_threshold, ensemble_size, max_keep_best)
if only_portfolio_runs:
fl_name += "_only_portfolio"
fl_name = os.path.join(ensemble_dir, fl_name)
if os.path.isfile(fl_name):
raise ValueError("Nothing left to do, %s already exists" % fl_name)
# figure out how many prediction files are in dir
if call_from_cmd:
pred_dir = os.path.join(ensemble_dir, "auto-sklearn-output",
".auto-sklearn", "predictions_ensemble")
n_models = glob.glob(pred_dir +
"/predictions_ensemble_%d_*.npy.gz" % seed)
else:
pred_dir = os.path.join(ensemble_dir, ".auto-sklearn",
"predictions_ensemble")
n_models = glob.glob(pred_dir +
"/predictions_ensemble_%d_*.npy" % seed)
n_models.sort(key=lambda x: int(float(x.split("_")[-2])))
print("\n".join(n_models))
print("Found %d ensemble predictions" % len(n_models))
if len(n_models) == 0:
raise ValueError("%s has no ensemble predictions" % pred_dir)
# Get start time of ensemble building: 1) load json 2) find key 3) get creation times
if call_from_cmd:
timestamps_fl = os.path.join(ensemble_dir, "auto-sklearn-output",
"timestamps.json")
else:
timestamps_fl = os.path.join(ensemble_dir, "timestamps.json")
with open(timestamps_fl, "r") as fh:
timestamps = json.load(fh)
model_timestamps = None
overall_start_time = None
for k in timestamps:
if "predictions_ensemble" in k:
model_timestamps = timestamps[k]
if "start_time_%d" % seed in timestamps[k]:
overall_start_time = timestamps[k]["start_time_%d" % seed]
timestamp_keys = list(model_timestamps.keys())
for timestamp_key in timestamp_keys:
if timestamp_key.endswith(
'lock') or 'predictions_ensemble' not in timestamp_key:
del model_timestamps[timestamp_key]
assert model_timestamps is not None and overall_start_time is not None
assert len(model_timestamps) == len(n_models), (len(model_timestamps),
len(n_models))
# Get overall timelimit
vanilla_results_fl = os.path.join(ensemble_dir, "result.json")
with open(vanilla_results_fl, "r") as fh:
vanilla_results = json.load(fh)
# If only portfolio configurations, read runhistory
if only_portfolio_runs:
if call_from_cmd:
runhistory_fl = os.path.join(ensemble_dir, "auto-sklearn-output",
"smac3-output", "run*",
"runhistory.json")
else:
runhistory_fl = os.path.join(ensemble_dir, "smac3-output", "run*",
"runhistory.json")
runhistory_fl = glob.glob(runhistory_fl)
assert len(runhistory_fl) == 1
with open(runhistory_fl[0], "r") as fh:
runhistory = json.load(fh)
init_design_num_runs = []
for i in runhistory["data"]:
if i[1][3]["configuration_origin"] == "Initial design":
if "error" in i[1][3]:
continue
init_design_num_runs.append(i[1][3]["num_run"])
print("Portfolio stopped after %s runs" % str(init_design_num_runs))
last_run = max(init_design_num_runs)
print("Cut down to only portfolio runs fom %d" % len(n_models))
for i, n in enumerate(n_models):
if int(float(n.split("_")[-2])) > last_run:
n_models = n_models[:i]
break
print("... to %d" % len(n_models))
# load data
X_train, y_train, X_test, y_test, cat = load_task(task_id)
if len(np.unique(y_test)) == 2:
task_type = BINARY_CLASSIFICATION
elif len(np.unique(y_test)) > 2:
task_type = MULTICLASS_CLASSIFICATION
else:
raise ValueError("Unknown task type for task %d" % task_id)
tmp_dir = tempfile.TemporaryDirectory()
loss_trajectory = []
# Construct ensemble builder
context = BackendContextMock(
temporary_directory=(ensemble_dir + "/auto-sklearn-output/"
if call_from_cmd else ensemble_dir),
output_directory=tmp_dir.name,
delete_tmp_folder_after_terminate=False,
delete_output_folder_after_terminate=False,
shared_mode=False)
backend = Backend(context)
ens_builder = EnsembleBuilder(
backend=backend,
dataset_name=str(task_id),
task_type=task_type,
metric=metric,
limit=np.inf,
ensemble_size=ensemble_size,
ensemble_nbest=max_keep_best,
performance_range_threshold=performance_range_threshold,
max_models_on_disc=None,
seed=seed,
shared_mode=False,
precision=precision,
max_iterations=1,
read_at_most=1,
memory_limit=memory_limit,
random_state=1,
sleep_duration=0)
try:
# iterate over all models, take construction time into account when creating new trajectory
current_ensemble_timestamp = 0
skipped = 1
for midx, model_path in enumerate(n_models):
tstamp = model_timestamps[model_path.split("/")[-1].replace(
'.gz', '')] - overall_start_time
if current_ensemble_timestamp > tstamp:
# while this model was built, the ensemble script was not yet done
skipped += 1
continue
# Do one ensemble building step
start = time.time()
ens_builder.random_state = check_random_state(1)
print("############## %d: Working on %s (skipped %d)" %
(midx + 1, model_path, skipped - 1))
logging.basicConfig(level=logging.DEBUG)
ens_builder.read_at_most = skipped
valid_pred, test_pred = ens_builder.main(return_pred=True)
last_dur = time.time() - start
current_ensemble_timestamp = tstamp + last_dur
if current_ensemble_timestamp >= vanilla_results["0"]["time_limit"]:
print("############## Went over time %f > %f; Stop here" %
(current_ensemble_timestamp,
vanilla_results["0"]["time_limit"]))
break
# Reset, since we have just read model files
skipped = 1
if test_pred is None:
# Adding this model did not change the ensemble, no new prediction
continue
if task_type == BINARY_CLASSIFICATION:
# Recreate nx2 array
test_pred = np.concatenate([
1 - test_pred.reshape([-1, 1]),
test_pred.reshape([-1, 1])
],
axis=1)
# Build trajectory entry
score = 1 - balanced_accuracy(y_true=y_test, y_pred=test_pred)
loss_trajectory.append((current_ensemble_timestamp, score))
print("############## Round %d took %g sec" %
(midx, time.time() - start))
except:
raise
finally:
tmp_dir.cleanup()
# Store results
result = dict()
result[ensemble_size] = {
'task_id': task_id,
'time_limit': vanilla_results["0"]["time_limit"],
'loss': loss_trajectory[-1][1],
'configuration': {
"n_models": n_models,
"performance_range_threshold": performance_range_threshold,
"ensemble_size": ensemble_size,
"max_keep_best": max_keep_best,
"seed": seed,
"memory_limit": memory_limit,
"precision": precision,
},
'n_models': len(n_models),
'trajectory': loss_trajectory,
}
with open(fl_name, 'wt') as fh:
json.dump(result, fh, indent=4)
print("Dumped to %s" % fl_name)
import shutil
import unittest
import numpy as np
from autosklearn.metrics import make_scorer
from autosklearn.ensemble_builder import (
EnsembleBuilder, AbstractEnsemble
)
def scorer_function(a, b):
return 0.9
MockMetric = make_scorer('mock', scorer_function)
class BackendMock(object):
def __init__(self, target_directory):
this_directory = os.path.abspath(
os.path.dirname(__file__)
)
shutil.copytree(os.path.join(this_directory, 'data'), os.path.join(target_directory))
self.temporary_directory = target_directory
self.internals_directory = os.path.join(self.temporary_directory, '.auto-sklearn')
def load_datamanager(self):
manager = unittest.mock.Mock()
manager.__reduce__ = lambda self: (unittest.mock.MagicMock, ())