def run(dataset: Dataset, config: TaskConfig):
log.info(
f"\n**** Gradient Boosting [sklearn v{sklearn.__version__}] ****\n")
save_metadata(config, version=sklearn.__version__)
is_classification = config.type == 'classification'
X_train, X_test = impute(dataset.train.X_enc, dataset.test.X_enc)
y_train, y_test = dataset.train.y, dataset.test.y
estimator = GradientBoostingClassifier if is_classification else GradientBoostingRegressor
predictor = estimator(random_state=config.seed, **config.framework_params)
with Timer() as training:
predictor.fit(X_train, y_train)
predictions = predictor.predict(X_test)
probabilities = predictor.predict_proba(
X_test) if is_classification else None
save_predictions(dataset=dataset,
output_file=config.output_predictions_file,
probabilities=probabilities,
predictions=predictions,
truth=y_test)
return dict(models_count=1, training_duration=training.duration)
def run(dataset: Dataset, config: TaskConfig):
from frameworks.shared.caller import run_in_venv
X_train_enc, X_test_enc = impute(dataset.train.X_enc, dataset.test.X_enc)
data = dict(
train=dict(
X_enc=X_train_enc,
y_enc=dataset.train.y_enc
),
test=dict(
X_enc=X_test_enc,
y_enc=dataset.test.y_enc
)
)
def process_results(results):
if results.probabilities is not None and not results.probabilities.shape: # numpy load always return an array
prob_format = results.probabilities.item()
if prob_format == "predictions":
target_values_enc = dataset.target.label_encoder.transform(dataset.target.values)
results.probabilities = Encoder('one-hot', target=False, encoded_type=float).fit(target_values_enc).transform(results.predictions)
else:
raise ValueError(f"Unknown probabilities format: {prob_format}")
return results
return run_in_venv(__file__, "exec.py",
input_data=data, dataset=dataset, config=config,
process_results=process_results)
def run(dataset, config):
log.info("\n**** Random Forest (sklearn %s) ****\n", sklearn.__version__)
is_classification = config.type == 'classification'
# Impute any missing data (can test using -t 146606)
X_train, X_test = impute(dataset.train.X_enc, dataset.test.X_enc)
y_train, y_test = dataset.train.y, dataset.test.y
log.info(
"Running RandomForest with a maximum time of {}s on {} cores.".format(
config.max_runtime_seconds, config.cores))
log.warning(
"We completely ignore the requirement to stay within the time limit.")
log.warning(
"We completely ignore the advice to optimize towards metric: {}.".
format(config.metric))
estimator = RandomForestClassifier if is_classification else RandomForestRegressor
rfc = estimator(n_jobs=config.cores, **config.framework_params)
rfc.fit(X_train, y_train)
predictions = rfc.predict(X_test)
probabilities = rfc.predict_proba(X_test) if is_classification else None
return ns(output_file=config.output_predictions_file,
probabilities=probabilities,
predictions=predictions,
truth=y_test,
target_is_encoded=False)
def run(dataset: Dataset, config: TaskConfig):
X_train_enc, X_test_enc = impute(dataset.train.X_enc, dataset.test.X_enc)
data = dict(train=dict(X_enc=X_train_enc, y_enc=dataset.train.y_enc),
test=dict(X_enc=X_test_enc, y_enc=dataset.test.y_enc))
return run_in_venv(__file__,
"exec.py",
input_data=data,
dataset=dataset,
config=config)
def run(dataset: Dataset, config: TaskConfig):
from amlb.datautils import impute
from frameworks.shared.caller import run_in_venv
X_train_enc, X_test_enc = impute(dataset.train.X_enc, dataset.test.X_enc)
data = dict(train=dict(X_enc=X_train_enc, y_enc=dataset.train.y_enc),
test=dict(X_enc=X_test_enc, y_enc=dataset.test.y_enc))
return run_in_venv(__file__,
"exec.py",
input_data=data,
dataset=dataset,
config=config)
def run(dataset: Dataset, config: TaskConfig):
log.info("\n**** Oboe ****\n")
is_classification = config.type == 'classification'
if not is_classification:
# regression currently fails (as of 26.02.2019: still under development state by oboe team)
raise ValueError('Regression is not yet supported (under development).')
X_train, X_test = impute(dataset.train.X_enc, dataset.test.X_enc)
y_train, y_test = dataset.train.y_enc, dataset.test.y_enc
training_params = {k: v for k, v in config.framework_params.items() if not k.startswith('_')}
n_cores = config.framework_params.get('_n_cores', config.cores)
log.info('Running oboe with a maximum time of {}s on {} cores.'.format(config.max_runtime_seconds, n_cores))
log.warning('We completely ignore the advice to optimize towards metric: {}.'.format(config.metric))
aml = AutoLearner(p_type='classification' if is_classification else 'regression',
n_cores=n_cores,
runtime_limit=config.max_runtime_seconds,
**training_params)
aml_models = lambda: [aml.ensemble, *aml.ensemble.base_learners] if len(aml.ensemble.base_learners) > 0 else []
with Timer() as training:
try:
aml.fit(X_train, y_train)
except IndexError as e:
if len(aml_models()) == 0: # incorrect handling of some IndexError in oboe if ensemble is empty
raise NoResultError("Oboe could not produce any model in the requested time.") from e
raise e
predictions = aml.predict(X_test).reshape(len(X_test))
if is_classification:
target_values_enc = dataset.target.label_encoder.transform(dataset.target.values)
probabilities = Encoder('one-hot', target=False, encoded_type=float).fit(target_values_enc).transform(predictions)
else:
probabilities = None
save_predictions_to_file(dataset=dataset,
output_file=config.output_predictions_file,
probabilities=probabilities,
predictions=predictions,
truth=y_test,
target_is_encoded=True)
return dict(
models_count=len(aml_models()),
training_duration=training.duration
)
def run_random_forest(dataset, config, tuner, log):
is_classification = config.type == 'classification'
X_train, X_test = impute(dataset.train.X, dataset.test.X)
y_train, y_test = dataset.train.y, dataset.test.y
estimator = RandomForestClassifier if is_classification else RandomForestRegressor
best_score, best_params, best_model = None, None, None
score_higher_better = True
tuner.update_search_space(SEARCH_SPACE)
start_time = time.time()
while True:
try:
param_idx, cur_params = tuner.generate_parameters()
cur_model = estimator(random_state=config.seed, **cur_params)
# Here score is the output of score() from the estimator
cur_score = cross_val_score(cur_model, X_train, y_train)
cur_score = sum(cur_score) / float(len(cur_score))
if best_score is None or (score_higher_better
and cur_score > best_score) or (
not score_higher_better
and cur_score < best_score):
best_score, best_params, best_model = cur_score, cur_params, cur_model
log.info("Trial {}: \n{}\nScore: {}\n".format(
param_idx, cur_params, cur_score))
tuner.receive_trial_result(param_idx, cur_params, cur_score)
current_time = time.time()
elapsed_time = current_time - start_time
if elapsed_time > config.max_runtime_seconds:
break
except:
break
# This line is required to fully terminate some advisors
tuner.handle_terminate()
log.info("Tuning done, the best parameters are:\n{}\n".format(best_params))
# retrain on the whole dataset
with Timer() as training:
best_model.fit(X_train, y_train)
predictions = best_model.predict(X_test)
probabilities = best_model.predict_proba(
X_test) if is_classification else None
return probabilities, predictions, training, y_test
def run(dataset: Dataset, config: TaskConfig):
log.info("\n**** Random Forest (sklearn) ****\n")
is_classification = config.type == 'classification'
X_train, X_test = impute(dataset.train.X_enc, dataset.test.X_enc)
y_train, y_test = dataset.train.y_enc, dataset.test.y_enc
training_params = {
k: v
for k, v in config.framework_params.items() if not k.startswith('_')
}
n_jobs = config.framework_params.get(
'_n_jobs', config.cores
) # useful to disable multicore, regardless of the dataset config
log.info(
"Running RandomForest with a maximum time of {}s on {} cores.".format(
config.max_runtime_seconds, n_jobs))
log.warning(
"We completely ignore the requirement to stay within the time limit.")
log.warning(
"We completely ignore the advice to optimize towards metric: {}.".
format(config.metric))
estimator = RandomForestClassifier if is_classification else RandomForestRegressor
rf = estimator(n_jobs=n_jobs, random_state=config.seed, **training_params)
with Timer() as training:
rf.fit(X_train, y_train)
predictions = rf.predict(X_test)
probabilities = rf.predict_proba(X_test) if is_classification else None
save_predictions_to_file(dataset=dataset,
output_file=config.output_predictions_file,
probabilities=probabilities,
predictions=predictions,
truth=y_test,
target_is_encoded=True)
return dict(models_count=len(rf), training_duration=training.duration)
def run(dataset: Dataset, config: TaskConfig):
log.info("****TabNet****")
save_metadata(config)
is_classification = config.type == 'classification'
X_train, X_test = dataset.train.X, dataset.test.X
X_train, X_test = impute(X_train, X_test)
X = np.concatenate((X_train, X_test), axis=0)
enc = OrdinalEncoder()
enc.fit(X)
X_train = enc.transform(X_train)
X_test = enc.transform(X_test)
y_train, y_test = dataset.train.y, dataset.test.y
estimator = TabNetClassifier if is_classification else TabNetRegressor
predictor = estimator() # you can change hyperparameters
if not is_classification:
y_train = np.reshape(y_train.astype(np.float32), (-1, 1))
y_test = np.reshape(y_test.astype(np.float32), (-1, 1))
with Timer() as training:
predictor.fit(X_train,
y_train,
eval_set=[(X_train, y_train), (X_test, y_test)])
with Timer() as predict:
predictions = predictor.predict(X_test)
probabilities = predictor.predict_proba(
X_test) if is_classification else None
save_predictions(dataset=dataset,
output_file=config.output_predictions_file,
probabilities=probabilities,
predictions=predictions,
truth=y_test)
return dict(models_count=1,
training_duration=training.duration,
predict_duration=predict.duration)
def run(dataset: Dataset, config: TaskConfig):
log.info("\n**** Decision Tree (sklearn) ****\n")
is_classification = config.type == 'classification'
X_train, X_test = impute(dataset.train.X_enc, dataset.test.X_enc)
y_train, y_test = dataset.train.y, dataset.test.y
estimator = DecisionTreeClassifier if is_classification else DecisionTreeRegressor
predictor = estimator(random_state=config.seed, **config.framework_params)
with Timer() as training:
predictor.fit(X_train, y_train)
predictions = predictor.predict(X_test)
probabilities = predictor.predict_proba(
X_test) if is_classification else None
save_predictions_to_file(dataset=dataset,
output_file=config.output_predictions_file,
probabilities=probabilities,
predictions=predictions,
truth=y_test)
return dict(models_count=1, training_duration=training.duration)
def run(dataset: Dataset, config: TaskConfig):
log.info("\n**** Hyperopt-sklearn ****\n")
is_classification = config.type == 'classification'
default = lambda: 0
metrics_to_loss_mapping = dict(
acc=(default, False), # lambda y, pred: 1.0 - accuracy_score(y, pred)
auc=(lambda y, pred: 1.0 - roc_auc_score(y, pred), False),
f1=(lambda y, pred: 1.0 - f1_score(y, pred), False),
# logloss=(log_loss, True),
mae=(mean_absolute_error, False),
mse=(mean_squared_error, False),
msle=(mean_squared_log_error, False),
r2=(default, False), # lambda y, pred: 1.0 - r2_score(y, pred)
)
loss_fn, continuous_loss_fn = metrics_to_loss_mapping[
config.metric] if config.metric in metrics_to_loss_mapping else (None,
False)
if loss_fn is None:
log.warning("Performance metric %s not supported: defaulting to %s.",
config.metric, 'accuracy' if is_classification else 'r2')
if loss_fn is default:
loss_fn = None
training_params = {
k: v
for k, v in config.framework_params.items() if not k.startswith('_')
}
log.warning("Ignoring cores constraint of %s cores.", config.cores)
log.info(
"Running hyperopt-sklearn with a maximum time of %ss on %s cores, optimizing %s.",
config.max_runtime_seconds, 'all', config.metric)
X_train, X_test = impute(dataset.train.X_enc, dataset.test.X_enc)
y_train, y_test = dataset.train.y_enc, dataset.test.y_enc
if is_classification:
classifier = any_classifier('clf')
regressor = None
else:
classifier = None
regressor = any_regressor('rgr')
estimator = HyperoptEstimator(classifier=classifier,
regressor=regressor,
algo=tpe.suggest,
loss_fn=loss_fn,
continuous_loss_fn=continuous_loss_fn,
trial_timeout=config.max_runtime_seconds,
seed=config.seed,
**training_params)
with InterruptTimeout(config.max_runtime_seconds * 4 / 3,
sig=signal.SIGQUIT):
with InterruptTimeout(config.max_runtime_seconds,
before_interrupt=ft.partial(
kill_proc_tree,
timeout=5,
include_parent=False)):
with Timer() as training:
estimator.fit(X_train, y_train)
predictions = estimator.predict(X_test)
if is_classification:
target_values_enc = dataset.target.label_encoder.transform(
dataset.target.values)
probabilities = Encoder(
'one-hot', target=False,
encoded_type=float).fit(target_values_enc).transform(predictions)
else:
probabilities = None
save_predictions_to_file(dataset=dataset,
output_file=config.output_predictions_file,
probabilities=probabilities,
predictions=predictions,
truth=y_test,
target_is_encoded=True)
return dict(models_count=len(estimator.trials),
training_duration=training.duration)
def run(dataset: Dataset, config: TaskConfig):
log.info("\n**** TPOT ****\n")
is_classification = config.type == 'classification'
# Mapping of benchmark metrics to TPOT metrics
metrics_mapping = dict(acc='accuracy',
auc='roc_auc',
f1='f1',
logloss='neg_log_loss',
mae='neg_mean_absolute_error',
mse='neg_mean_squared_error',
msle='neg_mean_squared_log_error',
r2='r2')
scoring_metric = metrics_mapping[
config.metric] if config.metric in metrics_mapping else None
if scoring_metric is None:
raise ValueError("Performance metric {} not supported.".format(
config.metric))
X_train, X_test = impute(dataset.train.X_enc, dataset.test.X_enc)
y_train, y_test = dataset.train.y_enc, dataset.test.y_enc
training_params = {
k: v
for k, v in config.framework_params.items() if not k.startswith('_')
}
n_jobs = config.framework_params.get(
'_n_jobs', config.cores
) # useful to disable multicore, regardless of the dataset config
log.info(
'Running TPOT with a maximum time of %ss on %s cores, optimizing %s.',
config.max_runtime_seconds, n_jobs, scoring_metric)
runtime_min = (config.max_runtime_seconds / 60)
estimator = TPOTClassifier if is_classification else TPOTRegressor
tpot = estimator(n_jobs=n_jobs,
max_time_mins=runtime_min,
scoring=scoring_metric,
random_state=config.seed,
**training_params)
with Timer() as training:
tpot.fit(X_train, y_train)
log.info('Predicting on the test set.')
predictions = tpot.predict(X_test)
try:
probabilities = tpot.predict_proba(
X_test) if is_classification else None
except RuntimeError:
# TPOT throws a RuntimeError if the optimized pipeline does not support `predict_proba`.
target_values_enc = dataset.target.label_encoder.transform(
dataset.target.values)
probabilities = Encoder(
'one-hot', target=False,
encoded_type=float).fit(target_values_enc).transform(predictions)
save_predictions_to_file(dataset=dataset,
output_file=config.output_predictions_file,
probabilities=probabilities,
predictions=predictions,
truth=y_test,
target_is_encoded=is_classification)
save_artifacts(tpot, config)
return dict(models_count=len(tpot.evaluated_individuals_),
training_duration=training.duration)
def run(dataset: Dataset, config: TaskConfig):
log.info("\n**** Tuned Random Forest (sklearn) ****\n")
is_classification = config.type == 'classification'
training_params = {
k: v
for k, v in config.framework_params.items() if not k.startswith('_')
}
tuning_params = config.framework_params.get('_tuning', training_params)
n_jobs = config.framework_params.get(
'_n_jobs', config.cores
) # useful to disable multicore, regardless of the dataset config
# Impute any missing data (can test using -t 146606)
X_train, X_test = impute(dataset.train.X_enc, dataset.test.X_enc)
y_train, y_test = dataset.train.y_enc, dataset.test.y_enc
log.info(
"Running RandomForest with a maximum time of {}s on {} cores.".format(
config.max_runtime_seconds, n_jobs))
estimator = RandomForestClassifier if is_classification else RandomForestRegressor
metric = dict(auc='roc_auc', logloss='neg_log_loss',
acc='accuracy')[config.metric]
n_features = X_train.shape[1]
default_value = max(1, int(math.sqrt(n_features)))
below_default = pick_values_uniform(start=1,
end=default_value,
length=5 + 1)[:-1] # 5 below
above_default = pick_values_uniform(start=default_value,
end=n_features,
length=10 + 1 -
len(below_default))[1:] # 5 above
# Mix up the order of `max_features` to try, so that a fair range is tried even if we have too little time
# to try all possible values. Order: [sqrt(p), 1, p, random order for remaining values]
# max_features_to_try = below_default[1:] + above_default[:-1]
# max_features_values = ([default_value, 1, n_features]
# + random.sample(max_features_to_try, k=len(max_features_to_try)))
max_features_values = [default_value] + below_default + above_default
# Define up to how much of total time we spend 'optimizing' `max_features`.
# (the remainder if used for fitting the final model).
safety_factor = 0.85
with stopit.ThreadingTimeout(seconds=int(config.max_runtime_seconds *
safety_factor)):
log.info("Evaluating multiple values for `max_features`: %s.",
max_features_values)
max_feature_scores = []
tuning_durations = []
for i, max_features_value in enumerate(max_features_values):
log.info("[{:2d}/{:2d}] Evaluating max_features={}".format(
i + 1, len(max_features_values), max_features_value))
imputation = Imputer()
random_forest = estimator(n_jobs=n_jobs,
random_state=config.seed,
max_features=max_features_value,
**tuning_params)
pipeline = Pipeline(steps=[('preprocessing',
imputation), ('learning',
random_forest)])
with Timer() as cv_scoring:
try:
scores = cross_val_score(estimator=pipeline,
X=dataset.train.X_enc,
y=dataset.train.y_enc,
scoring=metric,
cv=5)
max_feature_scores.append(
(statistics.mean(scores), max_features_value))
except stopit.utils.TimeoutException as toe:
log.error(
"Failed CV scoring for max_features=%s : Timeout",
max_features_value)
tuning_durations.append(
(max_features_value, cv_scoring.duration))
raise toe
except Exception as e:
log.error("Failed CV scoring for max_features=%s :\n%s",
max_features_value, e)
log.debug("Exception:", exc_info=True)
tuning_durations.append((max_features_value, cv_scoring.duration))
log.info("Tuning scores:\n%s", sorted(max_feature_scores))
log.info("Tuning durations:\n%s", sorted(tuning_durations))
_, best_max_features_value = max(
max_feature_scores) if len(max_feature_scores) > 0 else (math.nan,
'auto')
log.info("Training final model with `max_features={}`.".format(
best_max_features_value))
rf = estimator(n_jobs=n_jobs,
random_state=config.seed,
max_features=best_max_features_value,
**training_params)
with Timer() as training:
rf.fit(X_train, y_train)
predictions = rf.predict(X_test)
probabilities = rf.predict_proba(X_test) if is_classification else None
save_predictions_to_file(dataset=dataset,
output_file=config.output_predictions_file,
probabilities=probabilities,
predictions=predictions,
truth=y_test,
target_is_encoded=True)
return dict(models_count=len(rf),
training_duration=training.duration +
sum(map(lambda t: t[1], tuning_durations)))
def run(dataset: Dataset, config: TaskConfig):
log.info(
"\n**** Random Forest (sklearn) Tuned with NNI EvolutionTuner ****\n")
is_classification = config.type == 'classification'
X_train, X_test = impute(dataset.train.X, dataset.test.X)
y_train, y_test = dataset.train.y, dataset.test.y
estimator = RandomForestClassifier if is_classification else RandomForestRegressor
# model = estimator(random_state=config.seed, **config.framework_params)
best_score, best_params, best_model = None, None, None
score_higher_better = True
log.info(
"Tuning hyperparameters with NNI EvolutionTuner with a maximum time of {}s\n"
.format(config.max_runtime_seconds))
tuner = EvolutionTuner()
tuner.update_search_space(SEARCH_SPACE)
start_time = time.time()
param_idx = 0
while True:
try:
cur_params = tuner.generate_parameters(param_idx)
cur_model = estimator(random_state=config.seed,
**cur_params,
**config.framework_params)
# Here score is the output of score() from the estimator
cur_score = cross_val_score(cur_model, X_train, y_train)
cur_score = sum(cur_score) / float(len(cur_score))
if best_score is None or (score_higher_better
and cur_score > best_score) or (
not score_higher_better
and cur_score < best_score):
best_score, best_params, best_model = cur_score, cur_params, cur_model
log.info("Trial {}: \n{}\nScore: {}\n".format(
param_idx, cur_params, cur_score))
tuner.receive_trial_result(param_idx, cur_params, cur_score)
param_idx += 1
current_time = time.time()
elapsed_time = current_time - start_time
if elapsed_time > config.max_runtime_seconds:
break
except:
break
log.info("Tuning done, the best parameters are:\n{}\n".format(best_params))
# retrain on the whole dataset
with Timer() as training:
best_model.fit(X_train, y_train)
predictions = best_model.predict(X_test)
probabilities = best_model.predict_proba(
X_test) if is_classification else None
save_predictions_to_file(dataset=dataset,
output_file=config.output_predictions_file,
probabilities=probabilities,
predictions=predictions,
truth=y_test)
return dict(models_count=1, training_duration=training.duration)
