def _test_simple(method=None, size_ratio=1.0):
automl = AutoML()
automl.add_learner(learner_name='XGBoost2D', learner_class=XGBoost2D)
X, y = fetch_openml(name=dataset, return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.33,
random_state=42)
final_size = int(len(y_train) * size_ratio)
X_train = X_train[:final_size]
y_train = y_train[:final_size]
automl_settings = {
"estimator_list": ['XGBoost2D'],
# "metric": 'accuracy',
"task": 'classification',
"log_file_name": f"test/xgboost2d_{dataset}_{method}_{final_size}.log",
# "model_history": True,
# "log_training_metric": True,
# "split_type": split_type,
"n_jobs": 1,
"hpo_method": method,
"log_type": "all",
"time_budget": 3600,
}
automl.fit(X_train=X_train, y_train=y_train, **automl_settings)
def test_simple(method=None):
automl = AutoML()
automl.add_learner(learner_name = 'XGBoost2D',
learner_class = XGBoost2D)
automl_settings = {
"estimator_list": ['XGBoost2D'],
# "metric": 'accuracy',
"task": 'classification',
"log_file_name": f"test/xgboost2d_{dataset}_{method}.log",
# "model_history": True,
# "log_training_metric": True,
# "split_type": split_type,
"n_jobs": 1,
"hpo_method": method,
"log_type": "all",
"time_budget": 3#6000,
}
try:
X, y = fetch_openml(name=dataset, return_X_y=True)
except:
from sklearn.datasets import load_wine
X, y = load_wine(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33,
random_state=42)
automl.fit(X_train=X_train, y_train=y_train, **automl_settings)
def test_simple(method=None):
automl = AutoML()
automl.add_learner(learner_name="XGBoost2D", learner_class=XGBoost2D)
automl_settings = {
"estimator_list": ["XGBoost2D"],
"task": "classification",
"log_file_name": f"test/xgboost2d_{dataset}_{method}.log",
"n_jobs": 1,
"hpo_method": method,
"log_type": "all",
"retrain_full": "budget",
"keep_search_state": True,
"time_budget": 1,
}
from sklearn.externals._arff import ArffException
try:
X, y = fetch_openml(name=dataset, return_X_y=True)
except (ArffException, ValueError):
from sklearn.datasets import load_wine
X, y = load_wine(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.33,
random_state=42)
automl.fit(X_train=X_train, y_train=y_train, **automl_settings)
print(automl.estimator_list)
print(automl.search_space)
print(automl.points_to_evaluate)
config = automl.best_config.copy()
config["learner"] = automl.best_estimator
automl.trainable(config)
from flaml import tune
from flaml.automl import size
from functools import partial
analysis = tune.run(
automl.trainable,
automl.search_space,
metric="val_loss",
mode="min",
low_cost_partial_config=automl.low_cost_partial_config,
points_to_evaluate=automl.points_to_evaluate,
cat_hp_cost=automl.cat_hp_cost,
resource_attr=automl.resource_attr,
min_resource=automl.min_resource,
max_resource=automl.max_resource,
time_budget_s=automl._state.time_budget,
config_constraints=[(partial(size,
automl._state), "<=", automl._mem_thres)],
metric_constraints=automl.metric_constraints,
num_samples=5,
)
print(analysis.trials[-1])
def _test(split_type):
automl = AutoML()
automl_settings = {
"time_budget": 2,
# "metric": 'accuracy',
"task": 'classification',
"log_file_name": "test/{}.log".format(dataset),
"model_history": True,
"log_training_metric": True,
"split_type": split_type,
}
X, y = fetch_openml(name=dataset, return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33,
random_state=42)
automl.fit(X_train=X_train, y_train=y_train, **automl_settings)
pred = automl.predict(X_test)
acc = accuracy_score(y_test, pred)
print(acc)
def test_simple(method=None):
automl = AutoML()
automl.add_learner(learner_name='XGBoost2D', learner_class=XGBoost2D)
automl_settings = {
"estimator_list": ['XGBoost2D'],
"task": 'classification',
"log_file_name": f"test/xgboost2d_{dataset}_{method}.log",
"n_jobs": 1,
"hpo_method": method,
"log_type": "all",
"time_budget": 3
}
from sklearn.externals._arff import ArffException
try:
X, y = fetch_openml(name=dataset, return_X_y=True)
except (ArffException, ValueError):
from sklearn.datasets import load_wine
X, y = load_wine(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.33,
random_state=42)
automl.fit(X_train=X_train, y_train=y_train, **automl_settings)