def test_smac_timeout_regression(self):
import time
from sklearn.datasets import load_boston
from lale.lib.lale import SMAC
planned_pipeline = (MinMaxScaler | Normalizer) >> LinearRegression
X, y = load_boston(return_X_y=True)
max_opt_time = 2.0
opt = SMAC(
estimator=planned_pipeline,
scoring="r2",
max_evals=1,
max_opt_time=max_opt_time,
)
start = time.time()
_ = opt.fit(X[:500, :], y[:500])
end = time.time()
opt_time = end - start
rel_diff = (opt_time - max_opt_time) / max_opt_time
assert (
rel_diff < 0.5
), "Max time: {}, Actual time: {}, relative diff: {}".format(
max_opt_time, opt_time, rel_diff
)
def test_smac_timeout_zero_classification(self):
from lale.lib.lale import SMAC
planned_pipeline = (MinMaxScaler | Normalizer) >> (LogisticRegression | KNeighborsClassifier)
opt = SMAC(estimator=planned_pipeline, max_evals=1, max_opt_time=0.0)
# run optimizer
res = opt.fit(self.X_train, self.y_train)
assert res.get_pipeline() is None
def test_smac1(self):
from sklearn.metrics import accuracy_score
from lale.lib.lale import SMAC
planned_pipeline = (PCA | NoOp) >> LogisticRegression
opt = SMAC(estimator=planned_pipeline, max_evals=1)
# run optimizer
res = opt.fit(self.X_train, self.y_train)
predictions = res.predict(self.X_test)
def test_smac1(self):
from lale.lib.lale import SMAC
planned_pipeline = (PCA | NoOp) >> LogisticRegression
opt = SMAC(estimator=planned_pipeline, max_evals=1)
# run optimizer
res = opt.fit(self.X_train, self.y_train)
_ = res.predict(self.X_test)
def test_smac_timeout_zero_regression(self):
from lale.lib.lale import SMAC
planned_pipeline = (MinMaxScaler | Normalizer) >> LinearRegression
from sklearn.datasets import load_boston
X, y = load_boston(return_X_y=True)
opt = SMAC(estimator=planned_pipeline, scoring = 'r2', max_evals=1, max_opt_time=0.0)
# run optimizer
res = opt.fit(X[:500,:], y[:500])
assert res.get_pipeline() is None
def test_smac2(self):
from sklearn.metrics import accuracy_score
from lale.lib.lale import SMAC
planned_pipeline = (PCA | NoOp) >> KNeighborsClassifier(n_neighbors = 10000)
opt = SMAC(estimator=planned_pipeline, max_evals=1)
# run optimizer
res = opt.fit(self.X_train, self.y_train)
# Get the trials object and make sure that SMAC assigned cost_for_crash which is MAXINT by default to
#at least one trial (correspond to KNN).
trials = res._impl.get_trials()
assert 2147483647.0 in trials.cost_per_config.values()
def test_smac2(self):
from test.mock_module import BadClassifier
import lale.operators
from lale.lib.lale import SMAC
BadClf = lale.operators.make_operator(BadClassifier)
planned_pipeline = (PCA | NoOp) >> BadClf()
opt = SMAC(estimator=planned_pipeline, max_evals=1)
# run optimizer
res = opt.fit(self.X_train, self.y_train)
# Get the trials object and make sure that SMAC assigned cost_for_crash which is MAXINT by default to
# at least one trial (correspond to KNN).
trials = res._impl.get_trials()
assert 2147483647.0 in trials.cost_per_config.values()
def test_smac_timeout_classification(self):
from lale.lib.lale import SMAC
import time
planned_pipeline = (MinMaxScaler | Normalizer) >> (LogisticRegression | KNeighborsClassifier)
max_opt_time = 4.0
opt = SMAC(estimator=planned_pipeline, max_evals=1, max_opt_time=max_opt_time)
start = time.time()
res = opt.fit(self.X_train, self.y_train)
end = time.time()
opt_time = end - start
rel_diff = (opt_time - max_opt_time) / max_opt_time
assert rel_diff < 1.2, (
'Max time: {}, Actual time: {}, relative diff: {}'.format(max_opt_time, opt_time, rel_diff)
)
