def main():
p = Pipeline([('step1', MultiplyByN()), ('step2', MultiplyByN()),
Pipeline([Identity(),
Identity(),
PCA(n_components=4)])])
p.set_hyperparams_space(
HyperparameterSpace({
'step1__multiply_by': RandInt(42, 50),
'step2__multiply_by': RandInt(-10, 0),
'Pipeline__PCA__n_components': RandInt(2, 3)
}))
samples = p.get_hyperparams_space().rvs()
p.set_hyperparams(samples)
samples = p.get_hyperparams()
assert 42 <= samples['step1__multiply_by'] <= 50
assert -10 <= samples['step2__multiply_by'] <= 0
assert samples['Pipeline__PCA__n_components'] in [2, 3]
assert p['Pipeline']['PCA'].get_wrapped_sklearn_predictor(
).n_components in [2, 3]
def __init__(self,
wrapped: BaseTransformer,
enabled: bool = True,
nullified_return_value=None,
cache_folder_when_no_handle=None,
use_hyperparameter_space=True,
nullify_hyperparams=True):
hyperparameter_space = HyperparameterSpace(
{OPTIONAL_ENABLED_HYPERPARAM: Boolean()
}) if use_hyperparameter_space else {}
MetaStep.__init__(self,
hyperparams=HyperparameterSamples(
{OPTIONAL_ENABLED_HYPERPARAM: enabled}),
hyperparams_space=hyperparameter_space,
wrapped=wrapped)
ForceHandleOnlyMixin.__init__(self, cache_folder_when_no_handle)
if nullified_return_value is None:
nullified_return_value = []
self.nullified_return_value = nullified_return_value
self.nullify_hyperparams = nullify_hyperparams
def test_automl_with_kfold(tmpdir):
# Given
hp_repository = HyperparamsJSONRepository(cache_folder=str('caching'))
auto_ml = AutoML(
pipeline=Pipeline([
MultiplyByN(2).set_hyperparams_space(
HyperparameterSpace({'multiply_by': FixedHyperparameter(2)})),
NumpyReshape(new_shape=(-1, 1)),
linear_model.LinearRegression()
]),
validation_splitter=ValidationSplitter(0.20),
hyperparams_optimizer=RandomSearchHyperparameterSelectionStrategy(),
scoring_callback=ScoringCallback(mean_squared_error,
higher_score_is_better=False),
callbacks=[
MetricCallback('mse',
metric_function=mean_squared_error,
higher_score_is_better=False),
],
n_trials=1,
epochs=10,
refit_trial=True,
print_func=print,
hyperparams_repository=hp_repository,
continue_loop_on_error=False)
data_inputs = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
expected_outputs = data_inputs * 4
# When
auto_ml.fit(data_inputs=data_inputs, expected_outputs=expected_outputs)
# Then
p = auto_ml.get_best_model()
outputs = p.transform(data_inputs)
mse = mean_squared_error(expected_outputs, outputs)
assert mse < 1000
def test_automl_early_stopping_callback(tmpdir):
# Given
hp_repository = InMemoryHyperparamsRepository(cache_folder=str(tmpdir))
n_epochs = 10
max_epochs_without_improvement = 3
auto_ml = AutoML(
pipeline=Pipeline([
MultiplyByN(2).set_hyperparams_space(
HyperparameterSpace({'multiply_by': FixedHyperparameter(2)})),
NumpyReshape(new_shape=(-1, 1)),
]),
hyperparams_optimizer=RandomSearchHyperparameterSelectionStrategy(),
validation_splitter=ValidationSplitter(0.20),
scoring_callback=ScoringCallback(mean_squared_error,
higher_score_is_better=False),
callbacks=[
MetricCallback('mse',
metric_function=mean_squared_error,
higher_score_is_better=False),
EarlyStoppingCallback(max_epochs_without_improvement)
],
n_trials=1,
refit_trial=True,
epochs=n_epochs,
hyperparams_repository=hp_repository,
continue_loop_on_error=False)
# When
data_inputs = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
expected_outputs = data_inputs * 2
auto_ml.fit(data_inputs=data_inputs, expected_outputs=expected_outputs)
# Then
trial = hp_repository.trials[0]
assert len(trial.validation_splits) == 1
validation_scores = trial.validation_splits[0].get_validation_scores()
nepochs_executed = len(validation_scores)
assert nepochs_executed == max_epochs_without_improvement + 1
def test_automl_sequential_wrapper(tmpdir):
# Setting seed for reproducibility
np.random.seed(68)
# Given
data_inputs = np.array(range(100))
expected_outputs = np.array(range(100, 200))
hyperparameter_space = HyperparameterSpace({
'multiplication_1__multiply_by':
RandInt(1, 3),
'multiplication_2__multiply_by':
RandInt(1, 3),
'multiplication_3__multiply_by':
RandInt(1, 3),
})
pipeline = Pipeline(
[('multiplication_1', MultiplyByN()),
('multiplication_2', MultiplyByN()),
('multiplication_3', MultiplyByN())],
cache_folder=tmpdir).set_hyperparams_space(hyperparameter_space)
auto_ml = RandomSearch(
KFoldCrossValidationWrapper().set_step(pipeline),
hyperparams_repository=HyperparamsJSONRepository(tmpdir),
n_iter=10)
# When
auto_ml: AutoMLSequentialWrapper = auto_ml.fit(data_inputs,
expected_outputs)
best_model: Pipeline = auto_ml.get_best_model()
predicted_outputs = best_model.transform(data_inputs)
# Then
actual_mse = ((predicted_outputs - expected_outputs)**2).mean()
assert actual_mse < 20000
def test_hyperparam_space():
p = Pipeline([
AddFeatures([
SomeStep(hyperparams_space=HyperparameterSpace(
{"n_components": RandInt(1, 5)})),
SomeStep(hyperparams_space=HyperparameterSpace(
{"n_components": RandInt(1, 5)}))
]),
ModelStacking([
SomeStep(hyperparams_space=HyperparameterSpace(
{"n_estimators": RandInt(1, 1000)})),
SomeStep(hyperparams_space=HyperparameterSpace(
{"n_estimators": RandInt(1, 1000)})),
SomeStep(hyperparams_space=HyperparameterSpace(
{"max_depth": RandInt(1, 100)})),
SomeStep(hyperparams_space=HyperparameterSpace(
{"max_depth": RandInt(1, 100)}))
],
joiner=NumpyTranspose(),
judge=SomeStep(hyperparams_space=HyperparameterSpace(
{"alpha": LogUniform(0.1, 10.0)})))
])
rvsed = p.get_hyperparams_space()
p.set_hyperparams(rvsed)
hyperparams = p.get_hyperparams()
assert "AddFeatures" in hyperparams.keys()
assert "SomeStep" in hyperparams["AddFeatures"]
assert "n_components" in hyperparams["AddFeatures"]["SomeStep"]
assert "SomeStep1" in hyperparams["AddFeatures"]
assert "n_components" in hyperparams["AddFeatures"]["SomeStep1"]
assert "SomeStep" in hyperparams["ModelStacking"]
assert "n_estimators" in hyperparams["ModelStacking"]["SomeStep"]
assert "SomeStep1" in hyperparams["ModelStacking"]
assert "max_depth" in hyperparams["ModelStacking"]["SomeStep2"]
def main(tmpdir, sleep_time: float = 0, n_iter: int = 10):
DATA_INPUTS = np.array(range(100))
EXPECTED_OUTPUTS = np.array(range(100, 200))
HYPERPARAMETER_SPACE = HyperparameterSpace({
'multiplication_1__multiply_by':
RandInt(1, 2),
'multiplication_2__multiply_by':
RandInt(1, 2),
'multiplication_3__multiply_by':
RandInt(1, 2),
})
print('Classic Pipeline:')
pipeline = Pipeline([
('multiplication_1', MultiplyByN()),
('sleep_1', ForEachDataInput(Sleep(sleep_time))),
('multiplication_2', MultiplyByN()),
('sleep_2', ForEachDataInput(Sleep(sleep_time))),
('multiplication_3', MultiplyByN()),
]).set_hyperparams_space(HYPERPARAMETER_SPACE)
time_a = time.time()
best_model = RandomSearch(pipeline,
n_iter=n_iter,
higher_score_is_better=True).fit(
DATA_INPUTS, EXPECTED_OUTPUTS)
outputs = best_model.transform(DATA_INPUTS)
time_b = time.time()
actual_score = mean_squared_error(EXPECTED_OUTPUTS, outputs)
print('{0} seconds'.format(time_b - time_a))
print('output: {0}'.format(outputs))
print('smallest mse: {0}'.format(actual_score))
print('best hyperparams: {0}'.format(pipeline.get_hyperparams()))
assert isinstance(actual_score, float)
print('Resumable Pipeline:')
pipeline = ResumablePipeline(
[('multiplication_1', MultiplyByN()),
('ForEach(sleep_1)', ForEachDataInput(Sleep(sleep_time))),
('checkpoint1', ExpandDim(DefaultCheckpoint())),
('multiplication_2', MultiplyByN()),
('sleep_2', ForEachDataInput(Sleep(sleep_time))),
('checkpoint2', ExpandDim(DefaultCheckpoint())),
('multiplication_3', MultiplyByN())],
cache_folder=tmpdir).set_hyperparams_space(HYPERPARAMETER_SPACE)
time_a = time.time()
best_model = RandomSearch(pipeline,
n_iter=n_iter,
higher_score_is_better=True).fit(
DATA_INPUTS, EXPECTED_OUTPUTS)
outputs = best_model.transform(DATA_INPUTS)
time_b = time.time()
pipeline.flush_all_cache()
actual_score = mean_squared_error(EXPECTED_OUTPUTS, outputs)
print('{0} seconds'.format(time_b - time_a))
print('output: {0}'.format(outputs))
print('smallest mse: {0}'.format(actual_score))
print('best hyperparams: {0}'.format(pipeline.get_hyperparams()))
assert isinstance(actual_score, float)
def main(tmpdir, sleep_time: float = 0.001, n_iter: int = 10):
DATA_INPUTS = np.array(range(100))
EXPECTED_OUTPUTS = np.array(range(100, 200))
HYPERPARAMETER_SPACE = HyperparameterSpace({
'multiplication_1__multiply_by': RandInt(1, 2),
'multiplication_2__multiply_by': RandInt(1, 2),
'multiplication_3__multiply_by': RandInt(1, 2),
})
print('Classic Pipeline:')
classic_pipeline_folder = os.path.join(str(tmpdir), 'classic')
pipeline = Pipeline([
('multiplication_1', MultiplyByN()),
('sleep_1', ForEachDataInput(Sleep(sleep_time))),
('multiplication_2', MultiplyByN()),
('sleep_2', ForEachDataInput(Sleep(sleep_time))),
('multiplication_3', MultiplyByN()),
], cache_folder=classic_pipeline_folder).set_hyperparams_space(HYPERPARAMETER_SPACE)
time_a = time.time()
auto_ml = AutoML(
pipeline,
refit_trial=True,
n_trials=n_iter,
cache_folder_when_no_handle=classic_pipeline_folder,
validation_splitter=ValidationSplitter(0.2),
hyperparams_optimizer=RandomSearchHyperparameterSelectionStrategy(),
scoring_callback=ScoringCallback(mean_squared_error, higher_score_is_better=False),
callbacks=[
MetricCallback('mse', metric_function=mean_squared_error, higher_score_is_better=False)
],
)
auto_ml = auto_ml.fit(DATA_INPUTS, EXPECTED_OUTPUTS)
outputs = auto_ml.get_best_model().predict(DATA_INPUTS)
time_b = time.time()
actual_score = mean_squared_error(EXPECTED_OUTPUTS, outputs)
print('{0} seconds'.format(time_b - time_a))
print('output: {0}'.format(outputs))
print('smallest mse: {0}'.format(actual_score))
print('best hyperparams: {0}'.format(pipeline.get_hyperparams()))
assert isinstance(actual_score, float)
print('Resumable Pipeline:')
resumable_pipeline_folder = os.path.join(str(tmpdir), 'resumable')
pipeline = ResumablePipeline([
('multiplication_1', MultiplyByN()),
('ForEach(sleep_1)', ForEachDataInput(Sleep(sleep_time))),
('checkpoint1', ExpandDim(DefaultCheckpoint())),
('multiplication_2', MultiplyByN()),
('sleep_2', ForEachDataInput(Sleep(sleep_time))),
('checkpoint2', ExpandDim(DefaultCheckpoint())),
('multiplication_3', MultiplyByN())
], cache_folder=resumable_pipeline_folder).set_hyperparams_space(HYPERPARAMETER_SPACE)
time_a = time.time()
auto_ml = AutoML(
pipeline,
refit_trial=True,
n_trials=n_iter,
cache_folder_when_no_handle=resumable_pipeline_folder,
validation_splitter=ValidationSplitter(0.2),
hyperparams_optimizer=RandomSearchHyperparameterSelectionStrategy(),
scoring_callback=ScoringCallback(mean_squared_error, higher_score_is_better=False),
callbacks=[
MetricCallback('mse', metric_function=mean_squared_error, higher_score_is_better=False)
]
)
auto_ml = auto_ml.fit(DATA_INPUTS, EXPECTED_OUTPUTS)
outputs = auto_ml.get_best_model().predict(DATA_INPUTS)
time_b = time.time()
pipeline.flush_all_cache()
actual_score = mean_squared_error(EXPECTED_OUTPUTS, outputs)
print('{0} seconds'.format(time_b - time_a))
print('output: {0}'.format(outputs))
print('smallest mse: {0}'.format(actual_score))
print('best hyperparams: {0}'.format(pipeline.get_hyperparams()))
assert isinstance(actual_score, float)
def plot_distribution_space(hyperparameter_space: HyperparameterSpace,
num_bins=50):
for title, distribution in hyperparameter_space.items():
print(title + ":")
plot_histogram(title, distribution, num_bins=num_bins)
plot_pdf_cdf(title, distribution)
assert from_flat_dic.to_nested_dict() == expected_dic
HYPE_SPACE = HyperparameterSpace({
"a__test":
Boolean(),
"a__lr":
Choice([0, 1, False, "Test"]),
"a__b__c":
PriorityChoice([0, 1, False, "Test"]),
"a__b__q":
Quantized(Uniform(-10, 10)),
"d__param":
RandInt(-10, 10),
"d__u":
Uniform(-10, 10),
"e__other":
LogUniform(0.001, 10),
"e__alpha":
Normal(0.0, 1.0),
"e__f__g":
LogNormal(0.0, 2.0),
"p__other_nondistribution_params":
"hey",
"p__could_also_be_as_fixed":
FixedHyperparameter("also hey"),
"p__its_over_9k":
9001
})
def test_hyperparams_space_rvs_outputs_samples():
def main():
def accuracy(data_inputs, expected_outputs):
return np.mean(
np.argmax(np.array(data_inputs), axis=1) == np.argmax(
np.array(expected_outputs), axis=1))
# load the dataset
df = read_csv('data/winequality-white.csv', sep=';')
data_inputs = df.values
data_inputs[:, -1] = data_inputs[:, -1] - 1
n_features = data_inputs.shape[1] - 1
n_classes = 10
p = Pipeline([
TrainOnlyWrapper(DataShuffler()),
ColumnTransformerInputOutput(
input_columns=[(
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], ToNumpy(np.float32)
)],
output_columns=[(11, Identity())]
),
OutputTransformerWrapper(PlotDistribution(column=-1)),
MiniBatchSequentialPipeline([
Tensorflow2ModelStep(
create_model=create_model,
create_loss=create_loss,
create_optimizer=create_optimizer
) \
.set_hyperparams(HyperparameterSamples({
'n_dense_layers': 2,
'input_dim': n_features,
'optimizer': 'adam',
'activation': 'relu',
'kernel_initializer': 'he_uniform',
'learning_rate': 0.01,
'hidden_dim': 20,
'n_classes': 3
})).set_hyperparams_space(HyperparameterSpace({
'n_dense_layers': RandInt(2, 4),
'hidden_dim_layer_multiplier': Uniform(0.30, 1),
'input_dim': FixedHyperparameter(n_features),
'optimizer': Choice([
OPTIMIZERS.ADAM.value,
OPTIMIZERS.SGD.value,
OPTIMIZERS.ADAGRAD.value
]),
'activation': Choice([
ACTIVATIONS.RELU.value,
ACTIVATIONS.TANH.value,
ACTIVATIONS.SIGMOID.value,
ACTIVATIONS.ELU.value,
]),
'kernel_initializer': Choice([
KERNEL_INITIALIZERS.GLOROT_NORMAL.value,
KERNEL_INITIALIZERS.GLOROT_UNIFORM.value,
KERNEL_INITIALIZERS.HE_UNIFORM.value
]),
'learning_rate': LogUniform(0.005, 0.01),
'hidden_dim': RandInt(3, 80),
'n_classes': FixedHyperparameter(n_classes)
}))
], batch_size=33),
OutputTransformerWrapper(Pipeline([
ExpandDim(),
OneHotEncoder(nb_columns=n_classes, name='classes')
]))
])
auto_ml = AutoML(
pipeline=p,
hyperparams_repository=InMemoryHyperparamsRepository(
cache_folder='trials'),
hyperparams_optimizer=RandomSearchHyperparameterSelectionStrategy(),
validation_splitter=ValidationSplitter(test_size=0.30),
scoring_callback=ScoringCallback(accuracy,
higher_score_is_better=True),
callbacks=[
MetricCallback(
name='classification_report_imbalanced_metric',
metric_function=classificaiton_report_imbalanced_metric,
higher_score_is_better=True),
MetricCallback(name='f1',
metric_function=f1_score_weighted,
higher_score_is_better=True),
MetricCallback(name='recall',
metric_function=recall_score_weighted,
higher_score_is_better=True),
MetricCallback(name='precision',
metric_function=precision_score_weighted,
higher_score_is_better=True),
EarlyStoppingCallback(max_epochs_without_improvement=3)
],
n_trials=200,
refit_trial=True,
epochs=75)
auto_ml = auto_ml.fit(data_inputs=data_inputs)
def main():
boston = load_boston()
X, y = shuffle(boston.data, boston.target, random_state=13)
X = X.astype(np.float32)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.25,
shuffle=False)
# Note that the hyperparameter spaces are defined here during the pipeline definition, but it could be already set
# within the classes ar their definition if using custom classes, or also it could be defined after declaring the
# pipeline using a flat dict or a nested dict.
p = Pipeline([
AddFeatures([
SKLearnWrapper(
PCA(n_components=2),
HyperparameterSpace({"n_components": RandInt(1, 3)})),
SKLearnWrapper(
FastICA(n_components=2),
HyperparameterSpace({"n_components": RandInt(1, 3)})),
]),
ModelStacking(
[
SKLearnWrapper(
GradientBoostingRegressor(),
HyperparameterSpace({
"n_estimators": RandInt(50, 600),
"max_depth": RandInt(1, 10),
"learning_rate": LogUniform(0.07, 0.7)
})),
SKLearnWrapper(
KMeans(),
HyperparameterSpace({"n_clusters": RandInt(5, 10)})),
],
joiner=NumpyTranspose(),
judge=SKLearnWrapper(
Ridge(),
HyperparameterSpace({
"alpha": LogUniform(0.7, 1.4),
"fit_intercept": Boolean()
})),
)
])
print("Meta-fitting on train:")
p = p.meta_fit(X_train,
y_train,
metastep=RandomSearch(
n_iter=10,
higher_score_is_better=True,
validation_technique=KFoldCrossValidationWrapper(
scoring_function=r2_score, k_fold=10)))
# Here is an alternative way to do it, more "pipeliney":
# p = RandomSearch(
# p,
# n_iter=15,
# higher_score_is_better=True,
# validation_technique=KFoldCrossValidation(scoring_function=r2_score, k_fold=3)
# ).fit(X_train, y_train)
print("")
print("Transforming train and test:")
y_train_predicted = p.predict(X_train)
y_test_predicted = p.predict(X_test)
print("")
print("Evaluating transformed train:")
score_transform = r2_score(y_train_predicted, y_train)
print('R2 regression score:', score_transform)
print("")
print("Evaluating transformed test:")
score_test = r2_score(y_test_predicted, y_test)
print('R2 regression score:', score_test)
from neuraxle.base import MetaStepMixin, BaseStep, NonFittableMixin, NonTransformableMixin
from neuraxle.hyperparams.distributions import RandInt, Boolean
from neuraxle.hyperparams.space import HyperparameterSpace, HyperparameterSamples
from neuraxle.steps.loop import StepClonerForEachDataInput
from testing.test_pipeline import SomeStep
SOME_STEP_HP_KEY = 'somestep_hyperparam'
RAND_INT_SOME_STEP = RandInt(-10, 0)
RAND_INT_STEP_CLONER = RandInt(0, 10)
META_STEP_HP = 'metastep_hyperparam'
SOME_STEP_HP = "SomeStep__somestep_hyperparam"
META_STEP_HP_VALUE = 1
SOME_STEP_HP_VALUE = 2
HYPE_SPACE = HyperparameterSpace({"a__test": Boolean()})
HYPE_SAMPLE = HyperparameterSamples({"a__test": True})
class SomeMetaStepMixin(NonTransformableMixin, NonFittableMixin, MetaStepMixin,
BaseStep):
pass
class SomeStepInverseTransform(SomeStep):
def fit_transform(self, data_inputs, expected_outputs=None):
return self, 'fit_transform'
def inverse_transform(self, processed_outputs):
return 'inverse_transform'
def test_flat_to_dict_hyperparams_with_hyperparameter_space(
flat: dict, expected_dic: dict):
dic = HyperparameterSpace(flat).to_nested_dict_as_dict_primitive()
assert dict(dic) == dict(expected_dic)
def get_params_space(self, deep=True):
neuraxle_params = HyperparameterSpace(
self.p.get_hyperparams_space()).to_flat_as_dict_primitive(
)
return neuraxle_params
boston = load_boston()
X, y = shuffle(boston.data, boston.target, random_state=13)
X = X.astype(np.float32)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.25,
shuffle=False)
# Note that the hyperparameter spaces are defined here during the pipeline definition, but it could be already set
# within the classes ar their definition if using custom classes, or also it could be defined after declaring the
# pipeline using a flat dict or a nested dict.
p = Pipeline([
AddFeatures([
SKLearnWrapper(PCA(n_components=2),
HyperparameterSpace({"n_components": RandInt(1, 3)})),
SKLearnWrapper(FastICA(n_components=2),
HyperparameterSpace({"n_components": RandInt(1, 3)})),
]),
ModelStacking(
[
SKLearnWrapper(
GradientBoostingRegressor(),
HyperparameterSpace({
"n_estimators": RandInt(50, 600),
"max_depth": RandInt(1, 10),
"learning_rate": LogUniform(0.07, 0.7)
})),
SKLearnWrapper(
GradientBoostingRegressor(),
HyperparameterSpace({
def test_deep_learning_pipeline():
# Given
boston = load_boston()
data_inputs, expected_outputs = shuffle(boston.data,
boston.target,
random_state=13)
expected_outputs = expected_outputs.astype(np.float32)
data_inputs = data_inputs.astype(np.float32)
pipeline = Pipeline([
AddFeatures([
SKLearnWrapper(
PCA(n_components=2),
HyperparameterSpace({"n_components": RandInt(1, 3)})),
SKLearnWrapper(
FastICA(n_components=2),
HyperparameterSpace({"n_components": RandInt(1, 3)})),
]),
ModelStacking(
[
SKLearnWrapper(
GradientBoostingRegressor(),
HyperparameterSpace({
"n_estimators": RandInt(50, 600),
"max_depth": RandInt(1, 10),
"learning_rate": LogUniform(0.07, 0.7)
})),
SKLearnWrapper(
KMeans(n_clusters=7),
HyperparameterSpace({"n_clusters": RandInt(5, 10)})),
],
joiner=NumpyTranspose(),
judge=SKLearnWrapper(
Ridge(),
HyperparameterSpace({
"alpha": LogUniform(0.7, 1.4),
"fit_intercept": Boolean()
})),
)
])
p = DeepLearningPipeline(
pipeline,
validation_size=VALIDATION_SIZE,
batch_size=BATCH_SIZE,
batch_metrics={'mse': to_numpy_metric_wrapper(mean_squared_error)},
shuffle_in_each_epoch_at_train=True,
n_epochs=N_EPOCHS,
epochs_metrics={'mse': to_numpy_metric_wrapper(mean_squared_error)},
scoring_function=to_numpy_metric_wrapper(mean_squared_error),
)
# When
p, outputs = p.fit_transform(data_inputs, expected_outputs)
# Then
batch_mse_train = p.get_batch_metric_train('mse')
epoch_mse_train = p.get_epoch_metric_train('mse')
batch_mse_validation = p.get_batch_metric_validation('mse')
epoch_mse_validation = p.get_epoch_metric_validation('mse')
assert len(epoch_mse_train) == N_EPOCHS
assert len(epoch_mse_validation) == N_EPOCHS
expected_len_batch_mse_train = math.ceil(
(len(data_inputs) / BATCH_SIZE) * (1 - VALIDATION_SIZE)) * N_EPOCHS
expected_len_batch_mse_validation = math.ceil(
(len(data_inputs) / BATCH_SIZE) * VALIDATION_SIZE) * N_EPOCHS
assert len(batch_mse_train) == expected_len_batch_mse_train
assert len(batch_mse_validation) == expected_len_batch_mse_validation
last_batch_mse_validation = batch_mse_validation[-1]
last_batch_mse_train = batch_mse_train[-1]
last_epoch_mse_train = epoch_mse_train[-1]
last_epoch_mse_validation = epoch_mse_validation[-1]
assert last_batch_mse_train < last_batch_mse_validation
assert last_epoch_mse_train < last_epoch_mse_validation
assert last_batch_mse_train < 1
assert last_epoch_mse_train < 1
class TensorflowV1ModelStep(BaseTensorflowModelStep):
"""
Base class for tensorflow 1 steps.
It uses :class:`TensorflowV1StepSaver` for saving the model.
.. seealso::
`Using the saved model format <https://www.tensorflow.org/guide/checkpoint>`_,
:class:`~neuraxle.base.BaseStep`
"""
HYPERPARAMS = HyperparameterSamples({})
HYPERPARAMS_SPACE = HyperparameterSpace({})
def __init__(self,
create_graph,
create_loss,
create_optimizer,
create_feed_dict=None,
data_inputs_dtype=None,
expected_outputs_dtype=None,
variable_scope=None,
has_expected_outputs=True,
print_loss=False,
print_func=None):
BaseTensorflowModelStep.__init__(
self,
create_model=create_graph,
create_loss=create_loss,
create_optimizer=create_optimizer,
create_inputs=create_feed_dict,
data_inputs_dtype=data_inputs_dtype,
expected_outputs_dtype=expected_outputs_dtype,
step_saver=TensorflowV1StepSaver(),
print_loss=print_loss,
print_func=print_func)
if variable_scope is None:
variable_scope = self.name
self.variable_scope = variable_scope
self.has_expected_outputs = has_expected_outputs
self.create_feed_dict = create_feed_dict
def setup(self) -> BaseStep:
"""
Setup tensorflow 1 graph, and session using a variable scope.
:return: self
:rtype: BaseStep
"""
if self.is_initialized:
return self
self.graph = tf.Graph()
with self.graph.as_default():
with tf.variable_scope(self.variable_scope, reuse=tf.AUTO_REUSE):
self.session = tf.Session(
config=tf.ConfigProto(log_device_placement=True),
graph=self.graph)
model = self.create_model(self)
if not isinstance(model, tuple):
tf.identity(model, name='output')
else:
tf.identity(model[0], name='output')
tf.identity(model[1], name='inference_output')
tf.identity(self.create_loss(self), name='loss')
self.create_optimizer(self).minimize(self['loss'],
name='optimizer')
init = tf.global_variables_initializer()
self.session.run(init)
self.is_initialized = True
def teardown(self) -> BaseStep:
"""
Close session on teardown.
:return:
"""
if self.session is not None:
self.session.close()
self.is_initialized = False
return self
def strip(self):
"""
Strip tensorflow 1 properties from to step to make the step serializable.
:return: stripped step
:rtype: BaseStep
"""
self.graph = None
self.session = None
return self
def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep':
with tf.variable_scope(self.variable_scope, reuse=tf.AUTO_REUSE):
return self.fit_model(data_inputs, expected_outputs)
def fit_model(self, data_inputs, expected_outputs=None) -> BaseStep:
"""
Fit tensorflow model using the variable scope.
:param data_inputs: data inputs
:param expected_outputs: expected outputs to fit on
:return: fitted self
:rtype: BaseStep
"""
feed_dict = {self['data_inputs']: data_inputs}
if self.has_expected_outputs:
feed_dict.update({self['expected_outputs']: expected_outputs})
if self.create_inputs is not None:
additional_feed_dict_arguments = self.create_inputs(
self, data_inputs, expected_outputs)
feed_dict.update(additional_feed_dict_arguments)
results = self.session.run([self['optimizer'], self['loss']],
feed_dict=feed_dict)
loss = results[1]
self.add_new_loss(loss)
return self
def transform(self, data_inputs, expected_outputs=None) -> 'BaseStep':
with tf.variable_scope(self.variable_scope, reuse=tf.AUTO_REUSE):
return self.transform_model(data_inputs)
def transform_model(self, data_inputs):
"""
Transform tensorflow model using the variable scope.
:param data_inputs:
:return:
"""
inference_output_name = self._get_inference_output_name()
feed_dict = {self['data_inputs']: data_inputs}
results = self.session.run([self[inference_output_name], self['loss']],
feed_dict=feed_dict)
self.add_new_loss(results[1], test_only=True)
return results[0]
def _get_inference_output_name(self):
"""
Return the output tensor name for inference (transform).
In create_graph, the user can return a tuple of two elements : the output tensor for training, and the output tensor for inference.
:return:
"""
inference_output_name = 'output'
if len(self['inference_output'].get_shape().as_list()) > 0:
inference_output_name = 'inference_output'
return inference_output_name
def __getitem__(self, item):
"""
Get a graph tensor by name using get item.
:param item: tensor name
:type item: str
:return: tensor
:rtype: tf.Tensor
"""
if ":" in item:
split = item.split(":")
tensor_name = split[0]
device = split[1]
else:
tensor_name = item
device = "0"
try:
result = self.graph.get_tensor_by_name("{0}/{1}:{2}".format(
self.variable_scope, tensor_name, device))
except KeyError:
result = None
if result is None:
try:
result = self.graph.get_operation_by_name("{0}/{1}".format(
self.variable_scope, tensor_name))
except KeyError:
result = tf.get_variable(tensor_name, [])
return result
def main(tmpdir):
boston = load_boston()
X, y = shuffle(boston.data, boston.target, random_state=13)
X = X.astype(np.float32)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, shuffle=False)
# Note that the hyperparameter spaces are defined here during the pipeline definition, but it could be already set
# within the classes ar their definition if using custom classes, or also it could be defined after declaring the
# pipeline using a flat dict or a nested dict.
p = Pipeline([
AddFeatures([
SKLearnWrapper(
PCA(n_components=2),
HyperparameterSpace({"n_components": RandInt(1, 3)})
),
SKLearnWrapper(
FastICA(n_components=2),
HyperparameterSpace({"n_components": RandInt(1, 3)})
),
]),
ModelStacking([
SKLearnWrapper(
GradientBoostingRegressor(),
HyperparameterSpace({
"n_estimators": RandInt(50, 300), "max_depth": RandInt(1, 4),
"learning_rate": LogUniform(0.07, 0.7)
})
),
SKLearnWrapper(
KMeans(),
HyperparameterSpace({"n_clusters": RandInt(5, 10)})
),
],
joiner=NumpyTranspose(),
judge=SKLearnWrapper(
Ridge(),
HyperparameterSpace({"alpha": LogUniform(0.7, 1.4), "fit_intercept": Boolean()})
),
)
])
print("Meta-fitting on train:")
auto_ml = AutoML(
p,
validation_splitter=ValidationSplitter(0.20),
refit_trial=True,
n_trials=10,
epochs=1, # 1 epoc here due to using sklearn models that just fit once.
cache_folder_when_no_handle=str(tmpdir),
scoring_callback=ScoringCallback(mean_squared_error, higher_score_is_better=False),
callbacks=[MetricCallback('mse', metric_function=mean_squared_error, higher_score_is_better=False)],
hyperparams_repository=InMemoryHyperparamsRepository(cache_folder=str(tmpdir))
)
random_search = auto_ml.fit(X_train, y_train)
p = random_search.get_best_model()
print("")
print("Transforming train and test:")
y_train_predicted = p.predict(X_train)
y_test_predicted = p.predict(X_test)
print("")
print("Evaluating transformed train:")
score_transform = r2_score(y_train_predicted, y_train)
print('R2 regression score:', score_transform)
print("")
print("Evaluating transformed test:")
score_test = r2_score(y_test_predicted, y_test)
print('R2 regression score:', score_test)
def set_params(self, **params) -> BaseEstimator:
self.p.set_hyperparams(HyperparameterSpace(params))
return self
from neuraxle.base import BaseStep, TruncableSteps, NonFittableMixin, MetaStepMixin
from neuraxle.hyperparams.distributions import LogUniform, Quantized, RandInt, Boolean
from neuraxle.hyperparams.space import HyperparameterSpace, HyperparameterSamples
HYPERPARAMETERS_SPACE = HyperparameterSpace({
'learning_rate':
LogUniform(0.0001, 0.1),
'l2_weight_reg':
LogUniform(0.0001, 0.1),
'momentum':
LogUniform(0.01, 1.0),
'hidden_size':
Quantized(LogUniform(16, 512)),
'num_layers':
RandInt(1, 4),
'num_lstm_layers':
RandInt(1, 2),
'use_xavier_init':
Boolean(),
'use_max_pool_else_avg_pool':
Boolean(),
'dropout_drop_proba':
LogUniform(0.3, 0.7)
})
HYPERPARAMETERS = HyperparameterSamples({
'learning_rate': 0.1,
'l2_weight_reg': 0.001,
'hidden_size': 32,
'num_layers': 3,
'num_lstm_layers': 1,
def set_hyperparams_space(
self, hyperparams_space: HyperparameterSpace) -> 'BaseStep':
self.hyperparams_space = HyperparameterSpace(hyperparams_space)
return self
def main():
# Define classification models, and hyperparams.
# See also HyperparameterSpace documentation : https://www.neuraxle.org/stable/api/neuraxle.hyperparams.space.html#neuraxle.hyperparams.space.HyperparameterSpace
decision_tree_classifier = SKLearnWrapper(
DecisionTreeClassifier(),
HyperparameterSpace({
'criterion': Choice(['gini', 'entropy']),
'splitter': Choice(['best', 'random']),
'min_samples_leaf': RandInt(2, 5),
'min_samples_split': RandInt(2, 4)
}))
extra_tree_classifier = SKLearnWrapper(
ExtraTreeClassifier(),
HyperparameterSpace({
'criterion': Choice(['gini', 'entropy']),
'splitter': Choice(['best', 'random']),
'min_samples_leaf': RandInt(2, 5),
'min_samples_split': RandInt(2, 4)
}))
ridge_classifier = Pipeline([
OutputTransformerWrapper(NumpyRavel()),
SKLearnWrapper(
RidgeClassifier(),
HyperparameterSpace({
'alpha': Choice([0.0, 1.0, 10.0, 100.0]),
'fit_intercept': Boolean(),
'normalize': Boolean()
}))
]).set_name('RidgeClassifier')
logistic_regression = Pipeline([
OutputTransformerWrapper(NumpyRavel()),
SKLearnWrapper(
LogisticRegression(),
HyperparameterSpace({
'C': LogUniform(0.01, 10.0),
'fit_intercept': Boolean(),
'penalty': Choice(['none', 'l2']),
'max_iter': RandInt(20, 200)
}))
]).set_name('LogisticRegression')
random_forest_classifier = Pipeline([
OutputTransformerWrapper(NumpyRavel()),
SKLearnWrapper(
RandomForestClassifier(),
HyperparameterSpace({
'n_estimators': RandInt(50, 600),
'criterion': Choice(['gini', 'entropy']),
'min_samples_leaf': RandInt(2, 5),
'min_samples_split': RandInt(2, 4),
'bootstrap': Boolean()
}))
]).set_name('RandomForestClassifier')
# Define a classification pipeline that lets the AutoML loop choose one of the classifier.
# See also ChooseOneStepOf documentation : https://www.neuraxle.org/stable/api/neuraxle.steps.flow.html#neuraxle.steps.flow.ChooseOneStepOf
pipeline = Pipeline([
ChooseOneStepOf([
decision_tree_classifier, extra_tree_classifier, ridge_classifier,
logistic_regression, random_forest_classifier
])
])
# Create the AutoML loop object.
# See also AutoML documentation : https://www.neuraxle.org/stable/api/neuraxle.metaopt.auto_ml.html#neuraxle.metaopt.auto_ml.AutoML
auto_ml = AutoML(
pipeline=pipeline,
hyperparams_optimizer=RandomSearchHyperparameterSelectionStrategy(),
validation_splitter=ValidationSplitter(test_size=0.20),
scoring_callback=ScoringCallback(accuracy_score,
higher_score_is_better=True),
n_trials=7,
epochs=1,
hyperparams_repository=HyperparamsJSONRepository(cache_folder='cache'),
refit_trial=True,
continue_loop_on_error=False)
# Load data, and launch AutoML loop !
X_train, y_train, X_test, y_test = generate_classification_data()
auto_ml = auto_ml.fit(X_train, y_train)
# Get the model from the best trial, and make predictions using predict.
# See also predict documentation : https://www.neuraxle.org/stable/api/neuraxle.base.html#neuraxle.base.BaseStep.predict
best_pipeline = auto_ml.get_best_model()
y_pred = best_pipeline.predict(X_test)
accuracy = accuracy_score(y_true=y_test, y_pred=y_pred)
print("Test accuracy score:", accuracy)
shutil.rmtree('cache')
class Tensorflow2ModelStep(BaseTensorflowModelStep):
"""
Base class for tensorflow 2 steps.
It uses :class:`TensorflowV2StepSaver` for saving the model.
.. seealso::
`Using the checkpoint model format <https://www.tensorflow.org/guide/checkpoint>`_,
:class:`~neuraxle.base.BaseStep`
"""
HYPERPARAMS = HyperparameterSamples({})
HYPERPARAMS_SPACE = HyperparameterSpace({})
def __init__(
self,
create_model,
create_loss,
create_optimizer,
create_inputs=None,
data_inputs_dtype=None,
expected_outputs_dtype=None,
tf_model_checkpoint_folder=None,
print_loss=False,
print_func=None,
device_name=None
):
BaseTensorflowModelStep.__init__(
self,
create_model=create_model,
create_loss=create_loss,
create_optimizer=create_optimizer,
create_inputs=create_inputs,
data_inputs_dtype=data_inputs_dtype,
expected_outputs_dtype=expected_outputs_dtype,
step_saver=TensorflowV2StepSaver(),
print_loss=print_loss,
print_func=print_func
)
if device_name is None:
device_name = '/CPU:0'
self.device_name = device_name
if tf_model_checkpoint_folder is None:
tf_model_checkpoint_folder = 'tensorflow_ckpts'
self.tf_model_checkpoint_folder = tf_model_checkpoint_folder
def setup(self) -> BaseStep:
"""
Setup optimizer, model, and checkpoints for saving.
:return: step
:rtype: BaseStep
"""
if self.is_initialized:
return self
with tf.device(self.device_name):
self.optimizer = self.create_optimizer(self)
self.model = self.create_model(self)
self.checkpoint = tf.train.Checkpoint(step=tf.Variable(1), optimizer=self.optimizer, net=self.model)
self.checkpoint_manager = tf.train.CheckpointManager(
self.checkpoint,
self.tf_model_checkpoint_folder,
max_to_keep=3
)
self.is_initialized = True
return self
def strip(self):
"""
Strip tensorflow 2 properties from to step to make it serializable.
:return:
"""
self.optimizer = None
self.model = None
self.checkpoint = None
self.checkpoint_manager = None
def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep':
with tf.device(self.device_name):
self._fit_model(data_inputs, expected_outputs)
return self
def _fit_model(self, data_inputs, expected_outputs):
inputs = self._create_inputs(data_inputs, expected_outputs)
with tf.GradientTape() as tape:
output = self.model(inputs, training=True)
loss = self.create_loss(
self,
expected_outputs=tf.convert_to_tensor(expected_outputs, dtype=self.expected_outputs_dtype),
predicted_outputs=output
)
self.add_new_loss(loss)
self.model.losses.append(loss)
self.optimizer.apply_gradients(zip(
tape.gradient(loss, self.model.trainable_variables),
self.model.trainable_variables
))
def _transform_data_container(self, data_container: DataContainer, context: ExecutionContext) -> DataContainer:
data_inputs = data_container.data_inputs
expected_outputs = data_container.expected_outputs
with tf.device(self.device_name):
output = self._transform_model(data_inputs, expected_outputs)
data_container.set_data_inputs(output.numpy())
return data_container
def _transform_model(self, data_inputs, expected_outputs):
output = self.model(self._create_inputs(data_inputs), training=False)
if expected_outputs is not None:
loss = self.create_loss(
self,
expected_outputs=tf.convert_to_tensor(expected_outputs, dtype=self.expected_outputs_dtype),
predicted_outputs=output
)
self.add_new_loss(loss, test_only=True)
return output
def transform(self, data_inputs):
with tf.device(self.device_name):
output = self.model(self._create_inputs(data_inputs), training=False)
return output.numpy()
def _create_inputs(self, data_inputs, expected_outputs=None):
if self.create_inputs is not None:
inputs = self.create_inputs(self, data_inputs, expected_outputs)
else:
inputs = tf.convert_to_tensor(data_inputs, self.data_inputs_dtype)
return inputs
from neuraxle.metaopt.callbacks import MetricCallback, ScoringCallback
from neuraxle.metaopt.tpe import TreeParzenEstimatorHyperparameterSelectionStrategy
from neuraxle.pipeline import Pipeline
from neuraxle.steps.misc import FitTransformCallbackStep
from neuraxle.steps.numpy import AddN
import os
@pytest.mark.parametrize(
"expected_output_mult, pipeline",
[(3.5,
Pipeline([
FitTransformCallbackStep().set_name('callback'),
AddN(0.).set_hyperparams_space(
HyperparameterSpace({
'add':
Choice(choice_list=[0, 1.5, 2, 3.5, 4, 5, 6]),
})),
AddN(0.).set_hyperparams_space(
HyperparameterSpace({
'add':
Choice(choice_list=[0, 1.5, 2, 3.5, 4, 5, 6]),
}))
])),
(3.5,
Pipeline([
FitTransformCallbackStep().set_name('callback'),
AddN(0.).set_hyperparams_space(
HyperparameterSpace({
'add': Quantized(hd=Uniform(0, 10)),
})),
AddN(0.).set_hyperparams_space(