def get_pytorch_model(self):
"""Returns a pytorch sequential model of the current incumbent configuration. Not possible for all models.
Arguments:
Returns:
model -- PyTorch sequential model of the current incumbent configuration
"""
try:
if NetworkSelector.get_name() in self.pipeline:
return self.pipeline[
NetworkSelector.get_name()].fit_output["network"].layers
else:
return self.pipeline[NetworkSelectorDatasetInfo.get_name(
)].fit_output["network"].layers
except:
print(
"Can not get PyTorch Sequential model for incumbent config. Returning Auto-PyTorch model"
)
if NetworkSelector.get_name() in self.pipeline:
return self.pipeline[
NetworkSelector.get_name()].fit_output["network"]
else:
return self.pipeline[NetworkSelectorDatasetInfo.get_name(
)].fit_output["network"]
def test_optimizer_selector(self):
pipeline = Pipeline([NetworkSelector(), OptimizerSelector()])
net_selector = pipeline[NetworkSelector.get_name()]
net_selector.add_network("mlpnet", MlpNet)
net_selector.add_network("shapedmlpnet", ShapedMlpNet)
net_selector.add_final_activation('none', nn.Sequential())
opt_selector = pipeline[OptimizerSelector.get_name()]
opt_selector.add_optimizer("adam", AdamOptimizer)
opt_selector.add_optimizer("sgd", SgdOptimizer)
pipeline_config = pipeline.get_pipeline_config()
pipeline_config["random_seed"] = 42
hyper_config = pipeline.get_hyperparameter_search_space(
).sample_configuration()
pipeline.fit_pipeline(hyperparameter_config=hyper_config,
pipeline_config=pipeline_config,
X=torch.rand(3, 3),
Y=torch.rand(3, 2),
embedding=nn.Sequential())
sampled_optimizer = opt_selector.fit_output['optimizer']
self.assertIn(type(sampled_optimizer), [optim.Adam, optim.SGD])
def _apply_default_pipeline_settings(pipeline):
from autoPyTorch.pipeline.nodes.network_selector import NetworkSelector
from autoPyTorch.pipeline.nodes.loss_module_selector import LossModuleSelector
from autoPyTorch.pipeline.nodes.metric_selector import MetricSelector
from autoPyTorch.pipeline.nodes.train_node import TrainNode
from autoPyTorch.pipeline.nodes.cross_validation import CrossValidation
import torch.nn as nn
from autoPyTorch.components.metrics.standard_metrics import multilabel_accuracy
from autoPyTorch.components.preprocessing.loss_weight_strategies import LossWeightStrategyWeightedBinary
AutoNetFeatureData._apply_default_pipeline_settings(pipeline)
net_selector = pipeline[NetworkSelector.get_name()]
net_selector.add_final_activation('sigmoid', nn.Sigmoid())
loss_selector = pipeline[LossModuleSelector.get_name()]
loss_selector.add_loss_module('bce_with_logits', nn.BCEWithLogitsLoss,
None, False)
loss_selector.add_loss_module('bce_with_logits_weighted',
nn.BCEWithLogitsLoss,
LossWeightStrategyWeightedBinary(),
False)
metric_selector = pipeline[MetricSelector.get_name()]
metric_selector.add_metric('multilabel_accuracy', multilabel_accuracy)
train_node = pipeline[TrainNode.get_name()]
train_node.default_minimize_value = False
cv = pipeline[CrossValidation.get_name()]
cv.use_stratified_cv_split_default = False
def _apply_default_pipeline_settings(pipeline):
from autoPyTorch.pipeline.nodes.network_selector import NetworkSelector
from autoPyTorch.pipeline.nodes.loss_module_selector import LossModuleSelector
from autoPyTorch.pipeline.nodes.metric_selector import MetricSelector
from autoPyTorch.pipeline.nodes.train_node import TrainNode
from autoPyTorch.pipeline.nodes.cross_validation import CrossValidation
import torch.nn as nn
from autoPyTorch.components.metrics.standard_metrics import mean_distance
AutoNetFeatureData._apply_default_pipeline_settings(pipeline)
net_selector = pipeline[NetworkSelector.get_name()]
net_selector.add_final_activation('none', nn.Sequential())
loss_selector = pipeline[LossModuleSelector.get_name()]
loss_selector.add_loss_module('l1_loss', nn.L1Loss)
metric_selector = pipeline[MetricSelector.get_name()]
metric_selector.add_metric('mean_distance', mean_distance)
train_node = pipeline[TrainNode.get_name()]
train_node.default_minimize_value = True
cv = pipeline[CrossValidation.get_name()]
cv.use_stratified_cv_split_default = False
def _apply_default_pipeline_settings(pipeline):
from autoPyTorch.pipeline.nodes.network_selector import NetworkSelector
from autoPyTorch.pipeline.nodes.loss_module_selector import LossModuleSelector
from autoPyTorch.pipeline.nodes.metric_selector import MetricSelector
from autoPyTorch.pipeline.nodes.train_node import TrainNode
from autoPyTorch.pipeline.nodes.resampling_strategy_selector import ResamplingStrategySelector
from autoPyTorch.pipeline.nodes.cross_validation import CrossValidation
from autoPyTorch.pipeline.nodes.one_hot_encoding import OneHotEncoding
from autoPyTorch.pipeline.nodes.resampling_strategy_selector import ResamplingStrategySelector
from autoPyTorch.components.preprocessing.resampling import RandomOverSamplingWithReplacement, RandomUnderSamplingWithReplacement, SMOTE, \
TargetSizeStrategyAverageSample, TargetSizeStrategyDownsample, TargetSizeStrategyMedianSample, TargetSizeStrategyUpsample
import torch.nn as nn
from autoPyTorch.components.metrics.standard_metrics import accuracy
from autoPyTorch.components.preprocessing.loss_weight_strategies import LossWeightStrategyWeighted
AutoNetFeatureData._apply_default_pipeline_settings(pipeline)
net_selector = pipeline[NetworkSelector.get_name()]
net_selector.add_final_activation('softmax', nn.Softmax(1))
loss_selector = pipeline[LossModuleSelector.get_name()]
loss_selector.add_loss_module('cross_entropy', nn.CrossEntropyLoss,
None, True)
loss_selector.add_loss_module('cross_entropy_weighted',
nn.CrossEntropyLoss,
LossWeightStrategyWeighted(), True)
metric_selector = pipeline[MetricSelector.get_name()]
metric_selector.add_metric('accuracy', accuracy)
resample_selector = pipeline[ResamplingStrategySelector.get_name()]
resample_selector.add_over_sampling_method(
'random', RandomOverSamplingWithReplacement)
resample_selector.add_over_sampling_method('smote', SMOTE)
resample_selector.add_under_sampling_method(
'random', RandomUnderSamplingWithReplacement)
resample_selector.add_target_size_strategy('upsample',
TargetSizeStrategyUpsample)
resample_selector.add_target_size_strategy(
'downsample', TargetSizeStrategyDownsample)
resample_selector.add_target_size_strategy(
'average', TargetSizeStrategyAverageSample)
resample_selector.add_target_size_strategy(
'median', TargetSizeStrategyMedianSample)
train_node = pipeline[TrainNode.get_name()]
train_node.default_minimize_value = False
cv = pipeline[CrossValidation.get_name()]
cv.use_stratified_cv_split_default = True
one_hot_encoding_node = pipeline[OneHotEncoding.get_name()]
one_hot_encoding_node.encode_Y = True
return pipeline
def test_network_selector(self):
pipeline = Pipeline([
NetworkSelector()
])
selector = pipeline[NetworkSelector.get_name()]
selector.add_network("mlpnet", MlpNet)
selector.add_network("shapedmlpnet", ShapedMlpNet)
selector.add_final_activation('none', nn.Sequential())
pipeline_config = pipeline.get_pipeline_config()
pipeline_config["random_seed"] = 42
hyper_config = pipeline.get_hyperparameter_search_space().sample_configuration()
pipeline.fit_pipeline(hyperparameter_config=hyper_config, pipeline_config=pipeline_config,
X=torch.rand(3,3), Y=torch.rand(3, 2), embedding=nn.Sequential())
sampled_network = pipeline[selector.get_name()].fit_output['network']
self.assertIn(type(sampled_network), [MlpNet, ShapedMlpNet])
def test_lr_scheduler_selector(self):
pipeline = Pipeline([
NetworkSelector(),
OptimizerSelector(),
LearningrateSchedulerSelector(),
])
net_selector = pipeline[NetworkSelector.get_name()]
net_selector.add_network("mlpnet", MlpNet)
net_selector.add_network("shapedmlpnet", ShapedMlpNet)
net_selector.add_final_activation('none', nn.Sequential())
opt_selector = pipeline[OptimizerSelector.get_name()]
opt_selector.add_optimizer("adam", AdamOptimizer)
opt_selector.add_optimizer("sgd", SgdOptimizer)
lr_scheduler_selector = pipeline[
LearningrateSchedulerSelector.get_name()]
lr_scheduler_selector.add_lr_scheduler("step", SchedulerStepLR)
lr_scheduler_selector.add_lr_scheduler("exp", SchedulerExponentialLR)
pipeline_config = pipeline.get_pipeline_config()
pipeline_config["random_seed"] = 42
hyper_config = pipeline.get_hyperparameter_search_space(
).sample_configuration()
pipeline.fit_pipeline(hyperparameter_config=hyper_config,
pipeline_config=pipeline_config,
X=torch.rand(3, 3),
Y=torch.rand(3, 2),
embedding=nn.Sequential(),
training_techniques=[],
train_indices=np.array([0, 1, 2]))
sampled_lr_scheduler = pipeline[
lr_scheduler_selector.get_name()].fit_output[
'training_techniques'][0].training_components['lr_scheduler']
self.assertIn(type(sampled_lr_scheduler),
[lr_scheduler.ExponentialLR, lr_scheduler.StepLR])
def get_default_pipeline(cls):
from autoPyTorch.pipeline.base.pipeline import Pipeline
from autoPyTorch.pipeline.nodes.autonet_settings import AutoNetSettings
from autoPyTorch.pipeline.nodes.optimization_algorithm import OptimizationAlgorithm
from autoPyTorch.pipeline.nodes.cross_validation import CrossValidation
from autoPyTorch.pipeline.nodes.imputation import Imputation
from autoPyTorch.pipeline.nodes.normalization_strategy_selector import NormalizationStrategySelector
from autoPyTorch.pipeline.nodes.one_hot_encoding import OneHotEncoding
from autoPyTorch.pipeline.nodes.preprocessor_selector import PreprocessorSelector
from autoPyTorch.pipeline.nodes.resampling_strategy_selector import ResamplingStrategySelector
from autoPyTorch.pipeline.nodes.embedding_selector import EmbeddingSelector
from autoPyTorch.pipeline.nodes.network_selector import NetworkSelector
from autoPyTorch.pipeline.nodes.optimizer_selector import OptimizerSelector
from autoPyTorch.pipeline.nodes.lr_scheduler_selector import LearningrateSchedulerSelector
from autoPyTorch.pipeline.nodes.log_functions_selector import LogFunctionsSelector
from autoPyTorch.pipeline.nodes.metric_selector import MetricSelector
from autoPyTorch.pipeline.nodes.loss_module_selector import LossModuleSelector
from autoPyTorch.pipeline.nodes.train_node import TrainNode
# build the pipeline
pipeline = Pipeline([
AutoNetSettings(),
OptimizationAlgorithm([
CrossValidation([
Imputation(),
NormalizationStrategySelector(),
OneHotEncoding(),
PreprocessorSelector(),
ResamplingStrategySelector(),
EmbeddingSelector(),
NetworkSelector(),
OptimizerSelector(),
LearningrateSchedulerSelector(),
LogFunctionsSelector(),
MetricSelector(),
LossModuleSelector(),
TrainNode()
])
])
])
cls._apply_default_pipeline_settings(pipeline)
return pipeline
def _apply_default_pipeline_settings(pipeline):
from autoPyTorch.pipeline.nodes.normalization_strategy_selector import NormalizationStrategySelector
from autoPyTorch.pipeline.nodes.preprocessor_selector import PreprocessorSelector
from autoPyTorch.pipeline.nodes.embedding_selector import EmbeddingSelector
from autoPyTorch.pipeline.nodes.network_selector import NetworkSelector
from autoPyTorch.pipeline.nodes.optimizer_selector import OptimizerSelector
from autoPyTorch.pipeline.nodes.lr_scheduler_selector import LearningrateSchedulerSelector
from autoPyTorch.pipeline.nodes.train_node import TrainNode
from autoPyTorch.components.networks.feature import MlpNet, ResNet, ShapedMlpNet, ShapedResNet
from autoPyTorch.components.optimizer.optimizer import AdamOptimizer, SgdOptimizer
from autoPyTorch.components.lr_scheduler.lr_schedulers import SchedulerCosineAnnealingWithRestartsLR, SchedulerNone, \
SchedulerCyclicLR, SchedulerExponentialLR, SchedulerReduceLROnPlateau, SchedulerReduceLROnPlateau, SchedulerStepLR
from autoPyTorch.components.networks.feature import LearnedEntityEmbedding
from sklearn.preprocessing import MinMaxScaler, StandardScaler, MaxAbsScaler
from autoPyTorch.components.preprocessing.feature_preprocessing import \
TruncatedSVD, FastICA, RandomKitchenSinks, KernelPCA, Nystroem
from autoPyTorch.training.early_stopping import EarlyStopping
from autoPyTorch.training.mixup import Mixup
pre_selector = pipeline[PreprocessorSelector.get_name()]
pre_selector.add_preprocessor('truncated_svd', TruncatedSVD)
pre_selector.add_preprocessor('fast_ica', FastICA)
pre_selector.add_preprocessor('kitchen_sinks', RandomKitchenSinks)
pre_selector.add_preprocessor('kernel_pca', KernelPCA)
pre_selector.add_preprocessor('nystroem', Nystroem)
norm_selector = pipeline[NormalizationStrategySelector.get_name()]
norm_selector.add_normalization_strategy('minmax', MinMaxScaler)
norm_selector.add_normalization_strategy('standardize', StandardScaler)
norm_selector.add_normalization_strategy('maxabs', MaxAbsScaler)
emb_selector = pipeline[EmbeddingSelector.get_name()]
emb_selector.add_embedding_module('learned', LearnedEntityEmbedding)
net_selector = pipeline[NetworkSelector.get_name()]
net_selector.add_network('mlpnet', MlpNet)
net_selector.add_network('shapedmlpnet', ShapedMlpNet)
net_selector.add_network('resnet', ResNet)
net_selector.add_network('shapedresnet', ShapedResNet)
opt_selector = pipeline[OptimizerSelector.get_name()]
opt_selector.add_optimizer('adam', AdamOptimizer)
opt_selector.add_optimizer('sgd', SgdOptimizer)
lr_selector = pipeline[LearningrateSchedulerSelector.get_name()]
lr_selector.add_lr_scheduler('cosine_annealing',
SchedulerCosineAnnealingWithRestartsLR)
lr_selector.add_lr_scheduler('cyclic', SchedulerCyclicLR)
lr_selector.add_lr_scheduler('exponential', SchedulerExponentialLR)
lr_selector.add_lr_scheduler('step', SchedulerStepLR)
lr_selector.add_lr_scheduler('plateau', SchedulerReduceLROnPlateau)
lr_selector.add_lr_scheduler('none', SchedulerNone)
train_node = pipeline[TrainNode.get_name()]
train_node.add_training_technique("early_stopping", EarlyStopping)
train_node.add_batch_loss_computation_technique("mixup", Mixup)