def get_pytorch_model(self): """Returns a pytorch sequential model of the current incumbent configuration. Not possible for all models. Arguments: Returns: model -- PyTorch sequential model of the current incumbent configuration """ try: if NetworkSelector.get_name() in self.pipeline: return self.pipeline[ NetworkSelector.get_name()].fit_output["network"].layers else: return self.pipeline[NetworkSelectorDatasetInfo.get_name( )].fit_output["network"].layers except: print( "Can not get PyTorch Sequential model for incumbent config. Returning Auto-PyTorch model" ) if NetworkSelector.get_name() in self.pipeline: return self.pipeline[ NetworkSelector.get_name()].fit_output["network"] else: return self.pipeline[NetworkSelectorDatasetInfo.get_name( )].fit_output["network"]
def test_optimizer_selector(self): pipeline = Pipeline([NetworkSelector(), OptimizerSelector()]) net_selector = pipeline[NetworkSelector.get_name()] net_selector.add_network("mlpnet", MlpNet) net_selector.add_network("shapedmlpnet", ShapedMlpNet) net_selector.add_final_activation('none', nn.Sequential()) opt_selector = pipeline[OptimizerSelector.get_name()] opt_selector.add_optimizer("adam", AdamOptimizer) opt_selector.add_optimizer("sgd", SgdOptimizer) pipeline_config = pipeline.get_pipeline_config() pipeline_config["random_seed"] = 42 hyper_config = pipeline.get_hyperparameter_search_space( ).sample_configuration() pipeline.fit_pipeline(hyperparameter_config=hyper_config, pipeline_config=pipeline_config, X=torch.rand(3, 3), Y=torch.rand(3, 2), embedding=nn.Sequential()) sampled_optimizer = opt_selector.fit_output['optimizer'] self.assertIn(type(sampled_optimizer), [optim.Adam, optim.SGD])
def _apply_default_pipeline_settings(pipeline): from autoPyTorch.pipeline.nodes.network_selector import NetworkSelector from autoPyTorch.pipeline.nodes.loss_module_selector import LossModuleSelector from autoPyTorch.pipeline.nodes.metric_selector import MetricSelector from autoPyTorch.pipeline.nodes.train_node import TrainNode from autoPyTorch.pipeline.nodes.cross_validation import CrossValidation import torch.nn as nn from autoPyTorch.components.metrics.standard_metrics import multilabel_accuracy from autoPyTorch.components.preprocessing.loss_weight_strategies import LossWeightStrategyWeightedBinary AutoNetFeatureData._apply_default_pipeline_settings(pipeline) net_selector = pipeline[NetworkSelector.get_name()] net_selector.add_final_activation('sigmoid', nn.Sigmoid()) loss_selector = pipeline[LossModuleSelector.get_name()] loss_selector.add_loss_module('bce_with_logits', nn.BCEWithLogitsLoss, None, False) loss_selector.add_loss_module('bce_with_logits_weighted', nn.BCEWithLogitsLoss, LossWeightStrategyWeightedBinary(), False) metric_selector = pipeline[MetricSelector.get_name()] metric_selector.add_metric('multilabel_accuracy', multilabel_accuracy) train_node = pipeline[TrainNode.get_name()] train_node.default_minimize_value = False cv = pipeline[CrossValidation.get_name()] cv.use_stratified_cv_split_default = False
def _apply_default_pipeline_settings(pipeline): from autoPyTorch.pipeline.nodes.network_selector import NetworkSelector from autoPyTorch.pipeline.nodes.loss_module_selector import LossModuleSelector from autoPyTorch.pipeline.nodes.metric_selector import MetricSelector from autoPyTorch.pipeline.nodes.train_node import TrainNode from autoPyTorch.pipeline.nodes.cross_validation import CrossValidation import torch.nn as nn from autoPyTorch.components.metrics.standard_metrics import mean_distance AutoNetFeatureData._apply_default_pipeline_settings(pipeline) net_selector = pipeline[NetworkSelector.get_name()] net_selector.add_final_activation('none', nn.Sequential()) loss_selector = pipeline[LossModuleSelector.get_name()] loss_selector.add_loss_module('l1_loss', nn.L1Loss) metric_selector = pipeline[MetricSelector.get_name()] metric_selector.add_metric('mean_distance', mean_distance) train_node = pipeline[TrainNode.get_name()] train_node.default_minimize_value = True cv = pipeline[CrossValidation.get_name()] cv.use_stratified_cv_split_default = False
def _apply_default_pipeline_settings(pipeline): from autoPyTorch.pipeline.nodes.network_selector import NetworkSelector from autoPyTorch.pipeline.nodes.loss_module_selector import LossModuleSelector from autoPyTorch.pipeline.nodes.metric_selector import MetricSelector from autoPyTorch.pipeline.nodes.train_node import TrainNode from autoPyTorch.pipeline.nodes.resampling_strategy_selector import ResamplingStrategySelector from autoPyTorch.pipeline.nodes.cross_validation import CrossValidation from autoPyTorch.pipeline.nodes.one_hot_encoding import OneHotEncoding from autoPyTorch.pipeline.nodes.resampling_strategy_selector import ResamplingStrategySelector from autoPyTorch.components.preprocessing.resampling import RandomOverSamplingWithReplacement, RandomUnderSamplingWithReplacement, SMOTE, \ TargetSizeStrategyAverageSample, TargetSizeStrategyDownsample, TargetSizeStrategyMedianSample, TargetSizeStrategyUpsample import torch.nn as nn from autoPyTorch.components.metrics.standard_metrics import accuracy from autoPyTorch.components.preprocessing.loss_weight_strategies import LossWeightStrategyWeighted AutoNetFeatureData._apply_default_pipeline_settings(pipeline) net_selector = pipeline[NetworkSelector.get_name()] net_selector.add_final_activation('softmax', nn.Softmax(1)) loss_selector = pipeline[LossModuleSelector.get_name()] loss_selector.add_loss_module('cross_entropy', nn.CrossEntropyLoss, None, True) loss_selector.add_loss_module('cross_entropy_weighted', nn.CrossEntropyLoss, LossWeightStrategyWeighted(), True) metric_selector = pipeline[MetricSelector.get_name()] metric_selector.add_metric('accuracy', accuracy) resample_selector = pipeline[ResamplingStrategySelector.get_name()] resample_selector.add_over_sampling_method( 'random', RandomOverSamplingWithReplacement) resample_selector.add_over_sampling_method('smote', SMOTE) resample_selector.add_under_sampling_method( 'random', RandomUnderSamplingWithReplacement) resample_selector.add_target_size_strategy('upsample', TargetSizeStrategyUpsample) resample_selector.add_target_size_strategy( 'downsample', TargetSizeStrategyDownsample) resample_selector.add_target_size_strategy( 'average', TargetSizeStrategyAverageSample) resample_selector.add_target_size_strategy( 'median', TargetSizeStrategyMedianSample) train_node = pipeline[TrainNode.get_name()] train_node.default_minimize_value = False cv = pipeline[CrossValidation.get_name()] cv.use_stratified_cv_split_default = True one_hot_encoding_node = pipeline[OneHotEncoding.get_name()] one_hot_encoding_node.encode_Y = True return pipeline
def test_network_selector(self): pipeline = Pipeline([ NetworkSelector() ]) selector = pipeline[NetworkSelector.get_name()] selector.add_network("mlpnet", MlpNet) selector.add_network("shapedmlpnet", ShapedMlpNet) selector.add_final_activation('none', nn.Sequential()) pipeline_config = pipeline.get_pipeline_config() pipeline_config["random_seed"] = 42 hyper_config = pipeline.get_hyperparameter_search_space().sample_configuration() pipeline.fit_pipeline(hyperparameter_config=hyper_config, pipeline_config=pipeline_config, X=torch.rand(3,3), Y=torch.rand(3, 2), embedding=nn.Sequential()) sampled_network = pipeline[selector.get_name()].fit_output['network'] self.assertIn(type(sampled_network), [MlpNet, ShapedMlpNet])
def test_lr_scheduler_selector(self): pipeline = Pipeline([ NetworkSelector(), OptimizerSelector(), LearningrateSchedulerSelector(), ]) net_selector = pipeline[NetworkSelector.get_name()] net_selector.add_network("mlpnet", MlpNet) net_selector.add_network("shapedmlpnet", ShapedMlpNet) net_selector.add_final_activation('none', nn.Sequential()) opt_selector = pipeline[OptimizerSelector.get_name()] opt_selector.add_optimizer("adam", AdamOptimizer) opt_selector.add_optimizer("sgd", SgdOptimizer) lr_scheduler_selector = pipeline[ LearningrateSchedulerSelector.get_name()] lr_scheduler_selector.add_lr_scheduler("step", SchedulerStepLR) lr_scheduler_selector.add_lr_scheduler("exp", SchedulerExponentialLR) pipeline_config = pipeline.get_pipeline_config() pipeline_config["random_seed"] = 42 hyper_config = pipeline.get_hyperparameter_search_space( ).sample_configuration() pipeline.fit_pipeline(hyperparameter_config=hyper_config, pipeline_config=pipeline_config, X=torch.rand(3, 3), Y=torch.rand(3, 2), embedding=nn.Sequential(), training_techniques=[], train_indices=np.array([0, 1, 2])) sampled_lr_scheduler = pipeline[ lr_scheduler_selector.get_name()].fit_output[ 'training_techniques'][0].training_components['lr_scheduler'] self.assertIn(type(sampled_lr_scheduler), [lr_scheduler.ExponentialLR, lr_scheduler.StepLR])
def get_default_pipeline(cls): from autoPyTorch.pipeline.base.pipeline import Pipeline from autoPyTorch.pipeline.nodes.autonet_settings import AutoNetSettings from autoPyTorch.pipeline.nodes.optimization_algorithm import OptimizationAlgorithm from autoPyTorch.pipeline.nodes.cross_validation import CrossValidation from autoPyTorch.pipeline.nodes.imputation import Imputation from autoPyTorch.pipeline.nodes.normalization_strategy_selector import NormalizationStrategySelector from autoPyTorch.pipeline.nodes.one_hot_encoding import OneHotEncoding from autoPyTorch.pipeline.nodes.preprocessor_selector import PreprocessorSelector from autoPyTorch.pipeline.nodes.resampling_strategy_selector import ResamplingStrategySelector from autoPyTorch.pipeline.nodes.embedding_selector import EmbeddingSelector from autoPyTorch.pipeline.nodes.network_selector import NetworkSelector from autoPyTorch.pipeline.nodes.optimizer_selector import OptimizerSelector from autoPyTorch.pipeline.nodes.lr_scheduler_selector import LearningrateSchedulerSelector from autoPyTorch.pipeline.nodes.log_functions_selector import LogFunctionsSelector from autoPyTorch.pipeline.nodes.metric_selector import MetricSelector from autoPyTorch.pipeline.nodes.loss_module_selector import LossModuleSelector from autoPyTorch.pipeline.nodes.train_node import TrainNode # build the pipeline pipeline = Pipeline([ AutoNetSettings(), OptimizationAlgorithm([ CrossValidation([ Imputation(), NormalizationStrategySelector(), OneHotEncoding(), PreprocessorSelector(), ResamplingStrategySelector(), EmbeddingSelector(), NetworkSelector(), OptimizerSelector(), LearningrateSchedulerSelector(), LogFunctionsSelector(), MetricSelector(), LossModuleSelector(), TrainNode() ]) ]) ]) cls._apply_default_pipeline_settings(pipeline) return pipeline
def _apply_default_pipeline_settings(pipeline): from autoPyTorch.pipeline.nodes.normalization_strategy_selector import NormalizationStrategySelector from autoPyTorch.pipeline.nodes.preprocessor_selector import PreprocessorSelector from autoPyTorch.pipeline.nodes.embedding_selector import EmbeddingSelector from autoPyTorch.pipeline.nodes.network_selector import NetworkSelector from autoPyTorch.pipeline.nodes.optimizer_selector import OptimizerSelector from autoPyTorch.pipeline.nodes.lr_scheduler_selector import LearningrateSchedulerSelector from autoPyTorch.pipeline.nodes.train_node import TrainNode from autoPyTorch.components.networks.feature import MlpNet, ResNet, ShapedMlpNet, ShapedResNet from autoPyTorch.components.optimizer.optimizer import AdamOptimizer, SgdOptimizer from autoPyTorch.components.lr_scheduler.lr_schedulers import SchedulerCosineAnnealingWithRestartsLR, SchedulerNone, \ SchedulerCyclicLR, SchedulerExponentialLR, SchedulerReduceLROnPlateau, SchedulerReduceLROnPlateau, SchedulerStepLR from autoPyTorch.components.networks.feature import LearnedEntityEmbedding from sklearn.preprocessing import MinMaxScaler, StandardScaler, MaxAbsScaler from autoPyTorch.components.preprocessing.feature_preprocessing import \ TruncatedSVD, FastICA, RandomKitchenSinks, KernelPCA, Nystroem from autoPyTorch.training.early_stopping import EarlyStopping from autoPyTorch.training.mixup import Mixup pre_selector = pipeline[PreprocessorSelector.get_name()] pre_selector.add_preprocessor('truncated_svd', TruncatedSVD) pre_selector.add_preprocessor('fast_ica', FastICA) pre_selector.add_preprocessor('kitchen_sinks', RandomKitchenSinks) pre_selector.add_preprocessor('kernel_pca', KernelPCA) pre_selector.add_preprocessor('nystroem', Nystroem) norm_selector = pipeline[NormalizationStrategySelector.get_name()] norm_selector.add_normalization_strategy('minmax', MinMaxScaler) norm_selector.add_normalization_strategy('standardize', StandardScaler) norm_selector.add_normalization_strategy('maxabs', MaxAbsScaler) emb_selector = pipeline[EmbeddingSelector.get_name()] emb_selector.add_embedding_module('learned', LearnedEntityEmbedding) net_selector = pipeline[NetworkSelector.get_name()] net_selector.add_network('mlpnet', MlpNet) net_selector.add_network('shapedmlpnet', ShapedMlpNet) net_selector.add_network('resnet', ResNet) net_selector.add_network('shapedresnet', ShapedResNet) opt_selector = pipeline[OptimizerSelector.get_name()] opt_selector.add_optimizer('adam', AdamOptimizer) opt_selector.add_optimizer('sgd', SgdOptimizer) lr_selector = pipeline[LearningrateSchedulerSelector.get_name()] lr_selector.add_lr_scheduler('cosine_annealing', SchedulerCosineAnnealingWithRestartsLR) lr_selector.add_lr_scheduler('cyclic', SchedulerCyclicLR) lr_selector.add_lr_scheduler('exponential', SchedulerExponentialLR) lr_selector.add_lr_scheduler('step', SchedulerStepLR) lr_selector.add_lr_scheduler('plateau', SchedulerReduceLROnPlateau) lr_selector.add_lr_scheduler('none', SchedulerNone) train_node = pipeline[TrainNode.get_name()] train_node.add_training_technique("early_stopping", EarlyStopping) train_node.add_batch_loss_computation_technique("mixup", Mixup)