def test_get_name_error(self): dataset_properties = {'task_type': 'tabular_classification'} names = ['root_mean_sqaured_error', 'average_precision'] try: get_metrics(dataset_properties, names) except ValueError as msg: self.assertRegex(str(msg), r"Invalid name entered for task [a-z]+_[a-z]+, " r"currently supported metrics for task include .*")
def test_get_name_error(): dataset_properties = { 'task_type': TASK_TYPES_TO_STRING[TABULAR_CLASSIFICATION], 'output_type': OUTPUT_TYPES_TO_STRING[BINARY] } names = ['root_mean_sqaured_error', 'average_precision'] with pytest.raises(ValueError, match=r"Invalid name entered for task [a-z]+_[a-z]+, "): get_metrics(dataset_properties, names)
def score(self, X: np.ndarray, y: np.ndarray, batch_size: Optional[int] = None, metric_name: str = 'accuracy') -> float: """Scores the fitted estimator on (X, y) Args: X (np.ndarray): input to the pipeline, from which to guess targets batch_size (Optional[int]): batch_size controls whether the pipeline will be called on small chunks of the data. Useful when calling the predict method on the whole array X results in a MemoryError. y (np.ndarray): Ground Truth labels metric_name (str: default = 'accuracy'): name of the metric to be calculated Returns: float: score based on the metric name """ from autoPyTorch.pipeline.components.training.metrics.utils import get_metrics, calculate_score metrics = get_metrics(self.dataset_properties, [metric_name]) y_pred = self.predict(X, batch_size=batch_size) score = calculate_score(y, y_pred, task_type=STRING_TO_TASK_TYPES[str( self.dataset_properties['task_type'])], metrics=metrics)[metric_name] return score
def test_get_name(self): dataset_properties = {'task_type': 'tabular_classification'} names = ['accuracy', 'average_precision'] metrics = get_metrics(dataset_properties, names) for i in range(len(metrics)): self.assertTrue(isinstance(metrics[i], autoPyTorchMetric)) self.assertEqual(metrics[i].name.lower(), names[i].lower())
def test_get_no_name_regression(output_type): dataset_properties = { 'task_type': 'tabular_regression', 'output_type': output_type } metrics = get_metrics(dataset_properties) for metric in metrics: assert isinstance(metric, autoPyTorchMetric)
def test_get_name(metric): dataset_properties = { 'task_type': TASK_TYPES_TO_STRING[TABULAR_CLASSIFICATION], 'output_type': OUTPUT_TYPES_TO_STRING[BINARY] } metrics = get_metrics(dataset_properties, [metric]) for i in range(len(metrics)): assert isinstance(metrics[i], autoPyTorchMetric) assert metrics[i].name.lower() == metric.lower()
def test_metrics(self): # test of all classification metrics dataset_properties = {'task_type': 'tabular_classification'} y_target = np.array([0, 1, 0, 1]) y_pred = np.array([0, 0, 0, 1]) metrics = get_metrics(dataset_properties=dataset_properties, all_supported_metrics=True) score_dict = calculate_score(y_pred, y_target, STRING_TO_TASK_TYPES[dataset_properties['task_type']], metrics) self.assertIsInstance(score_dict, dict) for name, score in score_dict.items(): self.assertIsInstance(name, str) self.assertIsInstance(score, float) # test of all regression metrics dataset_properties = {'task_type': 'tabular_regression'} y_target = np.array([0.1, 0.6, 0.7, 0.4]) y_pred = np.array([0.6, 0.7, 0.4, 1]) metrics = get_metrics(dataset_properties=dataset_properties, all_supported_metrics=True) score_dict = calculate_score(y_pred, y_target, STRING_TO_TASK_TYPES[dataset_properties['task_type']], metrics) self.assertIsInstance(score_dict, dict) for name, score in score_dict.items(): self.assertIsInstance(name, str) self.assertIsInstance(score, float)
def test_classification_metrics(): # test of all classification metrics dataset_properties = { 'task_type': TASK_TYPES_TO_STRING[TABULAR_CLASSIFICATION], 'output_type': OUTPUT_TYPES_TO_STRING[BINARY] } y_target = np.array([0, 1, 0, 1]) y_pred = np.array([0, 0, 0, 1]) metrics = get_metrics(dataset_properties=dataset_properties, all_supported_metrics=True) score_dict = calculate_score( y_pred, y_target, STRING_TO_TASK_TYPES[dataset_properties['task_type']], metrics) assert isinstance(score_dict, dict) for name, score in score_dict.items(): assert isinstance(name, str) assert isinstance(score, float)
def test_regression_metrics(): # test of all regression metrics dataset_properties = { 'task_type': TASK_TYPES_TO_STRING[TABULAR_REGRESSION], 'output_type': OUTPUT_TYPES_TO_STRING[CONTINUOUS] } y_target = np.array([0.1, 0.6, 0.7, 0.4]) y_pred = np.array([0.6, 0.7, 0.4, 1]) metrics = get_metrics(dataset_properties=dataset_properties, all_supported_metrics=True) score_dict = calculate_score( y_pred, y_target, STRING_TO_TASK_TYPES[dataset_properties['task_type']], metrics) assert isinstance(score_dict, dict) for name, score in score_dict.items(): assert isinstance(name, str) assert isinstance(score, float)
def setUp(self): # Data self.X, self.y = make_classification(n_samples=5000, n_features=4, n_informative=3, n_redundant=1, n_repeated=0, n_classes=2, n_clusters_per_class=2, shuffle=True, random_state=0) self.X = torch.FloatTensor(self.X) self.y = torch.LongTensor(self.y) self.dataset = torch.utils.data.TensorDataset(self.X, self.y) self.loader = torch.utils.data.DataLoader(self.dataset, batch_size=20) self.dataset_properties = { 'task_type': 'tabular_classification', 'output_type': 'binary' } # training requirements layers = [] layers.append(torch.nn.Linear(4, 4)) layers.append(torch.nn.Sigmoid()) layers.append(torch.nn.Linear(4, 2)) self.model = torch.nn.Sequential(*layers) self.criterion = torch.nn.CrossEntropyLoss() self.optimizer = torch.optim.SGD(self.model.parameters(), lr=0.01) self.device = torch.device('cpu') self.logger = logging.getLogger('test') self.metrics = get_metrics(self.dataset_properties) self.epochs = 20 self.budget_tracker = BudgetTracker( budget_type='epochs', max_epochs=self.epochs, ) self.task_type = STRING_TO_TASK_TYPES[ self.dataset_properties['task_type']]
def score(self, X: np.ndarray, y: np.ndarray, batch_size: Optional[int] = None) -> np.ndarray: """score. Args: X (np.ndarray): input to the pipeline, from which to guess targets batch_size (Optional[int]): batch_size controls whether the pipeline will be called on small chunks of the data. Useful when calling the predict method on the whole array X results in a MemoryError. Returns: np.ndarray: coefficient of determination R^2 of the prediction """ from autoPyTorch.pipeline.components.training.metrics.utils import get_metrics, calculate_score metrics = get_metrics(self.dataset_properties, ['r2']) y_pred = self.predict(X, batch_size=batch_size) r2 = calculate_score(y, y_pred, task_type=STRING_TO_TASK_TYPES[ self.dataset_properties['task_type']], metrics=metrics)['r2'] return r2
def __init__(self, task_type: str, output_type: str, optimize_metric: Optional[str] = None, logger_port: int = logging.handlers.DEFAULT_TCP_LOGGING_PORT, random_state: Optional[np.random.RandomState] = None, name: Optional[str] = None): self.model: Optional[Union[CatBoost, BaseEstimator]] = None self.name = name if name is not None else self.__class__.__name__ self.logger_port = logger_port self.logger = get_named_client_logger( name=self.name, host='localhost', port=logger_port, ) if random_state is None: self.random_state = check_random_state(1) else: self.random_state = check_random_state(random_state) self.config = self.get_config() self.all_nan: Optional[np.ndarray] = None self.num_classes: Optional[int] = None self.is_classification = STRING_TO_TASK_TYPES[ task_type] not in REGRESSION_TASKS self.metric = get_metrics(dataset_properties={ 'task_type': task_type, 'output_type': output_type }, names=[optimize_metric] if optimize_metric is not None else None)[0]
def __init__( self, backend: Backend, queue: Queue, metric: autoPyTorchMetric, budget: float, configuration: Union[int, str, Configuration], budget_type: str = None, pipeline_config: Optional[Dict[str, Any]] = None, seed: int = 1, output_y_hat_optimization: bool = True, num_run: Optional[int] = None, include: Optional[Dict[str, Any]] = None, exclude: Optional[Dict[str, Any]] = None, disable_file_output: Union[bool, List[str]] = False, init_params: Optional[Dict[str, Any]] = None, logger_port: Optional[int] = None, all_supported_metrics: bool = True, search_space_updates: Optional[HyperparameterSearchSpaceUpdates] = None ) -> None: self.starttime = time.time() self.configuration = configuration self.backend: Backend = backend self.queue = queue self.datamanager: BaseDataset = self.backend.load_datamanager() assert self.datamanager.task_type is not None, \ "Expected dataset {} to have task_type got None".format(self.datamanager.__class__.__name__) self.task_type = STRING_TO_TASK_TYPES[self.datamanager.task_type] self.output_type = STRING_TO_OUTPUT_TYPES[self.datamanager.output_type] self.issparse = self.datamanager.issparse self.include = include self.exclude = exclude self.search_space_updates = search_space_updates self.X_train, self.y_train = self.datamanager.train_tensors if self.datamanager.val_tensors is not None: self.X_valid, self.y_valid = self.datamanager.val_tensors else: self.X_valid, self.y_valid = None, None if self.datamanager.test_tensors is not None: self.X_test, self.y_test = self.datamanager.test_tensors else: self.X_test, self.y_test = None, None self.metric = metric self.seed = seed # Flag to save target for ensemble self.output_y_hat_optimization = output_y_hat_optimization if isinstance(disable_file_output, bool): self.disable_file_output: bool = disable_file_output elif isinstance(disable_file_output, List): self.disabled_file_outputs: List[str] = disable_file_output else: raise ValueError( 'disable_file_output should be either a bool or a list') self.pipeline_class: Optional[Union[BaseEstimator, BasePipeline]] = None if self.task_type in REGRESSION_TASKS: if isinstance(self.configuration, int): self.pipeline_class = DummyRegressionPipeline elif isinstance(self.configuration, str): self.pipeline_class = MyTraditionalTabularRegressionPipeline elif isinstance(self.configuration, Configuration): self.pipeline_class = autoPyTorch.pipeline.tabular_regression.TabularRegressionPipeline else: raise ValueError('task {} not available'.format( self.task_type)) self.predict_function = self._predict_regression else: if isinstance(self.configuration, int): self.pipeline_class = DummyClassificationPipeline elif isinstance(self.configuration, str): if self.task_type in TABULAR_TASKS: self.pipeline_class = MyTraditionalTabularClassificationPipeline else: raise ValueError( "Only tabular tasks are currently supported with traditional methods" ) elif isinstance(self.configuration, Configuration): if self.task_type in TABULAR_TASKS: self.pipeline_class = autoPyTorch.pipeline.tabular_classification.TabularClassificationPipeline elif self.task_type in IMAGE_TASKS: self.pipeline_class = autoPyTorch.pipeline.image_classification.ImageClassificationPipeline else: raise ValueError('task {} not available'.format( self.task_type)) self.predict_function = self._predict_proba self.dataset_properties = self.datamanager.get_dataset_properties( get_dataset_requirements( info=self.datamanager.get_required_dataset_info(), include=self.include, exclude=self.exclude, search_space_updates=self.search_space_updates)) self.additional_metrics: Optional[List[autoPyTorchMetric]] = None metrics_dict: Optional[Dict[str, List[str]]] = None if all_supported_metrics: self.additional_metrics = get_metrics( dataset_properties=self.dataset_properties, all_supported_metrics=all_supported_metrics) # Update fit dictionary with metrics passed to the evaluator metrics_dict = {'additional_metrics': []} metrics_dict['additional_metrics'].append(self.metric.name) for metric in self.additional_metrics: metrics_dict['additional_metrics'].append(metric.name) self._init_params = init_params assert self.pipeline_class is not None, "Could not infer pipeline class" pipeline_config = pipeline_config if pipeline_config is not None \ else self.pipeline_class.get_default_pipeline_options() self.budget_type = pipeline_config[ 'budget_type'] if budget_type is None else budget_type self.budget = pipeline_config[ self.budget_type] if budget == 0 else budget self.num_run = 0 if num_run is None else num_run logger_name = '%s(%d)' % (self.__class__.__name__.split('.')[-1], self.seed) if logger_port is None: logger_port = logging.handlers.DEFAULT_TCP_LOGGING_PORT self.logger = get_named_client_logger( name=logger_name, port=logger_port, ) self._init_fit_dictionary(logger_port=logger_port, pipeline_config=pipeline_config, metrics_dict=metrics_dict) self.Y_optimization: Optional[np.ndarray] = None self.Y_actual_train: Optional[np.ndarray] = None self.pipelines: Optional[List[BaseEstimator]] = None self.pipeline: Optional[BaseEstimator] = None self.logger.debug("Fit dictionary in Abstract evaluator: {}".format( dict_repr(self.fit_dictionary))) self.logger.debug("Search space updates :{}".format( self.search_space_updates))
def test_get_no_name(self): dataset_properties = {'task_type': 'tabular_classification'} metrics = get_metrics(dataset_properties) for metric in metrics: self.assertTrue(isinstance(metric, autoPyTorchMetric))
def _fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> torch.nn.Module: """ Fits a component by using an input dictionary with pre-requisites Args: X (X: Dict[str, Any]): Dependencies needed by current component to perform fit y (Any): not used. To comply with sklearn API Returns: A instance of self """ # Comply with mypy # Notice that choice here stands for the component choice framework, # where we dynamically build the configuration space by selecting the available # component choices. In this case, is what trainer choices are available assert self.choice is not None # Setup a Logger and other logging support # Writer is not pickable -- make sure it is not saved in self writer = None if 'use_tensorboard_logger' in X and X['use_tensorboard_logger']: writer = SummaryWriter(log_dir=X['backend'].temporary_directory) if X["torch_num_threads"] > 0: torch.set_num_threads(X["torch_num_threads"]) budget_tracker = BudgetTracker( budget_type=X['budget_type'], max_runtime=X['runtime'] if 'runtime' in X else None, max_epochs=X['epochs'] if 'epochs' in X else None, ) # Support additional user metrics additional_metrics = X['additional_metrics'] if 'additional_metrics' in X else None additional_losses = X['additional_losses'] if 'additional_losses' in X else None self.choice.prepare( model=X['network'], metrics=get_metrics(dataset_properties=X['dataset_properties'], names=additional_metrics), criterion=get_loss_instance(X['dataset_properties'], name=additional_losses), budget_tracker=budget_tracker, optimizer=X['optimizer'], device=self.get_device(X), metrics_during_training=X['metrics_during_training'], scheduler=X['lr_scheduler'], task_type=STRING_TO_TASK_TYPES[X['dataset_properties']['task_type']] ) total_parameter_count, trainable_parameter_count = self.count_parameters(X['network']) self.run_summary = RunSummary( total_parameter_count, trainable_parameter_count, ) epoch = 1 while True: # prepare epoch start_time = time.time() self.choice.on_epoch_start(X=X, epoch=epoch) # training train_loss, train_metrics = self.choice.train_epoch( train_loader=X['train_data_loader'], epoch=epoch, logger=self.logger, writer=writer, ) val_loss, val_metrics, test_loss, test_metrics = None, {}, None, {} if self.eval_valid_each_epoch(X): val_loss, val_metrics = self.choice.evaluate(X['val_data_loader'], epoch, writer) if 'test_data_loader' in X and X['test_data_loader']: test_loss, test_metrics = self.choice.evaluate(X['test_data_loader'], epoch, writer) # Save training information self.run_summary.add_performance( epoch=epoch, start_time=start_time, end_time=time.time(), train_loss=train_loss, val_loss=val_loss, test_loss=test_loss, train_metrics=train_metrics, val_metrics=val_metrics, test_metrics=test_metrics, ) # Save the weights of the best model and, if patience # exhausted break training if self.early_stop_handler(X): break if self.choice.on_epoch_end(X=X, epoch=epoch): break self.logger.debug(self.run_summary.repr_last_epoch()) # Reached max epoch on next iter, don't even go there if budget_tracker.is_max_epoch_reached(epoch + 1): break epoch += 1 torch.cuda.empty_cache() # wrap up -- add score if not evaluating every epoch if not self.eval_valid_each_epoch(X): val_loss, val_metrics = self.choice.evaluate(X['val_data_loader']) if 'test_data_loader' in X and X['val_data_loader']: test_loss, test_metrics = self.choice.evaluate(X['test_data_loader']) self.run_summary.add_performance( epoch=epoch, start_time=start_time, end_time=time.time(), train_loss=train_loss, val_loss=val_loss, test_loss=test_loss, train_metrics=train_metrics, val_metrics=val_metrics, test_metrics=test_metrics, ) self.logger.debug(self.run_summary.repr_last_epoch()) self.save_model_for_ensemble() self.logger.info(f"Finished training with {self.run_summary.repr_last_epoch()}") # Tag as fitted self.fitted_ = True return X['network'].state_dict()
def prepare_trainer(self, n_samples: int, trainer: BaseTrainerComponent, task_type: int, epochs=50): # make this test reproducible torch.manual_seed(1) if task_type in CLASSIFICATION_TASKS: X, y = make_classification(n_samples=n_samples, n_features=4, n_informative=3, n_redundant=1, n_repeated=0, n_classes=2, n_clusters_per_class=2, class_sep=3.0, shuffle=True, random_state=0) X = StandardScaler().fit_transform(X) X = torch.tensor(X, dtype=torch.float) y = torch.tensor(y, dtype=torch.long) output_type = BINARY num_outputs = 2 criterion = torch.nn.CrossEntropyLoss elif task_type in REGRESSION_TASKS: X, y = make_regression(n_samples=n_samples, n_features=4, n_informative=3, n_targets=1, shuffle=True, random_state=0) X = StandardScaler().fit_transform(X) X = torch.tensor(X, dtype=torch.float) y = torch.tensor(y, dtype=torch.float) # normalize targets for regression since NNs are better when predicting small outputs y = ((y - y.mean()) / y.std()).unsqueeze(1) output_type = CONTINUOUS num_outputs = 1 criterion = torch.nn.MSELoss else: raise ValueError( f"task type {task_type} not supported for standard trainer test" ) dataset = torch.utils.data.TensorDataset(X, y) loader = torch.utils.data.DataLoader(dataset, batch_size=20) dataset_properties = { 'task_type': TASK_TYPES_TO_STRING[task_type], 'output_type': OUTPUT_TYPES_TO_STRING[output_type] } # training requirements model = torch.nn.Sequential(torch.nn.Linear(4, 4), torch.nn.Sigmoid(), torch.nn.Linear(4, num_outputs)) optimizer = torch.optim.SGD(model.parameters(), lr=0.01) device = torch.device('cpu') logger = logging.getLogger('StandardTrainer - test') metrics = get_metrics(dataset_properties) epochs = epochs budget_tracker = BudgetTracker( budget_type='epochs', max_epochs=epochs, ) trainer.prepare(scheduler=None, model=model, metrics=metrics, criterion=criterion, budget_tracker=budget_tracker, optimizer=optimizer, device=device, metrics_during_training=True, task_type=task_type, labels=y) return trainer, model, optimizer, loader, criterion, epochs, logger