def _hyperparameter_tune(self, X, y, X_val, y_val, scheduler_options, **kwargs): """ Performs HPO and sets self.params to best hyperparameter values """ try_import_mxnet() from .tabular_nn_trial import tabular_nn_trial from .tabular_nn_dataset import TabularNNDataset time_start = time.time() self.verbosity = kwargs.get('verbosity', 2) logger.log(15, "Beginning hyperparameter tuning for Neural Network...") self._set_default_searchspace() # changes non-specified default hyperparams from fixed values to search-spaces. scheduler_cls, scheduler_params = scheduler_options # Unpack tuple if scheduler_cls is None or scheduler_params is None: raise ValueError("scheduler_cls and scheduler_params cannot be None for hyperparameter tuning") num_cpus = scheduler_params['resource']['num_cpus'] params_copy = self._get_params() self.num_dataloading_workers = max(1, int(num_cpus/2.0)) self.batch_size = params_copy['batch_size'] train_dataset, val_dataset = self.generate_datasets(X=X, y=y, params=params_copy, X_val=X_val, y_val=y_val) train_path = self.path + "train" val_path = self.path + "validation" train_dataset.save(file_prefix=train_path) val_dataset.save(file_prefix=val_path) if not np.any([isinstance(params_copy[hyperparam], Space) for hyperparam in params_copy]): logger.warning("Warning: Attempting to do hyperparameter optimization without any search space (all hyperparameters are already fixed values)") else: logger.log(15, "Hyperparameter search space for Neural Network: ") for hyperparam in params_copy: if isinstance(params_copy[hyperparam], Space): logger.log(15, str(hyperparam)+ ": "+str(params_copy[hyperparam])) util_args = dict( train_path=train_path, val_path=val_path, model=self, time_start=time_start, time_limit=scheduler_params['time_out'], fit_kwargs=scheduler_params['resource'], ) tabular_nn_trial.register_args(util_args=util_args, **params_copy) scheduler = scheduler_cls(tabular_nn_trial, **scheduler_params) if ('dist_ip_addrs' in scheduler_params) and (len(scheduler_params['dist_ip_addrs']) > 0): # TODO: Ensure proper working directory setup on remote machines # This is multi-machine setting, so need to copy dataset to workers: logger.log(15, "Uploading preprocessed data to remote workers...") scheduler.upload_files([ train_path + TabularNNDataset.DATAOBJ_SUFFIX, train_path + TabularNNDataset.DATAVALUES_SUFFIX, val_path + TabularNNDataset.DATAOBJ_SUFFIX, val_path + TabularNNDataset.DATAVALUES_SUFFIX ]) # TODO: currently does not work. logger.log(15, "uploaded") scheduler.run() scheduler.join_jobs() return self._get_hpo_results(scheduler=scheduler, scheduler_params=scheduler_params, time_start=time_start)
def _fit(self, X: pd.DataFrame, y: pd.Series, X_val: Optional[pd.DataFrame] = None, y_val: Optional[pd.Series] = None, time_limit: Optional[int] = None, sample_weight=None, verbosity=2, **kwargs): try_import_mxnet() try_import_autogluon_vision() from autogluon.vision import ImagePredictor params = self._get_model_params() X = self.preprocess(X, fit=True) if X_val is not None: X_val = self.preprocess(X_val) if sample_weight is not None: # TODO: support logger.log( 15, "\tsample_weight not yet supported for ImagePredictorModel, this model will ignore them in training." ) X = X.reset_index(drop=True) y = y.reset_index(drop=True) if X_val is not None: X_val = X_val.reset_index(drop=True) y_val = y_val.reset_index(drop=True) X[self._label_column_name] = y if X_val is not None: X_val[self._label_column_name] = y_val verbosity_image = max(0, verbosity - 1) root_logger = logging.getLogger() root_log_level = root_logger.level # TODO: ImagePredictor doesn't use problem_type in any way at present. # It also doesn't error or warn if problem_type is not one it expects. self.model = ImagePredictor( problem_type=self.problem_type, path=self.path, # eval_metric=self.eval_metric, # TODO: multiclass/binary vision problem works only with accuracy, regression with rmse verbosity=verbosity_image) logger.log(15, f'\tHyperparameters: {params}') # FIXME: ImagePredictor crashes if given float time_limit if time_limit is not None: time_limit = int(time_limit) self.model.fit(train_data=X, tuning_data=X_val, time_limit=time_limit, hyperparameters=params, random_state=0) # self.model.set_verbosity(verbosity) # TODO: How to set verbosity of fit predictor? root_logger.setLevel(root_log_level) # Reset log level
def _fit(self, X, y, X_val=None, y_val=None, time_limit=None, sample_weight=None, num_cpus=1, num_gpus=0, reporter=None, **kwargs): """ X (pd.DataFrame): training data features (not necessarily preprocessed yet) X_val (pd.DataFrame): test data features (should have same column names as Xtrain) y (pd.Series): y_val (pd.Series): are pandas Series kwargs: Can specify amount of compute resources to utilize (num_cpus, num_gpus). """ start_time = time.time() try_import_mxnet() import mxnet as mx self.verbosity = kwargs.get('verbosity', 2) global _has_warned_mxnet_deprecation if not _has_warned_mxnet_deprecation: _has_warned_mxnet_deprecation = True logger.log(30, '\tWARNING: TabularNeuralNetMxnetModel (alias "NN" & "NN_MXNET") has been deprecated in v0.4.0.\n' '\t\tStarting in v0.6.0, calling TabularNeuralNetMxnetModel will raise an exception.\n' '\t\tConsider instead using TabularNeuralNetTorchModel via "NN_TORCH".') if sample_weight is not None: # TODO: support logger.log(15, "sample_weight not yet supported for TabularNeuralNetModel, this model will ignore them in training.") params = self._get_model_params() if num_cpus is not None: self.num_dataloading_workers = max(1, int(num_cpus/2.0)) else: self.num_dataloading_workers = 1 if self.num_dataloading_workers == 1: self.num_dataloading_workers = 0 # 0 is always faster and uses less memory than 1 self.batch_size = params['batch_size'] train_dataset, val_dataset = self.generate_datasets(X=X, y=y, params=params, X_val=X_val, y_val=y_val) logger.log(15, "Training data for neural network has: %d examples, %d features (%d vector, %d embedding)" % (train_dataset.num_examples, train_dataset.num_features, len(train_dataset.feature_groups['vector']), len(train_dataset.feature_groups['embed']) )) # self._save_preprocessor() # TODO: should save these things for hyperparam tunning. Need one HP tuner for network-specific HPs, another for preprocessing HPs. if num_gpus is not None and num_gpus >= 1: self.ctx = mx.gpu() # Currently cannot use more than 1 GPU else: self.ctx = mx.cpu() self.get_net(train_dataset, params=params) if time_limit is not None: time_elapsed = time.time() - start_time time_limit_orig = time_limit time_limit = time_limit - time_elapsed if time_limit <= time_limit_orig * 0.4: # if 60% of time was spent preprocessing, likely not enough time to train model raise TimeLimitExceeded self.train_net(train_dataset=train_dataset, params=params, val_dataset=val_dataset, initialize=True, setup_trainer=True, time_limit=time_limit, reporter=reporter) self.params_post_fit = params """
def _fit(self, X, y, X_val=None, y_val=None, time_limit=None, num_cpus=1, num_gpus=0, reporter=None, sample_weight=None, **kwargs): """ X (pd.DataFrame): training data features (not necessarily preprocessed yet) X_val (pd.DataFrame): test data features (should have same column names as Xtrain) y (pd.Series): y_val (pd.Series): are pandas Series kwargs: Can specify amount of compute resources to utilize (num_cpus, num_gpus). """ start_time = time.time() try_import_mxnet() import mxnet as mx self.verbosity = kwargs.get('verbosity', 2) if sample_weight is not None: # TODO: support logger.log(15, "sample_weight not yet supported for TabularNeuralNetModel, this model will ignore them in training.") params = self.params.copy() params = fixedvals_from_searchspaces(params) if self.feature_metadata is None: raise ValueError("Trainer class must set feature_metadata for this model") if num_cpus is not None: self.num_dataloading_workers = max(1, int(num_cpus/2.0)) else: self.num_dataloading_workers = 1 if self.num_dataloading_workers == 1: self.num_dataloading_workers = 0 # 0 is always faster and uses less memory than 1 self.batch_size = params['batch_size'] train_dataset, val_dataset = self.generate_datasets(X=X, y=y, params=params, X_val=X_val, y_val=y_val) logger.log(15, "Training data for neural network has: %d examples, %d features (%d vector, %d embedding, %d language)" % (train_dataset.num_examples, train_dataset.num_features, len(train_dataset.feature_groups['vector']), len(train_dataset.feature_groups['embed']), len(train_dataset.feature_groups['language']) )) # self._save_preprocessor() # TODO: should save these things for hyperparam tunning. Need one HP tuner for network-specific HPs, another for preprocessing HPs. if num_gpus is not None and num_gpus >= 1: self.ctx = mx.gpu() # Currently cannot use more than 1 GPU else: self.ctx = mx.cpu() self.get_net(train_dataset, params=params) if time_limit is not None: time_elapsed = time.time() - start_time time_limit_orig = time_limit time_limit = time_limit - time_elapsed if time_limit <= time_limit_orig * 0.4: # if 60% of time was spent preprocessing, likely not enough time to train model raise TimeLimitExceeded self.train_net(train_dataset=train_dataset, params=params, val_dataset=val_dataset, initialize=True, setup_trainer=True, time_limit=time_limit, reporter=reporter) self.params_post_fit = params """
def _fit(self, X: pd.DataFrame, y: pd.Series, X_val: Optional[pd.DataFrame] = None, y_val: Optional[pd.Series] = None, time_limit: Optional[int] = None, sample_weight=None, verbosity=2, **kwargs): try_import_mxnet() try_import_autogluon_vision() from autogluon.vision import ImagePredictor params = self._get_model_params() X = self.preprocess(X, fit=True) if X_val is not None: X_val = self.preprocess(X_val) if sample_weight is not None: # TODO: support logger.log(15, "\tsample_weight not yet supported for ImagePredictorModel, this model will ignore them in training.") X = X.reset_index(drop=True) y = y.reset_index(drop=True) if X_val is not None: X_val = X_val.reset_index(drop=True) y_val = y_val.reset_index(drop=True) X[self._label_column_name] = y if X_val is not None: X_val[self._label_column_name] = y_val null_indices = X['image'] == '' # TODO: Consider some kind of weighting of the two options so there isn't a harsh cutoff at 50 # FIXME: What if all rows in a class are null? Will probably crash. if null_indices.sum() > 50: self._dummy_pred_proba = self._compute_dummy_pred_proba(y[null_indices]) # FIXME: Do this one for better results else: # Not enough null to get a confident estimate of null label average, instead use all data average self._dummy_pred_proba = self._compute_dummy_pred_proba(y) if null_indices.sum() > 0: X = X[~null_indices] if X_val is not None: null_indices_val = X_val['image'] == '' if null_indices_val.sum() > 0: X_val = X_val[~null_indices_val] verbosity_image = max(0, verbosity - 1) root_logger = logging.getLogger() root_log_level = root_logger.level # TODO: ImagePredictor doesn't use problem_type in any way at present. # It also doesn't error or warn if problem_type is not one it expects. self.model = ImagePredictor( problem_type=self.problem_type, path=self.path, # eval_metric=self.eval_metric, # TODO: multiclass/binary vision problem works only with accuracy, regression with rmse verbosity=verbosity_image ) logger.log(15, f'\tHyperparameters: {params}') # FIXME: ImagePredictor crashes if given float time_limit if time_limit is not None: time_limit = int(time_limit) self.model.fit(train_data=X, tuning_data=X_val, time_limit=time_limit, hyperparameters=params, random_state=0) # self.model.set_verbosity(verbosity) # TODO: How to set verbosity of fit predictor? root_logger.setLevel(root_log_level) # Reset log level
def _fit(self, X: pd.DataFrame, y: pd.Series, X_val: Optional[pd.DataFrame] = None, y_val: Optional[pd.Series] = None, time_limit: Optional[int] = None, sample_weight=None, **kwargs): """The internal fit function Parameters ---------- X Features of the training dataset y Labels of the training dataset X_val Features of the validation dataset y_val Labels of the validation dataset time_limit The time limits for the fit function kwargs Other keyword arguments """ try_import_mxnet() try_import_autogluon_text() from autogluon.text import TextPredictor # Decide name of the label column if 'label' in X.columns: label_col_id = 0 while True: self._label_column_name = 'label{}'.format(label_col_id) if self._label_column_name not in X.columns: break label_col_id += 1 else: self._label_column_name = 'label' X_train = self.preprocess(X, fit=True) if X_val is not None: X_val = self.preprocess(X_val) # Get arguments from kwargs verbosity = kwargs.get('verbosity', 2) num_cpus = kwargs.get('num_cpus', None) num_gpus = kwargs.get('num_gpus', None) if sample_weight is not None: # TODO: support logger.log( 15, "sample_weight not yet supported for TextPredictorModel, this model will ignore them in training." ) X_train.insert(len(X_train.columns), self._label_column_name, y) if X_val is not None: X_val.insert(len(X_val.columns), self._label_column_name, y_val) assert self.params['tune_kwargs']['num_trials'] == 1 \ or self.params['tune_kwargs']['num_trials'] is None,\ 'Currently, you cannot nest the hyperparameter search in text neural network ' \ 'and the AutoGluon Tabular.' verbosity_text = max(0, verbosity - 1) root_logger = logging.getLogger() root_log_level = root_logger.level self.model = TextPredictor(label=self._label_column_name, problem_type=self.problem_type, path=self.path, eval_metric=self.eval_metric, verbosity=verbosity_text) self.model.fit(train_data=X_train, tuning_data=X_val, time_limit=time_limit, num_gpus=num_gpus, num_cpus=num_cpus, hyperparameters=self.params, seed=self.params.get('seed', 0)) self.model.set_verbosity(verbosity) root_logger.setLevel(root_log_level) # Reset log level