def __init__(self, label, problem_type=None, eval_metric=None, path=None, verbosity=2, **kwargs): self.verbosity = verbosity set_logger_verbosity(self.verbosity, logger=logger) self._validate_init_kwargs(kwargs) path = setup_outputdir(path) learner_type = kwargs.pop('learner_type', DefaultLearner) learner_kwargs = kwargs.pop( 'learner_kwargs', dict()) # TODO: id_columns -> ignored_columns +1 self._learner: AbstractLearner = learner_type( path_context=path, label=label, feature_generator=None, eval_metric=eval_metric, problem_type=problem_type, **learner_kwargs) self._learner_type = type(self._learner) self._trainer = None
def __init__(self, label, problem_type=None, eval_metric=None, path=None, verbosity=2, warn_if_exist=True): self.verbosity = verbosity set_logger_verbosity(self.verbosity, logger=logger) self._label = label self._problem_type = problem_type self._eval_metric = eval_metric self._path = setup_outputdir(path, warn_if_exist=warn_if_exist) self._model = None self._fit_called = False self._backend = None
def load(cls, path, verbosity=2): set_logger_verbosity( verbosity, logger=logger ) # Reset logging after load (may be in new Python session) if path is None: raise ValueError("output_directory cannot be None in load()") path = setup_outputdir( path, warn_if_exist=False) # replace ~ with absolute path if it exists predictor: TabularPredictor = load_pkl.load(path=path + cls.predictor_file_name) learner = predictor._learner_type.load(path) predictor._set_post_fit_vars(learner=learner) try: from ...version import __version__ version_inference = __version__ except: version_inference = None # TODO: v0.1 Move version var to predictor object in the case where learner does not exist try: version_fit = predictor._learner.version except: version_fit = None if version_fit is None: version_fit = 'Unknown (Likely <=0.0.11)' if version_inference != version_fit: logger.warning('') logger.warning( '############################## WARNING ##############################' ) logger.warning( 'WARNING: AutoGluon version differs from the version used during the original model fit! This may lead to instability and it is highly recommended the model be loaded with the exact AutoGluon version it was fit with.' ) logger.warning(f'\tFit Version: {version_fit}') logger.warning(f'\tCurrent Version: {version_inference}') logger.warning( '############################## WARNING ##############################' ) logger.warning('') return predictor
def fit(self, train_data, tuning_data=None, time_limit='auto', presets=None, hyperparameters=None, **kwargs): """Automatic fit process for object detection. Tip: if you observe very slow training speed only happening at the first epoch and your overall time budget is not large, you may disable `CUDNN_AUTOTUNE` by setting the environment variable `export MXNET_CUDNN_AUTOTUNE_DEFAULT=0` before running your python script or insert `import os; os.environ['MXNET_CUDNN_AUTOTUNE_DEFAULT'] = '0'` before any code block. The tuning is beneficial in terms of training speed in the long run, but may cost your noticeble overhead at the begining of each trial. Parameters ---------- train_data : pd.DataFrame or str Training data, can be a dataframe like image dataset. For more details of how to construct a object detection dataset, please checkout: `http://preview.d2l.ai/d8/main/object_detection/getting_started.html`. If a string is provided, will search for d8 datasets. tuning_data : pd.DataFrame or str, default = None Holdout tuning data for validation, reserved for model selection and hyperparameter-tuning, can be a dataframe like image dataset. If a string is provided, will search for k8 datasets. If `None`, the validation dataset will be randomly split from `train_data` according to `holdout_frac`. time_limit : int, default = 'auto'(defaults to 2 hours if no presets detected) Time limit in seconds, if `None`, will run until all tuning and training finished. If `time_limit` is hit during `fit`, the HPO process will interrupt and return the current best configuration. presets : list or str or dict, default = ['medium_quality_faster_train'] List of preset configurations for various arguments in `fit()`. Can significantly impact predictive accuracy, memory-footprint, and inference latency of trained models, and various other properties of the returned `predictor`. It is recommended to specify presets and avoid specifying most other `fit()` arguments or model hyperparameters prior to becoming familiar with AutoGluon. As an example, to get the most accurate overall predictor (regardless of its efficiency), set `presets='best_quality'`. To get good quality with faster inference speed, set `presets='good_quality_faster_inference'` Any user-specified arguments in `fit()` will override the values used by presets. If specifying a list of presets, later presets will override earlier presets if they alter the same argument. For precise definitions of the provided presets, see file: `autogluon/vision/configs/presets_configs.py`. Users can specify custom presets by passing in a dictionary of argument values as an element to the list. Available Presets: ['best_quality', 'high_quality_fast_inference', 'good_quality_faster_inference', 'medium_quality_faster_train'] It is recommended to only use one `quality` based preset in a given call to `fit()` as they alter many of the same arguments and are not compatible with each-other. Note that depending on your specific hardware limitation(# gpu, size of gpu memory...) your mileage may vary a lot, you may choose lower quality presets if necessary, and try to reduce `batch_size` if OOM("RuntimeError: CUDA error: out of memory") happens frequently during the `fit`. In-depth Preset Info: best_quality={ 'hyperparameters': { 'transfer': Categorical('faster_rcnn_fpn_resnet101_v1d_coco'), 'lr': Real(1e-5, 1e-3, log=True), 'batch_size': Categorical(4, 8), 'epochs': 30 }, 'hyperparameter_tune_kwargs': { 'num_trials': 128, 'search_strategy': 'bayesopt'}, 'time_limit': 24*3600,} Best predictive accuracy with little consideration to training/inference time or model size. Achieve even better results by specifying a large time_limit value. Recommended for applications that benefit from the best possible model accuracy and be prepared with the extremly long training time. good_quality_fast_inference={ 'hyperparameters': { 'transfer': Categorical('ssd_512_resnet50_v1_coco', 'yolo3_darknet53_coco', 'center_net_resnet50_v1b_coco'), 'lr': Real(1e-4, 1e-2, log=True), 'batch_size': Categorical(8, 16, 32, 64), 'epochs': 50 }, 'hyperparameter_tune_kwargs': { 'num_trials': 512, 'search_strategy': 'bayesopt'}, 'time_limit': 12*3600,} Good predictive accuracy with fast inference. Recommended for applications that require reasonable inference speed and/or model size. medium_quality_faster_train={ 'hyperparameters': { 'transfer': Categorical('ssd_512_resnet50_v1_coco'), 'lr': 0.01, 'batch_size': Categorical(8, 16), 'epochs': 30 }, 'hyperparameter_tune_kwargs': { 'num_trials': 16, 'search_strategy': 'random'}, 'time_limit': 2*3600,} Medium predictive accuracy with very fast inference and very fast training time. This is the default preset in AutoGluon, but should generally only be used for quick prototyping. medium_quality_faster_inference={ 'hyperparameters': { 'transfer': Categorical('center_net_resnet18_v1b_coco', 'yolo3_mobilenet1.0_coco'), 'lr': Categorical(0.01, 0.005, 0.001), 'batch_size': Categorical(32, 64, 128), 'epochs': Categorical(30, 50), }, 'hyperparameter_tune_kwargs': { 'num_trials': 32, 'search_strategy': 'bayesopt'}, 'time_limit': 4*3600,} Medium predictive accuracy with very fast inference. Comparing with `medium_quality_faster_train` it uses faster model but explores more hyperparameters. hyperparameters : dict, default = None Extra hyperparameters for specific models. Accepted args includes(not limited to): epochs : int, default value based on network The `epochs` for model training. batch_size : int Mini batch size lr : float Trainer learning rate for optimization process. You can get the list of accepted hyperparameters in `config.yaml` saved by this predictor. **kwargs : holdout_frac : float, default = 0.1 The random split ratio for `tuning_data` if `tuning_data==None`. random_state : int, default = None The random_state(seed) for shuffling data, only used if `tuning_data==None`. Note that the `random_state` only affect the splitting process, not model training. If not specified(None), will leave the original random sampling intact. nthreads_per_trial : int, default = (# cpu cores) Number of CPU threads for each trial, if `None`, will detect the # cores on current instance. ngpus_per_trial : int, default = (# gpus) Number of GPUs to use for each trial, if `None`, will detect the # gpus on current instance. hyperparameter_tune_kwargs: dict, default = None num_trials : int, default = 1 The limit of HPO trials that can be performed within `time_limit`. The HPO process will be terminated when `num_trials` trials have finished or wall clock `time_limit` is reached, whichever comes first. search_strategy : str, default = 'random' Searcher strategy for HPO, 'random' by default. Options include: ‘random’ (random search), ‘bayesopt’ (Gaussian process Bayesian optimization), ‘grid’ (grid search). max_reward : float, default = None The reward threashold for stopping criteria. If `max_reward` is reached during HPO, the scheduler will terminate earlier to reduce time cost. scheduler_options : dict, default = None Extra options for HPO scheduler, please refer to :class:`autogluon.core.Searcher` for details. """ # init/validate kwargs kwargs = self._validate_kwargs(kwargs) # unpack num_trials = kwargs['hyperparameter_tune_kwargs']['num_trials'] nthreads_per_trial = kwargs['nthreads_per_trial'] ngpus_per_trial = kwargs['ngpus_per_trial'] holdout_frac = kwargs['holdout_frac'] random_state = kwargs['random_state'] search_strategy = kwargs['hyperparameter_tune_kwargs'][ 'search_strategy'] max_reward = kwargs['hyperparameter_tune_kwargs']['max_reward'] scheduler_options = kwargs['hyperparameter_tune_kwargs'][ 'scheduler_options'] log_level = verbosity2loglevel(self._verbosity) set_logger_verbosity(self._verbosity, logger=logger) if presets: if not isinstance(presets, list): presets = [presets] logger.log(20, f'Presets specified: {presets}') if time_limit == 'auto': # no presets, no user specified time_limit time_limit = 7200 logger.log(20, f'`time_limit=auto` set to `time_limit={time_limit}`.') if self._detector is not None: self._detector._logger.setLevel(log_level) self._detector._logger.propagate = True self._fit_summary = self._detector.fit(train_data, tuning_data, 1 - holdout_frac, random_state, resume=False) if hasattr(self._classifier, 'fit_history'): self._fit_summary[ 'fit_history'] = self._classifier.fit_history() return self # new HPO task if time_limit is not None and num_trials is None: num_trials = 99999 if time_limit is None and num_trials is None: raise ValueError( "`time_limit` and kwargs['hyperparameter_tune_kwargs']['num_trials'] can not be `None` at the same time, " "otherwise the training will not be terminated gracefully.") config = { 'log_dir': self._log_dir, 'num_trials': 99999 if num_trials is None else max(1, num_trials), 'time_limits': 2147483647 if time_limit is None else max(1, time_limit), 'search_strategy': search_strategy, } if max_reward is not None: config['max_reward'] = max_reward if nthreads_per_trial is not None: config['nthreads_per_trial'] = nthreads_per_trial if ngpus_per_trial is not None: config['ngpus_per_trial'] = ngpus_per_trial if isinstance(hyperparameters, dict): if 'batch_size' in hyperparameters: bs = hyperparameters['batch_size'] _check_gpu_memory_presets(bs, ngpus_per_trial, 4, 1280) # 1280MB per sample # check if hyperparameters overwriting existing config for k, v in hyperparameters.items(): if k in config: raise ValueError( f'Overwriting {k} = {config[k]} to {v} by hyperparameters is ambiguous.' ) config.update(hyperparameters) if scheduler_options is not None: config.update(scheduler_options) # verbosity if log_level > logging.INFO: logging.getLogger( 'gluoncv.auto.tasks.object_detection').propagate = False for logger_name in ('SSDEstimator', 'CenterNetEstimator', 'YOLOv3Estimator', 'FasterRCNNEstimator'): logging.getLogger(logger_name).setLevel(log_level) logging.getLogger(logger_name).propagate = False task = _ObjectDetection(config=config) task._logger.setLevel(log_level) task._logger.propagate = True with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") with MXNetErrorCatcher() as err: self._detector = task.fit(train_data, tuning_data, 1 - holdout_frac, random_state) if err.exc_value is not None: raise RuntimeError(err.exc_value) self._detector._logger.setLevel(log_level) self._detector._logger.propagate = True self._fit_summary = task.fit_summary() if hasattr(task, 'fit_history'): self._fit_summary['fit_history'] = task.fit_history() return self
def fit(self, train_data, tuning_data=None, time_limit='auto', presets=None, hyperparameters=None, **kwargs): """Automatic fit process for image prediction. Parameters ---------- train_data : pd.DataFrame or str Training data, can be a dataframe like image dataset. For dataframe like datasets, `image` and `label` columns are required. `image`: raw image paths. `label`: categorical integer id, starting from 0. For more details of how to construct a dataset for image predictor, check out: `http://preview.d2l.ai/d8/main/image_classification/getting_started.html`. If a string is provided, will search for d8 built-in datasets. tuning_data : pd.DataFrame or str, default = None Another dataset containing validation data reserved for model selection and hyperparameter-tuning, can be a dataframe like image dataset. If a string is provided, will search for k8 datasets. If `None`, the validation dataset will be randomly split from `train_data` according to `holdout_frac`. time_limit : int, default = 'auto' (defaults to 2 hours if no presets detected) Time limit in seconds, if `None`, will run until all tuning and training finished. If `time_limit` is hit during `fit`, the HPO process will interrupt and return the current best configuration. presets : list or str or dict, default = ['medium_quality_faster_train'] List of preset configurations for various arguments in `fit()`. Can significantly impact predictive accuracy, memory-footprint, and inference latency of trained models, and various other properties of the returned `predictor`. It is recommended to specify presets and avoid specifying most other `fit()` arguments or model hyperparameters prior to becoming familiar with AutoGluon. As an example, to get the most accurate overall predictor (regardless of its efficiency), set `presets='best_quality'`. To get good quality with faster inference speed, set `presets='good_quality_faster_inference'` Any user-specified arguments in `fit()` will override the values used by presets. If specifying a list of presets, later presets will override earlier presets if they alter the same argument. For precise definitions of the provided presets, see file: `autogluon/vision/configs/presets_configs.py`. Users can specify custom presets by passing in a dictionary of argument values as an element to the list. Available Presets: ['best_quality', 'high_quality_fast_inference', 'good_quality_faster_inference', 'medium_quality_faster_train'] It is recommended to only use one `quality` based preset in a given call to `fit()` as they alter many of the same arguments and are not compatible with each-other. Note that depending on your specific hardware limitation(# gpu, size of gpu memory...) your mileage may vary a lot, you may choose lower quality presets if necessary, and try to reduce `batch_size` if OOM("RuntimeError: CUDA error: out of memory") happens frequently during the `fit`. In-depth Preset Info: # Best predictive accuracy with little consideration to inference time or model size. Achieve even better results by specifying a large time_limit value. # Recommended for applications that benefit from the best possible model accuracy. best_quality={ 'hyperparameters': { 'model': Categorical('resnet50_v1b', 'resnet101_v1d', 'resnest200'), 'lr': Real(1e-5, 1e-2, log=True), 'batch_size': Categorical(8, 16, 32, 64, 128), 'epochs': 200, 'early_stop_patience': 50 }, 'hyperparameter_tune_kwargs': { 'num_trials': 1024, 'searcher': 'random', }, 'time_limit': 12*3600, }, # Good predictive accuracy with fast inference. # Recommended for applications that require reasonable inference speed and/or model size. good_quality_fast_inference={ 'hyperparameters': { 'model': Categorical('resnet50_v1b', 'resnet34_v1b'), 'lr': Real(1e-4, 1e-2, log=True), 'batch_size': Categorical(8, 16, 32, 64, 128), 'epochs': 150, 'early_stop_patience': 20 }, 'hyperparameter_tune_kwargs': { 'num_trials': 512, 'searcher': 'random', }, 'time_limit': 8*3600, }, # Medium predictive accuracy with very fast inference and very fast training time. # This is the default preset in AutoGluon, but should generally only be used for quick prototyping. medium_quality_faster_train={ 'hyperparameters': { 'model': 'resnet50_v1b', 'lr': 0.01, 'batch_size': 64, 'epochs': 50, 'early_stop_patience': 5 }, 'time_limit': 1*3600, }, # Medium predictive accuracy with very fast inference. # Comparing with `medium_quality_faster_train` it uses faster model but explores more hyperparameters. medium_quality_faster_inference={ 'hyperparameters': { 'model': Categorical('resnet18_v1b', 'mobilenetv3_small'), 'lr': Categorical(0.01, 0.005, 0.001), 'batch_size': Categorical(64, 128), 'epochs': Categorical(50, 100), 'early_stop_patience': 10 }, 'hyperparameter_tune_kwargs': { 'num_trials': 32, 'searcher': 'random', }, 'time_limit': 2*3600, }, hyperparameters : dict, default = None Extra hyperparameters for specific models. Accepted args includes(not limited to): epochs : int, default value based on network The `epochs` for model training. net : mx.gluon.Block The custom network. If defined, the model name in config will be ignored so your custom network will be used for training rather than pulling it from model zoo. optimizer : mx.Optimizer The custom optimizer object. If defined, the optimizer will be ignored in config but this object will be used in training instead. batch_size : int Mini batch size lr : float Trainer learning rate for optimization process. early_stop_patience : int, default=10 Number of epochs with no improvement after which train is early stopped. Use `None` to disable. early_stop_min_delta : float, default=1e-4 The small delta value to ignore when evaluating the metric. A large delta helps stablize the early stopping strategy against tiny fluctuation, e.g. 0.5->0.49->0.48->0.499->0.500001 is still considered as a good timing for early stopping. early_stop_baseline : float, default=None The minimum(baseline) value to trigger early stopping. For example, with `early_stop_baseline=0.5`, early stopping won't be triggered if the metric is less than 0.5 even if plateau is detected. Use `None` to disable. early_stop_max_value : float, default=None The max value for metric, early stop training instantly once the max value is achieved. Use `None` to disable. You can get the list of accepted hyperparameters in `config.yaml` saved by this predictor. **kwargs : holdout_frac : float, default = 0.1 The random split ratio for `tuning_data` if `tuning_data==None`. random_state : int, default = None The random_state(seed) for shuffling data, only used if `tuning_data==None`. Note that the `random_state` only affect the splitting process, not model training. If not specified(None), will leave the original random sampling intact. nthreads_per_trial : int, default = (# cpu cores) Number of CPU threads for each trial, if `None`, will detect the # cores on current instance. ngpus_per_trial : int, default = (# gpus) Number of GPUs to use for each trial, if `None`, will detect the # gpus on current instance. hyperparameter_tune_kwargs: dict, default = None num_trials : int, default = 1 The limit of HPO trials that can be performed within `time_limit`. The HPO process will be terminated when `num_trials` trials have finished or wall clock `time_limit` is reached, whichever comes first. search_strategy : str, default = 'random' Searcher strategy for HPO, 'random' by default. Options include: ‘random’ (random search), ‘bayesopt’ (Gaussian process Bayesian optimization), ‘grid’ (grid search). max_reward : float, default = None The reward threashold for stopping criteria. If `max_reward` is reached during HPO, the scheduler will terminate earlier to reduce time cost. scheduler_options : dict, default = None Extra options for HPO scheduler, please refer to :class:`autogluon.core.Searcher` for details. """ if self._problem_type is None: # options: multiclass, binary, regression self._problem_type = MULTICLASS assert self._problem_type in ( MULTICLASS, BINARY, REGRESSION), f"Invalid problem_type: {self._problem_type}" if self._eval_metric is None: if self._problem_type == REGRESSION: # options: rmse self._eval_metric = 'rmse' logger.log( 20, 'ImagePredictor sets rmse as default eval_metric for regression problems.' ) else: # options: accuracy self._eval_metric = 'accuracy' logger.log( 20, 'ImagePredictor sets accuracy as default eval_metric for classification problems.' ) # init/validate kwargs kwargs = self._validate_kwargs(kwargs) # unpack num_trials = kwargs['hyperparameter_tune_kwargs']['num_trials'] nthreads_per_trial = kwargs['nthreads_per_trial'] ngpus_per_trial = kwargs['ngpus_per_trial'] holdout_frac = kwargs['holdout_frac'] random_state = kwargs['random_state'] scheduler = kwargs['hyperparameter_tune_kwargs']['scheduler'] searcher = kwargs['hyperparameter_tune_kwargs']['searcher'] max_reward = kwargs['hyperparameter_tune_kwargs']['max_reward'] scheduler_options = kwargs['hyperparameter_tune_kwargs'][ 'scheduler_options'] # deep copy to avoid inplace overwrite train_data = copy.deepcopy(train_data) tuning_data = copy.deepcopy(tuning_data) log_level = verbosity2loglevel(self._verbosity) set_logger_verbosity(self._verbosity, logger=logger) if presets: if not isinstance(presets, list): presets = [presets] logger.log(20, f'Presets specified: {presets}') if time_limit == 'auto': # no presets, no user specified time_limit time_limit = 7200 logger.log(20, f'`time_limit=auto` set to `time_limit={time_limit}`.') use_rec = False if isinstance(train_data, str) and train_data == 'imagenet': # FIXME: imagenet does not work, crashes in validating data due to empty DataFrames. logger.warning( 'ImageNet is a huge dataset which cannot be downloaded directly, ' + 'please follow the data preparation tutorial in GluonCV.' + 'The following record files(symlinks) will be used: \n' + 'rec_train : ~/.mxnet/datasets/imagenet/rec/train.rec\n' + 'rec_train_idx : ~/.mxnet/datasets/imagenet/rec/train.idx\n' + 'rec_val : ~/.mxnet/datasets/imagenet/rec/val.rec\n' + 'rec_val_idx : ~/.mxnet/datasets/imagenet/rec/val.idx\n') train_data = pd.DataFrame({'image': [], self._label_inner: []}) tuning_data = pd.DataFrame({'image': [], self._label_inner: []}) use_rec = True if isinstance(train_data, str): from d8.image_classification import Dataset as D8D names = D8D.list() if train_data.lower() in names: train_data = D8D.get(train_data) else: valid_names = '\n'.join(names) raise ValueError( f'`train_data` {train_data} is not among valid list {valid_names}' ) if tuning_data is None: train_data, tuning_data = train_data.split(1 - holdout_frac) if isinstance(tuning_data, str): from d8.image_classification import Dataset as D8D names = D8D.list() if tuning_data.lower() in names: tuning_data = D8D.get(tuning_data) else: valid_names = '\n'.join(names) raise ValueError( f'`tuning_data` {tuning_data} is not among valid list {valid_names}' ) # data sanity check train_data = self._validate_data(train_data) train_labels = _get_valid_labels(train_data) self._label_cleaner = LabelCleaner.construct( problem_type=self._problem_type, y=train_labels, y_uncleaned=train_labels) train_labels_cleaned = self._label_cleaner.transform(train_labels) # converting to internal label set _set_valid_labels(train_data, train_labels_cleaned) tuning_data_validated = False if tuning_data is None: train_data, tuning_data, _, _ = generate_train_test_split( X=train_data, y=train_data[self._label_inner], problem_type=self._problem_type, test_size=holdout_frac) logger.info( 'Randomly split train_data into train[%d]/validation[%d] splits.', len(train_data), len(tuning_data)) train_data = train_data.reset_index(drop=True) tuning_data = tuning_data.reset_index(drop=True) tuning_data_validated = True train_data = self._validate_data(train_data) if isinstance(train_data, self.Dataset): train_data = self.Dataset(train_data, classes=train_data.classes) if tuning_data is not None and not tuning_data_validated: tuning_data = self._validate_data(tuning_data) # converting to internal label set _set_valid_labels( tuning_data, self._label_cleaner.transform(_get_valid_labels(tuning_data))) if isinstance(tuning_data, self.Dataset): tuning_data = self.Dataset(tuning_data, classes=tuning_data.classes) if self._classifier is not None: logging.getLogger("ImageClassificationEstimator").propagate = True self._classifier._logger.setLevel(log_level) self._fit_summary = self._classifier.fit(train_data, tuning_data, 1 - holdout_frac, random_state, resume=False) if hasattr(self._classifier, 'fit_history'): self._fit_summary[ 'fit_history'] = self._classifier.fit_history() return self # new HPO task if time_limit is not None and num_trials is None: num_trials = 99999 if time_limit is None and num_trials is None: raise ValueError( '`time_limit` and `num_trials` can not be `None` at the same time, ' 'otherwise the training will not be terminated gracefully.') config = { 'log_dir': self._log_dir, 'num_trials': 99999 if num_trials is None else max(1, num_trials), 'time_limits': 2147483647 if time_limit is None else max(1, time_limit), 'searcher': searcher, # needed for gluon-cv TODO: remove after gluon-cv is updated https://github.com/dmlc/gluon-cv/issues/1633 'search_strategy': searcher, 'scheduler': scheduler, } if max_reward is not None: config['max_reward'] = max_reward if nthreads_per_trial is not None: config['nthreads_per_trial'] = nthreads_per_trial elif is_fork_enabled(): # This is needed to address multiprocessing.context.TimeoutError in fork mode config['nthreads_per_trial'] = 0 if ngpus_per_trial is not None: config['ngpus_per_trial'] = ngpus_per_trial if isinstance(hyperparameters, dict): if 'batch_size' in hyperparameters: bs = hyperparameters['batch_size'] _check_gpu_memory_presets(bs, ngpus_per_trial, 4, 256) # 256MB per sample net = hyperparameters.pop('net', None) if net is not None: config['custom_net'] = net optimizer = hyperparameters.pop('optimizer', None) if optimizer is not None: config['custom_optimizer'] = optimizer # check if hyperparameters overwriting existing config for k, v in hyperparameters.items(): if k in config: raise ValueError( f'Overwriting {k} = {config[k]} to {v} by hyperparameters is ambiguous.' ) config.update(hyperparameters) if scheduler_options is not None: config.update(scheduler_options) if use_rec == True: config['use_rec'] = True if 'early_stop_patience' not in config: config['early_stop_patience'] = 10 if config['early_stop_patience'] == None: config['early_stop_patience'] = -1 # TODO(zhreshold): expose the transform function(or sign function) for converting custom metrics if 'early_stop_baseline' not in config or config[ 'early_stop_baseline'] == None: config['early_stop_baseline'] = -np.Inf if 'early_stop_max_value' not in config or config[ 'early_stop_max_value'] == None: config['early_stop_max_value'] = np.Inf # batch size cannot be larger than dataset size if ngpus_per_trial is not None and ngpus_per_trial > 1: min_value = ngpus_per_trial else: min_value = 1 bs = sanitize_batch_size(config.get('batch_size', 16), min_value=min_value, max_value=len(train_data)) config['batch_size'] = bs # verbosity if log_level > logging.INFO: logging.getLogger( 'gluoncv.auto.tasks.image_classification').propagate = False logging.getLogger("ImageClassificationEstimator").propagate = False logging.getLogger("ImageClassificationEstimator").setLevel( log_level) task = _ImageClassification(config=config, problem_type=self._problem_type) # GluonCV can't handle these separately - patching created config task.search_strategy = scheduler task.scheduler_options['searcher'] = searcher task._logger.setLevel(log_level) task._logger.propagate = True self._train_classes = train_data.classes with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") with MXNetErrorCatcher() as err: self._classifier = task.fit(train_data, tuning_data, 1 - holdout_frac, random_state) if err.exc_value is not None: raise RuntimeError(err.exc_value + err.hint) self._classifier._logger.setLevel(log_level) self._classifier._logger.propagate = True self._fit_summary = task.fit_summary() if hasattr(task, 'fit_history'): self._fit_summary['fit_history'] = task.fit_history() return self
def fit(self, train_data, tuning_data=None, time_limit=None, presets=None, hyperparameters=None, **kwargs): """Automatic fit process for image prediction. Parameters ---------- train_data : pd.DataFrame or str Training data, can be a dataframe like image dataset. For dataframe like datasets, `image` and `label` columns are required. `image`: raw image paths. `label`: categorical integer id, starting from 0. For more details of how to construct a dataset for image predictor, check out: `http://preview.d2l.ai/d8/main/image_classification/getting_started.html`. If a string is provided, will search for k8 built-in datasets. tuning_data : pd.DataFrame or str, default = None Another dataset containing validation data reserved for model selection and hyperparameter-tuning, can be a dataframe like image dataset. If a string is provided, will search for k8 datasets. If `None`, the validation dataset will be randomly split from `train_data` according to `holdout_frac`. time_limit : int, default = None Time limit in seconds, if not specified, will run until all tuning and training finished. If `time_limit` is hit during `fit`, the HPO process will interrupt and return the current best configuration. presets : list or str or dict, default = ['medium_quality_faster_train'] List of preset configurations for various arguments in `fit()`. Can significantly impact predictive accuracy, memory-footprint, and inference latency of trained models, and various other properties of the returned `predictor`. It is recommended to specify presets and avoid specifying most other `fit()` arguments or model hyperparameters prior to becoming familiar with AutoGluon. As an example, to get the most accurate overall predictor (regardless of its efficiency), set `presets='best_quality'`. To get good quality with faster inference speed, set `presets='good_quality_faster_inference'` Any user-specified arguments in `fit()` will override the values used by presets. If specifying a list of presets, later presets will override earlier presets if they alter the same argument. For precise definitions of the provided presets, see file: `autogluon/vision/configs/presets_configs.py`. Users can specify custom presets by passing in a dictionary of argument values as an element to the list. Available Presets: ['best_quality', 'high_quality_fast_inference', 'good_quality_faster_inference', 'medium_quality_faster_train'] It is recommended to only use one `quality` based preset in a given call to `fit()` as they alter many of the same arguments and are not compatible with each-other. Note that depending on your specific hardware limitation(# gpu, size of gpu memory...) your mileage may vary a lot, you may choose lower quality presets if necessary, and try to reduce `batch_size` if OOM("RuntimeError: CUDA error: out of memory") happens frequently during the `fit`. In-depth Preset Info: best_quality={ 'hyperparameters': { 'model': Categorical('resnet50_v1b', 'resnet101_v1d', 'resnest200'), 'lr': Real(1e-5, 1e-2, log=True), 'batch_size': Categorical(8, 16, 32, 64, 128), 'epochs': 200 }, 'hyperparameter_tune_kwargs': { 'num_trials': 1024, 'search_strategy': 'bayesopt'}} Best predictive accuracy with little consideration to inference time or model size. Achieve even better results by specifying a large time_limit value. Recommended for applications that benefit from the best possible model accuracy. good_quality_fast_inference={ 'hyperparameters': { 'model': Categorical('resnet50_v1b', 'resnet34_v1b'), 'lr': Real(1e-4, 1e-2, log=True), 'batch_size': Categorical(8, 16, 32, 64, 128), 'epochs': 150 }, 'hyperparameter_tune_kwargs': { 'num_trials': 512, 'search_strategy': 'bayesopt'}} Good predictive accuracy with fast inference. Recommended for applications that require reasonable inference speed and/or model size. medium_quality_faster_train={ 'hyperparameters': { 'model': 'resnet50_v1b', 'lr': 0.01, 'batch_size': 64, 'epochs': 50 }, 'hyperparameter_tune_kwargs': { 'num_trials': 8, 'search_strategy': 'random'}} Medium predictive accuracy with very fast inference and very fast training time. This is the default preset in AutoGluon, but should generally only be used for quick prototyping. medium_quality_faster_inference={ 'hyperparameters': { 'model': Categorical('resnet18_v1b', 'mobilenetv3_small'), 'lr': Categorical(0.01, 0.005, 0.001), 'batch_size': Categorical(64, 128), 'epochs': Categorical(50, 100), }, 'hyperparameter_tune_kwargs': { 'num_trials': 32, 'search_strategy': 'bayesopt'}} Medium predictive accuracy with very fast inference. Comparing with `medium_quality_faster_train` it uses faster model but explores more hyperparameters. hyperparameters : dict, default = None Extra hyperparameters for specific models. Accepted args includes(not limited to): epochs : int, default value based on network The `epochs` for model training. net : mx.gluon.Block The custom network. If defined, the model name in config will be ignored so your custom network will be used for training rather than pulling it from model zoo. optimizer : mx.Optimizer The custom optimizer object. If defined, the optimizer will be ignored in config but this object will be used in training instead. batch_size : int Mini batch size lr : float Trainer learning rate for optimization process. You can get the list of accepted hyperparameters in `config.yaml` saved by this predictor. **kwargs : holdout_frac : float, default = 0.1 The random split ratio for `tuning_data` if `tuning_data==None`. random_state : int, default = None The random_state(seed) for shuffling data, only used if `tuning_data==None`. Note that the `random_state` only affect the splitting process, not model training. If not specified(None), will leave the original random sampling intact. nthreads_per_trial : int, default = (# cpu cores) Number of CPU threads for each trial, if `None`, will detect the # cores on current instance. ngpus_per_trial : int, default = (# gpus) Number of GPUs to use for each trial, if `None`, will detect the # gpus on current instance. hyperparameter_tune_kwargs: dict, default = None num_trials : int, default = 1 The limit of HPO trials that can be performed within `time_limit`. The HPO process will be terminated when `num_trials` trials have finished or wall clock `time_limit` is reached, whichever comes first. search_strategy : str, default = 'random' Searcher strategy for HPO, 'random' by default. Options include: ‘random’ (random search), ‘bayesopt’ (Gaussian process Bayesian optimization), ‘grid’ (grid search). max_reward : float, default = None The reward threashold for stopping criteria. If `max_reward` is reached during HPO, the scheduler will terminate earlier to reduce time cost. scheduler_options : dict, default = None Extra options for HPO scheduler, please refer to :class:`autogluon.core.Searcher` for details. """ if self._problem_type is None: # options: multiclass self._problem_type = 'multiclass' if self._eval_metric is None: # options: accuracy, self._eval_metric = 'accuracy' # init/validate kwargs kwargs = self._validate_kwargs(kwargs) # unpack num_trials = kwargs['hyperparameter_tune_kwargs']['num_trials'] nthreads_per_trial = kwargs['nthreads_per_trial'] ngpus_per_trial = kwargs['ngpus_per_trial'] holdout_frac = kwargs['holdout_frac'] random_state = kwargs['random_state'] search_strategy = kwargs['hyperparameter_tune_kwargs'][ 'search_strategy'] max_reward = kwargs['hyperparameter_tune_kwargs']['max_reward'] scheduler_options = kwargs['hyperparameter_tune_kwargs'][ 'scheduler_options'] log_level = verbosity2loglevel(self._verbosity) set_logger_verbosity(self._verbosity, logger=logger) if presets: if not isinstance(presets, list): presets = [presets] logger.log(20, f'Presets specified: {presets}') use_rec = False if isinstance(train_data, str) and train_data == 'imagenet': logging.warn( 'ImageNet is a huge dataset which cannot be downloaded directly, ' + 'please follow the data preparation tutorial in GluonCV.' + 'The following record files(symlinks) will be used: \n' + 'rec_train : ~/.mxnet/datasets/imagenet/rec/train.rec\n' + 'rec_train_idx : ~/.mxnet/datasets/imagenet/rec/train.idx\n' + 'rec_val : ~/.mxnet/datasets/imagenet/rec/val.rec\n' + 'rec_val_idx : ~/.mxnet/datasets/imagenet/rec/val.idx\n') train_data = pd.DataFrame({'image': [], 'label': []}) tuning_data = pd.DataFrame({'image': [], 'label': []}) use_rec = True if isinstance(train_data, str): from d8.image_classification import Dataset as D8D names = D8D.list() if train_data.lower() in names: train_data = D8D.get(train_data) else: valid_names = '\n'.join(names) raise ValueError( f'`train_data` {train_data} is not among valid list {valid_names}' ) if tuning_data is None: train_data, tuning_data = train_data.split(1 - holdout_frac) if isinstance(tuning_data, str): from d8.image_classification import Dataset as D8D names = D8D.list() if tuning_data.lower() in names: tuning_data = D8D.get(tuning_data) else: valid_names = '\n'.join(names) raise ValueError( f'`tuning_data` {tuning_data} is not among valid list {valid_names}' ) if self._classifier is not None: logging.getLogger("ImageClassificationEstimator").propagate = True self._classifier._logger.setLevel(log_level) self._fit_summary = self._classifier.fit(train_data, tuning_data, 1 - holdout_frac, random_state, resume=False) return # new HPO task if time_limit is not None and num_trials is None: num_trials = 99999 if time_limit is None and num_trials is None: raise ValueError( '`time_limit` and `num_trials` can not be `None` at the same time, ' 'otherwise the training will not be terminated gracefully.') config = { 'log_dir': self._log_dir, 'num_trials': 99999 if num_trials is None else max(1, num_trials), 'time_limits': 2147483647 if time_limit is None else max(1, time_limit), 'search_strategy': search_strategy, } if max_reward is not None: config['max_reward'] = max_reward if nthreads_per_trial is not None: config['nthreads_per_trial'] = nthreads_per_trial if ngpus_per_trial is not None: config['ngpus_per_trial'] = ngpus_per_trial if isinstance(hyperparameters, dict): if 'batch_size' in hyperparameters: bs = hyperparameters['batch_size'] if ngpus_per_trial is not None and ngpus_per_trial > 1 and bs > 64: # using gpus, check batch size vs. available gpu memory free_gpu_memory = get_gpu_free_memory() if not free_gpu_memory: warnings.warn( 'Unable to detect free GPU memory, we are unable to verify ' 'whether your data mini-batches will fit on the GPU for the specified batch_size.' ) elif len(free_gpu_memory) < ngpus_per_trial: warnings.warn( f'Detected GPU memory for {len(free_gpu_memory)} gpus but {ngpus_per_trial} is requested.' ) elif sum(free_gpu_memory[:ngpus_per_trial]) / bs < 128: warnings.warn( f'batch-size: {bs} is potentially larger than what your gpus can support ' + f'free memory: {free_gpu_memory[:ngpus_per_trial]} ' + 'Try reducing "batch_size" if you encounter memory issues' ) net = hyperparameters.pop('net', None) if net is not None: config['custom_net'] = net optimizer = hyperparameters.pop('optimizer', None) if optimizer is not None: config['custom_optimizer'] = optimizer # check if hyperparameters overwriting existing config for k, v in hyperparameters.items(): if k in config: raise ValueError( f'Overwriting {k} = {config[k]} to {v} by hyperparameters is ambiguous.' ) config.update(hyperparameters) if scheduler_options is not None: config.update(scheduler_options) if use_rec == True: config['use_rec'] = True # verbosity if log_level > logging.INFO: logging.getLogger( 'gluoncv.auto.tasks.image_classification').propagate = False logging.getLogger("ImageClassificationEstimator").propagate = False logging.getLogger("ImageClassificationEstimator").setLevel( log_level) task = _ImageClassification(config=config) task._logger.setLevel(log_level) task._logger.propagate = True with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") self._classifier = task.fit(train_data, tuning_data, 1 - holdout_frac, random_state) self._classifier._logger.setLevel(log_level) self._classifier._logger.propagate = True self._fit_summary = task.fit_summary() return self
def fit(self, train_data, tuning_data=None, time_limit=None, presets=None, hyperparameters=None, feature_metadata=None, **kwargs): """ Fit models to predict a column of data table based on the other columns. # TODO: Move documentation from TabularPrediction.fit to here # TODO: Move num_cpu/num_gpu to AG_args_fit # TODO: AG_args -> ag_args? +1 -> Will change after replacing original TabularPredictor to avoid extra API breaks. # TODO: consider adding kwarg option for data which has already been preprocessed by feature generator to skip feature generation. """ if self._learner.is_fit: raise AssertionError( 'Predictor is already fit! To fit additional models, refer to `predictor.fit_extra`.' ) kwargs_orig = kwargs.copy() kwargs = self._validate_fit_kwargs(kwargs) verbosity = kwargs.get('verbosity', self.verbosity) set_logger_verbosity(verbosity, logger=logger) if verbosity >= 3: logger.log(20, '============ fit kwarg info ============') logger.log(20, 'User Specified kwargs:') logger.log(20, f'{pprint.pformat(kwargs_orig)}') logger.log(20, 'Full kwargs:') logger.log(20, f'{pprint.pformat(kwargs)}') logger.log(20, '========================================') holdout_frac = kwargs['holdout_frac'] num_bag_folds = kwargs['num_bag_folds'] num_bag_sets = kwargs['num_bag_sets'] num_stack_levels = kwargs['num_stack_levels'] auto_stack = kwargs['auto_stack'] hyperparameter_tune_kwargs = kwargs['hyperparameter_tune_kwargs'] num_cpus = kwargs['num_cpus'] num_gpus = kwargs['num_gpus'] feature_generator = kwargs['feature_generator'] unlabeled_data = kwargs['unlabeled_data'] save_bagged_folds = kwargs['save_bagged_folds'] ag_args = kwargs['AG_args'] ag_args_fit = kwargs['AG_args_fit'] ag_args_ensemble = kwargs['AG_args_ensemble'] excluded_model_types = kwargs['excluded_model_types'] self._set_feature_generator(feature_generator=feature_generator, feature_metadata=feature_metadata) train_data, tuning_data, unlabeled_data = self._validate_fit_data( train_data=train_data, tuning_data=tuning_data, unlabeled_data=unlabeled_data) if hyperparameters is None: hyperparameters = 'default' if isinstance(hyperparameters, str): hyperparameters = get_hyperparameter_config(hyperparameters) # Process kwargs to create trainer, schedulers, searchers: num_bag_folds, num_bag_sets, num_stack_levels = self._sanitize_stack_args( num_bag_folds=num_bag_folds, num_bag_sets=num_bag_sets, num_stack_levels=num_stack_levels, time_limit=time_limit, auto_stack=auto_stack, num_train_rows=len(train_data), ) if hyperparameter_tune_kwargs is not None: scheduler_options = self._init_scheduler( hyperparameter_tune_kwargs, time_limit, hyperparameters, num_cpus, num_gpus, num_bag_folds, num_stack_levels) else: scheduler_options = None hyperparameter_tune = scheduler_options is not None if hyperparameter_tune: logger.log( 30, 'Warning: hyperparameter tuning is currently experimental and may cause the process to hang. Setting `auto_stack=True` instead is recommended to achieve maximum quality models.' ) if holdout_frac is None: holdout_frac = default_holdout_frac(len(train_data), hyperparameter_tune) if ag_args_fit is None: ag_args_fit = dict() # TODO: v0.1: Update to be 'auto' or None by default to give full control to individual models. if 'num_cpus' not in ag_args_fit and num_cpus is not None: ag_args_fit['num_cpus'] = num_cpus if 'num_gpus' not in ag_args_fit and num_gpus is not None: ag_args_fit['num_gpus'] = num_gpus # TODO: v0.1: make core_kwargs a kwargs argument to predictor.fit, add aux_kwargs to predictor.fit core_kwargs = { 'ag_args': ag_args, 'ag_args_ensemble': ag_args_ensemble, 'ag_args_fit': ag_args_fit, 'excluded_model_types': excluded_model_types } self._learner.fit(X=train_data, X_val=tuning_data, X_unlabeled=unlabeled_data, hyperparameter_tune_kwargs=scheduler_options, holdout_frac=holdout_frac, num_bagging_folds=num_bag_folds, num_bagging_sets=num_bag_sets, stack_ensemble_levels=num_stack_levels, hyperparameters=hyperparameters, core_kwargs=core_kwargs, time_limit=time_limit, save_bagged_folds=save_bagged_folds, verbosity=verbosity) self._set_post_fit_vars() self._post_fit( keep_only_best=kwargs['keep_only_best'], refit_full=kwargs['refit_full'], set_best_to_refit_full=kwargs['set_best_to_refit_full'], save_space=kwargs['save_space'], ) self.save() return self
def fit_extra( self, hyperparameters, time_limit=None, base_model_names=None, fit_new_weighted_ensemble=True, relative_stack=True, # kwargs # core_kwargs=None, aux_kwargs=None, **kwargs): # TODO: Allow disable aux (default to disabled) time_start = time.time() kwargs_orig = kwargs.copy() kwargs = self._validate_fit_extra_kwargs(kwargs) verbosity = kwargs.get('verbosity', self.verbosity) set_logger_verbosity(verbosity, logger=logger) if verbosity >= 3: logger.log(20, '============ fit kwarg info ============') logger.log(20, 'User Specified kwargs:') logger.log(20, f'{pprint.pformat(kwargs_orig)}') logger.log(20, 'Full kwargs:') logger.log(20, f'{pprint.pformat(kwargs)}') logger.log(20, '========================================') # TODO: num_bag_sets num_stack_levels = kwargs['num_stack_levels'] hyperparameter_tune_kwargs = kwargs['hyperparameter_tune_kwargs'] num_cpus = kwargs['num_cpus'] num_gpus = kwargs['num_gpus'] # save_bagged_folds = kwargs['save_bagged_folds'] # TODO: Enable ag_args = kwargs['AG_args'] ag_args_fit = kwargs['AG_args_fit'] ag_args_ensemble = kwargs['AG_args_ensemble'] excluded_model_types = kwargs['excluded_model_types'] if isinstance(hyperparameters, str): hyperparameters = get_hyperparameter_config(hyperparameters) if num_stack_levels is None: hyperparameter_keys = list(hyperparameters.keys()) highest_level = 0 for key in hyperparameter_keys: if isinstance(key, int): highest_level = max(key, highest_level) num_stack_levels = highest_level if hyperparameter_tune_kwargs is not None: scheduler_options = self._init_scheduler( hyperparameter_tune_kwargs, time_limit, hyperparameters, num_cpus, num_gpus, self._trainer.k_fold, num_stack_levels) else: scheduler_options = None hyperparameter_tune = scheduler_options is not None if hyperparameter_tune: raise ValueError( 'Hyperparameter Tuning is not allowed in `fit_extra`.' ) # FIXME: Change this # logger.log(30, 'Warning: hyperparameter tuning is currently experimental and may cause the process to hang.') # TODO: v0.1: make core_kwargs a kwargs argument to predictor.fit, add aux_kwargs to predictor.fit core_kwargs = { 'ag_args': ag_args, 'ag_args_ensemble': ag_args_ensemble, 'ag_args_fit': ag_args_fit, 'excluded_model_types': excluded_model_types } # TODO: Add special error message if called and training/val data was not cached. X_train, y_train, X_val, y_val = self._trainer.load_data() fit_models = self._trainer.train_multi_levels( X_train=X_train, y_train=y_train, hyperparameters=hyperparameters, X_val=X_val, y_val=y_val, base_model_names=base_model_names, time_limit=time_limit, relative_stack=relative_stack, level_end=num_stack_levels, core_kwargs=core_kwargs, aux_kwargs=aux_kwargs) if time_limit is not None: time_limit = time_limit - (time.time() - time_start) if fit_new_weighted_ensemble: if time_limit is not None: time_limit_weighted = max(time_limit, 60) else: time_limit_weighted = None fit_models += self.fit_weighted_ensemble( time_limits=time_limit_weighted) self._post_fit( keep_only_best=kwargs['keep_only_best'], refit_full=kwargs['refit_full'], set_best_to_refit_full=kwargs['set_best_to_refit_full'], save_space=kwargs['save_space'], ) self.save() return self
def set_verbosity(self, verbosity: int): self.verbosity = verbosity set_logger_verbosity(self.verbosity, logger=logger)
def set_verbosity(self, target_verbosity): self.verbosity = target_verbosity set_logger_verbosity(self.verbosity, logger=logger)