def test_dali(dataset_config, transform): augmentations = [ EasyDict({ 'module': 'nvidia_dali', 'args': { 'transforms': [transform] } }) ] dataset_config.train.augmentations = augmentations if dataset_config == lndmrks_dataset_config and transform[ 'transform'] in excpected_raise_error_transform: with pytest.raises(RuntimeError): dataloader = create_dataloader( dataloader_config=dali_loader, dataset_config=dataset_config, preprocess_config=preprocess_args, collate_fn=dataset_config.collate_fn) else: dataloader = create_dataloader(dataloader_config=dali_loader, dataset_config=dataset_config, preprocess_config=preprocess_args, collate_fn=dataset_config.collate_fn) for data in dataloader: fetched_data = data break
def test_dataloader(dataloader_cfg): dataloader = create_dataloader(dataloader_config=dataloader_cfg, dataset_config=classification_config, preprocess_config=preprocess_args, collate_fn=None) for data in dataloader: fetched_data = data break assert isinstance(fetched_data[0], torch.Tensor) assert len(fetched_data[0].shape) == 4 # N,C,H,W assert fetched_data[0].shape[ 2] == preprocess_args.input_size # Assume square input assert isinstance(len(dataloader), int) assert hasattr(dataloader, 'dataset') assert isinstance(dataloader.dataset, BasicDatasetWrapper)
def test_dali_loader_with_no_dali_aug(): augmentations = [ EasyDict({ 'module': 'albumentations', 'args': { 'transforms': [transforms[0]] } }) ] class_dataset_config.train.augmentations = augmentations dataloader = create_dataloader(dataloader_config=dali_loader, dataset_config=class_dataset_config, preprocess_config=preprocess_args, collate_fn=class_dataset_config.collate_fn)
def test_neg_dali_aug_and_pytorch_data_loader(): augmentations = [ EasyDict({ 'module': 'nvidia_dali', 'args': { 'transforms': [transforms[0]] } }) ] class_dataset_config.train.augmentations = augmentations # Expect RuntimeError if nvidia dali augmentation module used with pytorch dataloader with pytest.raises(RuntimeError): dataloader = create_dataloader( dataloader_config=pytorch_loader, dataset_config=class_dataset_config, preprocess_config=preprocess_args, collate_fn=class_dataset_config.collate_fn)
def test_neg_dali_aug_not_the_first_aug(): augmentations = [ EasyDict({ 'module': 'albumentations', 'args': { 'transforms': [transforms[0]] } }), EasyDict({ 'module': 'nvidia_dali', 'args': { 'transforms': [transforms[0]] } }) ] class_dataset_config.train.augmentations = augmentations # Expect RuntimeError if dali augmentations is not in the first order of the augmentation list with pytest.raises(RuntimeError): dataloader = create_dataloader( dataloader_config=dali_loader, dataset_config=class_dataset_config, preprocess_config=preprocess_args, collate_fn=class_dataset_config.collate_fn)
accumulation_step = config.trainer.driver.args.accumulation_step except: pass # Get steps per epoch steps_per_epoch = 100 ## Try to get accumulation_step from argparse or experiment config if args.steps_per_epoch is not None: steps_per_epoch = args.steps_per_epoch else: try: dataloader = create_dataloader( dataloader_config=config.dataloader, dataset_config=config.dataset, preprocess_config=config.model.preprocess_args, collate_fn=None) steps_per_epoch = len(dataloader) except: pass lrs = calculate_lr(lr_scheduler_cfg, epochs=epochs, optimizer_config=optim_cfg, accumulation_step=accumulation_step, steps_per_epoch=steps_per_epoch) plt.plot(lrs) plt.grid(True, linestyle='--', alpha=0.2) plt.ylabel("learning rate")
def __init__(self, config:EasyDict, config_path: Union[str,Path,None] = None, hypopt: bool = False, resume: bool = False): """Class initialization Args: config (EasyDict): dictionary parsed from Vortex experiment file config_path (Union[str,Path,None], optional): path to experiment file. Need to be provided for backup **experiment file**. Defaults to None. hypopt (bool, optional): flag for hypopt, disable several pipeline process. Defaults to False. resume (bool, optional): flag to resume training. Defaults to False. Raises: Exception: raise undocumented error if exist Example: ```python from vortex.development.utils.parser import load_config from vortex.development.core.pipelines import TrainingPipeline # Parse config config_path = 'experiments/config/example.yml' config = load_config(config_path) train_executor = TrainingPipeline(config=config, config_path=config_path, hypopt=False) ``` """ self.start_epoch = 0 checkpoint, state_dict = None, None if resume or ('checkpoint' in config and config.checkpoint is not None): if 'checkpoint' not in config: raise RuntimeError("You specify to resume but 'checkpoint' is not configured " "in the config file. Please specify 'checkpoint' option in the top level " "of your config file pointing to model path used for resume.") if resume or os.path.exists(config.checkpoint): checkpoint = torch.load(config.checkpoint, map_location=torch.device('cpu')) state_dict = checkpoint['state_dict'] if resume: self.start_epoch = checkpoint['epoch'] model_config = EasyDict(checkpoint['config']) if config.model.name != model_config.model.name: raise RuntimeError("Model name configuration specified in config file ({}) is not " "the same as saved in model checkpoint ({}).".format(config.model.name, model_config.model.name)) if config.model.network_args != model_config.model.network_args: raise RuntimeError("'network_args' configuration specified in config file ({}) is " "not the same as saved in model checkpoint ({}).".format(config.model.network_args, model_config.model.network_args)) if 'name' in config.dataset.train: cfg_dataset_name = config.dataset.train.name elif 'dataset' in config.dataset.train: cfg_dataset_name = config.dataset.train.dataset else: raise RuntimeError("dataset name is not found in config. Please specify in " "'config.dataset.train.name'.") model_dataset_name = None if 'name' in model_config.dataset.train: model_dataset_name = model_config.dataset.train.name elif 'dataset' in model_config.dataset.train: model_dataset_name = model_config.dataset.train.dataset if cfg_dataset_name != model_dataset_name: raise RuntimeError("Dataset specified in config file ({}) is not the same as saved " "in model checkpoint ({}).".format(cfg_dataset_name, model_dataset_name)) if ('n_classes' in config.model.network_args and (config.model.network_args.n_classes != model_config.model.network_args.n_classes)): raise RuntimeError("Number of classes configuration specified in config file ({}) " "is not the same as saved in model checkpoint ({}).".format( config.model.network_args.n_classes, model_config.model.network_args.n_classes)) self.config = config self.hypopt = hypopt # Check experiment config validity self._check_experiment_config(config) if not self.hypopt: # Create experiment logger self.experiment_logger = create_experiment_logger(config) # Output directory creation # If config_path is provided, it will duplicate the experiment file into the run directory self.experiment_directory,self.run_directory=check_and_create_output_dir(config, self.experiment_logger, config_path) # Create local experiments run log file self._create_local_runs_log(self.config, self.experiment_logger, self.experiment_directory, self.run_directory) else: self.experiment_logger=None # Training components creation if 'device' in config: self.device = config.device elif 'device' in config.trainer: self.device = config.trainer.device else: raise RuntimeError("'device' field not found in config. Please specify properly in main level.") model_components = create_model(model_config=config.model, state_dict=state_dict) if not isinstance(model_components, EasyDict): model_components = EasyDict(model_components) # not working for easydict # model_components.setdefault('collate_fn',None) if not 'collate_fn' in model_components: model_components.collate_fn = None self.model_components = model_components self.model_components.network = self.model_components.network.to(self.device) self.criterion = self.model_components.loss.to(self.device) param_groups = None if 'param_groups' in self.model_components: param_groups = self.model_components.param_groups if 'dataloader' in config: dataloader_config = config.dataloader elif 'dataloader' in config.dataset: dataloader_config = config.dataset.dataloader else: raise RuntimeError("Dataloader config field not found in config.") self.dataloader = create_dataloader(dataloader_config=dataloader_config, dataset_config=config.dataset, preprocess_config=config.model.preprocess_args, collate_fn=self.model_components.collate_fn, stage='train') self.trainer = engine.create_trainer( config.trainer, criterion=self.criterion, model=self.model_components.network, experiment_logger=self.experiment_logger, param_groups=param_groups ) if resume: self.trainer.optimizer.load_state_dict(checkpoint['optimizer_state']) if self.trainer.scheduler is not None: scheduler_args = self.config.trainer.lr_scheduler.args if isinstance(scheduler_args, dict): for name, v in scheduler_args.items(): if name in checkpoint["scheduler_state"]: checkpoint["scheduler_state"][name] = v self.trainer.scheduler.load_state_dict(checkpoint["scheduler_state"]) has_save = False self.save_best_metrics, self.save_best_type = None, None self.best_metrics = None if 'save_best_metrics' in self.config.trainer and self.config.trainer.save_best_metrics is not None: has_save = self.config.trainer.save_best_metrics is not None self.save_best_metrics = self.config.trainer.save_best_metrics if not isinstance(self.save_best_metrics, (list, tuple)): self.save_best_metrics = [self.save_best_metrics] self.save_best_type = list({'loss' if m == 'loss' else 'val_metric' for m in self.save_best_metrics}) self.best_metrics = {name: float('inf') if name == 'loss' else float('-inf') for name in self.save_best_metrics} if 'loss' in self.save_best_metrics: self.save_best_metrics.remove('loss') if resume: best_metrics_ckpt = checkpoint['best_metrics'] if isinstance(best_metrics_ckpt, dict): self.best_metrics.update(best_metrics_ckpt) self.save_epoch, self.save_last_epoch = None, None if 'save_epoch' in self.config.trainer and self.config.trainer.save_epoch is not None: self.save_epoch = self.config.trainer.save_epoch has_save = has_save or self.config.trainer.save_epoch is not None if not has_save: warnings.warn("No model checkpoint saving configuration is specified, the training would still " "work but will only save the last epoch model.\nYou can configure either one of " "'config.trainer.save_epoch' or 'config.trainer.save_best_metric") # Validation components creation try: if 'validator' in config: validator_cfg = config.validator elif 'device' in config.trainer: validator_cfg = config.trainer.validator else: raise RuntimeError("'validator' field not found in config. Please specify properly in main level.") val_dataset = create_dataset(config.dataset, config.model.preprocess_args, stage='validate') ## use same batch-size as training by default validation_args = EasyDict({'batch_size' : self.dataloader.batch_size}) validation_args.update(validator_cfg.args) self.validator = engine.create_validator( self.model_components, val_dataset, validation_args, device=self.device ) self.val_epoch = validator_cfg.val_epoch self.valid_for_validation = True except AttributeError as e: warnings.warn('validation step not properly configured, will be skipped') self.valid_for_validation = False except Exception as e: raise Exception(str(e)) # Reproducibility settings check if hasattr(config, 'seed') : _set_seed(config.seed) if not self.hypopt: print("\nexperiment directory:", self.run_directory) self._has_cls_names = hasattr(self.dataloader.dataset, "class_names")