def test_create_validator(): softmax = dict(network_args=dict( backbone='shufflenetv2_x1.0', n_classes=10, freeze_backbone=False, ), preprocess_args=dict(input_size=32, input_normalization=dict( mean=[0.4914, 0.4822, 0.4465], std=[0.2023, 0.1994, 0.2010])), loss_args=dict(reduction='mean'), postprocess_args={}) model = create_model(EasyDict(name='softmax', **softmax)) validator = engine.create_validator( model, validation_args={}, dataset=DummyDataset(), ) assert isinstance(validator, engine.validator.get_validator('classification')) mean_std = dict( mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], ) RetinaFace = dict( preprocess_args=dict( input_size=640, input_normalization=mean_std, ), network_args=dict( n_classes=1, backbone='shufflenetv2_x1.0', pyramid_channels=64, aspect_ratios=[1, 2., 3.], ), loss_args=dict( overlap_thresh=0.35, cls=2.0, box=1.0, ldm=1.0, neg_pos=7, ), postprocess_args=dict(nms=True, ), ) model = create_model(EasyDict(name='RetinaFace', **RetinaFace)) validation_args = dict( score_threshold=0.2, iou_threshold=0.2, ) validator = engine.create_validator( model, dataset=DummyDataset(), validation_args=validation_args, ) assert isinstance(validator, engine.validator.get_validator('detection'))
def export_check(model_name : str, model_arg : dict, export_arg : dict, suffix : str, predictor_args : dict={}, example_image=None) : output_directory = Path(test_output_directory) output_directory.mkdir(exist_ok=True,parents=True) model_config = create_model_cfg(model_name, model_arg) model_components = create_model(model_config) ## TODO : remove if model_name in tmp_output_format : predictor_args['metainfo'] = tmp_output_format[model_name] predictor = create_predictor( model_components=model_components, **predictor_args ).eval() experiment_name = get_test_experiment_name(model_name, model_arg.network_args.backbone, suffix) image_size = model_config.preprocess_args.input_size exporter = create_exporter( config=export_arg, experiment_name=experiment_name, image_size=image_size, output_directory=output_directory ) ## TODO : read export error msg result = exporter(predictor, example_image_path=example_image) torch.save(predictor.model.state_dict(), output_directory / '{}.pth'.format(experiment_name)) del predictor, model_components return result
def __init__(self, config: EasyDict, weights: Union[str, Path, None] = None): """Class initialization Args: config (EasyDict): dictionary parsed from Vortex experiment file weights (Union[str,Path,None], optional): path to selected Vortex model's weight. If set to None, it will \ assume that final model weights exist in **experiment directory**. \ Defaults to None. Example: ```python from vortex.utils.parser import load_config from vortex.core.pipelines import GraphExportPipeline # Parse config config = load_config('experiments/config/example.yml') graph_exporter = GraphExportPipeline(config=config, weights='experiments/outputs/example/example.pth') ``` """ # Configure output directory self.experiment_directory, _ = check_and_create_output_dir(config) self.experiment_name = config.experiment_name # Initialize Pytorch model if weights is None: state_dict = self.experiment_directory / '{}.pth'.format( self.experiment_name) else: state_dict = weights model_components = create_model(config.model, state_dict=state_dict, stage='validate') model_components.network = model_components.network.eval() self.predictor = create_predictor(model_components).eval() self.image_size = config.model.preprocess_args.input_size # Initialize dataset train to get class_names dataset = create_dataset( config.dataset, preprocess_config=config.model.preprocess_args, stage='train') self.class_names = dataset.dataset.class_names if hasattr( dataset.dataset, 'class_names') else None # Initialize export config self.export_configs = [config.exporter] \ if not isinstance(config.exporter, list) \ else config.exporter
def test_create_softmax_model(): softmax_model_conf = EasyDict({ 'name' : 'softmax', 'network_args' : { 'backbone' : 'shufflenetv2_x1.0', 'n_classes' : 10, 'pretrained_backbone': True, 'freeze_backbone': False }, 'preprocess_args': { 'input_size': 32, 'input_normalization': { 'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2023, 0.1994, 0.2010] } }, 'loss_args': { 'reduction': 'mean' }, 'postprocess_args': {} }) softmax_model_components=create_model(model_config=softmax_model_conf,stage='train') assert isinstance(softmax_model_components,EasyDict) assert 'network' in softmax_model_components.keys() assert hasattr(softmax_model_components.network,'task') assert hasattr(softmax_model_components.network,'output_format') assert 'postprocess' in softmax_model_components.keys() assert 'preprocess' in softmax_model_components.keys() assert 'loss' in softmax_model_components.keys() assert 'collate_fn' in softmax_model_components.keys() softmax_model_components=create_model(model_config=softmax_model_conf,stage='validate') assert isinstance(softmax_model_components,EasyDict) assert 'network' in softmax_model_components.keys() assert hasattr(softmax_model_components.network,'task') assert hasattr(softmax_model_components.network,'output_format') assert 'postprocess' in softmax_model_components.keys() assert 'preprocess' in softmax_model_components.keys()
def __init__(self, config: EasyDict, config_path: Union[str, Path, None] = None, hypopt: bool = False): """Class initialization Args: config (EasyDict): dictionary parsed from Vortex experiment file config_path (Union[str,Path,None], optional): path to experiment file. Need to be provided for \ backup **experiment file**. Defaults to None. hypopt (bool, optional): flag for hypopt, disable several pipeline process. Defaults to False. Raises: Exception: raise undocumented error if exist Example: ```python from vortex.utils.parser import load_config from vortex.core.pipelines import TrainingPipeline # Parse config config_path = 'experiments/config/example.yml' config = load_config(config_path) train_executor = TrainingPipeline(config=config, config_path=config_path, hypopt=False) ``` """ self.config = config self.hypopt = hypopt # Check experiment config validity self._check_experiment_config(config) if not self.hypopt: # Create experiment logger self.experiment_logger = create_experiment_logger(config) # Output directory creation # If config_path is provided, it will duplicate the experiment file into the run directory self.experiment_directory, self.run_directory = check_and_create_output_dir( config, self.experiment_logger, config_path) # Create local experiments run log file self._create_local_runs_log(self.config, self.experiment_logger, self.experiment_directory, self.run_directory) else: self.experiment_logger = None # Training components creation self.device = config.trainer.device self.model_components = create_model(model_config=config.model) self.dataloader = create_dataloader( dataset_config=config.dataset, preprocess_config=config.model.preprocess_args, collate_fn=self.model_components.collate_fn, stage='train') self.model_components.network = self.model_components.network.to( self.device) self.criterion = self.model_components.loss self.criterion = self.criterion.to(self.device) self.trainer = engine.create_trainer( config.trainer, criterion=self.criterion, model=self.model_components.network, experiment_logger=self.experiment_logger, ) # Validation components creation try: val_dataset = create_dataset(config.dataset, config.model.preprocess_args, stage='validate') ## use same batch-size as training by default validation_args = EasyDict( {'batch_size': self.dataloader.batch_size}) validation_args.update(config.trainer.validation.args) self.validator = engine.create_validator(self.model_components, val_dataset, validation_args, device=self.device) self.val_epoch = config.trainer.validation.val_epoch self.valid_for_validation = True except AttributeError as e: warnings.warn( 'validation step not properly configured, will be skipped') self.valid_for_validation = False except Exception as e: raise Exception(str(e)) # Reproducibility settings check if hasattr(config, 'seed'): _set_seed(config.seed)
def eval_check(runtime : str, model_name : str, model_arg : dict, export_arg : dict, suffix : str, predictor_args : dict={}) : ## torch predictor output_directory = Path(test_output_directory) output_directory.mkdir(exist_ok=True,parents=True) model_config = create_model_cfg(model_name, model_arg) model_components = create_model(model_config) experiment_name = get_test_experiment_name(model_name, model_arg.network_args.backbone, suffix) onnx_model_path = output_directory / '{}.onnx'.format(experiment_name) pth_path = output_directory / '{}.pth'.format(experiment_name) if not (onnx_model_path.exists() and pth_path.exists()) : return EvalResult(Status.ERROR, msg='file not found, export might have failed') model_components.network.load_state_dict( torch.load(pth_path) ) if model_name in tmp_output_format : predictor_args['metainfo'] = tmp_output_format[model_name] predictor = create_predictor( model_components=model_components, **predictor_args ).eval() image_size = model_config.preprocess_args.input_size ## onnx model try : ## TODO : check for fallback onnx_model = create_runtime_model(str(onnx_model_path), runtime) except Exception as e: print(e) return EvalResult(Status.ERROR, msg='RuntimeErorr : {}'.format(str(e))) ## predict check input_test = (np.random.rand(1,image_size,image_size,3) * 255).astype(np.uint8) ## TODO : read additional input from predictor or onnx input spec additional_args = dict( score_threshold=0.0, iou_threshold=1.0, ) torch_results = torch_predict(predictor, input_test, **additional_args) onnx_results = onnx_predict(onnx_model, input_test, **additional_args) ok = len(torch_results) == len(onnx_results) # print("len(torch_results) == len(onnx_results)", len(torch_results) == len(onnx_results)) status = EvalResult(status=ok, msg="len(torch_results) != len(onnx_results)" if not ok else "") for torch_result, onnx_result in zip(torch_results, onnx_results) : if not status : break ok = len(torch_result.keys()) == len(onnx_result.keys()) # print("len(torch_result.keys()) == len(onnx_result.keys())", len(torch_result.keys()) == len(onnx_result.keys())) status = EvalResult(status=ok, msg="len(torch_result.keys()) != len(onnx_result.keys())" if not ok else "") if not status : break ok = all(key in onnx_result.keys() for key in torch_result.keys()) # print("all(key in onnx_result.keys() for key in torch_result.keys())", all(key in onnx_result.keys() for key in torch_result.keys())) status = EvalResult(status=ok, msg="not all(key in onnx_result.keys() for key in torch_result.keys())" if not ok else "") if not status : break ok = all(isclose(onnx_value, torch_value, **isclose_config) for key in onnx_result.keys() for onnx_value, torch_value in zip(onnx_result[key].flatten(), torch_result[key].flatten())) ## TODO : collect diff across batch # diff = sum(np.sum(np.abs(onnx_result[key] - torch_result[key]).flatten()).item() / len(onnx_result[key].flatten()) for key in onnx_result.keys()) diff = sum(((abs(onnx_value-torch_value) / len(onnx_result[key].flatten())) if len(onnx_result[key].flatten()) else 0) for key in onnx_result.keys() for onnx_value, torch_value in zip(onnx_result[key].flatten(), torch_result[key].flatten())) # print("all(np.isclose(onnx_result[key], torch_result[key], **isclose_config) for key in onnx_result.keys())", all(np.isclose(onnx_result[key], torch_result[key], **isclose_config) for key in onnx_result.keys())) s = Status.FAILED if not ok else Status.SUCCESS status = EvalResult(status=s, diff=diff, msg="isclose failed" if not ok else "") if not status : break del onnx_model, predictor, onnx_results, torch_results, input_test, model_components return status
def __init__( self, config: EasyDict, weights: Union[str, Path, None] = None, device: Union[str, None] = None, ): """Class initialization Args: config (EasyDict): dictionary parsed from Vortex experiment file weights (Union[str,Path,None], optional): path to selected Vortex model's weight. If set to None, it will \ assume that final model weights exist in **experiment directory**. \ Defaults to None. device (Union[str,None], optional): selected device for model's computation. If None, it will use the device \ described in **experiment file**. Defaults to None. Raises: FileNotFoundError: raise error if selected 'weights' file is not found Example: ```python from vortex.core.pipelines import PytorchPredictionPipeline from vortex.utils.parser import load_config # Parse config config_path = 'experiments/config/example.yml' config = load_config(config_path) weights_file = 'experiments/outputs/example/example.pth' device = 'cuda' vortex_predictor = PytorchPredictionPipeline(config = config, weights = weights_file, device = device) ``` """ self.config = config self.output_file_prefix = 'prediction' # Configure experiment directory experiment_directory, _ = check_and_create_output_dir(config) # Initialize dataset to get class_names dataset = create_dataset( config.dataset, stage='train', preprocess_config=config.model.preprocess_args) self.class_names = dataset.dataset.class_names if hasattr( dataset.dataset, 'class_names') else None # Set compute device if device is None: device = config.trainer.device device = torch.device(device) # Initialize model if weights is None: filename = Path(experiment_directory) / ('%s.pth' % config.experiment_name) else: filename = Path(weights) if not filename.exists(): raise FileNotFoundError( 'Selected weights file {} is not found'.format(filename)) model_components = create_model(config.model, state_dict=str(filename), stage='validate') model_components.network = model_components.network.to(device) self.predictor = create_predictor(model_components) self.predictor.to(device) # Configure input size for image self.input_size = config.model.preprocess_args.input_size