def test_create_validator():
softmax = dict(network_args=dict(
backbone='shufflenetv2_x1.0',
n_classes=10,
freeze_backbone=False,
),
preprocess_args=dict(input_size=32,
input_normalization=dict(
mean=[0.4914, 0.4822, 0.4465],
std=[0.2023, 0.1994, 0.2010])),
loss_args=dict(reduction='mean'),
postprocess_args={})
model = create_model(EasyDict(name='softmax', **softmax))
validator = engine.create_validator(
model,
validation_args={},
dataset=DummyDataset(),
)
assert isinstance(validator,
engine.validator.get_validator('classification'))
mean_std = dict(
mean=[0.5, 0.5, 0.5],
std=[0.5, 0.5, 0.5],
)
RetinaFace = dict(
preprocess_args=dict(
input_size=640,
input_normalization=mean_std,
),
network_args=dict(
n_classes=1,
backbone='shufflenetv2_x1.0',
pyramid_channels=64,
aspect_ratios=[1, 2., 3.],
),
loss_args=dict(
overlap_thresh=0.35,
cls=2.0,
box=1.0,
ldm=1.0,
neg_pos=7,
),
postprocess_args=dict(nms=True, ),
)
model = create_model(EasyDict(name='RetinaFace', **RetinaFace))
validation_args = dict(
score_threshold=0.2,
iou_threshold=0.2,
)
validator = engine.create_validator(
model,
dataset=DummyDataset(),
validation_args=validation_args,
)
assert isinstance(validator, engine.validator.get_validator('detection'))
def export_check(model_name : str, model_arg : dict, export_arg : dict, suffix : str, predictor_args : dict={}, example_image=None) :
output_directory = Path(test_output_directory)
output_directory.mkdir(exist_ok=True,parents=True)
model_config = create_model_cfg(model_name, model_arg)
model_components = create_model(model_config)
## TODO : remove
if model_name in tmp_output_format :
predictor_args['metainfo'] = tmp_output_format[model_name]
predictor = create_predictor(
model_components=model_components,
**predictor_args
).eval()
experiment_name = get_test_experiment_name(model_name, model_arg.network_args.backbone, suffix)
image_size = model_config.preprocess_args.input_size
exporter = create_exporter(
config=export_arg,
experiment_name=experiment_name,
image_size=image_size,
output_directory=output_directory
)
## TODO : read export error msg
result = exporter(predictor, example_image_path=example_image)
torch.save(predictor.model.state_dict(), output_directory / '{}.pth'.format(experiment_name))
del predictor, model_components
return result
def __init__(self,
config: EasyDict,
weights: Union[str, Path, None] = None):
"""Class initialization
Args:
config (EasyDict): dictionary parsed from Vortex experiment file
weights (Union[str,Path,None], optional): path to selected Vortex model's weight. If set to None, it will \
assume that final model weights exist in **experiment directory**. \
Defaults to None.
Example:
```python
from vortex.utils.parser import load_config
from vortex.core.pipelines import GraphExportPipeline
# Parse config
config = load_config('experiments/config/example.yml')
graph_exporter = GraphExportPipeline(config=config,
weights='experiments/outputs/example/example.pth')
```
"""
# Configure output directory
self.experiment_directory, _ = check_and_create_output_dir(config)
self.experiment_name = config.experiment_name
# Initialize Pytorch model
if weights is None:
state_dict = self.experiment_directory / '{}.pth'.format(
self.experiment_name)
else:
state_dict = weights
model_components = create_model(config.model,
state_dict=state_dict,
stage='validate')
model_components.network = model_components.network.eval()
self.predictor = create_predictor(model_components).eval()
self.image_size = config.model.preprocess_args.input_size
# Initialize dataset train to get class_names
dataset = create_dataset(
config.dataset,
preprocess_config=config.model.preprocess_args,
stage='train')
self.class_names = dataset.dataset.class_names if hasattr(
dataset.dataset, 'class_names') else None
# Initialize export config
self.export_configs = [config.exporter] \
if not isinstance(config.exporter, list) \
else config.exporter
def test_create_softmax_model():
softmax_model_conf = EasyDict({
'name' : 'softmax',
'network_args' : {
'backbone' : 'shufflenetv2_x1.0',
'n_classes' : 10,
'pretrained_backbone': True,
'freeze_backbone': False
},
'preprocess_args': {
'input_size': 32,
'input_normalization': {
'mean': [0.4914, 0.4822, 0.4465],
'std': [0.2023, 0.1994, 0.2010]
}
},
'loss_args': {
'reduction': 'mean'
},
'postprocess_args': {}
})
softmax_model_components=create_model(model_config=softmax_model_conf,stage='train')
assert isinstance(softmax_model_components,EasyDict)
assert 'network' in softmax_model_components.keys()
assert hasattr(softmax_model_components.network,'task')
assert hasattr(softmax_model_components.network,'output_format')
assert 'postprocess' in softmax_model_components.keys()
assert 'preprocess' in softmax_model_components.keys()
assert 'loss' in softmax_model_components.keys()
assert 'collate_fn' in softmax_model_components.keys()
softmax_model_components=create_model(model_config=softmax_model_conf,stage='validate')
assert isinstance(softmax_model_components,EasyDict)
assert 'network' in softmax_model_components.keys()
assert hasattr(softmax_model_components.network,'task')
assert hasattr(softmax_model_components.network,'output_format')
assert 'postprocess' in softmax_model_components.keys()
assert 'preprocess' in softmax_model_components.keys()
def __init__(self,
config: EasyDict,
config_path: Union[str, Path, None] = None,
hypopt: bool = False):
"""Class initialization
Args:
config (EasyDict): dictionary parsed from Vortex experiment file
config_path (Union[str,Path,None], optional): path to experiment file. Need to be provided for \
backup **experiment file**. Defaults to None.
hypopt (bool, optional): flag for hypopt, disable several pipeline process. Defaults to False.
Raises:
Exception: raise undocumented error if exist
Example:
```python
from vortex.utils.parser import load_config
from vortex.core.pipelines import TrainingPipeline
# Parse config
config_path = 'experiments/config/example.yml'
config = load_config(config_path)
train_executor = TrainingPipeline(config=config,
config_path=config_path,
hypopt=False)
```
"""
self.config = config
self.hypopt = hypopt
# Check experiment config validity
self._check_experiment_config(config)
if not self.hypopt:
# Create experiment logger
self.experiment_logger = create_experiment_logger(config)
# Output directory creation
# If config_path is provided, it will duplicate the experiment file into the run directory
self.experiment_directory, self.run_directory = check_and_create_output_dir(
config, self.experiment_logger, config_path)
# Create local experiments run log file
self._create_local_runs_log(self.config, self.experiment_logger,
self.experiment_directory,
self.run_directory)
else:
self.experiment_logger = None
# Training components creation
self.device = config.trainer.device
self.model_components = create_model(model_config=config.model)
self.dataloader = create_dataloader(
dataset_config=config.dataset,
preprocess_config=config.model.preprocess_args,
collate_fn=self.model_components.collate_fn,
stage='train')
self.model_components.network = self.model_components.network.to(
self.device)
self.criterion = self.model_components.loss
self.criterion = self.criterion.to(self.device)
self.trainer = engine.create_trainer(
config.trainer,
criterion=self.criterion,
model=self.model_components.network,
experiment_logger=self.experiment_logger,
)
# Validation components creation
try:
val_dataset = create_dataset(config.dataset,
config.model.preprocess_args,
stage='validate')
## use same batch-size as training by default
validation_args = EasyDict(
{'batch_size': self.dataloader.batch_size})
validation_args.update(config.trainer.validation.args)
self.validator = engine.create_validator(self.model_components,
val_dataset,
validation_args,
device=self.device)
self.val_epoch = config.trainer.validation.val_epoch
self.valid_for_validation = True
except AttributeError as e:
warnings.warn(
'validation step not properly configured, will be skipped')
self.valid_for_validation = False
except Exception as e:
raise Exception(str(e))
# Reproducibility settings check
if hasattr(config, 'seed'):
_set_seed(config.seed)
def eval_check(runtime : str, model_name : str, model_arg : dict, export_arg : dict, suffix : str, predictor_args : dict={}) :
## torch predictor
output_directory = Path(test_output_directory)
output_directory.mkdir(exist_ok=True,parents=True)
model_config = create_model_cfg(model_name, model_arg)
model_components = create_model(model_config)
experiment_name = get_test_experiment_name(model_name, model_arg.network_args.backbone, suffix)
onnx_model_path = output_directory / '{}.onnx'.format(experiment_name)
pth_path = output_directory / '{}.pth'.format(experiment_name)
if not (onnx_model_path.exists() and pth_path.exists()) :
return EvalResult(Status.ERROR, msg='file not found, export might have failed')
model_components.network.load_state_dict(
torch.load(pth_path)
)
if model_name in tmp_output_format :
predictor_args['metainfo'] = tmp_output_format[model_name]
predictor = create_predictor(
model_components=model_components,
**predictor_args
).eval()
image_size = model_config.preprocess_args.input_size
## onnx model
try :
## TODO : check for fallback
onnx_model = create_runtime_model(str(onnx_model_path), runtime)
except Exception as e:
print(e)
return EvalResult(Status.ERROR, msg='RuntimeErorr : {}'.format(str(e)))
## predict check
input_test = (np.random.rand(1,image_size,image_size,3) * 255).astype(np.uint8)
## TODO : read additional input from predictor or onnx input spec
additional_args = dict(
score_threshold=0.0,
iou_threshold=1.0,
)
torch_results = torch_predict(predictor, input_test, **additional_args)
onnx_results = onnx_predict(onnx_model, input_test, **additional_args)
ok = len(torch_results) == len(onnx_results)
# print("len(torch_results) == len(onnx_results)", len(torch_results) == len(onnx_results))
status = EvalResult(status=ok, msg="len(torch_results) != len(onnx_results)" if not ok else "")
for torch_result, onnx_result in zip(torch_results, onnx_results) :
if not status :
break
ok = len(torch_result.keys()) == len(onnx_result.keys())
# print("len(torch_result.keys()) == len(onnx_result.keys())", len(torch_result.keys()) == len(onnx_result.keys()))
status = EvalResult(status=ok, msg="len(torch_result.keys()) != len(onnx_result.keys())" if not ok else "")
if not status :
break
ok = all(key in onnx_result.keys() for key in torch_result.keys())
# print("all(key in onnx_result.keys() for key in torch_result.keys())", all(key in onnx_result.keys() for key in torch_result.keys()))
status = EvalResult(status=ok, msg="not all(key in onnx_result.keys() for key in torch_result.keys())" if not ok else "")
if not status :
break
ok = all(isclose(onnx_value, torch_value, **isclose_config) for key in onnx_result.keys() for onnx_value, torch_value in zip(onnx_result[key].flatten(), torch_result[key].flatten()))
## TODO : collect diff across batch
# diff = sum(np.sum(np.abs(onnx_result[key] - torch_result[key]).flatten()).item() / len(onnx_result[key].flatten()) for key in onnx_result.keys())
diff = sum(((abs(onnx_value-torch_value) / len(onnx_result[key].flatten())) if len(onnx_result[key].flatten()) else 0) for key in onnx_result.keys() for onnx_value, torch_value in zip(onnx_result[key].flatten(), torch_result[key].flatten()))
# print("all(np.isclose(onnx_result[key], torch_result[key], **isclose_config) for key in onnx_result.keys())", all(np.isclose(onnx_result[key], torch_result[key], **isclose_config) for key in onnx_result.keys()))
s = Status.FAILED if not ok else Status.SUCCESS
status = EvalResult(status=s, diff=diff, msg="isclose failed" if not ok else "")
if not status :
break
del onnx_model, predictor, onnx_results, torch_results, input_test, model_components
return status
def __init__(
self,
config: EasyDict,
weights: Union[str, Path, None] = None,
device: Union[str, None] = None,
):
"""Class initialization
Args:
config (EasyDict): dictionary parsed from Vortex experiment file
weights (Union[str,Path,None], optional): path to selected Vortex model's weight. If set to None, it will \
assume that final model weights exist in **experiment directory**. \
Defaults to None.
device (Union[str,None], optional): selected device for model's computation. If None, it will use the device \
described in **experiment file**. Defaults to None.
Raises:
FileNotFoundError: raise error if selected 'weights' file is not found
Example:
```python
from vortex.core.pipelines import PytorchPredictionPipeline
from vortex.utils.parser import load_config
# Parse config
config_path = 'experiments/config/example.yml'
config = load_config(config_path)
weights_file = 'experiments/outputs/example/example.pth'
device = 'cuda'
vortex_predictor = PytorchPredictionPipeline(config = config,
weights = weights_file,
device = device)
```
"""
self.config = config
self.output_file_prefix = 'prediction'
# Configure experiment directory
experiment_directory, _ = check_and_create_output_dir(config)
# Initialize dataset to get class_names
dataset = create_dataset(
config.dataset,
stage='train',
preprocess_config=config.model.preprocess_args)
self.class_names = dataset.dataset.class_names if hasattr(
dataset.dataset, 'class_names') else None
# Set compute device
if device is None:
device = config.trainer.device
device = torch.device(device)
# Initialize model
if weights is None:
filename = Path(experiment_directory) / ('%s.pth' %
config.experiment_name)
else:
filename = Path(weights)
if not filename.exists():
raise FileNotFoundError(
'Selected weights file {} is not found'.format(filename))
model_components = create_model(config.model,
state_dict=str(filename),
stage='validate')
model_components.network = model_components.network.to(device)
self.predictor = create_predictor(model_components)
self.predictor.to(device)
# Configure input size for image
self.input_size = config.model.preprocess_args.input_size