def test_quantize_model_post_training_mnist():
# Prepare model paths
mnist_model_path = Zoo.search_models(
domain="cv",
sub_domain="classification",
architecture="mnistnet",
framework="pytorch",
)[0].onnx_file.downloaded_path()
quant_model_path = tempfile.NamedTemporaryFile(suffix=".onnx", delete=False).name
# Prepare sample validation dataset
batch_size = 1
val_dataset = MNISTDataset(train=False)
input_dict = [{"input": img.numpy()} for (img, _) in val_dataset]
data_loader = DataLoader(input_dict, None, batch_size)
# Run calibration and quantization
quantize_model_post_training(
mnist_model_path, data_loader, quant_model_path, show_progress=False
)
# Verify that ResNet identity has no affect
_test_resnet_identity_quant(quant_model_path, False, False)
# Verify Convs and MatMuls are quantized
_test_model_is_quantized(mnist_model_path, quant_model_path)
# Verify quant model accuracy
test_data_loader = DataLoader(input_dict, None, 1) # initialize a new generator
_test_quant_model_output(
mnist_model_path, quant_model_path, test_data_loader, [0], batch_size
)
# Clean up
os.remove(quant_model_path)
def validate_model_data(data_path: str, model_path: str):
model_dataloader = DataLoader.from_model_random(model_path, batch_size=1)
file_dataloader = DataLoader(data_path, None, 1)
input_from_data, _ = next(file_dataloader)
input_from_model, _ = next(model_dataloader)
for data_key, model_key in zip(input_from_data.keys(),
input_from_model.keys()):
if input_from_data[data_key].shape != input_from_model[model_key].shape:
raise ValidationError(
"Data shape from input does not match model input shape.")
if input_from_data[data_key].dtype != input_from_model[model_key].dtype:
raise ValidationError(
"Data type from input does not match model input type.")
def test_dataloader(
data_shape: Dict[str, Tuple[int, ...]],
label_shape: Union[None, Dict[str, Tuple[int, ...]]],
samples: int,
batch_size: int,
iter_steps: int,
):
with tempfile.TemporaryDirectory() as tempdir:
data_glob = os.path.join(tempdir, "inp_*.npz")
label_glob = (os.path.join(tempdir, "out_*.npz")
if label_shape is not None else None)
for i in range(samples):
data_path = os.path.join(tempdir, "inp_{}.npz".format(i))
data = {}
for key in data_shape:
data[key] = numpy.random.randn(*data_shape[key])
numpy.savez(data_path, **data)
if label_shape is not None:
label_path = os.path.join(tempdir, "out_{}.npz".format(i))
label = {}
for key in label_shape:
label[key] = numpy.random.randn(*label_shape[key])
numpy.savez(label_path, **label)
dataloader = DataLoader(data_glob, label_glob, batch_size, iter_steps)
_test_dataloader(dataloader, data_shape, label_shape, batch_size,
iter_steps, samples)
def test_quantize_model_post_training_resnet50_imagenette():
# Prepare model paths
resnet50_imagenette_path = Zoo.load_model(
domain="cv",
sub_domain="classification",
architecture="resnet_v1",
sub_architecture="50",
framework="pytorch",
repo="sparseml",
dataset="imagenette",
training_scheme=None,
sparse_name="base",
sparse_category="none",
sparse_target=None,
).onnx_file.downloaded_path()
quant_model_path = tempfile.NamedTemporaryFile(suffix=".onnx", delete=False).name
# Prepare sample validation dataset
batch_size = 1
val_dataset = ImagenetteDataset(train=False, dataset_size=ImagenetteSize.s320)
input_dict = [{"input": img.numpy()} for (img, _) in val_dataset]
data_loader = DataLoader(input_dict, None, batch_size)
# Run calibration and quantization
quantize_model_post_training(
resnet50_imagenette_path,
data_loader,
quant_model_path,
show_progress=False,
run_extra_opt=False,
)
# Verify that ResNet identity optimization is successful and save output for testing
_test_resnet_identity_quant(quant_model_path, True, True)
# Verify Convs and MatMuls are quantized
_test_model_is_quantized(resnet50_imagenette_path, quant_model_path)
# Verify quant model accuracy
test_data_loader = DataLoader(input_dict, None, 1) # initialize a new generator
_test_quant_model_output(
resnet50_imagenette_path, quant_model_path, test_data_loader, [1], batch_size
)
# Clean up
os.remove(quant_model_path)
def _run_benchmark(
self,
benchmark: ProjectBenchmark,
model: Union[str, ModelProto],
runner: ModelRunner,
core_count: int,
batch_size: int,
inference_engine: str,
inference_model_optimization: Union[str, None],
num_steps: int,
step_index: int,
):
data_iter = DataLoader.from_model_random(model,
batch_size=batch_size,
iter_steps=-1)
measurements = []
total_iterations = self.warmup_iterations_per_check + self.iterations_per_check
iterations = 0
for _, current_measurements in runner.run_iter(
data_iter,
show_progress=False,
max_steps=total_iterations,
):
measurements.append(current_measurements)
iteration_percent = (iterations + 1) / (total_iterations)
iter_val = (step_index + iteration_percent) / num_steps
yield iter_val
iterations += 1
if self.warmup_iterations_per_check > 0:
measurements = measurements[self.warmup_iterations_per_check:]
result = data_dump_and_validation(
ProjectBenchmarkResultSchema(),
{
"core_count": core_count,
"batch_size": batch_size,
"inference_engine": inference_engine,
"inference_model_optimization": inference_model_optimization,
"measurements": measurements,
},
)
benchmark.result["benchmarks"].append(result)
def test_dataloader_from_random(
data_shapes: Dict[str, Tuple[int, ...]],
label_shapes: Union[None, Dict[str, Tuple[int, ...]]],
batch_size: int,
iter_steps: int,
num_samples: int,
data_types: Dict[str, numpy.dtype],
):
dataloader = DataLoader.from_random(data_shapes, label_shapes, batch_size,
iter_steps, num_samples, data_types)
_test_dataloader(
dataloader,
data_shapes,
label_shapes,
batch_size,
iter_steps,
num_samples,
data_types,
)
def test_dataloader_from_model(
dataloader_models: DataloaderModelFixture,
batch_size: int,
iter_steps: int,
num_samples: int,
create_labels: bool,
strip_first_dim: bool,
):
dataloader = DataLoader.from_model_random(
dataloader_models.model_path,
batch_size,
iter_steps,
num_samples,
create_labels,
strip_first_dim,
)
data_shapes = dict(dataloader_models.data_shape)
label_shapes = dict(dataloader_models.label_shape)
if strip_first_dim:
for key in data_shapes:
data_shapes[key] = data_shapes[key][1:]
for key in label_shapes:
label_shapes[key] = label_shapes[key][1:]
if not create_labels:
label_shapes = None
_test_dataloader(
dataloader,
data_shapes,
label_shapes,
batch_size,
iter_steps,
num_samples,
dataloader_models.data_types,
)
def run(self) -> Iterator[Dict[str, Any]]:
"""
Perform the work for the job.
Runs and saves the appropriate perf profile based on the configuration
:return: an iterator containing progress update information
"""
_LOGGER.info(
("running perf profile for project_id {} and "
"model_id {} and profile_id {} with "
"batch_size:{}, core_count:{}, "
"pruning_estimations:{}, quantized_estimations:{}, "
"iterations_per_check:{}, warmup_iterations_per_check:{}").format(
self.project_id,
self.model_id,
self.profile_id,
self.batch_size,
self.core_count,
self.pruning_estimations,
self.quantized_estimations,
self.iterations_per_check,
self.warmup_iterations_per_check,
))
model = self.get_project_model()
model.validate_filesystem()
profile = self._get_project_perf_profile()
profile.analysis = {}
data_loader = DataLoader.from_model_random(model.file_path,
self.batch_size)
num_steps = 1
if self.pruning_estimations:
num_steps += 1
if self.quantized_estimations:
num_steps += 1
for progress in self._run_baseline_perf(model, data_loader, profile,
num_steps):
_LOGGER.debug(
("perf profile baseline analysis for project_id {} and "
"model_id {} and profile_id {}: {}").format(
self.project_id, self.model_id, self.profile_id,
progress))
yield progress
if self.pruning_estimations:
for progress in self._run_pruning_sensitivity(
model, data_loader, profile, num_steps):
_LOGGER.debug(
("perf profile pruning weight magnitude analysis for "
"project_id {} and model_id {} and profile_id {}: {}"
).format(self.project_id, self.model_id, self.profile_id,
progress))
yield progress
else:
profile.analysis["pruning"] = None
if self.quantized_estimations:
raise NotImplementedError(
"quantized estimations are currently not available")
else:
profile.analysis["quantization"] = None
profile.save()
def load_data(
data: Any,
model: Any = None,
batch_size: int = 1,
total_iterations: int = 0,
**kwargs,
) -> Iterable[Tuple[Dict[str, Any], Any]]:
"""
Creates a iteratable data loader for the given data.
Acceptable types for data are:
- a folder path containing numpy files
- a list of file paths
- a SparseML DataLoader
- a SparseZoo DataLoader
- an iterable
- None type, in which case model must be passed
:param data: data to use for benchmarking
:param model: model to use for generating data
:param batch_size: batch size
:param total_iterations: total number of iterations
:param kwargs: additional arguments to pass to the DataLoader
:return: an iterable of data and labels
"""
# Creates random data from model input shapes if data is not provided
if not data:
if not model:
raise ValueError("must provide model or data")
model = load_model(model)
return DataLoader.from_model_random(
model, batch_size, iter_steps=total_iterations
)
# If data is a SparseZoo stub, downloads model data
if isinstance(data, str) and data.startswith("zoo:"):
model_from_zoo = Zoo.load_model_from_stub(data)
data = model_from_zoo.data_inputs.loader(
batch_size, total_iterations, batch_as_list=False
)
# Imediately return the data if it is already a DataLoader
if isinstance(data, DataLoader):
return data
# If data is a SparseZoo DataLoader, unbatches the dataloader and creates
# DataLoader from it
elif isinstance(data, SparseZooDataLoader):
datasets = [
SparseZooDataset(name, dataset) for name, dataset in data.datasets.items()
]
data = SparseZooDataLoader(*datasets, batch_size=1, batch_as_list=False)
data = [
OrderedDict(
[
(element, value.reshape(value.shape[1:]))
for element, value in entry.items()
]
)
for entry in data
]
# If data is a dictionary of data shapes, creates DataLoader from random data
elif isinstance(data, dict):
is_dict_of_shapes = True
for _, value in data.items():
is_dict_of_shapes = is_dict_of_shapes and isinstance(value, tuple)
if is_dict_of_shapes:
return DataLoader.from_random(
data,
None,
batch_size=batch_size,
iter_steps=total_iterations,
**kwargs,
)
# If data is a list of data shapes, creates DataLoader from random data
elif isinstance(data, Iterable):
element = next(iter(data))
if isinstance(element, tuple):
data_shapes = OrderedDict(
(f"{index:04}", shape) for index, shape in enumerate(data)
)
return DataLoader.from_random(
data_shapes,
None,
batch_size=batch_size,
iter_steps=total_iterations,
**kwargs,
)
return DataLoader(
data, None, batch_size=batch_size, iter_steps=total_iterations, **kwargs
)
def _run_iter(
self,
**kwargs,
) -> Generator[Tuple[AnalyzerProgress, PruningSensitivityResult], None, None]:
sparsity_levels = (
kwargs["pruning_perf_analysis_sparsity_levels"]
if "pruning_perf_analysis_sparsity_levels" in kwargs
else default_pruning_sparsities_perf()
)
num_steps = len(sparsity_levels)
model = kwargs["model"]
data_loader = DataLoader.from_model_random(model, self._batch_size, -1)
# build map of possible layer identifiers to prunable param name
id_to_param_name = {}
param_names = self._model_info.get_prunable_param_names()
for param_name in param_names:
layer_info = self._model_info.layer_info[param_name]
# by output id
output_id = layer_info.attributes.get("node_output_id")
if output_id is not None:
id_to_param_name[output_id] = param_name
# by node name
node_name = layer_info.attributes.get("node_name")
if node_name is not None:
id_to_param_name[node_name] = param_name
# directly match to param name
id_to_param_name[param_name] = param_names
runner = DeepSparseAnalyzeModelRunner(model, self._batch_size, self._num_cores)
for idx, sparsity in enumerate(sparsity_levels):
if sparsity <= 1e-9:
sparsity = None # to enforce dense execution
yield AnalyzerProgress(step=idx, total_steps=num_steps), self.result
results = runner.run(
data_loader,
show_progress=False,
num_iterations=self._iterations_per_check,
num_warmup_iterations=self._warmup_iterations_per_check,
imposed_ks=sparsity,
max_steps=1,
)[0][0]
_LOGGER.debug(
"measured perf results for one shot sparsity {}".format(sparsity)
)
# model sparsity -> average time in seconds
self.result.add_model_sparsity_result(
sparsity or 0.0, results["average_total_time"] / 1000.0
)
for layer in results["layer_info"]:
layer_name = id_to_param_name.get(
layer["canonical_name"],
id_to_param_name.get(layer["name"]), # fallback to internal name
)
if layer_name is not None:
self.result.add_layer_sparsity_result(
layer_name,
sparsity if sparsity is not None else 0.0,
layer["average_run_time_in_ms"] / 1000.0,
)
yield AnalyzerProgress(step=num_steps, total_steps=num_steps), self.result