def test_scale_and_sign_init_for_quant_algo__after_load_state(
self, wrap_dataloader):
config = create_config()
data_loader = self.create_dataloader(wrap_dataloader, config)
config.register_extra_structs([QuantizationRangeInitArgs(data_loader)])
_, compressed_model = self.create_algo_and_compressed_model(config)
ref_loaded_scale_val = torch.ones((1, 1, 1, 1)) * 100
load_state(
compressed_model,
{
'module.features.0.0.pre_ops.0.op.signed_tensor':
torch.tensor([0.]), # quantizer of 1st conv's weights
'module.features.1.0.pre_ops.0.op.scale':
ref_loaded_scale_val # quantizer of 2nd conv's weights
})
self.check_sign_and_scale(
compressed_model, {
'.*Sequential\\[0\\].*UpdateWeight.*':
(False, torch.ones(2, 1, 1, 1)),
'.*Sequential\\[1\\].*UpdateWeight. *':
(True, ref_loaded_scale_val),
'.*activation_quantizers.*Sequential\\[0\\].*': (True, 4),
'.*activation_quantizers.*nncf_model_input*': (False, 1)
})
def auto_register_extra_structs(self, args, train_dataset, data_collator):
if self.get("log_dir") is None:
self["log_dir"] = args.output_dir
if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
os.makedirs(self["log_dir"])
if args.do_train:
if train_dataset.info.builder_name == "conll2003":
train_dataset = deepcopy(train_dataset)
train_dataset = filter_columns(
train_dataset,
keep_columns=[
"labels", "input_ids", "attention_mask",
"token_type_ids"
],
remove_columns=[
"ner_tags", "pos_tags", "tokens", "id", "chunk_tags"
],
)
train_dataloader = get_train_dataloader_for_init(
args, train_dataset, data_collator)
initializing_data_loader_cls = get_data_loader_cls(
args, train_dataset)
self.register_extra_structs([
QuantizationRangeInitArgs(
initializing_data_loader_cls(train_dataloader)),
BNAdaptationInitArgs(
initializing_data_loader_cls(train_dataloader)),
])
def test_ad_hoc_range_init_does_not_replace_parameter_tensors(
self, wrap_dataloader, quant_type):
config = create_config()
config["compression"].update({
"activations": {
"mode": quant_type
},
"weights": {
"mode": quant_type
}
})
data_loader = self.create_dataloader(wrap_dataloader, config)
config.register_extra_structs([QuantizationRangeInitArgs(data_loader)])
model = TwoConvTestModel()
quant_model, quant_ctrl = create_compressed_model_and_algo_for_test(
model, config)
param_name_vs_id = {
name: id(tnsr)
for name, tnsr in quant_model.named_parameters()
}
quant_ctrl.init_range()
for name, param in quant_model.named_parameters():
assert param_name_vs_id[name] == id(param)
def register_default_init_args(nncf_config: 'NNCFConfig',
train_loader: torch.utils.data.DataLoader,
criterion: _Loss = None,
criterion_fn: Callable[[Any, Any, _Loss], torch.Tensor] = None,
train_steps_fn: Callable[[torch.utils.data.DataLoader, torch.nn.Module,
torch.optim.Optimizer, 'CompressionAlgorithmController',
Optional[int]], type(None)] = None,
validate_fn: Callable[[torch.nn.Module, torch.utils.data.DataLoader],
Tuple[float, float]] = None,
val_loader: torch.utils.data.DataLoader = None,
autoq_eval_fn: Callable[[torch.nn.Module, torch.utils.data.DataLoader], float] = None,
model_eval_fn: Callable[[torch.nn.Module, torch.utils.data.DataLoader], float] = None,
distributed_callbacks: Tuple[Callable, Callable] = None,
execution_parameters: 'ExecutionParameters' = None,
legr_train_optimizer: torch.optim.Optimizer = None,
device: str = None, ) -> 'NNCFConfig':
nncf_config.register_extra_structs([QuantizationRangeInitArgs(data_loader=wrap_dataloader_for_init(train_loader),
device=device),
BNAdaptationInitArgs(data_loader=wrap_dataloader_for_init(train_loader),
device=device),
])
if train_loader and train_steps_fn and val_loader and validate_fn:
nncf_config.register_extra_structs([LeGRInitArgs(
train_loader=train_loader,
train_fn=train_steps_fn,
val_loader=val_loader,
val_fn=validate_fn,
train_optimizer=legr_train_optimizer,
nncf_config=nncf_config,
)])
if criterion is not None:
if not criterion_fn:
criterion_fn = default_criterion_fn
nncf_config.register_extra_structs([QuantizationPrecisionInitArgs(criterion_fn=criterion_fn,
criterion=criterion,
data_loader=train_loader,
device=device)])
if autoq_eval_fn is not None:
if not val_loader:
val_loader = train_loader
nncf_config.register_extra_structs([AutoQPrecisionInitArgs(data_loader=val_loader,
eval_fn=autoq_eval_fn,
nncf_config=nncf_config)])
if model_eval_fn is not None:
nncf_config.register_extra_structs([ModelEvaluationArgs(eval_fn=model_eval_fn)])
if distributed_callbacks is None:
if execution_parameters is None:
nncf_logger.info('Please, provide execution parameters for optimal model initialization')
distributed_callbacks = (partial(default_distributed_wrapper, execution_parameters=execution_parameters),
default_distributed_unwrapper)
nncf_config.register_extra_structs([DistributedCallbacksArgs(*distributed_callbacks)])
return nncf_config
def test_scope_overrides(self, wrap_dataloader):
config = create_config()
config['target_device'] = 'TRIAL'
config["compression"]["scope_overrides"] = {
"weights": {
r"{re}NNCFConv2d\[[0-9]*\]/conv2d_0": {
"bits": 7,
"mode": "asymmetric",
},
},
"activations": {
r"{re}NNCFConv2d\[[0-9]*\]/conv2d_0": {
"bits": 7,
"signed": False,
}
}
}
data_loader = self.create_dataloader(wrap_dataloader, config)
config.register_extra_structs([QuantizationRangeInitArgs(data_loader)])
_, compressed_model = self.create_algo_and_compressed_model(config)
quantizers = get_all_modules_by_type(
compressed_model, ['SymmetricQuantizer', 'AsymmetricQuantizer'])
quantizer_str_dict = {str(k): v for k, v in quantizers.items()}
group_1 = [
quantizer_str_dict[
"NNCFNetwork/TwoConvTestModel[nncf_module]/Sequential[features]/"
"Sequential[0]/NNCFConv2d[0]/ModuleDict[pre_ops]/UpdateWeight[0]/"
"AsymmetricQuantizer[op]"],
quantizer_str_dict[
"NNCFNetwork/TwoConvTestModel[nncf_module]/Sequential[features]/"
"Sequential[1]/NNCFConv2d[0]/ModuleDict[pre_ops]/UpdateWeight[0]/"
"AsymmetricQuantizer[op]"]
]
group_2 = [
quantizer_str_dict[
f"NNCFNetwork/ModuleDict[{EXTERNAL_QUANTIZERS_STORAGE_NAME}]/"
"SymmetricQuantizer[TwoConvTestModel/Sequential[features]"
"/Sequential[0]/NNCFConv2d[0]/conv2d_0|OUTPUT]"],
quantizer_str_dict[
f"NNCFNetwork/ModuleDict[{EXTERNAL_QUANTIZERS_STORAGE_NAME}]/SymmetricQuantizer"
"[/nncf_model_input_0|OUTPUT]"],
]
for quantizer in group_1:
assert isinstance(quantizer, AsymmetricQuantizer)
assert quantizer.levels == 2**7
for quantizer in group_2:
assert isinstance(quantizer, SymmetricQuantizer)
assert not quantizer.signed
def test_scale_and_sign_init_for_quant_algo__without_init_section(
self, wrap_dataloader, config_creator):
config = config_creator()
data_loader = self.create_dataloader(wrap_dataloader, config)
config.register_extra_structs([QuantizationRangeInitArgs(data_loader)])
_, compressed_model = self.create_algo_and_compressed_model(config)
self.check_sign_and_scale(
compressed_model, {
'.*Sequential\\[0\\].*UpdateWeight.*':
(True, torch.ones(2, 1, 1, 1)),
'.*Sequential\\[1\\].*UpdateWeight. *': (True, 1),
'.*activation_quantizers.*Sequential\\[0\\].*': (True, 4),
'.*activation_quantizers.*nncf_model_input*': (False, 1)
})
def add_range_init(config):
for compression in config['compression']:
if compression['algorithm'] == 'quantization':
if 'initializer' not in compression:
compression['initializer'] = {}
compression['initializer'].update(
{'range': {
'num_init_samples': 1
}})
data_loader = create_ones_mock_dataloader(config)
config = NNCFConfig.from_dict(config)
config.register_extra_structs([
QuantizationRangeInitArgs(
wrap_dataloader_for_init(data_loader))
])
return config
def scale_signed_dumping_worker(gpu, ngpus_per_node, config, tmp_path):
distributed_init_test_default(gpu, ngpus_per_node, config)
data_loader = create_rank_dataloader(config, gpu)
model = safe_thread_call(partial(squeezenet1_1, pretrained=True))
config.register_extra_structs(
[QuantizationRangeInitArgs(wrap_dataloader_for_init(data_loader))])
quant_model, compression_ctrl = create_compressed_model_and_algo_for_test(
model, config)
compression_scheduler = compression_ctrl.scheduler
quant_model = post_compression_test_distr_init(compression_ctrl, config,
ngpus_per_node, quant_model)
criterion = torch.nn.MSELoss().cuda(config.gpu)
optimizer = torch.optim.Adam(quant_model.parameters(), lr=0.01)
torch.backends.cudnn.benchmark = True
# just to reproduce the same scale values without Dropout
quant_model.eval()
act_sum = 0
for layer in get_all_modules_by_type(quant_model,
"SymmetricQuantizer").values():
act_sum += layer.scale.sum()
ref_sum = 3720.864
assert act_sum.item() == approx(ref_sum, 0.01), \
'sum of scales is not expected {} vs {} rank {}'.format(act_sum.item(), ref_sum, config.rank)
out_file_path = get_path_after_broadcast(tmp_path, config.rank)
save_params(quant_model, out_file_path)
compression_scheduler.step()
for i, (input_, _) in enumerate(data_loader):
if i > 5:
break
output = quant_model(input_)
optimizer.zero_grad()
dummy_target = torch.randn(1000).cuda(config.gpu, non_blocking=True)
loss = criterion(output, dummy_target)
compression_scheduler.step()
loss.backward()
optimizer.step()
compression_scheduler.step()
out_file_path = get_path_path_after_train_iters(tmp_path, config.rank)
save_params(quant_model, out_file_path)
def test_scale_and_sign_init_for_quant_algo__with_zero_init_steps(
self, wrap_dataloader):
config = create_config()
config['compression']['initializer']['range']['num_init_samples'] = 0
data_loader = self.create_dataloader(wrap_dataloader, config)
config.register_extra_structs([QuantizationRangeInitArgs(data_loader)])
_, compressed_model = self.create_algo_and_compressed_model(config)
self.check_sign_and_scale(
compressed_model, {
'.*Sequential\\[0\\].*UpdateWeight.*':
(True, torch.ones(2, 1, 1, 1)),
'.*Sequential\\[1\\].*UpdateWeight. *': (True, 1),
'.*activation_quantizers.*Sequential\\[0\\].*': (False, 1),
'.*activation_quantizers.*nncf_model_input*': (False, 1)
})
def test_staged_scheduler_with_range_init():
config = get_squeezenet_quantization_config()
config['compression'].update({
'params': {
"activations_quant_start_epoch": 1,
"weights_quant_start_epoch": 2,
},
'initializer': {
'range': {
'num_init_samples': 1
}
}
})
register_bn_adaptation_init_args(config)
model = squeezenet1_1(num_classes=10, dropout=0)
input_infos_list = create_input_infos(config)
input_sample_size = input_infos_list[0].shape
data_loader = DataLoader(OnesDatasetMock(input_sample_size[1:]),
batch_size=1,
num_workers=0, # Workaround for PyTorch MultiprocessingDataLoader issues
shuffle=False)
config.register_extra_structs([QuantizationRangeInitArgs(wrap_dataloader_for_init(data_loader))])
model, algo = create_compressed_model_and_algo_for_test(model, config)
scheduler = algo.scheduler
for module in algo.all_quantizations.values():
assert not module.is_enabled_quantization()
scheduler.epoch_step()
for module in algo.all_quantizations.values():
assert not module.is_enabled_quantization()
scheduler.epoch_step()
for wq_info in algo.weight_quantizers.values():
assert not wq_info.quantizer_module_ref.is_enabled_quantization()
for aq_info in algo.non_weight_quantizers.values():
assert aq_info.quantizer_module_ref.is_enabled_quantization()
scheduler.epoch_step()
for module in algo.all_quantizations.values():
assert module.is_enabled_quantization()
def register_default_init_args(nncf_config: NNCFConfig,
data_loader: tf.data.Dataset,
batch_size: int,
device: str = None) -> NNCFConfig:
"""
Register extra structures in the NNCFConfig. Initialization of some
compression algorithms requires certain extra structures.
:param nncf_config: An instance of the NNCFConfig class without extra structures.
:param data_loader: Dataset used for initialization.
:param batch_size: Batch size used for initialization.
:param device: Device to perform initialization. If `device` is `None` then the device
of the model parameters will be used.
:return: An instance of the NNCFConfig class with extra structures.
"""
nncf_config.register_extra_structs([
QuantizationRangeInitArgs(data_loader=TFInitializingDataLoader(data_loader, batch_size),
device=device),
BNAdaptationInitArgs(data_loader=TFInitializingDataLoader(data_loader, batch_size),
device=device)
])
return nncf_config
def test_per_layer_range_init_collectors_are_called_the_required_number_of_times(
range_init_call_count_test_struct, mocker):
config = create_config()
config['compression']['initializer'][
'range'] = range_init_call_count_test_struct.range_init_config
data_loader = TestRangeInit.create_dataloader(True, config, 10)
config.register_extra_structs([QuantizationRangeInitArgs(data_loader)])
range_minmax_init_create_spy = mocker.spy(PTMinMaxStatisticCollector,
'__init__')
range_meanminmax_init_create_spy = mocker.spy(
PTMeanMinMaxStatisticCollector, '__init__')
range_threesigma_init_create_spy = mocker.spy(
PTMedianMADStatisticCollector, '__init__')
range_minmax_init_register_input_spy = mocker.spy(
PTMinMaxStatisticCollector, '_register_input')
range_meanminmax_init_register_input_spy = mocker.spy(
PTMeanMinMaxStatisticCollector, '_register_input')
range_threesigma_init_register_input_spy = mocker.spy(
PTMedianMADStatisticCollector, '_register_input')
TestRangeInit.create_algo_and_compressed_model(config)
assert range_minmax_init_create_spy.call_count == \
range_init_call_count_test_struct.expected_call_count_initializer_create['min_max']
assert range_meanminmax_init_create_spy.call_count == \
range_init_call_count_test_struct.expected_call_count_initializer_create['mean_min_max']
assert range_threesigma_init_create_spy.call_count == \
range_init_call_count_test_struct.expected_call_count_initializer_create['three_sigma']
assert range_minmax_init_register_input_spy.call_count == \
range_init_call_count_test_struct.expected_call_count_register_input['min_max']
assert range_meanminmax_init_register_input_spy.call_count == \
range_init_call_count_test_struct.expected_call_count_register_input['mean_min_max']
assert range_threesigma_init_register_input_spy.call_count == \
range_init_call_count_test_struct.expected_call_count_register_input['three_sigma']
def test_init_ranges_are_set(quantization_mode: str, per_channel: bool,
range_init_type_vs_ref_vals: Tuple[str, float,
float, float]):
class SyntheticDataset(torch.utils.data.Dataset):
def __init__(self):
super().__init__()
self._length = 1
def __getitem__(self, idx):
if idx >= self._length:
raise StopIteration
test_input_sample = torch.zeros([3, 100, 100])
for i in range(0, 100):
for j in range(0, 100):
test_input_sample[0][i][j] = i * 100 + j
test_input_sample[1] = test_input_sample[0]
test_input_sample[2] = test_input_sample[0]
return test_input_sample, test_input_sample
def __len__(self):
return self._length
data_loader = torch.utils.data.DataLoader(SyntheticDataset(),
batch_size=1,
drop_last=True)
range_init_type = range_init_type_vs_ref_vals[0]
config_with_init = NNCFConfig()
config_with_init.update({
"input_info": {
"sample_size": [1, 3, 100, 100]
},
"target_device": "TRIAL",
"compression": {
"algorithm": "quantization",
"activations": {
"mode": quantization_mode,
"per_channel": per_channel
},
"weights": {
"mode": quantization_mode,
"per_channel": per_channel
},
"initializer": {
"range": {
"num_init_samples": 1,
"type": range_init_type
}
}
}
})
if range_init_type == "percentile":
config_with_init["compression"]["initializer"]["range"]["params"] = {
"min_percentile": 32.10,
"max_percentile": 67.89
}
# Activations init check
id_model = SingleConv2dIdentityModel()
config_with_init.register_extra_structs(
[QuantizationRangeInitArgs(wrap_dataloader_for_init(data_loader))])
register_bn_adaptation_init_args(config_with_init)
_, compression_ctrl = create_compressed_model_and_algo_for_test(
id_model, config_with_init)
act_quantizer_info = next(
iter(compression_ctrl.non_weight_quantizers.values()))
ref_scale = range_init_type_vs_ref_vals[1]
ref_input_low = range_init_type_vs_ref_vals[2]
ref_input_high = range_init_type_vs_ref_vals[3]
def check_scales(quantizer: BaseQuantizer, per_channel: bool):
# Absolute tolerance is 1.0 due to percentile value interpolation
if quantization_mode == 'symmetric':
assert torch.allclose(quantizer.scale,
torch.ones_like(quantizer.scale) * ref_scale,
atol=1.0)
if per_channel:
assert quantizer.scale.numel() == 3
else:
assert quantizer.scale.numel() == 1
else:
assert torch.allclose(quantizer.input_low,
torch.ones_like(quantizer.input_low) *
ref_input_low,
atol=1.0)
assert torch.allclose(quantizer.input_range,
torch.ones_like(quantizer.input_low) *
ref_input_high,
atol=1.0)
if per_channel:
assert quantizer.input_low.numel() == 3
assert quantizer.input_range.numel() == 3
else:
assert quantizer.input_low.numel() == 1
assert quantizer.input_range.numel() == 1
check_scales(act_quantizer_info.quantizer_module_ref, per_channel)
# Weight init check
synth_weight_model = SingleConv2dSyntheticWeightModel()
_, compression_ctrl = create_compressed_model_and_algo_for_test(
synth_weight_model, config_with_init)
weight_quantizer_info = next(
iter(compression_ctrl.weight_quantizers.values()))
check_scales(weight_quantizer_info.quantizer_module_ref, per_channel)