def test_can_not_set_sparsity_more_than_one_for_magnitude_sparse_algo():
config = get_basic_magnitude_sparsity_config()
_, compression_ctrl = create_compressed_model_and_algo_for_test(
MagnitudeTestModel(), config)
with pytest.raises(AttributeError):
compression_ctrl.set_sparsity_level(1)
compression_ctrl.set_sparsity_level(1.2)
def test_magnitude_algo_set_independently_sparsity_level_for_one_module():
module_name_conv1 = 'MagnitudeTestModel/NNCFConv2d[conv1]'
module_name_conv2 = 'MagnitudeTestModel/NNCFConv2d[conv2]'
config = get_basic_magnitude_sparsity_config()
config['compression']['params'] = {"sparsity_level_setting_mode": 'local'}
sparse_model, compression_ctrl = create_compressed_model_and_algo_for_test(MagnitudeTestModel(), config)
sparse_info_conv1 = [sparse_info for sparse_info in compression_ctrl.sparsified_module_info\
if sparse_info.module_name == module_name_conv1]
sparse_info_conv2 = [sparse_info for sparse_info in compression_ctrl.sparsified_module_info\
if sparse_info.module_name == module_name_conv2]
compression_ctrl.set_sparsity_level(0.5, sparse_info_conv1[0])
weights_conv1 = sparse_model.conv1.weight
weights_conv2 = sparse_model.conv2.weight
count_nonzero_conv1 = sparse_model.conv1.pre_ops['0'].operand.apply_binary_mask(weights_conv1).nonzero().size(0)
count_param_conv1 = weights_conv1.view(-1).size(0)
assert count_param_conv1 - count_nonzero_conv1 == 4 # 8 * 0.5
compression_ctrl.set_sparsity_level(0.3, sparse_info_conv2[0])
count_nonzero_conv1 = sparse_model.conv1.pre_ops['0'].operand.apply_binary_mask(weights_conv1).nonzero().size(0)
count_param_conv1 = weights_conv1.view(-1).size(0)
count_nonzero_conv2 = sparse_model.conv2.pre_ops['0'].operand.apply_binary_mask(weights_conv2).nonzero().size(0)
count_param_conv2 = weights_conv2.view(-1).size(0)
assert count_param_conv1 - count_nonzero_conv1 == 4 # 8 * 0.5
assert count_param_conv2 - count_nonzero_conv2 == 6 # ~ 18 * 0.3
def test_can_create_magnitude_sparse_algo__with_defaults():
model = MagnitudeTestModel()
config = get_basic_magnitude_sparsity_config()
config['compression']['params'] = \
{'schedule': 'multistep'}
sparse_model, compression_ctrl = create_compressed_model_and_algo_for_test(
deepcopy(model), config)
assert isinstance(compression_ctrl, MagnitudeSparsityController)
assert compression_ctrl.sparsity_level == approx(0.1)
assert len(list(sparse_model.modules())) == 12
_, sparse_model_conv = check_correct_nncf_modules_replacement(
model, sparse_model)
i = 0
for sparse_module in sparse_model_conv.values():
store = []
ref_mask = torch.ones_like(
sparse_module.weight) if i == 0 else ref_mask_2
i += 1
for op in sparse_module.pre_ops.values():
if isinstance(op, UpdateWeight) and isinstance(
op.operand, BinaryMask):
assert compression_ctrl.threshold == approx(0.24, 0.1)
assert torch.allclose(op.operand.binary_mask, ref_mask)
assert isinstance(compression_ctrl.weight_importance,
type(normed_magnitude))
assert op.__class__.__name__ not in store
store.append(op.__class__.__name__)
def test_can_not_set_sparsity_more_than_one_for_magnitude_sparse_algo():
model = MagnitudeTestModel()
config = get_basic_magnitude_sparsity_config()
compression_algo = create_compression_algorithm(model, config)
with pytest.raises(AttributeError):
compression_algo.set_sparsity_level(1)
compression_algo.set_sparsity_level(1.2)
def test_can_create_magnitude_sparse_algo__with_defaults():
model = MagnitudeTestModel()
config = get_basic_magnitude_sparsity_config()
config['compression']['params'] = \
{'schedule': 'multistep'}
compression_algo = create_compression_algorithm(deepcopy(model), config)
assert isinstance(compression_algo, MagnitudeSparsity)
sparse_model = compression_algo.model
assert compression_algo.sparsity_level == approx(0.1)
assert len(list(sparse_model.modules())) == 11
model_conv = get_all_modules_by_type(model, 'Conv2d')
sparse_model_conv = get_all_modules_by_type(sparse_model, 'NNCFConv2d')
assert len(model_conv) == len(sparse_model_conv)
i = 0
for module_name in model_conv:
scope = module_name.split('/')
scope[-1] = scope[-1].replace('Conv2d', 'NNCFConv2d')
sparse_module_name = '/'.join(scope)
assert sparse_module_name in sparse_model_conv
store = []
sparse_module = sparse_model_conv[sparse_module_name]
ref_mask = torch.ones_like(sparse_module.weight) if i == 0 else ref_mask_2
i += 1
for op in sparse_module.pre_ops.values():
if isinstance(op, UpdateWeight) and isinstance(op.operand, BinaryMask):
assert compression_algo.threshold == approx(0.24, 0.1)
assert torch.allclose(op.operand.binary_mask, ref_mask)
assert isinstance(compression_algo.weight_importance, type(normed_magnitude))
assert op.__class__.__name__ not in store
store.append(op.__class__.__name__)
def test_can_not_create_magnitude_algo__without_steps():
config = get_basic_magnitude_sparsity_config()
config['compression']['params'] = {
'schedule': 'multistep',
'multistep_sparsity_levels': [0.1]
}
with pytest.raises(AttributeError):
_, _ = create_compressed_model_and_algo_for_test(MockModel(), config)
def test_can_create_magnitude_algo__without_levels():
config = get_basic_magnitude_sparsity_config()
config['compression']['params'] = {
'schedule': 'multistep',
'multistep_steps': [1]
}
_, compression_ctrl = create_compressed_model_and_algo_for_test(
MockModel(), config)
assert compression_ctrl.sparsity_level == approx(0.1)
def test_magnitude_sparse_algo_sets_threshold(weight_importance, sparsity_level, threshold):
model = MagnitudeTestModel()
config = get_basic_magnitude_sparsity_config()
config['compression']['weight_importance'] = weight_importance
config['compression']['params'] = {'schedule': 'multistep'}
compression_algo = create_compression_algorithm(model, config)
if sparsity_level:
compression_algo.set_sparsity_level(sparsity_level)
assert compression_algo.threshold == pytest.approx(threshold, 0.01)
def test_can_not_create_magnitude_algo__without_steps():
model = MagnitudeTestModel()
config = get_basic_magnitude_sparsity_config()
config['compression']['params'] = {
'schedule': 'multistep',
'sparsity_levels': [0.1]
}
with pytest.raises(AttributeError):
create_compression_algorithm(model, config)
def get_multistep_normed_abs_config():
config = get_basic_magnitude_sparsity_config()
compression_config = config['compression']
compression_config['params'] = {
'schedule': 'multistep',
'weight_importance': 'normed_abs',
'multistep_steps': [1, 3],
'multistep_sparsity_levels': [0.1, 0.5, 0.9]
}
return config
def test_magnitude_sparse_algo_sets_threshold(weight_importance, sparsity_level, threshold):
model = MagnitudeTestModel()
config = get_basic_magnitude_sparsity_config()
config['compression']['params'] = {'schedule': 'multistep',
'weight_importance': weight_importance}
_, compression_ctrl = create_compressed_model_and_algo_for_test(model, config)
if sparsity_level:
compression_ctrl.set_sparsity_level(sparsity_level)
stats = compression_ctrl.statistics()
assert stats["sparsity_threshold"] == pytest.approx(threshold, 0.01)
def test_magnitude_algo_set_binary_mask_on_forward():
config = get_basic_magnitude_sparsity_config()
config['compression']['weight_importance'] = 'abs'
sparse_model, compression_ctrl = create_compressed_model_and_algo_for_test(MagnitudeTestModel(), config)
compression_ctrl.set_sparsity_level(0.3)
with torch.no_grad():
sparse_model(torch.ones([1, 1, 10, 10]))
op = sparse_model.conv1.pre_ops['0']
check_equal(ref_mask_1, op.operand.binary_mask)
op = sparse_model.conv2.pre_ops['0']
check_equal(ref_mask_2, op.operand.binary_mask)
def test_magnitude_algo_set_binary_mask_on_forward():
model = MagnitudeTestModel()
config = get_basic_magnitude_sparsity_config()
config['compression']['weight_importance'] = 'abs'
compression_algo = create_compression_algorithm(model, config)
compression_algo.set_sparsity_level(0.3)
model = compression_algo.model
with torch.no_grad():
model(torch.ones([1, 1, 10, 10]))
op = model.conv1.pre_ops['0']
check_equal(ref_mask_1, op.operand.binary_mask)
op = model.conv2.pre_ops['0']
check_equal(ref_mask_2, op.operand.binary_mask)
def test_can_create_magnitude_algo__without_levels():
model = MagnitudeTestModel()
config = get_basic_magnitude_sparsity_config()
config['compression']['params'] = {'schedule': 'multistep', 'steps': [1]}
compression_algo = create_compression_algorithm(model, config)
assert compression_algo.sparsity_level == approx(0.1)
