def test_disable_loss(self, module, frozen):
model = sparse_model(module, frozen)
sw = model.sparsifier
assert sw.frozen is (True if frozen is None else frozen)
loss = SparseLoss([model.sparsifier])
loss.disable()
assert sw.frozen
def test_disable_loss(self, module, sparsify):
model = sparse_model(module, sparsify)
sw = model.sparsifier
assert sw.sparsify is (False if sparsify is None else sparsify)
loss = SparseLoss([model.sparsifier])
loss.disable()
assert not sw.sparsify
def __init__(self, target_model: NNCFNetwork,
sparsified_module_info: List[SparseModuleInfo], params):
super().__init__(target_model, sparsified_module_info)
self._distributed = False
self._loss = SparseLoss() # type: SparseLoss
scheduler_cls = SPARSITY_SCHEDULERS.get(
params.get("schedule", "exponential"))
self._scheduler = scheduler_cls(self, params)
sparsify_operations = [m.operand for m in self.sparsified_module_info]
self._loss.set_layers(sparsify_operations)
self._check_sparsity_masks = params.get("check_sparsity_masks", False)
def test_get_target_sparsity_rate(self, module, target, expected_rate):
model = sparse_model(module, None)
loss = SparseLoss([model.sparsifier])
if target is not None:
loss.target = target
actual_rate = None
try:
actual_rate = loss.target_sparsity_rate
if expected_rate is None:
pytest.fail("Exception should be raised")
except IndexError:
if expected_rate is not None:
pytest.fail("Exception is not expected")
if expected_rate is not None:
assert actual_rate == expected_rate
def __init__(self):
self.set_sparsity_level = mocker.stub()
self.freeze = mocker.stub()
from nncf.sparsity.rb.loss import SparseLoss
self.loss = SparseLoss()
self.loss.current_sparsity = 0.3
self.sparsity_init = 0
def __init__(self, target_model: NNCFNetwork,
sparsified_module_info: List[SparseModuleInfo],
params):
super().__init__(target_model, sparsified_module_info)
self._scheduler = None
self._distributed = False
sparsity_level_mode = params.get("sparsity_level_setting_mode", "global")
sparsify_operations = [m.operand for m in self.sparsified_module_info]
if sparsity_level_mode == 'local':
self._loss = SparseLossForPerLayerSparsity(sparsify_operations)
else:
self._loss = SparseLoss(sparsify_operations) # type: SparseLoss
schedule_type = params.get("schedule", "exponential")
scheduler_cls = SPARSITY_SCHEDULERS.get(schedule_type)
self._scheduler = scheduler_cls(self, params)
self._check_sparsity_masks = params.get("check_sparsity_masks", False)
def test_calc_loss(self, module, frozen, raising):
model = sparse_model(module, frozen)
sw = model.sparsifier
assert sw.frozen is (True if frozen is None else frozen)
loss = SparseLoss([model.sparsifier])
try:
assert loss() == 0
except ZeroDivisionError:
pytest.fail("Division by zero")
except AssertionError:
if not raising:
pytest.fail("Exception is not expected")
def test_create_loss__with_defaults(self, module):
model = sparse_model(module, None)
loss = SparseLoss([model.sparsifier])
assert not loss.disabled
assert loss.target_sparsity_rate == 0
assert loss.p == 0.05
class RBSparsityController(BaseSparsityAlgoController):
def __init__(self, target_model: NNCFNetwork,
sparsified_module_info: List[SparseModuleInfo], params):
super().__init__(target_model, sparsified_module_info)
self._distributed = False
self._loss = SparseLoss() # type: SparseLoss
scheduler_cls = SPARSITY_SCHEDULERS.get(
params.get("schedule", "exponential"))
self._scheduler = scheduler_cls(self, params)
sparsify_operations = [m.operand for m in self.sparsified_module_info]
self._loss.set_layers(sparsify_operations)
self._check_sparsity_masks = params.get("check_sparsity_masks", False)
def set_sparsity_level(self, sparsity_level):
self._loss.target = 1 - sparsity_level
def freeze(self):
self._loss.disable()
def distributed(self):
if not dist.is_initialized():
raise KeyError(
'Could not set distributed mode for the compression algorithm '
'because the default process group has not been initialized.')
if next(self._model.parameters()).is_cuda:
state = torch.cuda.get_rng_state()
if dist.get_backend() == dist.Backend.NCCL:
state = state.cuda()
torch.distributed.broadcast(state, src=0)
torch.cuda.set_rng_state(state.cpu())
else:
state = torch.get_rng_state()
torch.distributed.broadcast(state, src=0)
torch.set_rng_state(state)
self._distributed = True
def check_distributed_masks(self):
if not self._distributed or get_world_size() == 1:
return 1
nvalues = 0
ncor_values = 0
eps = 1e-4
for minfo in self.sparsified_module_info:
mask = minfo.operand.mask
mask_list = [
torch.empty_like(mask) for _ in range(get_world_size())
]
# nccl does not support gather, send, recv operations
dist.all_gather(mask_list, mask)
for i in range(1, len(mask_list)):
rel_error = (mask_list[0] - mask_list[i]) / mask_list[0]
ncor_values = ncor_values + (rel_error.abs() < eps).sum(
dtype=mask.dtype)
nvalues = nvalues + mask_list[i].numel()
return ncor_values / nvalues
def add_algo_specific_stats(self, stats):
stats["target_sparsity_rate"] = self.loss.target_sparsity_rate
if self._distributed and self._check_sparsity_masks:
stats["masks_consistents"] = self.check_distributed_masks()
return stats
