def resnet_cifar_remove_layers(model):
"""Remove layers from ResNet-Cifar.
Search for convolution layers which have 100% sparse weight tensors and remove
them from the model. This ugly code is specific to ResNet for Cifar, using the
layer gating mechanism that we added in order to remove layers from the network.
"""
# Create a list of all the layers that have their weights tensor 100% sparse
layers_to_remove = [param_name for param_name, param in model.named_parameters()
if distiller.density(param) == 0]
for param_name in layers_to_remove:
parts = param_name.split('.')
layer = int(parts[1][-1]) - 1
block = int(parts[2])
if parts[3] == 'downsample':
downsample = int(parts[4][0])
msglogger.info("Removing layer: %s [layer=%d block=%d downsample=%d]" %
(param_name[:param_name.find('.weight')], layer, block, downsample))
else:
conv = int(parts[3][-1]) - 1
msglogger.info("Removing layer: %s [layer=%d block=%d conv=%d]" %
(param_name[:param_name.find('.weight')], layer, block, conv))
model.module.layer_gates[layer][block][conv] = False
def test_sparsity():
zeros = torch.zeros(2, 3, 5, 6)
print(distiller.sparsity(zeros))
assert distiller.sparsity(zeros) == 1.0
assert distiller.sparsity_3D(zeros) == 1.0
assert distiller.density_3D(zeros) == 0.0
ones = torch.ones(12, 43, 4, 6)
assert distiller.sparsity(ones) == 0.0
x = torch.tensor([[1., 2., 0, 4., 0], [1., 2., 0, 4., 0]])
assert distiller.density(x) == 0.6
assert distiller.density_cols(x, transposed=False) == 0.6
assert distiller.sparsity_rows(x, transposed=False) == 0
x = torch.tensor([[0., 0., 0], [1., 4., 0], [1., 2., 0], [0., 0., 0]])
assert distiller.density(x) == 4 / 12
assert distiller.sparsity_rows(x, transposed=False) == 0.5
assert common.almost_equal(distiller.sparsity_cols(x, transposed=False),
1 / 3)
assert common.almost_equal(distiller.sparsity_rows(x), 1 / 3)
def weights_sparsity_summary(model,
return_total_sparsity=False,
param_dims=[2, 4]):
df = pd.DataFrame(columns=[
'Name', 'Shape', 'NNZ (dense)', 'NNZ (sparse)', 'Cols (%)', 'Rows (%)',
'Ch (%)', '2D (%)', '3D (%)', 'Fine (%)', 'Std', 'Mean', 'Abs-Mean'
])
pd.set_option('precision', 2)
params_size = 0
sparse_params_size = 0
summary_param_types = ['weight', 'bias']
for name, param in model.state_dict().items():
# Extract just the actual parameter's name, which in this context we treat as its "type"
curr_param_type = name.split('.')[-1]
if param.dim(
) in param_dims and curr_param_type in summary_param_types:
_density = distiller.density(param)
params_size += torch.numel(param)
sparse_params_size += param.numel() * _density
df.loc[len(df.index)] = ([
name,
distiller.size_to_str(param.size()),
torch.numel(param),
int(_density * param.numel()),
distiller.sparsity_cols(param) * 100,
distiller.sparsity_rows(param) * 100,
distiller.sparsity_ch(param) * 100,
distiller.sparsity_2D(param) * 100,
distiller.sparsity_3D(param) * 100, (1 - _density) * 100,
param.std().item(),
param.mean().item(),
param.abs().mean().item()
])
total_sparsity = (1 - sparse_params_size / params_size) * 100
df.loc[len(df.index)] = ([
'Total sparsity:', '-', params_size,
int(sparse_params_size), 0, 0, 0, 0, 0, total_sparsity, 0, 0, 0
])
if return_total_sparsity:
return df, total_sparsity
return df
def weights_sparsity_summary(model, return_total_sparsity=False, param_dims=[2,4]):
df = pd.DataFrame(columns=['Name', 'Shape', 'NNZ (dense)', 'NNZ (sparse)',
'Cols (%)','Rows (%)', 'Ch (%)', '2D (%)', '3D (%)',
'Fine (%)', 'Std', 'Mean', 'Abs-Mean'])
pd.set_option('precision', 2)
params_size = 0
sparse_params_size = 0
for name, param in model.state_dict().items():
if (param.dim() in param_dims) and any(type in name for type in ['weight', 'bias']):
_density = distiller.density(param)
params_size += torch.numel(param)
sparse_params_size += param.numel() * _density
df.loc[len(df.index)] = ([
name,
distiller.size_to_str(param.size()),
torch.numel(param),
int(_density * param.numel()),
distiller.sparsity_cols(param)*100,
distiller.sparsity_rows(param)*100,
distiller.sparsity_ch(param)*100,
distiller.sparsity_2D(param)*100,
distiller.sparsity_3D(param)*100,
(1-_density)*100,
param.std().item(),
param.mean().item(),
param.abs().mean().item()
])
total_sparsity = (1 - sparse_params_size/params_size)*100
df.loc[len(df.index)] = ([
'Total sparsity:',
'-',
params_size,
int(sparse_params_size),
0, 0, 0, 0, 0,
total_sparsity,
0, 0, 0])
if return_total_sparsity:
return df, total_sparsity
return df
def masks_sparsity_summary(model, scheduler, param_dims=[2, 4]):
df = pd.DataFrame(columns=['Name', 'Fine (%)'])
pd.set_option('precision', 2)
params_size = 0
sparse_params_size = 0
for name, param in model.state_dict().items():
# Extract just the actual parameter's name, which in this context we treat as its "type"
if param.dim() in param_dims and any(type in name for type in ['weight', 'bias']):
mask = scheduler.zeros_mask_dict[name].mask
if mask is None:
_density = 1
else:
_density = distiller.density(mask)
params_size += torch.numel(param)
sparse_params_size += param.numel() * _density
df.loc[len(df.index)] = ([name, (1-_density)*100])
assert params_size != 0
total_sparsity = (1 - sparse_params_size/params_size)*100
df.loc[len(df.index)] = (['Total sparsity:', total_sparsity])
return df
def weights_sparsity_summary(model,
return_total_sparsity=False,
param_dims=[2, 4]):
df = pd.DataFrame(columns=[
"Name",
"Shape",
"NNZ (dense)",
"NNZ (sparse)",
"Cols (%)",
"Rows (%)",
"Ch (%)",
"2D (%)",
"3D (%)",
"Fine (%)",
"Std",
"Mean",
"Abs-Mean",
])
pd.set_option("precision", 2)
params_size = 0
sparse_params_size = 0
for name, param in model.state_dict().items():
# Extract just the actual parameter's name, which in this context we treat as its "type"
if param.dim() in param_dims and any(type in name
for type in ["weight", "bias"]):
_density = distiller.density(param)
params_size += torch.numel(param)
sparse_params_size += param.numel() * _density
df.loc[len(df.index)] = [
name,
distiller.size_to_str(param.size()),
torch.numel(param),
int(_density * param.numel()),
distiller.sparsity_cols(param) * 100,
distiller.sparsity_rows(param) * 100,
distiller.sparsity_ch(param) * 100,
distiller.sparsity_2D(param) * 100,
distiller.sparsity_3D(param) * 100,
(1 - _density) * 100,
param.std().item(),
param.mean().item(),
param.abs().mean().item(),
]
total_sparsity = (1 - sparse_params_size / params_size) * 100
df.loc[len(df.index)] = [
"Total sparsity:",
"-",
params_size,
int(sparse_params_size),
0,
0,
0,
0,
0,
total_sparsity,
0,
0,
0,
]
if return_total_sparsity:
return df, total_sparsity
return df
