def iter_desparsify(m): for name, child in m.named_children(): #print(name) iter_desparsify(child) if type(child) == SparseConv: conv = child.conv w = conv.weight.data mean = 0.0 #w.mean() s = (w - mean).std() r = (s * torch.clamp(child.r, 0, 5)).item() w = F.hardshrink(w - mean, r + 1e-6) conv.weight.data = w m.__setattr__(name, conv) if type(child) == SparseFc: fc = child.fc w = fc.weight.data mean = 0.0 # w.mean() s = (w - mean).std() r = (s * torch.clamp(child.r, 0, 5)).item() w = F.hardshrink(w - mean, r + 1e-6) fc.weight.data = w m.__setattr__(name, fc)
def sparsity(model, print_per_layer=False): zeros_cnt = 0 cnt = 0 for name, layer in model.named_modules(): if ('Sparse' in layer.__class__.__name__): if 'Conv' in layer.__class__.__name__: w = layer.conv.weight elif 'Fc' in layer.__class__.__name__: w = layer.fc.weight else: print(" Not Recognized Sparse Module ") m = 0.0 #w.mean() a = torch.clamp(layer.r, 0, 5) l = a * (w - m).std() nw = F.hardshrink((w - m) / (l + 1e-6), 1) tsparsity = (nw == 0).float().sum().item() tnum = nw.numel() zeros_cnt += tsparsity cnt += tnum if print_per_layer: print("{} sparsity: {}%".format( name, round(100.0 * tsparsity / tnum, 2))) return 100 * float(zeros_cnt) / float(cnt), zeros_cnt
def test_activate(): a = torch.randn(3, 4) b = copy.deepcopy(a) a = engine.activate(a, 'hardshrink') b = F.hardshrink(b) assert torch.equal(a, b), 'activate with str spec did not work correctly.' a = engine.activate(a, 'relu') b = F.relu(b) assert torch.equal(a, b), 'activate with str spec did not work correctly.'
def sparsify(param, sparsity): return F.hardshrink(param, sparsity).data total = 0 correct = 0 for batch_idx, (inputs, targets) in enumerate(testloader): inputs, targets = Variable(inputs, volatile=True), Variable(targets, volatile=True) outputs = model2(inputs) _, predicted = torch.max(outputs.data, 1) total += targets.size(0) correct += predicted.eq(targets.data).cpu().sum() print('Epoch : %d Test Acc : %.3f' % (epoch, 100. * correct / total)) print('--------------------------------------------------------------') model.train()
def test_hardshrink(self): inp = torch.randn(1, 3, 32, 32, device='cuda', dtype=self.dtype) output = F.hardshrink(inp, lambd=0.5)
def out(self, x, pow=torch.ones(1)): if self.training: return F.hardshrink(x, lambd=self.thres**pow.item()) else: return F.hardshrink(F.relu(x), lambd=self.thres**pow.item())
def hardfix(inp, tresh, fix_val=None): out = F.hardshrink(inp, lambd=tresh) out = F.hardtanh(out, min_val=-tresh - 1e-6, max_val=tresh + 1e-6) if fix_val: out *= fix_val / tresh return out
pic(x, y, (2,4,2), 'elu') y = F.softplus(x) pic(x, y, (2,4,2), 'softplus') ''' tanh/ hardtanh/ softsign ''' x = torch.arange(-5,5,0.1).view(-1,1) y = F.tanh(x) pic(x, y, (2,4,3), 'tanh') y = F.hardtanh(x) pic(x, y, (2,4,3), 'hardtanh') y = F.softsign(x) pic(x, y, (2,4,3), 'softsign') ''' hardshrink/ tanhshrink/ tanhshrink ''' x = torch.arange(-3,3,0.1).view(-1,1) y = F.hardshrink(x) pic(x, y, (2,4,4), 'hardshrink') y = F.tanhshrink(x) pic(x, y, (2,4,4), 'tanhshrink') y = F.softshrink(x) pic(x, y, (2,4,4), 'tanhshrink') ''' sigmoid ''' x = torch.arange(-5,5,0.1).view(-1,1) y = F.sigmoid(x) pic(x, y, (2,4,5), 'sigmoid') ''' relu6 ''' x = torch.arange(-5,10,0.1).view(-1,1) y = F.relu6(x) pic(x, y, (2,4,6), 'relu6')
def hardshrink(input, *args, **kwargs): return _wrap_tensor(input, F.hardshrink(input.F, *args, **kwargs))
def forward(self, input): out = F.linear(input, self.weight, self.bias) #print('safasfa=',self.lam_lin.data[0]) return F.relu(F.hardshrink(out, self.lam_lin.data[0]))
def forward(ctx, w): w = F.hardshrink(w, 1) return w
def forward(self, x, y, z, w): x = F.hardshrink(x) y = F.hardshrink(y, 0.1) z = F.hardshrink(z, 0.22) w = F.hardshrink(w, 0) return x, y, z, w