def applyGroupLassoStages(model, lr, lambda_1, emb_dim): softShrink = nn.Softshrink(lr * lambda_1) n = emb_dim with torch.no_grad(): for name, param in model.named_parameters(): if "first_stage_0_1" in name or "first_stage_1_0" in name: if "weight" in name: normTensor = torch.norm(param[:, :n], p=2, keepdim=True) param[:, :n] = param[:, :n] * softShrink( normTensor) / torch.clamp(normTensor, min=lr * lambda_1 * 0.1) normTensor = torch.norm(param[:, n:], p=2, keepdim=True) param[:, n:] = param[:, n:] * softShrink( normTensor) / torch.clamp(normTensor, min=lr * lambda_1 * 0.1) # if "second_stage" in name: # if "weight" in name: # normTensor = torch.norm(param[:,:n], p=2, keepdim = True) # param[:,:n] = param[:,:n]*softShrink(normTensor)/torch.clamp(normTensor, min=lr*lambda_1*0.1) # normTensor = torch.norm(param[:,n:], p=2, keepdim = True) # param[:,n:] = param[:,n:]*softShrink(normTensor)/torch.clamp(normTensor, min=lr*lambda_1*0.1) '''
def __init__(self, alpha=1.0): super().__init__() self.activations = [ nn.ELU(), nn.Hardshrink(), nn.Hardtanh(), nn.LeakyReLU(), nn.LogSigmoid(), nn.ReLU(), nn.PReLU(), nn.SELU(), nn.CELU(), nn.Sigmoid(), nn.Softplus(), nn.Softshrink(), nn.Softsign(), nn.Tanh(), nn.Tanhshrink() ] self.P = [ torch.nn.Parameter(torch.randn(1, requires_grad=True)) for _ in self.activations ] for activation, param in zip(self.activations, self.P): activation_name = str(activation).split("(")[0] self.add_module(name=activation_name, module=activation) self.register_parameter(name=activation_name + "p", param=param)
def create_str_to_activations_converter(self): """Creates a dictionary which converts strings to activations""" str_to_activations_converter = { "elu": nn.ELU(), "hardshrink": nn.Hardshrink(), "hardtanh": nn.Hardtanh(), "leakyrelu": nn.LeakyReLU(), "logsigmoid": nn.LogSigmoid(), "prelu": nn.PReLU(), "relu": nn.ReLU(), "relu6": nn.ReLU6(), "rrelu": nn.RReLU(), "selu": nn.SELU(), "sigmoid": nn.Sigmoid(), "softplus": nn.Softplus(), "logsoftmax": nn.LogSoftmax(), "softshrink": nn.Softshrink(), "softsign": nn.Softsign(), "tanh": nn.Tanh(), "tanhshrink": nn.Tanhshrink(), "softmin": nn.Softmin(), "softmax": nn.Softmax(dim=1), "none": None } return str_to_activations_converter
def forward(self, signal): """ Algorithm 1 from the paper, runs convolutional learned FISTA. The signals is denoted 'y' in the paper.""" torch.set_default_tensor_type(torch.DoubleTensor) x = torch.zeros(self.code_size) prev_x = torch.zeros(self.code_size) s = 0 prev_s = 0 pad = self.kernel_size//2 #TODO: figure padding soft_threshold = nn.Softshrink(self.lam / self.L) for t in range(self.T): s = (1+(1+4*prev_s**2)**0.5)/2 # line 3 in Algorithm 1 w = x + ((prev_s - 1)/s)*(x - prev_x) # line 4 # line 5 print("H's shape: {}, w's shape: {}".format(self.H.shape, w.shape)) v = torch.mm(self.H, w) print("v's shape: {}, signal's shape: {}, x: {}, local dictionary's shape: {}".format(v.shape, signal.shape, x.shape, self.local_dictionary.shape)) v = signal - v c = w + (1/self.L)*F.conv_transpose1d(v, self.local_dictionary, padding=pad) # TODO: check translation prev_x = x # line 6 x = soft_threshold(c) return x # maybe return F.conv1d(z, self.H)?
def get_activation(activation_type): if activation_type == "relu": return nn.ReLU() elif activation_type == "relu6": return nn.ReLU6() elif activation_type == "prelu": return nn.PReLU() elif activation_type == "selu": return nn.SELU() elif activation_type == "celu": return nn.CELU() elif activation_type == "gelu": return nn.GELU() elif activation_type == "sigmoid": return nn.Sigmoid() elif activation_type == "softplus": return nn.Softplus() elif activation_type == "softshrink": return nn.Softshrink() elif activation_type == "softsign": return nn.Softsign() elif activation_type == "tanh": return nn.Tanh() elif activation_type == "tanhshrink": return nn.Tanhshrink() else: raise ValueError("Unknown activation type {}".format(activation_type))
def fista(D, Y, lambd, maxIter, gpu_id): DtD = torch.matmul(torch.t(D), D) L = torch.norm(DtD, 2) linv = 1 / L DtY = torch.matmul(torch.t(D), Y) x_old = Variable(torch.zeros(DtD.shape[1], DtY.shape[2]).cuda(gpu_id), requires_grad=True) t = 1 y_old = x_old # inicialize x and y at 0 lambd = lambd * (linv.data.cpu().numpy()) A = Variable(torch.eye(DtD.shape[1]).cuda(gpu_id), requires_grad=True) - torch.mul(DtD, linv) DtY = torch.mul(DtY, linv) Softshrink = nn.Softshrink(lambd) with torch.no_grad(): for ii in range(maxIter): Ay = torch.matmul(A, y_old) #y = gamma_t del y_old with torch.enable_grad(): x_new = Softshrink((Ay + DtY)) #,lambd,gpu_id t_new = (1 + np.sqrt(1 + 4 * t**2)) / 2. tt = (t - 1) / t_new y_old = torch.mul(x_new, (1 + tt)) y_old -= torch.mul(x_old, tt) if torch.norm((x_old - x_new), p=2) / x_old.shape[1] < 1e-4: x_old = x_new break t = t_new x_old = x_new del x_new return x_old
def str2act(s): if s is 'none': return None elif s is 'hardtanh': return nn.Hardtanh() elif s is 'sigmoid': return nn.Sigmoid() elif s is 'relu6': return nn.ReLU6() elif s is 'tanh': return nn.Tanh() elif s is 'tanhshrink': return nn.Tanhshrink() elif s is 'hardshrink': return nn.Hardshrink() elif s is 'leakyrelu': return nn.LeakyReLU() elif s is 'softshrink': return nn.Softshrink() elif s is 'softsign': return nn.Softsign() elif s is 'relu': return nn.ReLU() elif s is 'prelu': return nn.PReLU() elif s is 'softplus': return nn.Softplus() elif s is 'elu': return nn.ELU() elif s is 'selu': return nn.SELU() else: raise ValueError("[!] Invalid activation function.")
def parse_activation(act): if act is None: return lambda x: x act, kwargs = parse_str(act) if act == 'sigmoid': return nn.Sigmoid(**kwargs) if act == 'tanh': return nn.Tanh(**kwargs) if act == 'relu': return nn.ReLU(**kwargs, inplace=True) if act == 'relu6': return nn.ReLU6(**kwargs, inplace=True) if act == 'elu': return nn.ELU(**kwargs, inplace=True) if act == 'selu': return nn.SELU(**kwargs, inplace=True) if act == 'prelu': return nn.PReLU(**kwargs) if act == 'leaky_relu': return nn.LeakyReLU(**kwargs, inplace=True) if act == 'threshold': return nn.Threshold(**kwargs, inplace=True) if act == 'hardtanh': return nn.Hardtanh(**kwargs, inplace=True) if act == 'log_sigmoid': return nn.LogSigmoid(**kwargs) if act == 'softplus': return nn.Softplus(**kwargs) if act == 'softshrink': return nn.Softshrink(**kwargs) if act == 'tanhshrink': return nn.Tanhshrink(**kwargs) if act == 'softmin': return nn.Softmin(**kwargs) if act == 'softmax': return nn.Softmax(**kwargs) if act == 'softmax2d': return nn.Softmax2d(**kwargs) if act == 'log_softmax': return nn.LogSoftmax(**kwargs) raise ValueError(f'unknown activation: {repr(act)}')
def __init__(self): super(NNActivationModule, self).__init__() self.activations = nn.ModuleList([ nn.ELU(), nn.Hardshrink(), nn.Hardsigmoid(), nn.Hardtanh(), nn.Hardswish(), nn.LeakyReLU(), nn.LogSigmoid(), # nn.MultiheadAttention(), nn.PReLU(), nn.ReLU(), nn.ReLU6(), nn.RReLU(), nn.SELU(), nn.CELU(), nn.GELU(), nn.Sigmoid(), nn.SiLU(), nn.Mish(), nn.Softplus(), nn.Softshrink(), nn.Softsign(), nn.Tanh(), nn.Tanhshrink(), # nn.Threshold(0.1, 20), nn.GLU(), nn.Softmin(), nn.Softmax(), nn.Softmax2d(), nn.LogSoftmax(), # nn.AdaptiveLogSoftmaxWithLoss(), ])
def __init__(self, in_size, hidden_size, num_classes, bias=True): super(playNet, self).__init__() # This part of the code is always run when you create # a new instance of this class. Give it attributes # that you want it to have. self.A1 = nn.Linear(in_size, hidden_size, bias=bias) self.sigma = nn.Softshrink() self.A2 = nn.Linear(hidden_size, num_classes, bias=bias)
def test_softshink(self): set_seed(16) lambd = 0.1 softshrink = Softshrink(n_features=1) softshrink.lambd.data[:] = lambd softshrink_gt = nn.Softshrink(lambd=lambd) tensor = torch.randn(10, 20) assert_array_almost_equal( softshrink(tensor).detach(), softshrink_gt(tensor))
def sthreshold(z, sval): """soft threshold a tensor if element(z) > sval, element(z)=sval if element(z) < -sval, element(z)=-sval """ with torch.no_grad(): T = nn.Softshrink(sval) # if z_i < -sval, z_i -> z_i +sval , ... z = T(z) return z
def __init__(self,lams,n_list=[],use_cuda=False): super(soft_thresh, self).__init__() self.use_cuda = use_cuda self.shrink = [] self.n_list = n_list if type(lams)==float: self.lams = [lams] self.shrink.append( nn.Softshrink(lams) ) elif type(lams)==list: self.lams = lams for lam in lams: self.shrink.append( nn.Softshrink(lam) ) if use_cuda: for s,shr in enumerate(self.shrink): self.shrink[s]= shr.cuda()
def forward(self, signal): """Applies convolutional LISTA on input signal. The signal is denoted 'x' in Raja's paper.""" z = torch.zeros(1, 1, self.code_size) soft_threshold = nn.Softshrink(0.5) # TODO: problem, lambda has to be a number and not a parameter! pad = math.ceil((self.kernel_size + self.code_size - self.input_size - 1)/2) for t in range(self.K): v = F.conv1d(z, self.decoder) # v is a temporary value for computation v = signal - v v = F.conv_transpose1d(v, self.encoder, padding=pad) z = soft_threshold(z + v) print(z) return z # maybe return F.conv1d(z, self.decoder)?
def __init__(self, in_size, hidden_size, num_classes, nloops=10, bias=True): super(playNet_recurrent, self).__init__() # This part of the code is always run when you create # a new instance of this class. Give it attributes # that you want it to have. self.input_layer = nn.Linear(in_size, hidden_size, bias=bias) self.recurrent_layer = nn.Linear(hidden_size, hidden_size, bias=False) self.final_layer = nn.Linear(hidden_size, num_classes, bias=bias) self.sigma = nn.Softshrink() self.nloops = nloops
def fista(D, Y, lambd, maxIter, gpu_id): DtD = torch.matmul( torch.t(D), D) # product of tensor t(D)-->D' and D --> D = dictionary L = torch.norm(DtD, 2) linv = 1 / L # inverse of the norm DtY = torch.matmul(torch.t(D), Y) x_old = Variable(torch.zeros(DtD.shape[1], DtY.shape[2]).cuda(gpu_id), requires_grad=True) t = 1 #initial state y_old = x_old # inicialize x and y at 0 lambd = lambd * (linv.data.cpu().numpy()) A = Variable(torch.eye(DtD.shape[1]).cuda(gpu_id), requires_grad=True) - torch.mul(DtD, linv) # A = I(_eye_) - 1/L(DtD) DtY = torch.mul(DtY, linv) # b = DtY - lambd? Softshrink = nn.Softshrink( lambd) # applies shrinkage function elementwise - act. function with torch.no_grad(): ## # model.eval(): will notify all your layers that you are in eval mode, # that way, batchnorm or dropout layers will work in eval model instead of training mode. # torch.no_grad(): impacts the autograd engine and deactivate it. It will reduce memory # usage and speed up computations but you wont be able to backprop # (which you dont want in an eval script). for ii in range(maxIter): Ay = torch.matmul(A, y_old) #y = gamma_t del y_old with torch.enable_grad(): x_new = Softshrink((Ay + DtY)) #x = c_t t_new = (1 + np.sqrt(1 + 4 * t**2)) / 2. #t = s_t tt = (t - 1) / t_new # = (s_t - 1)/(s_(t+1)) y_old = torch.mul(x_new, (1 + tt)) y_old -= torch.mul( x_old, tt ) # gamma_(t+1) = c_(t+1) + (s_t -1)(c_(t+1) - y_t)/s_(t+1) --> y_t = x_old, c_(t+1) = x_new if torch.norm((x_old - x_new), p=2) / x_old.shape[1] < 1e-4: x_old = x_new break t = t_new x_old = x_new del x_new #print(x_old.shape) # retorna 1x161xn_pixels. Es a dir: un vector sparce code per cada pixel. x_old = c*p o c? return x_old
def applyGroupLassoTM(model, lr, lambda_1, emb_dim): softShrink = nn.Softshrink(lr * lambda_1) n = emb_dim with torch.no_grad(): for name, param in model.named_parameters(): if "transition_model_" in name: if "weight" in name: normTensor = torch.norm(param[:, :n], p=2, keepdim=True) param[:, :n] = param[:, :n] * softShrink( normTensor) / torch.clamp(normTensor, min=lr * lambda_1 * 0.1) normTensor = torch.norm(param[:, n:], p=2, keepdim=True) param[:, n:] = param[:, n:] * softShrink( normTensor) / torch.clamp(normTensor, min=lr * lambda_1 * 0.1)
def get_activation_function(activation_name): if activation_name == 'elu': return nn.ELU(alpha=1.0, inplace=False) elif activation_name == 'hardshrink': return nn.Hardshrink(lambd=0.5) elif activation_name == 'hardtanh': return nn.Hardtanh(min_val=-1, max_val=1, inplace=False, min_value=None, max_value=None) elif activation_name == 'leaky_relu': return nn.LeakyReLU(negative_slope=0.03, inplace=False) elif activation_name == 'logsigmoid': return nn.LogSigmoid() elif activation_name == 'prelu': return nn.PReLU(num_parameters=1, init=0.25) elif activation_name == 'relu': return nn.ReLU(inplace=False) elif activation_name == 'relu6': return nn.ReLU6(inplace=False) elif activation_name == 'rrelu': return nn.RReLU(lower=0.125, upper=0.3333333333333333, inplace=False) elif activation_name == 'selu': return nn.SELU(inplace=False) elif activation_name == 'sigmoid': return nn.Sigmoid() elif activation_name == 'softplus': return nn.Softplus(beta=1, threshold=20) elif activation_name == 'softshrink': return nn.Softshrink(lambd=0.5) elif activation_name == 'softsign': return nn.Softsign() elif activation_name == 'tanh': return nn.Tanh() elif activation_name == 'tanhshrink': return nn.Tanhshrink() elif activation_name == 'swish': def swish(x): return x * F.sigmoid(x) return swish else: print('Activation function name is not recognized.') sys.exit()
def activations(self): return { "celu": nn.CELU(), "gelu": nn.GELU(), "lrelu": nn.LeakyReLU(0.2), "prelu": nn.PReLU(), "relu": nn.ReLU(), "relu6": nn.ReLU6(), "selu": nn.SELU(), "sigmoid": nn.Sigmoid(), "softplus": nn.Softplus(), "softshrink": nn.Softshrink(), "softsign": nn.Softsign(), "hardtanh": nn.Hardtanh(), "tanh": nn.Tanh(), "tanhshrink": nn.Tanhshrink() }
def get_activation_func(activation_func_name: str = "relu"): if activation_func_name is "none": return None elif activation_func_name == "relu": return nn.ReLU() elif activation_func_name == "relu6": return nn.ReLU6() elif activation_func_name == "prelu": return nn.PReLU() elif activation_func_name == "elu": return nn.ELU() elif activation_func_name == "gelu": return nn.GELU() elif activation_func_name == "selu": return nn.SELU() elif activation_func_name == "leakyrelu": return nn.LeakyReLU() elif activation_func_name == "sigmoid": return nn.Sigmoid() elif activation_func_name == "tanh": return nn.Tanh() elif activation_func_name == "hardtanh": return nn.Hardtanh() elif activation_func_name == "tanhshrink": return nn.Tanhshrink() elif activation_func_name == "hardshrink": return nn.Hardshrink() elif activation_func_name == "softshrink": return nn.Softshrink() elif activation_func_name == "softsign": return nn.Softsign() elif activation_func_name == "softplus": return nn.Softplus() elif activation_func_name == "mish": return Mish() elif activation_func_name == "ftswishplus": return FTSwishPlus() elif activation_func_name == "lightrelu": return LightRelu() elif activation_func_name == "trelu": return TRelu() else: raise ValueError("[!] Invalid activation function.")
def __init__(self, m, n, W_e, L, theta, max_iter): """ # Arguments n: int, dimensions of the measurement m: int, dimensions of the sparse signal W_e: array, dictionary max_iter:int, max number of internal iteration L: Lipschitz const, L=2 theta: Thresholding = a/L, a is sparsity """ super(LISTA, self).__init__() self._W = nn.Linear(in_features=m, out_features=n, bias=False) self._S = nn.Linear(in_features=n, out_features=n, bias=False) self.shrinkage = nn.Softshrink(theta) self.theta = theta self.max_iter = max_iter self.A = W_e self.L = L
def __init__(self, ksize=7): super(DCT, self).__init__() #self.sigma = nn.Parameter(torch.Tensor(1).fill_(1)) channels = ksize**2 self.direct_trans = nn.Conv2d(in_channels=1, out_channels=channels, kernel_size=ksize, stride=1, padding=ksize // 2) self.shrinkage = nn.Softshrink() self.inverse_trans_and_aggregation = nn.ConvTranspose2d( in_channels=channels, out_channels=1, kernel_size=ksize, stride=1, padding=ksize // 2)
def __init__(self, numClasses: int, validUsers: list, names: list = [], optimizer: None = None, criterion: nn.Module = nn.CrossEntropyLoss(), savedModelPath: str = "voiceModel.bin", audioDirectory: str = "AudioData", unlockPhrase: str = ""): """ input numClasses: integer for the number of classes for classification input validUsers: list of strings of valid users for unlocking the safe input optimizer: optimizer used for training (don't set this here set this after creating the object and before calling trainNetwork) input criterion: criterion for the loss function input savedModelPath: path where to save/load the model to/from input audioDirectory: path where the folders containing training audio can be found input unlockPhrase: recognized word must match this phrase, if the unlockPhrase is not set the recognizer will not utilize this authentication method """ super(VoiceRecognizer, self).__init__() # calling nn.module.__init__() self.dtype = torch.float self.device = torch.device("cpu") self.savedModelPath = savedModelPath self.names = names self.criterion = criterion self.optimizer = None # optimizer generally needs VoiceCNNInstance.parameters to initialize self.unlockPhrase = unlockPhrase self.validUsers = validUsers self.layer1 = nn.Sequential( nn.Conv1d(1, 10, kernel_size=3, stride=1, padding=2), nn.InstanceNorm1d(10), nn.ReLU(), nn.MaxPool1d(kernel_size=3, stride=2)) self.layer2 = nn.Sequential( nn.Conv1d(10, 70, kernel_size=3, stride=1, padding=2), nn.BatchNorm1d(70), nn.Softshrink(), nn.Dropout(), nn.AvgPool1d(kernel_size=2, stride=2)) self.layer3 = nn.Sequential( nn.Conv1d(70, 3, kernel_size=5, stride=1, padding=2), nn.InstanceNorm1d(3), nn.PReLU(), nn.Dropout(), nn.MaxPool1d(kernel_size=4, stride=2)) self.fc = nn.Linear(285, numClasses)
def fista(D, Y, lambd, maxIter): DtD = torch.matmul(torch.t(D), D) L = torch.norm(DtD, 2) linv = 1/L # print("D: ", D.shape) # print("Dt: ", torch.t(D).shape) # print("Y: ", Y.shape) DtY = torch.matmul(torch.t(D), Y) x_old = Variable(torch.zeros( DtD.shape[1], DtY.shape[1]).cuda(), requires_grad=True) t = 1 y_old = x_old #lambd = lambd*(linv.cpu().numpy()) lambd = lambd*(linv.detach().cpu().numpy()) A = Variable(torch.eye(DtD.shape[1]).cuda(), requires_grad=False) - torch.mul(DtD, linv) DtY = torch.mul(DtY, linv) Softshrink = nn.Softshrink(lambd) with torch.no_grad(): for ii in range(maxIter): Ay = torch.matmul(A, y_old) del y_old with torch.enable_grad(): x_new = Softshrink((Ay + DtY)) t_new = (1 + np.sqrt(1 + 4 * t ** 2)) / 2. tt = (t-1)/t_new y_old = torch.mul(x_new, (1 + tt)) y_old -= torch.mul(x_old, tt) if torch.norm((x_old - x_new), p=2)/x_old.shape[1] < 1e-4: x_old = x_new break t = t_new x_old = x_new del x_new #noiseLev = torch.sum(torch.abs(Y - D@x_old))/Y.shape[0] #noiseLev = torch.sum((Y - D@x_old)**2)/Y.shape[0] return x_old
def applyGroupLassoBaseLine(model, lr, lambda_1, emb_dim): softShrink = nn.Softshrink(lr * lambda_1) n = emb_dim size = model.transition_model[0].weight.data.shape assert size[0] % n == 0 assert size[1] % n == 0 assert model.transition_model[2].weight.data.shape[ 0] == model.transition_model[2].weight.data.shape[1] O, I = size[0] // n, size[1] // n with torch.no_grad(): for name, param in model.named_parameters(): if "transition_model.0" in name: if "weight" in name: for i in range(O): for j in range(I): normTensor = torch.norm(param[i * n:(i + 1) * n, j * n:(j + 1) * n], p=2, keepdim=True) param[i * n:(i + 1) * n, j * n:(j + 1) * n] = param[i * n:(i + 1) * n, j * n:(j + 1) * n] * softShrink( normTensor) / torch.clamp( normTensor, min=lr * lambda_1 * 0.1) if "transition_model.2" in name: if "weight" in name: for i in range(O): for j in range(O): normTensor = torch.norm(param[i * n:(i + 1) * n, j * n:(j + 1) * n], p=2, keepdim=True) param[i * n:(i + 1) * n, j * n:(j + 1) * n] = param[i * n:(i + 1) * n, j * n:(j + 1) * n] * softShrink( normTensor) / torch.clamp( normTensor, min=lr * lambda_1 * 0.1)
def fista(D, Y, lam, maxIter, gpu_id): DtD = torch.matmul( torch.t(D), D) # product of tensor t(D)-->D' and D --> D = dictionary L = torch.norm(DtD, 2) linv = 1 / L # inverse of the norm DtY = torch.matmul(torch.t(D), Y) x_old = Variable(torch.zeros(DtD.shape[1], DtY.shape[2]).cuda(gpu_id), requires_grad=True) t = 1 #initial state y_old = x_old # inicialize x and y at 0 lambd = lam * (linv.data.cpu().numpy()) A = Variable(torch.eye(DtD.shape[1]).cuda(gpu_id), requires_grad=True) - torch.mul(DtD, linv) # A = I(_eye_) - 1/L(DtD) DtY = torch.mul(DtY, linv) # b = DtY - lambd? Softshrink = nn.Softshrink( lambd) # applies shrinkage function elementwise - act. function with torch.no_grad(): for ii in range(maxIter): Ay = torch.matmul(A, y_old) #y = gamma_t del y_old with torch.enable_grad(): x_new = Softshrink((Ay + DtY)) #x = c_t t_new = (1 + np.sqrt(1 + 4 * t**2)) / 2. #t = s_t tt = (t - 1) / t_new # = (s_t - 1)/(s_(t+1)) y_old = torch.mul(x_new, (1 + tt)) y_old -= torch.mul( x_old, tt ) # gamma_(t+1) = c_(t+1) + (s_t -1)(c_(t+1) - y_t)/s_(t+1) --> y_t = x_old, c_(t+1) = x_new if torch.norm((x_old - x_new), p=2) / x_old.shape[1] < 1e-4: x_old = x_new break t = t_new x_old = x_new del x_new # print(x_old.shape) # retorna 1x161xn_pixels. Es a dir: un vector sparce code per cada pixel. x_old = c*p o c? return x_old
def applyGroupLassoDecoder(model, lr, lambda_2, emb_dim, num_objects): softShrink = nn.Softshrink(lr * lambda_2) n = emb_dim size = model.decoder.weight.data.shape assert size[0] == num_objects assert size[1] % n == 0 K = size[1] // n with torch.no_grad(): for i in range(K): normTensor = torch.norm(model.decoder.weight.data[:, i * n:(i + 1) * n], p=2, keepdim=True) model.decoder.weight.data[:, i * n:( i + 1) * n] = model.decoder.weight.data[:, i * n:( i + 1) * n] * softShrink(normTensor) / torch.clamp( normTensor, min=lr * lambda_2 * 0.1)
def get_activation_(act): if act is None or act == 'relu': act_fn = nn.ReLU(inplace=True) # relu as default elif act == 'mish': act_fn = Mish() elif act == 'selu': act_fn = Selu() elif act == 'elu': act_fn = nn.ELU() elif act == 'hardshrink': act_fn = nn.Hardshrink() elif act == 'hardtanh': act_fn = nn.Hardtanh() elif act == 'leakyrelu': act_fn = nn.LeakyReLU() elif act == 'logsigmoid': act_fn = nn.LogSigmoid() elif act == 'prelu': act_fn = nn.PReLU() elif act == 'relu6': act_fn = nn.ReLU6() elif act == 'rrelu': act_fn = nn.RReLU() elif act == 'celu': act_fn = nn.CELU() elif act == 'sigmoid': act_fn = nn.Sigmoid() elif act == 'softplus': act_fn = nn.Softplus() elif act == 'softshrink': act_fn = nn.Softshrink() elif act == 'softsign': act_fn = nn.Softsign() elif act == 'tanhshrink': act_fn = nn.Tanhshrink() else: raise ValueError('Act is not properly defined: check activations list') return act_fn
def __init__(self, P): super(AAF, self).__init__() self.P = P self.n = P.size()[0] self.F = [ nn.ELU(), nn.Hardshrink(), nn.Hardtanh(), nn.LeakyReLU(), nn.LogSigmoid(), nn.ReLU(), nn.ReLU6(), nn.RReLU(), nn.SELU(), nn.CELU(), nn.Sigmoid(), nn.Softplus(), nn.Softshrink(), nn.Softsign(), nn.Tanh(), nn.Tanhshrink() ]
def __init__(self): super().__init__() self.in_layer = nn.Sequential( nn.Conv3d(1, 2, kernel_size=3, padding=1), nn.BatchNorm3d(2), nn.Tanh(), nn.Conv3d(2, 1, kernel_size=3, padding=1), nn.BatchNorm3d(1), nn.Tanh()) self.conv = nn.Sequential( nn.Conv3d(1, 9, kernel_size=3, padding=1), nn.BatchNorm3d(9), #27 nn.Tanh()) self.out_layer = nn.Sequential( nn.Conv3d(1, 2, kernel_size=3, padding=1), nn.BatchNorm3d(2), nn.Softmax(dim=1), #NCDHW nn.Conv3d(2, 1, kernel_size=3, padding=1), nn.BatchNorm3d(1), nn.Tanh()) self.conv3d_1 = nn.Conv3d(1, 1, kernel_size=3, padding=1) self.conv3d_3 = nn.Conv3d(1, 3, kernel_size=3, padding=1) self.maxpool = nn.MaxPool3d(kernel_size=3, stride=1, padding=1) self.Softshrink = nn.Softshrink(lambd=0.5) self.relu = nn.ReLU() #inplace=True self.tanh = nn.Tanh()