def applyGroupLassoStages(model, lr, lambda_1, emb_dim):
softShrink = nn.Softshrink(lr * lambda_1)
n = emb_dim
with torch.no_grad():
for name, param in model.named_parameters():
if "first_stage_0_1" in name or "first_stage_1_0" in name:
if "weight" in name:
normTensor = torch.norm(param[:, :n], p=2, keepdim=True)
param[:, :n] = param[:, :n] * softShrink(
normTensor) / torch.clamp(normTensor,
min=lr * lambda_1 * 0.1)
normTensor = torch.norm(param[:, n:], p=2, keepdim=True)
param[:, n:] = param[:, n:] * softShrink(
normTensor) / torch.clamp(normTensor,
min=lr * lambda_1 * 0.1)
# if "second_stage" in name:
# if "weight" in name:
# normTensor = torch.norm(param[:,:n], p=2, keepdim = True)
# param[:,:n] = param[:,:n]*softShrink(normTensor)/torch.clamp(normTensor, min=lr*lambda_1*0.1)
# normTensor = torch.norm(param[:,n:], p=2, keepdim = True)
# param[:,n:] = param[:,n:]*softShrink(normTensor)/torch.clamp(normTensor, min=lr*lambda_1*0.1)
'''
def __init__(self, alpha=1.0):
super().__init__()
self.activations = [
nn.ELU(),
nn.Hardshrink(),
nn.Hardtanh(),
nn.LeakyReLU(),
nn.LogSigmoid(),
nn.ReLU(),
nn.PReLU(),
nn.SELU(),
nn.CELU(),
nn.Sigmoid(),
nn.Softplus(),
nn.Softshrink(),
nn.Softsign(),
nn.Tanh(),
nn.Tanhshrink()
]
self.P = [
torch.nn.Parameter(torch.randn(1, requires_grad=True))
for _ in self.activations
]
for activation, param in zip(self.activations, self.P):
activation_name = str(activation).split("(")[0]
self.add_module(name=activation_name, module=activation)
self.register_parameter(name=activation_name + "p", param=param)
def create_str_to_activations_converter(self):
"""Creates a dictionary which converts strings to activations"""
str_to_activations_converter = {
"elu": nn.ELU(),
"hardshrink": nn.Hardshrink(),
"hardtanh": nn.Hardtanh(),
"leakyrelu": nn.LeakyReLU(),
"logsigmoid": nn.LogSigmoid(),
"prelu": nn.PReLU(),
"relu": nn.ReLU(),
"relu6": nn.ReLU6(),
"rrelu": nn.RReLU(),
"selu": nn.SELU(),
"sigmoid": nn.Sigmoid(),
"softplus": nn.Softplus(),
"logsoftmax": nn.LogSoftmax(),
"softshrink": nn.Softshrink(),
"softsign": nn.Softsign(),
"tanh": nn.Tanh(),
"tanhshrink": nn.Tanhshrink(),
"softmin": nn.Softmin(),
"softmax": nn.Softmax(dim=1),
"none": None
}
return str_to_activations_converter
def forward(self, signal):
""" Algorithm 1 from the paper, runs convolutional learned FISTA. The signals is denoted 'y' in the paper."""
torch.set_default_tensor_type(torch.DoubleTensor)
x = torch.zeros(self.code_size)
prev_x = torch.zeros(self.code_size)
s = 0
prev_s = 0
pad = self.kernel_size//2 #TODO: figure padding
soft_threshold = nn.Softshrink(self.lam / self.L)
for t in range(self.T):
s = (1+(1+4*prev_s**2)**0.5)/2 # line 3 in Algorithm 1
w = x + ((prev_s - 1)/s)*(x - prev_x) # line 4
# line 5
print("H's shape: {}, w's shape: {}".format(self.H.shape, w.shape))
v = torch.mm(self.H, w)
print("v's shape: {}, signal's shape: {}, x: {}, local dictionary's shape: {}".format(v.shape, signal.shape, x.shape, self.local_dictionary.shape))
v = signal - v
c = w + (1/self.L)*F.conv_transpose1d(v, self.local_dictionary, padding=pad) # TODO: check translation
prev_x = x # line 6
x = soft_threshold(c)
return x # maybe return F.conv1d(z, self.H)?
def get_activation(activation_type):
if activation_type == "relu":
return nn.ReLU()
elif activation_type == "relu6":
return nn.ReLU6()
elif activation_type == "prelu":
return nn.PReLU()
elif activation_type == "selu":
return nn.SELU()
elif activation_type == "celu":
return nn.CELU()
elif activation_type == "gelu":
return nn.GELU()
elif activation_type == "sigmoid":
return nn.Sigmoid()
elif activation_type == "softplus":
return nn.Softplus()
elif activation_type == "softshrink":
return nn.Softshrink()
elif activation_type == "softsign":
return nn.Softsign()
elif activation_type == "tanh":
return nn.Tanh()
elif activation_type == "tanhshrink":
return nn.Tanhshrink()
else:
raise ValueError("Unknown activation type {}".format(activation_type))
def fista(D, Y, lambd, maxIter, gpu_id):
DtD = torch.matmul(torch.t(D), D)
L = torch.norm(DtD, 2)
linv = 1 / L
DtY = torch.matmul(torch.t(D), Y)
x_old = Variable(torch.zeros(DtD.shape[1], DtY.shape[2]).cuda(gpu_id),
requires_grad=True)
t = 1
y_old = x_old # inicialize x and y at 0
lambd = lambd * (linv.data.cpu().numpy())
A = Variable(torch.eye(DtD.shape[1]).cuda(gpu_id),
requires_grad=True) - torch.mul(DtD, linv)
DtY = torch.mul(DtY, linv)
Softshrink = nn.Softshrink(lambd)
with torch.no_grad():
for ii in range(maxIter):
Ay = torch.matmul(A, y_old) #y = gamma_t
del y_old
with torch.enable_grad():
x_new = Softshrink((Ay + DtY)) #,lambd,gpu_id
t_new = (1 + np.sqrt(1 + 4 * t**2)) / 2.
tt = (t - 1) / t_new
y_old = torch.mul(x_new, (1 + tt))
y_old -= torch.mul(x_old, tt)
if torch.norm((x_old - x_new), p=2) / x_old.shape[1] < 1e-4:
x_old = x_new
break
t = t_new
x_old = x_new
del x_new
return x_old
def str2act(s):
if s is 'none':
return None
elif s is 'hardtanh':
return nn.Hardtanh()
elif s is 'sigmoid':
return nn.Sigmoid()
elif s is 'relu6':
return nn.ReLU6()
elif s is 'tanh':
return nn.Tanh()
elif s is 'tanhshrink':
return nn.Tanhshrink()
elif s is 'hardshrink':
return nn.Hardshrink()
elif s is 'leakyrelu':
return nn.LeakyReLU()
elif s is 'softshrink':
return nn.Softshrink()
elif s is 'softsign':
return nn.Softsign()
elif s is 'relu':
return nn.ReLU()
elif s is 'prelu':
return nn.PReLU()
elif s is 'softplus':
return nn.Softplus()
elif s is 'elu':
return nn.ELU()
elif s is 'selu':
return nn.SELU()
else:
raise ValueError("[!] Invalid activation function.")
def parse_activation(act):
if act is None:
return lambda x: x
act, kwargs = parse_str(act)
if act == 'sigmoid': return nn.Sigmoid(**kwargs)
if act == 'tanh': return nn.Tanh(**kwargs)
if act == 'relu': return nn.ReLU(**kwargs, inplace=True)
if act == 'relu6': return nn.ReLU6(**kwargs, inplace=True)
if act == 'elu': return nn.ELU(**kwargs, inplace=True)
if act == 'selu': return nn.SELU(**kwargs, inplace=True)
if act == 'prelu': return nn.PReLU(**kwargs)
if act == 'leaky_relu': return nn.LeakyReLU(**kwargs, inplace=True)
if act == 'threshold': return nn.Threshold(**kwargs, inplace=True)
if act == 'hardtanh': return nn.Hardtanh(**kwargs, inplace=True)
if act == 'log_sigmoid': return nn.LogSigmoid(**kwargs)
if act == 'softplus': return nn.Softplus(**kwargs)
if act == 'softshrink': return nn.Softshrink(**kwargs)
if act == 'tanhshrink': return nn.Tanhshrink(**kwargs)
if act == 'softmin': return nn.Softmin(**kwargs)
if act == 'softmax': return nn.Softmax(**kwargs)
if act == 'softmax2d': return nn.Softmax2d(**kwargs)
if act == 'log_softmax': return nn.LogSoftmax(**kwargs)
raise ValueError(f'unknown activation: {repr(act)}')
def __init__(self):
super(NNActivationModule, self).__init__()
self.activations = nn.ModuleList([
nn.ELU(),
nn.Hardshrink(),
nn.Hardsigmoid(),
nn.Hardtanh(),
nn.Hardswish(),
nn.LeakyReLU(),
nn.LogSigmoid(),
# nn.MultiheadAttention(),
nn.PReLU(),
nn.ReLU(),
nn.ReLU6(),
nn.RReLU(),
nn.SELU(),
nn.CELU(),
nn.GELU(),
nn.Sigmoid(),
nn.SiLU(),
nn.Mish(),
nn.Softplus(),
nn.Softshrink(),
nn.Softsign(),
nn.Tanh(),
nn.Tanhshrink(),
# nn.Threshold(0.1, 20),
nn.GLU(),
nn.Softmin(),
nn.Softmax(),
nn.Softmax2d(),
nn.LogSoftmax(),
# nn.AdaptiveLogSoftmaxWithLoss(),
])
def __init__(self, in_size, hidden_size, num_classes, bias=True):
super(playNet, self).__init__()
# This part of the code is always run when you create
# a new instance of this class. Give it attributes
# that you want it to have.
self.A1 = nn.Linear(in_size, hidden_size, bias=bias)
self.sigma = nn.Softshrink()
self.A2 = nn.Linear(hidden_size, num_classes, bias=bias)
def test_softshink(self):
set_seed(16)
lambd = 0.1
softshrink = Softshrink(n_features=1)
softshrink.lambd.data[:] = lambd
softshrink_gt = nn.Softshrink(lambd=lambd)
tensor = torch.randn(10, 20)
assert_array_almost_equal(
softshrink(tensor).detach(), softshrink_gt(tensor))
def sthreshold(z, sval):
"""soft threshold a tensor
if element(z) > sval, element(z)=sval
if element(z) < -sval, element(z)=-sval
"""
with torch.no_grad():
T = nn.Softshrink(sval) # if z_i < -sval, z_i -> z_i +sval , ...
z = T(z)
return z
def __init__(self,lams,n_list=[],use_cuda=False):
super(soft_thresh, self).__init__()
self.use_cuda = use_cuda
self.shrink = []
self.n_list = n_list
if type(lams)==float:
self.lams = [lams]
self.shrink.append( nn.Softshrink(lams) )
elif type(lams)==list:
self.lams = lams
for lam in lams:
self.shrink.append( nn.Softshrink(lam) )
if use_cuda:
for s,shr in enumerate(self.shrink):
self.shrink[s]= shr.cuda()
def forward(self, signal):
"""Applies convolutional LISTA on input signal. The signal is denoted 'x' in Raja's paper."""
z = torch.zeros(1, 1, self.code_size)
soft_threshold = nn.Softshrink(0.5) # TODO: problem, lambda has to be a number and not a parameter!
pad = math.ceil((self.kernel_size + self.code_size - self.input_size - 1)/2)
for t in range(self.K):
v = F.conv1d(z, self.decoder) # v is a temporary value for computation
v = signal - v
v = F.conv_transpose1d(v, self.encoder, padding=pad)
z = soft_threshold(z + v)
print(z)
return z # maybe return F.conv1d(z, self.decoder)?
def __init__(self,
in_size,
hidden_size,
num_classes,
nloops=10,
bias=True):
super(playNet_recurrent, self).__init__()
# This part of the code is always run when you create
# a new instance of this class. Give it attributes
# that you want it to have.
self.input_layer = nn.Linear(in_size, hidden_size, bias=bias)
self.recurrent_layer = nn.Linear(hidden_size, hidden_size, bias=False)
self.final_layer = nn.Linear(hidden_size, num_classes, bias=bias)
self.sigma = nn.Softshrink()
self.nloops = nloops
def fista(D, Y, lambd, maxIter, gpu_id):
DtD = torch.matmul(
torch.t(D), D) # product of tensor t(D)-->D' and D --> D = dictionary
L = torch.norm(DtD, 2)
linv = 1 / L # inverse of the norm
DtY = torch.matmul(torch.t(D), Y)
x_old = Variable(torch.zeros(DtD.shape[1], DtY.shape[2]).cuda(gpu_id),
requires_grad=True)
t = 1 #initial state
y_old = x_old # inicialize x and y at 0
lambd = lambd * (linv.data.cpu().numpy())
A = Variable(torch.eye(DtD.shape[1]).cuda(gpu_id),
requires_grad=True) - torch.mul(DtD, linv)
# A = I(_eye_) - 1/L(DtD)
DtY = torch.mul(DtY, linv)
# b = DtY - lambd?
Softshrink = nn.Softshrink(
lambd) # applies shrinkage function elementwise - act. function
with torch.no_grad():
##
# model.eval(): will notify all your layers that you are in eval mode,
# that way, batchnorm or dropout layers will work in eval model instead of training mode.
# torch.no_grad(): impacts the autograd engine and deactivate it. It will reduce memory
# usage and speed up computations but you wont be able to backprop
# (which you dont want in an eval script).
for ii in range(maxIter):
Ay = torch.matmul(A, y_old) #y = gamma_t
del y_old
with torch.enable_grad():
x_new = Softshrink((Ay + DtY)) #x = c_t
t_new = (1 + np.sqrt(1 + 4 * t**2)) / 2. #t = s_t
tt = (t - 1) / t_new # = (s_t - 1)/(s_(t+1))
y_old = torch.mul(x_new, (1 + tt))
y_old -= torch.mul(
x_old, tt
) # gamma_(t+1) = c_(t+1) + (s_t -1)(c_(t+1) - y_t)/s_(t+1) --> y_t = x_old, c_(t+1) = x_new
if torch.norm((x_old - x_new), p=2) / x_old.shape[1] < 1e-4:
x_old = x_new
break
t = t_new
x_old = x_new
del x_new
#print(x_old.shape) # retorna 1x161xn_pixels. Es a dir: un vector sparce code per cada pixel. x_old = c*p o c?
return x_old
def applyGroupLassoTM(model, lr, lambda_1, emb_dim):
softShrink = nn.Softshrink(lr * lambda_1)
n = emb_dim
with torch.no_grad():
for name, param in model.named_parameters():
if "transition_model_" in name:
if "weight" in name:
normTensor = torch.norm(param[:, :n], p=2, keepdim=True)
param[:, :n] = param[:, :n] * softShrink(
normTensor) / torch.clamp(normTensor,
min=lr * lambda_1 * 0.1)
normTensor = torch.norm(param[:, n:], p=2, keepdim=True)
param[:, n:] = param[:, n:] * softShrink(
normTensor) / torch.clamp(normTensor,
min=lr * lambda_1 * 0.1)
def get_activation_function(activation_name):
if activation_name == 'elu':
return nn.ELU(alpha=1.0, inplace=False)
elif activation_name == 'hardshrink':
return nn.Hardshrink(lambd=0.5)
elif activation_name == 'hardtanh':
return nn.Hardtanh(min_val=-1,
max_val=1,
inplace=False,
min_value=None,
max_value=None)
elif activation_name == 'leaky_relu':
return nn.LeakyReLU(negative_slope=0.03, inplace=False)
elif activation_name == 'logsigmoid':
return nn.LogSigmoid()
elif activation_name == 'prelu':
return nn.PReLU(num_parameters=1, init=0.25)
elif activation_name == 'relu':
return nn.ReLU(inplace=False)
elif activation_name == 'relu6':
return nn.ReLU6(inplace=False)
elif activation_name == 'rrelu':
return nn.RReLU(lower=0.125, upper=0.3333333333333333, inplace=False)
elif activation_name == 'selu':
return nn.SELU(inplace=False)
elif activation_name == 'sigmoid':
return nn.Sigmoid()
elif activation_name == 'softplus':
return nn.Softplus(beta=1, threshold=20)
elif activation_name == 'softshrink':
return nn.Softshrink(lambd=0.5)
elif activation_name == 'softsign':
return nn.Softsign()
elif activation_name == 'tanh':
return nn.Tanh()
elif activation_name == 'tanhshrink':
return nn.Tanhshrink()
elif activation_name == 'swish':
def swish(x):
return x * F.sigmoid(x)
return swish
else:
print('Activation function name is not recognized.')
sys.exit()
def activations(self):
return {
"celu": nn.CELU(),
"gelu": nn.GELU(),
"lrelu": nn.LeakyReLU(0.2),
"prelu": nn.PReLU(),
"relu": nn.ReLU(),
"relu6": nn.ReLU6(),
"selu": nn.SELU(),
"sigmoid": nn.Sigmoid(),
"softplus": nn.Softplus(),
"softshrink": nn.Softshrink(),
"softsign": nn.Softsign(),
"hardtanh": nn.Hardtanh(),
"tanh": nn.Tanh(),
"tanhshrink": nn.Tanhshrink()
}
def get_activation_func(activation_func_name: str = "relu"):
if activation_func_name is "none":
return None
elif activation_func_name == "relu":
return nn.ReLU()
elif activation_func_name == "relu6":
return nn.ReLU6()
elif activation_func_name == "prelu":
return nn.PReLU()
elif activation_func_name == "elu":
return nn.ELU()
elif activation_func_name == "gelu":
return nn.GELU()
elif activation_func_name == "selu":
return nn.SELU()
elif activation_func_name == "leakyrelu":
return nn.LeakyReLU()
elif activation_func_name == "sigmoid":
return nn.Sigmoid()
elif activation_func_name == "tanh":
return nn.Tanh()
elif activation_func_name == "hardtanh":
return nn.Hardtanh()
elif activation_func_name == "tanhshrink":
return nn.Tanhshrink()
elif activation_func_name == "hardshrink":
return nn.Hardshrink()
elif activation_func_name == "softshrink":
return nn.Softshrink()
elif activation_func_name == "softsign":
return nn.Softsign()
elif activation_func_name == "softplus":
return nn.Softplus()
elif activation_func_name == "mish":
return Mish()
elif activation_func_name == "ftswishplus":
return FTSwishPlus()
elif activation_func_name == "lightrelu":
return LightRelu()
elif activation_func_name == "trelu":
return TRelu()
else:
raise ValueError("[!] Invalid activation function.")
def __init__(self, m, n, W_e, L, theta, max_iter):
"""
# Arguments
n: int, dimensions of the measurement
m: int, dimensions of the sparse signal
W_e: array, dictionary
max_iter:int, max number of internal iteration
L: Lipschitz const, L=2
theta: Thresholding = a/L, a is sparsity
"""
super(LISTA, self).__init__()
self._W = nn.Linear(in_features=m, out_features=n, bias=False)
self._S = nn.Linear(in_features=n, out_features=n, bias=False)
self.shrinkage = nn.Softshrink(theta)
self.theta = theta
self.max_iter = max_iter
self.A = W_e
self.L = L
def __init__(self, ksize=7):
super(DCT, self).__init__()
#self.sigma = nn.Parameter(torch.Tensor(1).fill_(1))
channels = ksize**2
self.direct_trans = nn.Conv2d(in_channels=1,
out_channels=channels,
kernel_size=ksize,
stride=1,
padding=ksize // 2)
self.shrinkage = nn.Softshrink()
self.inverse_trans_and_aggregation = nn.ConvTranspose2d(
in_channels=channels,
out_channels=1,
kernel_size=ksize,
stride=1,
padding=ksize // 2)
def __init__(self,
numClasses: int,
validUsers: list,
names: list = [],
optimizer: None = None,
criterion: nn.Module = nn.CrossEntropyLoss(),
savedModelPath: str = "voiceModel.bin",
audioDirectory: str = "AudioData",
unlockPhrase: str = ""):
"""
input numClasses: integer for the number of classes for classification
input validUsers: list of strings of valid users for unlocking the safe
input optimizer: optimizer used for training (don't set this here set
this after creating the object and before calling trainNetwork)
input criterion: criterion for the loss function
input savedModelPath: path where to save/load the model to/from
input audioDirectory: path where the folders containing training audio can be found
input unlockPhrase: recognized word must match this phrase, if the unlockPhrase is not set the
recognizer will not utilize this authentication method
"""
super(VoiceRecognizer, self).__init__() # calling nn.module.__init__()
self.dtype = torch.float
self.device = torch.device("cpu")
self.savedModelPath = savedModelPath
self.names = names
self.criterion = criterion
self.optimizer = None # optimizer generally needs VoiceCNNInstance.parameters to initialize
self.unlockPhrase = unlockPhrase
self.validUsers = validUsers
self.layer1 = nn.Sequential(
nn.Conv1d(1, 10, kernel_size=3, stride=1, padding=2),
nn.InstanceNorm1d(10), nn.ReLU(),
nn.MaxPool1d(kernel_size=3, stride=2))
self.layer2 = nn.Sequential(
nn.Conv1d(10, 70, kernel_size=3, stride=1, padding=2),
nn.BatchNorm1d(70), nn.Softshrink(), nn.Dropout(),
nn.AvgPool1d(kernel_size=2, stride=2))
self.layer3 = nn.Sequential(
nn.Conv1d(70, 3, kernel_size=5, stride=1, padding=2),
nn.InstanceNorm1d(3), nn.PReLU(), nn.Dropout(),
nn.MaxPool1d(kernel_size=4, stride=2))
self.fc = nn.Linear(285, numClasses)
def fista(D, Y, lambd, maxIter):
DtD = torch.matmul(torch.t(D), D)
L = torch.norm(DtD, 2)
linv = 1/L
# print("D: ", D.shape)
# print("Dt: ", torch.t(D).shape)
# print("Y: ", Y.shape)
DtY = torch.matmul(torch.t(D), Y)
x_old = Variable(torch.zeros(
DtD.shape[1], DtY.shape[1]).cuda(), requires_grad=True)
t = 1
y_old = x_old
#lambd = lambd*(linv.cpu().numpy())
lambd = lambd*(linv.detach().cpu().numpy())
A = Variable(torch.eye(DtD.shape[1]).cuda(),
requires_grad=False) - torch.mul(DtD, linv)
DtY = torch.mul(DtY, linv)
Softshrink = nn.Softshrink(lambd)
with torch.no_grad():
for ii in range(maxIter):
Ay = torch.matmul(A, y_old)
del y_old
with torch.enable_grad():
x_new = Softshrink((Ay + DtY))
t_new = (1 + np.sqrt(1 + 4 * t ** 2)) / 2.
tt = (t-1)/t_new
y_old = torch.mul(x_new, (1 + tt))
y_old -= torch.mul(x_old, tt)
if torch.norm((x_old - x_new), p=2)/x_old.shape[1] < 1e-4:
x_old = x_new
break
t = t_new
x_old = x_new
del x_new
#noiseLev = torch.sum(torch.abs(Y - D@x_old))/Y.shape[0]
#noiseLev = torch.sum((Y - D@x_old)**2)/Y.shape[0]
return x_old
def applyGroupLassoBaseLine(model, lr, lambda_1, emb_dim):
softShrink = nn.Softshrink(lr * lambda_1)
n = emb_dim
size = model.transition_model[0].weight.data.shape
assert size[0] % n == 0
assert size[1] % n == 0
assert model.transition_model[2].weight.data.shape[
0] == model.transition_model[2].weight.data.shape[1]
O, I = size[0] // n, size[1] // n
with torch.no_grad():
for name, param in model.named_parameters():
if "transition_model.0" in name:
if "weight" in name:
for i in range(O):
for j in range(I):
normTensor = torch.norm(param[i * n:(i + 1) * n,
j * n:(j + 1) * n],
p=2,
keepdim=True)
param[i * n:(i + 1) * n, j * n:(j + 1) *
n] = param[i * n:(i + 1) * n,
j * n:(j + 1) * n] * softShrink(
normTensor) / torch.clamp(
normTensor,
min=lr * lambda_1 * 0.1)
if "transition_model.2" in name:
if "weight" in name:
for i in range(O):
for j in range(O):
normTensor = torch.norm(param[i * n:(i + 1) * n,
j * n:(j + 1) * n],
p=2,
keepdim=True)
param[i * n:(i + 1) * n, j * n:(j + 1) *
n] = param[i * n:(i + 1) * n,
j * n:(j + 1) * n] * softShrink(
normTensor) / torch.clamp(
normTensor,
min=lr * lambda_1 * 0.1)
def fista(D, Y, lam, maxIter, gpu_id):
DtD = torch.matmul(
torch.t(D), D) # product of tensor t(D)-->D' and D --> D = dictionary
L = torch.norm(DtD, 2)
linv = 1 / L # inverse of the norm
DtY = torch.matmul(torch.t(D), Y)
x_old = Variable(torch.zeros(DtD.shape[1], DtY.shape[2]).cuda(gpu_id),
requires_grad=True)
t = 1 #initial state
y_old = x_old # inicialize x and y at 0
lambd = lam * (linv.data.cpu().numpy())
A = Variable(torch.eye(DtD.shape[1]).cuda(gpu_id),
requires_grad=True) - torch.mul(DtD, linv)
# A = I(_eye_) - 1/L(DtD)
DtY = torch.mul(DtY, linv)
# b = DtY - lambd?
Softshrink = nn.Softshrink(
lambd) # applies shrinkage function elementwise - act. function
with torch.no_grad():
for ii in range(maxIter):
Ay = torch.matmul(A, y_old) #y = gamma_t
del y_old
with torch.enable_grad():
x_new = Softshrink((Ay + DtY)) #x = c_t
t_new = (1 + np.sqrt(1 + 4 * t**2)) / 2. #t = s_t
tt = (t - 1) / t_new # = (s_t - 1)/(s_(t+1))
y_old = torch.mul(x_new, (1 + tt))
y_old -= torch.mul(
x_old, tt
) # gamma_(t+1) = c_(t+1) + (s_t -1)(c_(t+1) - y_t)/s_(t+1) --> y_t = x_old, c_(t+1) = x_new
if torch.norm((x_old - x_new), p=2) / x_old.shape[1] < 1e-4:
x_old = x_new
break
t = t_new
x_old = x_new
del x_new
# print(x_old.shape) # retorna 1x161xn_pixels. Es a dir: un vector sparce code per cada pixel. x_old = c*p o c?
return x_old
def applyGroupLassoDecoder(model, lr, lambda_2, emb_dim, num_objects):
softShrink = nn.Softshrink(lr * lambda_2)
n = emb_dim
size = model.decoder.weight.data.shape
assert size[0] == num_objects
assert size[1] % n == 0
K = size[1] // n
with torch.no_grad():
for i in range(K):
normTensor = torch.norm(model.decoder.weight.data[:, i *
n:(i + 1) * n],
p=2,
keepdim=True)
model.decoder.weight.data[:, i * n:(
i + 1) * n] = model.decoder.weight.data[:, i * n:(
i + 1) * n] * softShrink(normTensor) / torch.clamp(
normTensor, min=lr * lambda_2 * 0.1)
def get_activation_(act):
if act is None or act == 'relu':
act_fn = nn.ReLU(inplace=True) # relu as default
elif act == 'mish':
act_fn = Mish()
elif act == 'selu':
act_fn = Selu()
elif act == 'elu':
act_fn = nn.ELU()
elif act == 'hardshrink':
act_fn = nn.Hardshrink()
elif act == 'hardtanh':
act_fn = nn.Hardtanh()
elif act == 'leakyrelu':
act_fn = nn.LeakyReLU()
elif act == 'logsigmoid':
act_fn = nn.LogSigmoid()
elif act == 'prelu':
act_fn = nn.PReLU()
elif act == 'relu6':
act_fn = nn.ReLU6()
elif act == 'rrelu':
act_fn = nn.RReLU()
elif act == 'celu':
act_fn = nn.CELU()
elif act == 'sigmoid':
act_fn = nn.Sigmoid()
elif act == 'softplus':
act_fn = nn.Softplus()
elif act == 'softshrink':
act_fn = nn.Softshrink()
elif act == 'softsign':
act_fn = nn.Softsign()
elif act == 'tanhshrink':
act_fn = nn.Tanhshrink()
else:
raise ValueError('Act is not properly defined: check activations list')
return act_fn
def __init__(self, P):
super(AAF, self).__init__()
self.P = P
self.n = P.size()[0]
self.F = [
nn.ELU(),
nn.Hardshrink(),
nn.Hardtanh(),
nn.LeakyReLU(),
nn.LogSigmoid(),
nn.ReLU(),
nn.ReLU6(),
nn.RReLU(),
nn.SELU(),
nn.CELU(),
nn.Sigmoid(),
nn.Softplus(),
nn.Softshrink(),
nn.Softsign(),
nn.Tanh(),
nn.Tanhshrink()
]
def __init__(self):
super().__init__()
self.in_layer = nn.Sequential(
nn.Conv3d(1, 2, kernel_size=3, padding=1), nn.BatchNorm3d(2),
nn.Tanh(), nn.Conv3d(2, 1, kernel_size=3, padding=1),
nn.BatchNorm3d(1), nn.Tanh())
self.conv = nn.Sequential(
nn.Conv3d(1, 9, kernel_size=3, padding=1),
nn.BatchNorm3d(9), #27
nn.Tanh())
self.out_layer = nn.Sequential(
nn.Conv3d(1, 2, kernel_size=3, padding=1),
nn.BatchNorm3d(2),
nn.Softmax(dim=1), #NCDHW
nn.Conv3d(2, 1, kernel_size=3, padding=1),
nn.BatchNorm3d(1),
nn.Tanh())
self.conv3d_1 = nn.Conv3d(1, 1, kernel_size=3, padding=1)
self.conv3d_3 = nn.Conv3d(1, 3, kernel_size=3, padding=1)
self.maxpool = nn.MaxPool3d(kernel_size=3, stride=1, padding=1)
self.Softshrink = nn.Softshrink(lambd=0.5)
self.relu = nn.ReLU() #inplace=True
self.tanh = nn.Tanh()
