def __init__(self):
self.lr = 0.00042307692307692304
self.hidden_sizes = [400, 400, 400, 400, 400, 400, 400]
self.momentum = 0.7923
self.activations = [
nn.Hardshrink(1.1),
nn.LeakyReLU(negative_slope=0.01),
nn.LeakyReLU(negative_slope=0.01),
nn.LeakyReLU(negative_slope=0.01),
nn.LeakyReLU(negative_slope=0.01),
nn.LeakyReLU(negative_slope=0.01),
nn.Hardshrink(1.1),
nn.Sigmoid()
]
# NOTE: Grids
# self.activations_grid = [
# [
# nn.Hardshrink(k),
# nn.LeakyReLU(negative_slope=0.01),
# nn.LeakyReLU(negative_slope=0.01),
# nn.LeakyReLU(negative_slope=0.01),
# nn.LeakyReLU(negative_slope=0.01),
# nn.LeakyReLU(negative_slope=0.01),
# nn.Hardshrink(k),
# nn.Sigmoid()
# ]
# for k in np.linspace(0.05, 2.5, 15)
# ]
self.lr_grid = list(np.linspace(0.0001, 0.0007, 40))
# self.hidden_sizes_grid = [
# [i, j, k, l, m, o]
# for i in [400]
# for j in [400]
# for k in [400]
# for l in [400]
# for m in [400]
# for o in [400]
# for p in [400]
# ]
# self.momentum_grid = list(np.linspace(0.7,0.9,40))
self.grid_search = GridSearch(self)
self.grid_search.set_enabled(False)
def __init__(self):
super().__init__()
self.out = nn.Sequential(nn.Linear(7, 2), nn.PReLU(), nn.Linear(2, 6),
nn.SELU(), nn.Linear(6, 2), nn.ELU(),
nn.Linear(2, 5), nn.Hardshrink(),
nn.Linear(5, 2), nn.Tanh(), nn.Linear(2, 2))
def str2act(s):
if s is 'none':
return None
elif s is 'hardtanh':
return nn.Hardtanh()
elif s is 'sigmoid':
return nn.Sigmoid()
elif s is 'relu6':
return nn.ReLU6()
elif s is 'tanh':
return nn.Tanh()
elif s is 'tanhshrink':
return nn.Tanhshrink()
elif s is 'hardshrink':
return nn.Hardshrink()
elif s is 'leakyrelu':
return nn.LeakyReLU()
elif s is 'softshrink':
return nn.Softshrink()
elif s is 'softsign':
return nn.Softsign()
elif s is 'relu':
return nn.ReLU()
elif s is 'prelu':
return nn.PReLU()
elif s is 'softplus':
return nn.Softplus()
elif s is 'elu':
return nn.ELU()
elif s is 'selu':
return nn.SELU()
else:
raise ValueError("[!] Invalid activation function.")
def create_str_to_activations_converter(self):
"""Creates a dictionary which converts strings to activations"""
str_to_activations_converter = {
"elu": nn.ELU(),
"hardshrink": nn.Hardshrink(),
"hardtanh": nn.Hardtanh(),
"leakyrelu": nn.LeakyReLU(),
"logsigmoid": nn.LogSigmoid(),
"prelu": nn.PReLU(),
"relu": nn.ReLU(),
"relu6": nn.ReLU6(),
"rrelu": nn.RReLU(),
"selu": nn.SELU(),
"sigmoid": nn.Sigmoid(),
"softplus": nn.Softplus(),
"logsoftmax": nn.LogSoftmax(),
"softshrink": nn.Softshrink(),
"softsign": nn.Softsign(),
"tanh": nn.Tanh(),
"tanhshrink": nn.Tanhshrink(),
"softmin": nn.Softmin(),
"softmax": nn.Softmax(dim=1),
"none": None
}
return str_to_activations_converter
def __init__(self, alpha=1.0):
super().__init__()
self.activations = [
nn.ELU(),
nn.Hardshrink(),
nn.Hardtanh(),
nn.LeakyReLU(),
nn.LogSigmoid(),
nn.ReLU(),
nn.PReLU(),
nn.SELU(),
nn.CELU(),
nn.Sigmoid(),
nn.Softplus(),
nn.Softshrink(),
nn.Softsign(),
nn.Tanh(),
nn.Tanhshrink()
]
self.P = [
torch.nn.Parameter(torch.randn(1, requires_grad=True))
for _ in self.activations
]
for activation, param in zip(self.activations, self.P):
activation_name = str(activation).split("(")[0]
self.add_module(name=activation_name, module=activation)
self.register_parameter(name=activation_name + "p", param=param)
def __init__(self):
self.best_step = sys.maxsize
self.sols = {}
self.solsSum = {}
self.hidden_size = 50
self.lr = 0.01
self.activation_hidden = 'relu6'
self.activation_output = 'sigmoid'
self.activations = {
'sigmoid': nn.Sigmoid(),
'relu': nn.ReLU(),
'relu6': nn.ReLU6(),
'rrelu0103': nn.RReLU(0.1, 0.3),
'rrelu0205': nn.RReLU(0.2, 0.5),
'htang1': nn.Hardtanh(-1, 1),
'htang2': nn.Hardtanh(-2, 2),
'htang3': nn.Hardtanh(-3, 3),
'tanh': nn.Tanh(),
'elu': nn.ELU(),
'selu': nn.SELU(),
'hardshrink': nn.Hardshrink(),
'leakyrelu01': nn.LeakyReLU(0.1),
'leakyrelu001': nn.LeakyReLU(0.01),
'logsigmoid': nn.LogSigmoid(),
'prelu': nn.PReLU(),
}
self.hidden_size_grid = [16, 20, 26, 32, 36, 40, 45, 50, 54]
self.lr_grid = [0.0001, 0.001, 0.005, 0.01, 0.1, 1]
# self.lr_grid = [0.1, .5, 1, 1.5, 2, 3, 5, 10]
# self.activation_hidden_grid = list(self.activations.keys())
# self.activation_output_grid = list(self.activations.keys())
self.grid_search = GridSearch(self)
self.grid_search.set_enabled(False)
def __init__(self):
super(NNActivationModule, self).__init__()
self.activations = nn.ModuleList([
nn.ELU(),
nn.Hardshrink(),
nn.Hardsigmoid(),
nn.Hardtanh(),
nn.Hardswish(),
nn.LeakyReLU(),
nn.LogSigmoid(),
# nn.MultiheadAttention(),
nn.PReLU(),
nn.ReLU(),
nn.ReLU6(),
nn.RReLU(),
nn.SELU(),
nn.CELU(),
nn.GELU(),
nn.Sigmoid(),
nn.SiLU(),
nn.Mish(),
nn.Softplus(),
nn.Softshrink(),
nn.Softsign(),
nn.Tanh(),
nn.Tanhshrink(),
# nn.Threshold(0.1, 20),
nn.GLU(),
nn.Softmin(),
nn.Softmax(),
nn.Softmax2d(),
nn.LogSoftmax(),
# nn.AdaptiveLogSoftmaxWithLoss(),
])
def __init__(self, p, tr, step, phi_step, thresh, dev):
super(Instance_block, self).__init__()
self.p = p
self.dev = dev
self.training = tr
self.step = nn.Parameter(step, requires_grad=self.training)
self.phi_step = nn.Parameter(phi_step, requires_grad=self.training)
self.thresh = torch.as_tensor(thresh)
self.thresh_func = nn.Hardshrink(self.thresh)
def __init__(self,
input_shape,
num_layers,
neurons,
activator_id=5,
optimizer_id=0):
super().__init__()
self.num_layers = num_layers # number of layers
self.neurons = neurons # number of neurons in each layer e.g. for 2 layers, neurons=[10,20]
self.activator_id = activator_id # activation function, can be one of the following: ELU, Hardshrink, LeakyReLU, LogSigmoid, PReLU, ReLU, ReLU6, RReLU, SELU, CELU, Sigmoid
self.optimizer_id = optimizer_id # optimizer id, can be one of the following: Adadelta, Adagrad, Adam, Adamax, ASGD, RMSprop, Rprop, SGD
# set activation function
if (activator_id == 0):
self.activator = nn.ELU()
elif (activator_id == 1):
self.activator = nn.Hardshrink()
elif (activator_id == 2):
self.activator = nn.LeakyReLU()
elif (activator_id == 3):
self.activator = nn.LogSigmoid()
elif (activator_id == 4):
self.activator = nn.PReLU()
elif (activator_id == 5):
self.activator = nn.ReLU()
elif (activator_id == 6):
self.activator = nn.ReLU6()
elif (activator_id == 7):
self.activator = nn.RReLU()
elif (activator_id == 8):
self.activator = nn.SELU()
elif (activator_id == 9):
self.activator = nn.CELU()
# network architecture
if (num_layers == 1):
self.layers = nn.Sequential(
nn.Linear(input_shape, self.neurons[0]), self.activator,
nn.Linear(self.neurons[0], 1))
elif (num_layers == 2):
self.layers = nn.Sequential(
nn.Linear(input_shape, self.neurons[0]), self.activator,
nn.Linear(self.neurons[0], self.neurons[1]), self.activator,
nn.Linear(self.neurons[1], 1))
elif (num_layers == 3):
self.layers = nn.Sequential(
nn.Linear(input_shape, self.neurons[0]), self.activator,
nn.Linear(self.neurons[0], self.neurons[1]), self.activator,
nn.Linear(self.neurons[1], self.neurons[2]), self.activator,
nn.Linear(self.neurons[2], 1))
elif (num_layers == 4):
self.layers = nn.Sequential(
nn.Linear(input_shape, self.neurons[0]), self.activator,
nn.Linear(self.neurons[0], self.neurons[1]), self.activator,
nn.Linear(self.neurons[1], self.neurons[2]), self.activator,
nn.Linear(self.neurons[2], self.neurons[3]), self.activator,
nn.Linear(self.neurons[3], 1))
def __init__(self):
self.activations = {
'sigmoid': nn.Sigmoid(),
'relu': nn.ReLU(),
'relu6': nn.ReLU6(),
'rrelu0103': nn.RReLU(0.1, 0.3),
'rrelu0205': nn.RReLU(0.2, 0.5),
'htang1': nn.Hardtanh(-1, 1),
'htang2': nn.Hardtanh(-2, 2),
'htang3': nn.Hardtanh(-3, 3),
'tanh': nn.Tanh(),
'elu': nn.ELU(),
'selu': nn.SELU(),
'hardshrink': nn.Hardshrink(),
'leakyrelu01': nn.LeakyReLU(0.1),
'leakyrelu001': nn.LeakyReLU(0.01),
'logsigmoid': nn.LogSigmoid(),
'prelu': nn.PReLU(),
}
self.loss_functions = {
'binary_cross_entropy': nn.BCELoss(),
'binary_cross_entropy_with_logits': nn.BCEWithLogitsLoss(),
'poisson_nll_loss': nn.PoissonNLLLoss(),
# 'cosine_embedding_loss': nn.CosineEmbeddingLoss(),
# 'cross_entropy': nn.CrossEntropyLoss(),
# 'ctc_loss': nn.CTCLoss(),
'hinge_embedding_loss': nn.HingeEmbeddingLoss(),
'kl_div': nn.KLDivLoss(),
'l1_loss': nn.L1Loss(),
'mse_loss': nn.MSELoss(),
# 'margin_ranking_loss': nn.MarginRankingLoss(),
# 'multilabel_margin_loss': nn.MultiLabelMarginLoss(),
'multilabel_soft_margin_loss': nn.MultiLabelSoftMarginLoss(),
# 'multi_margin_loss': nn.MultiMarginLoss(),
# 'nll_loss': nn.NLLLoss(),
'smooth_l1_loss': nn.SmoothL1Loss(),
'soft_margin_loss': nn.SoftMarginLoss(),
# 'triplet_margin_loss': nn.TripletMarginLoss(),
}
self.learning_rate = 2.8
self.momentum = 0.8
self.hidden_size = 10
self.activation_hidden = 'relu'
self.loss_function = 'binary_cross_entropy'
self.sols = {}
self.solsSum = {}
self.random = 3
self.random_grid = [_ for _ in range(10)]
# self.hidden_size_grid = [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39]
# self.hidden_size_grid = [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
# self.learning_rate_grid = [0.1, 1.0, 2.0, 3.0, 5.0]
# self.activation_hidden_grid = list(self.activations.keys())
# self.activation_hidden_grid = list(self.activations.keys())
# self.loss_function_grid = list(self.loss_functions.keys())
self.grid_search = GridSearch(self)
self.grid_search.set_enabled(False)
def __init__(self, size):#,wav,wav_inv): super().__init__() # wavelet operator must be defined for specific resolution self.space1 = odl.uniform_discr([-128, -128], [128, 128], [2*size, 2*size],dtype='float32') self.wav = OperatorAsModule(odl.trafos.WaveletTransform(self.space1,'haar',1)) self.wav_inv = OperatorAsModule(odl.trafos.WaveletTransform(self.space1,'haar',1).inverse) self.size = size self.fact = 2**(np.log2(512/self.size)-1) # scaling factor self.thresh = nn.Hardshrink(0.01)
def __init__(self, num_classes=17):
super(NN_model, self).__init__()
self.linear1 = nn.Linear(376, 100)
self.linear2 = nn.Linear(100, 50)
self.linear3 = nn.Linear(50, num_classes)
# self.m = nn.SELU()
self.hard_tanh = nn.Hardtanh(min_val=-.4, max_val=0.4)
self.tanh = nn.Tanh()
self.shrink = nn.Hardshrink()
self.norm = nn.LayerNorm(((1, 376)))
def __init__(
self,
dim,
num_heads=8,
qkv_bias=False,
qk_scale=None,
attn_drop=0.0,
proj_drop=0.0,
lambd=0.0,
):
super().__init__(dim, num_heads, qkv_bias, qk_scale, attn_drop, proj_drop)
self.hard_shrink = nn.Hardshrink(lambd=lambd)
def __init__(self, input_size, hidden_size, output_size, dropout=0.5):
super(Net, self).__init__()
#self.fc1 = nn.Linear(input_size, hidden_size)
self.fc1 = nn.Linear(input_size, hidden_size)
self.l1 = nn.ReLU()
self.l2 = nn.Sigmoid()
self.l3 = nn.Tanh()
self.l4 = nn.ELU()
self.l5 = nn.Hardshrink()
self.ln = nn.Linear(hidden_size, hidden_size)
self.fc2 = nn.Linear(hidden_size, output_size)
self.dp = nn.Dropout(dropout)
def __init__(self, num_classes=10):
super(Model, self).__init__()
self.features = nn.Sequential(
nn.Conv2d(1, 32, kernel_size=3, stride=1),
nn.Hardshrink(lambd=0.3),
#nn.Softshrink(),
#nn.ReLU(),
#nn.Tanh(),
#nn.ELU(),
nn.Conv2d(32, 32, kernel_size=3, stride=1),
nn.Hardshrink(lambd=0.3),
#nn.Softshrink(),
#nn.ReLU(),
#nn.Tanh(),
#nn.ELU(),
nn.Conv2d(32, 32, kernel_size=3, stride=1),
nn.ReLU(inplace=True),
nn.MaxPool2d(2))
self.classifier = nn.Sequential(nn.Dropout(),
nn.Linear(3872, num_classes),
nn.Softmax(dim=1))
def get_activation_function(activation_name):
if activation_name == 'elu':
return nn.ELU(alpha=1.0, inplace=False)
elif activation_name == 'hardshrink':
return nn.Hardshrink(lambd=0.5)
elif activation_name == 'hardtanh':
return nn.Hardtanh(min_val=-1,
max_val=1,
inplace=False,
min_value=None,
max_value=None)
elif activation_name == 'leaky_relu':
return nn.LeakyReLU(negative_slope=0.03, inplace=False)
elif activation_name == 'logsigmoid':
return nn.LogSigmoid()
elif activation_name == 'prelu':
return nn.PReLU(num_parameters=1, init=0.25)
elif activation_name == 'relu':
return nn.ReLU(inplace=False)
elif activation_name == 'relu6':
return nn.ReLU6(inplace=False)
elif activation_name == 'rrelu':
return nn.RReLU(lower=0.125, upper=0.3333333333333333, inplace=False)
elif activation_name == 'selu':
return nn.SELU(inplace=False)
elif activation_name == 'sigmoid':
return nn.Sigmoid()
elif activation_name == 'softplus':
return nn.Softplus(beta=1, threshold=20)
elif activation_name == 'softshrink':
return nn.Softshrink(lambd=0.5)
elif activation_name == 'softsign':
return nn.Softsign()
elif activation_name == 'tanh':
return nn.Tanh()
elif activation_name == 'tanhshrink':
return nn.Tanhshrink()
elif activation_name == 'swish':
def swish(x):
return x * F.sigmoid(x)
return swish
else:
print('Activation function name is not recognized.')
sys.exit()
def get_activation_func(activation_func_name: str = "relu"):
if activation_func_name is "none":
return None
elif activation_func_name == "relu":
return nn.ReLU()
elif activation_func_name == "relu6":
return nn.ReLU6()
elif activation_func_name == "prelu":
return nn.PReLU()
elif activation_func_name == "elu":
return nn.ELU()
elif activation_func_name == "gelu":
return nn.GELU()
elif activation_func_name == "selu":
return nn.SELU()
elif activation_func_name == "leakyrelu":
return nn.LeakyReLU()
elif activation_func_name == "sigmoid":
return nn.Sigmoid()
elif activation_func_name == "tanh":
return nn.Tanh()
elif activation_func_name == "hardtanh":
return nn.Hardtanh()
elif activation_func_name == "tanhshrink":
return nn.Tanhshrink()
elif activation_func_name == "hardshrink":
return nn.Hardshrink()
elif activation_func_name == "softshrink":
return nn.Softshrink()
elif activation_func_name == "softsign":
return nn.Softsign()
elif activation_func_name == "softplus":
return nn.Softplus()
elif activation_func_name == "mish":
return Mish()
elif activation_func_name == "ftswishplus":
return FTSwishPlus()
elif activation_func_name == "lightrelu":
return LightRelu()
elif activation_func_name == "trelu":
return TRelu()
else:
raise ValueError("[!] Invalid activation function.")
def get_activation_(act):
if act is None or act == 'relu':
act_fn = nn.ReLU(inplace=True) # relu as default
elif act == 'mish':
act_fn = Mish()
elif act == 'selu':
act_fn = Selu()
elif act == 'elu':
act_fn = nn.ELU()
elif act == 'hardshrink':
act_fn = nn.Hardshrink()
elif act == 'hardtanh':
act_fn = nn.Hardtanh()
elif act == 'leakyrelu':
act_fn = nn.LeakyReLU()
elif act == 'logsigmoid':
act_fn = nn.LogSigmoid()
elif act == 'prelu':
act_fn = nn.PReLU()
elif act == 'relu6':
act_fn = nn.ReLU6()
elif act == 'rrelu':
act_fn = nn.RReLU()
elif act == 'celu':
act_fn = nn.CELU()
elif act == 'sigmoid':
act_fn = nn.Sigmoid()
elif act == 'softplus':
act_fn = nn.Softplus()
elif act == 'softshrink':
act_fn = nn.Softshrink()
elif act == 'softsign':
act_fn = nn.Softsign()
elif act == 'tanhshrink':
act_fn = nn.Tanhshrink()
else:
raise ValueError('Act is not properly defined: check activations list')
return act_fn
def __init__(self, P):
super(AAF, self).__init__()
self.P = P
self.n = P.size()[0]
self.F = [
nn.ELU(),
nn.Hardshrink(),
nn.Hardtanh(),
nn.LeakyReLU(),
nn.LogSigmoid(),
nn.ReLU(),
nn.ReLU6(),
nn.RReLU(),
nn.SELU(),
nn.CELU(),
nn.Sigmoid(),
nn.Softplus(),
nn.Softshrink(),
nn.Softsign(),
nn.Tanh(),
nn.Tanhshrink()
]
def AdaptiveActivationFunction16(self, x):
sz = x.size()
res = torch.zeros(sz).cuda()
af = [0] * 16
af[0] = nn.ELU()(x)
af[1] = nn.Hardshrink()(x)
af[2] = nn.Hardtanh()(x)
af[3] = nn.LeakyReLU()(x)
af[4] = nn.LogSigmoid()(x)
af[5] = nn.ReLU()(x)
af[6] = nn.ReLU6()(x)
af[7] = nn.RReLU()(x)
af[8] = nn.SELU()(x)
af[9] = nn.CELU()(x)
af[10] = nn.Sigmoid()(x)
af[11] = nn.Softplus()(x)
af[12] = nn.Softshrink()(x)
af[13] = nn.Softsign()(x)
af[14] = nn.Tanh()(x)
af[15] = nn.Tanhshrink()(x)
for i in range(11):
res += self.adaparam[i] * af[i]
return res
def __init__(self,
activation_function=0,
image_size=(140, 140),
kernel_size=3,
pool_var=False):
super().__init__()
self.pool_var = pool_var
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#Define the activation function
self.act = []
self.act.append(nn.Sigmoid()) #0
self.act.append(nn.ReLU()) #1
self.act.append(nn.LeakyReLU()) #2
self.act.append(nn.Tanh()) #3
self.act.append(nn.SELU()) #4
self.act.append(nn.Hardshrink()) #5
self.act.append(nn.Hardtanh()) #6
self.activation = self.act[activation_function]
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
self.conv1 = nn.Conv2d(3, 6, kernel_size)
self.conv2 = nn.Conv2d(6, 16, kernel_size)
self.conv3 = nn.Conv2d(16, 32, 5)
self.pool = nn.MaxPool2d(2, 2)
self.n_neurons_fc = self.evaluateNeuronFCLayer(image_size)
# self.n_neurons_fc = 25088
self.fc1 = nn.Linear(self.n_neurons_fc, 2048)
self.fc2 = nn.Linear(2048, 256)
# self.fc3 = nn.Linear(1024, 256)
self.fc4 = nn.Linear(256, 1)
self.softmax = nn.Softmax(dim=1)
def __init__(self):
self.best_step = 1000
self.activations = {
'sigmoid': nn.Sigmoid(),
'custom': self.custom,
'relu': nn.ReLU(),
'relu6': nn.ReLU6(),
'rrelu0103': nn.RReLU(0.1, 0.3),
'rrelu0205': nn.RReLU(0.2, 0.5),
'htang1': nn.Hardtanh(-1, 1),
'htang2': nn.Hardtanh(-2, 2),
'htang3': nn.Hardtanh(-3, 3),
'tanh': nn.Tanh(),
'elu': nn.ELU(),
'selu': nn.SELU(),
'hardshrink': nn.Hardshrink(),
'leakyrelu01': nn.LeakyReLU(0.1),
'leakyrelu001': nn.LeakyReLU(0.01),
'logsigmoid': nn.LogSigmoid(),
'prelu': nn.PReLU(),
}
self.learning_rate = 1.0
self.hidden_size = 11
self.activation_hidden = 'relu'
self.activation_output = 'sigmoid'
self.sols = {}
self.solsSum = {}
self.random = 0
self.random_grid = [_ for _ in range(10)]
self.hidden_size_grid = [3, 5, 7, 11]
self.learning_rate_grid = [0.001, 0.01, 0.1, 1.0, 10.0, 100.0]
#self.learning_rate_grid = [1.0 + i/100.0 for i in range(10)]
self.activation_hidden_grid = self.activations.keys()
#self.activation_output_grid = self.activations.keys()
self.grid_search = GridSearch(self)
self.grid_search.set_enabled(True)
def get_layers_for_network_module(nnpt_params, task_type, first_layer_units):
layers = []
nnpt_params["hidden_layer_info"] = {
int(key): val
for key, val in nnpt_params['hidden_layer_info'].items()
}
layers_list = sorted(tuple(nnpt_params["hidden_layer_info"]))
print("=" * 45)
print(nnpt_params)
print("n_layers - ", len(layers_list))
print("task_type - ", task_type)
print("layers_tuple - ", layers_list)
if task_type == "CLASSIFICATION":
for val in layers_list:
print(val)
layer_dict = nnpt_params["hidden_layer_info"][val]
layer_name = layer_dict["layer"]
if val == 1:
layer_units_ip = first_layer_units
else:
layer_units_ip = layer_dict["units_ip"]
kernel_weight_init = layer_dict["weight_init"]
kernel_bias_init = layer_dict["bias_init"]
kernel_weight_constraint = layer_dict["weight_constraint"]
layer_units_op = layer_dict["units_op"]
#layer_bias = layer_dict["bias"]
layer_activation = layer_dict["activation"]
layer_batchnormalization = layer_dict["batchnormalization"]
layer_dropout = layer_dict["dropout"]
print("~" * 50)
print("Layer ID - ", val)
print("Layer Name - ", layer_name)
print("~" * 50)
if layer_name == "Linear":
main_layer = nn.Linear(in_features=layer_units_ip,
out_features=layer_units_op)
get_kernel_weights(kernel_weight_init, main_layer,
layer_units_ip, layer_units_op)
get_kernel_bias(kernel_bias_init, main_layer, layer_units_ip,
layer_units_op)
get_kernel_weight_constraint(kernel_weight_constraint,
main_layer)
layers.append(main_layer)
if layer_activation != None:
if layer_activation["name"] == "ELU":
activation = nn.ELU(alpha=layer_activation["alpha"],
inplace=False)
layers.append(activation)
if layer_activation["name"] == "Hardshrink":
activation = nn.Hardshrink(
lambd=layer_activation["lambd"])
layers.append(activation)
if layer_activation["name"] == "Hardtanh":
activation = nn.Hardtanh(
min_val=layer_activation["min_val"],
max_val=layer_activation["max_val"],
inplace=False)
layers.append(activation)
if layer_activation["name"] == "LeakyReLU":
activation = nn.LeakyReLU(
negative_slope=layer_activation["negative_slope"],
inplace=False)
layers.append(activation)
if layer_activation["name"] == "LogSigmoid":
activation = nn.LogSigmoid()
layers.append(activation)
if layer_activation["name"] == "MultiheadAttention":
activation = nn.MultiheadAttention(
embed_dim=layer_activation["embed_dim"],
num_heads=layer_activation["num_heads"],
dropout=layer_activation["dropout"],
bias=layer_activation["bias"],
add_bias_kv=layer_activation["add_bias_kv"],
add_zero_attn=layer_activation["add_zero_attn"],
kdim=layer_activation["kdim"],
vdim=layer_activation["vdim"])
layers.append(activation)
if layer_activation["name"] == "PreLU":
activation = nn.PreLU(
num_parameters=layer_activation["num_parameters"],
init=layer_activation["init"])
layers.append(activation)
if layer_activation["name"] == "ReLU":
activation = nn.ReLU()
layers.append(activation)
if layer_activation["name"] == "ReLU6":
activation = nn.ReLU6()
layers.append(activation)
if layer_activation["name"] == "RreLU":
activation = nn.RreLU(lower=layer_activation["lower"],
upper=layer_activation["upper"],
inplace=False)
layers.append(activation)
if layer_activation["name"] == "SELU":
activation = nn.SELU()
layers.append(activation)
if layer_activation["name"] == "CELU":
activation = nn.CELU(alpha=layer_activation["alpha"],
inplace=False)
layers.append(activation)
if layer_activation["name"] == "GELU":
activation = nn.GELU()
layers.append(activation)
if layer_activation["name"] == "Sigmoid":
activation = nn.Sigmoid()
layers.append(activation)
if layer_activation["name"] == "Softplus":
activation = nn.Softplus(
beta=layer_activation["beta"],
threshold=layer_activation["threshold"])
layers.append(activation)
if layer_activation["name"] == "Softshrink":
activation = nn.Softshrink(
lambd=layer_activation["lambd"])
layers.append(activation)
if layer_activation["name"] == "Softsign":
activation = nn.Softsign()
layers.append(activation)
if layer_activation["name"] == "Tanh":
activation = nn.Tanh()
layers.append(activation)
if layer_activation["name"] == "Tanhshrink":
activation = nn.Tanhshrink()
layers.append(activation)
if layer_activation["name"] == "Threshold":
activation = nn.Threshold(
threshold=layer_activation["threshold"],
value=layer_activation["value"])
layers.append(activation)
if layer_activation["name"] == "Softmin":
activation = nn.Softmin(dim=layer_activation["dim"])
layers.append(activation)
if layer_activation["name"] == "Softmax":
activation = nn.Softmax(dim=layer_activation["dim"])
layers.append(activation)
if layer_activation["name"] == "Softmax2d":
activation = nn.Softmax2d()
layers.append(activation)
if layer_activation["name"] == "LogSoftmax":
activation = nn.LogSoftmax(dim=layer_activation["dim"])
layers.append(activation)
if layer_activation[
"name"] == "AdaptiveLogSoftmaxWithLoss":
activation = nn.AdaptiveLogSoftmaxWithLoss(
n_classes=layer_activation["n_classes"],
cutoffs=layer_activation["cutoffs"],
div_value=layer_activation["div_value"],
head_bias=layer_activation["head_bias"])
layers.append(activation)
else:
pass
if layer_batchnormalization != None:
if layer_batchnormalization["name"] == "BatchNorm1d":
batch_normalization = nn.BatchNorm1d(
num_features=layer_batchnormalization[
"num_features"],
eps=layer_batchnormalization["eps"],
momentum=layer_batchnormalization["momentum"],
affine=layer_batchnormalization["affine"],
track_running_stats=layer_batchnormalization[
"track_running_stats"])
layers.append(batch_normalization)
if layer_batchnormalization["name"] == "BatchNorm2d":
batch_normalization = nn.BatchNorm2d(
num_features=layer_batchnormalization[
"num_features"],
eps=layer_batchnormalization["eps"],
momentum=layer_batchnormalization["momentum"],
affine=layer_batchnormalization["affine"],
track_running_stats=layer_batchnormalization[
"track_running_stats"])
layers.append(batch_normalization)
if layer_batchnormalization["name"] == "BatchNorm3d":
batch_normalization = nn.BatchNorm3d(
num_features=layer_batchnormalization[
"num_features"],
eps=layer_batchnormalization["eps"],
momentum=layer_batchnormalization["momentum"],
affine=layer_batchnormalization["affine"],
track_running_stats=layer_batchnormalization[
"track_running_stats"])
layers.append(batch_normalization)
if layer_batchnormalization["name"] == "SyncBatchNorm":
batch_normalization = nn.SyncBatchNorm(
num_features=layer_batchnormalization[
"num_features"],
eps=layer_batchnormalization["eps"],
momentum=layer_batchnormalization["momentum"],
affine=layer_batchnormalization["affine"],
track_running_stats=layer_batchnormalization[
"track_running_stats"],
process_group=layer_batchnormalization[
"process_group"])
layers.append(batch_normalization)
if layer_batchnormalization["name"] == "InstanceNorm1d":
batch_normalization = nn.InstanceNorm1d(
num_features=layer_batchnormalization[
"num_features"],
eps=layer_batchnormalization["eps"],
momentum=layer_batchnormalization["momentum"],
affine=layer_batchnormalization["affine"],
track_running_stats=layer_batchnormalization[
"track_running_stats"])
layers.append(batch_normalization)
if layer_batchnormalization["name"] == "InstanceNorm2d":
batch_normalization = nn.InstanceNorm2d(
num_features=layer_batchnormalization[
"num_features"],
eps=layer_batchnormalization["eps"],
momentum=layer_batchnormalization["momentum"],
affine=layer_batchnormalization["affine"],
track_running_stats=layer_batchnormalization[
"track_running_stats"])
layers.append(batch_normalization)
if layer_batchnormalization["name"] == "InstanceNorm3d":
batch_normalization = nn.InstanceNorm3d(
num_features=layer_batchnormalization[
"num_features"],
eps=layer_batchnormalization["eps"],
momentum=layer_batchnormalization["momentum"],
affine=layer_batchnormalization["affine"],
track_running_stats=layer_batchnormalization[
"track_running_stats"])
layers.append(batch_normalization)
if layer_batchnormalization["name"] == "GroupNorm":
batch_normalization = nn.GroupNorm(
num_groups=layer_batchnormalization["num_groups"],
num_channels=layer_batchnormalization[
"num_channels"],
eps=layer_batchnormalization["eps"],
affine=layer_batchnormalization["affine"])
layers.append(batch_normalization)
if layer_batchnormalization["name"] == "LayerNorm":
batch_normalization = nn.LayerNorm(
normalized_shape=layer_batchnormalization[
"normalized_shape"],
eps=layer_batchnormalization["eps"],
elementwise_affine=layer_batchnormalization[
"elementwise_affine"])
layers.append(batch_normalization)
if layer_batchnormalization["name"] == "LocalResponseNorm":
batch_normalization = nn.LocalResponseNorm(
size=layer_batchnormalization["size"],
alpha=layer_batchnormalization["alpha"],
beta=layer_batchnormalization["beta"],
k=layer_batchnormalization["k"])
layers.append(batch_normalization)
else:
pass
if layer_dropout != None:
if layer_dropout["name"] == "Dropout":
dropout = nn.Dropout(p=layer_dropout["p"],
inplace=False)
layers.append(dropout)
if layer_dropout["name"] == "Dropout2d":
dropout = nn.Dropout2d(p=layer_dropout["p"],
inplace=False)
layers.append(dropout)
if layer_dropout["name"] == "Dropout3d":
dropout = nn.Dropout3d(p=layer_dropout["p"],
inplace=False)
layers.append(dropout)
if layer_dropout["name"] == "AlphaDropout":
dropout = nn.AlphaDropout(p=layer_dropout["p"],
inplace=False)
layers.append(dropout)
else:
pass
print("~" * 50)
print("FINAL LAYERS FOR NETWORK - ", layers)
print("~" * 50)
if task_type == "REGRESSION":
for val in layers_list:
layer_dict = nnpt_params["hidden_layer_info"][val]
layer_name = layer_dict["layer"]
if val == 1:
layer_units_ip = first_layer_units
else:
layer_units_ip = layer_dict["units_ip"]
layer_units_op = layer_dict["units_op"]
layer_bias = layer_dict["bias"]
layer_activation = layer_dict["activation"]
layer_batchnormalization = layer_dict["batchnormalization"]
layer_dropout = layer_dict["dropout"]
print("~" * 50)
print("Layer ID - ", val)
print("Layer Name - ", layer_name)
print("~" * 50)
if layer_name == "Linear":
main_layer = nn.Linear(in_features=layer_units_ip,
out_features=layer_units_op,
bias=layer_bias)
layers.append(main_layer)
if layer_activation != None:
if layer_activation["name"] == "ELU":
activation = nn.ELU(alpha=layer_activation["alpha"],
inplace=False)
if layer_activation["name"] == "Hardshrink":
activation = nn.Hardshrink(
lambd=layer_activation["lambd"])
if layer_activation["name"] == "Hardtanh":
activation = nn.Hardtanh(
min_val=layer_activation["min_val"],
max_val=layer_activation["max_val"],
inplace=False)
if layer_activation["name"] == "LeakyReLU":
activation = nn.LeakyReLU(
negative_slope=layer_activation["negative_slope"],
inplace=False)
if layer_activation["name"] == "LogSigmoid":
activation = nn.LogSigmoid()
if layer_activation["name"] == "MultiheadAttention":
activation = nn.MultiheadAttention(
embed_dim=layer_activation["embed_dim"],
num_heads=layer_activation["num_heads"],
dropout=layer_activation["dropout"],
bias=layer_activation["bias"],
add_bias_kv=layer_activation["add_bias_kv"],
add_zero_attn=layer_activation["add_zero_attn"],
kdim=layer_activation["kdim"],
vdim=layer_activation["vdim"])
if layer_activation["name"] == "PreLU":
activation = nn.PreLU(
num_parameters=layer_activation["num_parameters"],
init=layer_activation["init"])
if layer_activation["name"] == "ReLU":
activation = nn.ReLU()
if layer_activation["name"] == "ReLU6":
activation = nn.ReLU6()
if layer_activation["name"] == "RreLU":
activation = nn.RreLU(lower=layer_activation["lower"],
upper=layer_activation["upper"],
inplace=False)
if layer_activation["name"] == "SELU":
activation = nn.SELU()
if layer_activation["name"] == "CELU":
activation = nn.CELU(alpha=layer_activation["alpha"],
inplace=False)
if layer_activation["name"] == "GELU":
activation = nn.GELU()
if layer_activation["name"] == "Sigmoid":
activation = nn.Sigmoid()
if layer_activation["name"] == "Softplus":
activation = nn.Softplus(
beta=layer_activation["beta"],
threshold=layer_activation["threshold"])
if layer_activation["name"] == "Softshrink":
activation = nn.Softshrink(
lambd=layer_activation["lambd"])
if layer_activation["name"] == "Softsign":
activation = nn.Softsign()
if layer_activation["name"] == "Tanh":
activation = nn.Tanh()
if layer_activation["name"] == "Tanhshrink":
activation = nn.Tanhshrink()
if layer_activation["name"] == "Threshold":
activation = nn.Threshold(
threshold=layer_activation["threshold"],
value=layer_activation["value"])
if layer_activation["name"] == "Softmin":
activation = nn.Softmin(dim=layer_activation["dim"])
if layer_activation["name"] == "Softmax":
activation = nn.Softmax(dim=layer_activation["dim"])
if layer_activation["name"] == "Softmax2d":
activation = nn.Softmax2d()
if layer_activation["name"] == "LogSoftmax":
activation = nn.LogSoftmax(dim=layer_activation["dim"])
if layer_activation[
"name"] == "AdaptiveLogSoftmaxWithLoss":
activation = nn.AdaptiveLogSoftmaxWithLoss(
n_classes=layer_activation["n_classes"],
cutoffs=layer_activation["cutoffs"],
div_value=layer_activation["div_value"],
head_bias=layer_activation["head_bias"])
layers.append(activation)
else:
pass
if layer_batchnormalization != None:
if layer_batchnormalization["name"] == "BatchNorm1d":
batch_normalization = nn.BatchNorm1d(
num_features=layer_batchnormalization[
"num_features"],
eps=layer_batchnormalization["eps"],
momentum=layer_batchnormalization["momentum"],
affine=layer_batchnormalization["affine"],
track_running_stats=layer_batchnormalization[
"track_running_stats"])
if layer_batchnormalization["name"] == "BatchNorm2d":
batch_normalization = nn.BatchNorm2d(
num_features=layer_batchnormalization[
"num_features"],
eps=layer_batchnormalization["eps"],
momentum=layer_batchnormalization["momentum"],
affine=layer_batchnormalization["affine"],
track_running_stats=layer_batchnormalization[
"track_running_stats"])
if layer_batchnormalization["name"] == "BatchNorm3d":
batch_normalization = nn.BatchNorm3d(
num_features=layer_batchnormalization[
"num_features"],
eps=layer_batchnormalization["eps"],
momentum=layer_batchnormalization["momentum"],
affine=layer_batchnormalization["affine"],
track_running_stats=layer_batchnormalization[
"track_running_stats"])
if layer_batchnormalization["name"] == "SyncBatchNorm":
batch_normalization = nn.SyncBatchNorm(
num_features=layer_batchnormalization[
"num_features"],
eps=layer_batchnormalization["eps"],
momentum=layer_batchnormalization["momentum"],
affine=layer_batchnormalization["affine"],
track_running_stats=layer_batchnormalization[
"track_running_stats"],
process_group=layer_batchnormalization[
"process_group"])
if layer_batchnormalization["name"] == "InstanceNorm1d":
batch_normalization = nn.InstanceNorm1d(
num_features=layer_batchnormalization[
"num_features"],
eps=layer_batchnormalization["eps"],
momentum=layer_batchnormalization["momentum"],
affine=layer_batchnormalization["affine"],
track_running_stats=layer_batchnormalization[
"track_running_stats"])
if layer_batchnormalization["name"] == "InstanceNorm2d":
batch_normalization = nn.InstanceNorm2d(
num_features=layer_batchnormalization[
"num_features"],
eps=layer_batchnormalization["eps"],
momentum=layer_batchnormalization["momentum"],
affine=layer_batchnormalization["affine"],
track_running_stats=layer_batchnormalization[
"track_running_stats"])
if layer_batchnormalization["name"] == "InstanceNorm3d":
batch_normalization = nn.InstanceNorm3d(
num_features=layer_batchnormalization[
"num_features"],
eps=layer_batchnormalization["eps"],
momentum=layer_batchnormalization["momentum"],
affine=layer_batchnormalization["affine"],
track_running_stats=layer_batchnormalization[
"track_running_stats"])
if layer_batchnormalization["name"] == "GroupNorm":
batch_normalization = nn.GroupNorm(
num_groups=layer_batchnormalization["num_groups"],
num_channels=layer_batchnormalization[
"num_channels"],
eps=layer_batchnormalization["eps"],
affine=layer_batchnormalization["affine"])
if layer_batchnormalization["name"] == "LayerNorm":
batch_normalization = nn.LayerNorm(
normalized_shape=layer_batchnormalization[
"normalized_shape"],
eps=layer_batchnormalization["eps"],
elementwise_affine=layer_batchnormalization[
"elementwise_affine"])
if layer_batchnormalization["name"] == "LocalResponseNorm":
batch_normalization = nn.LocalResponseNorm(
size=layer_batchnormalization["size"],
alpha=layer_batchnormalization["alpha"],
beta=layer_batchnormalization["beta"],
k=layer_batchnormalization["k"])
layers.append(batch_normalization)
else:
pass
if layer_dropout != None:
if layer_dropout["name"] == "Dropout":
dropout = nn.Dropout(p=layer_dropout["p"],
inplace=False)
if layer_dropout["name"] == "Dropout2d":
dropout = nn.Dropout2d(p=layer_dropout["p"],
inplace=False)
if layer_dropout["name"] == "Dropout3d":
dropout = nn.Dropout3d(p=layer_dropout["p"],
inplace=False)
if layer_dropout["name"] == "AlphaDropout":
dropout = nn.AlphaDropout(p=layer_dropout["p"],
inplace=False)
layers.append(dropout)
else:
pass
return layers
def __append_layer(self, net_style, args_dict):
args_values_list = list(args_dict.values())
if net_style == "Conv2d":
self.layers.append(
nn.Conv2d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4], args_values_list[5],
args_values_list[6], args_values_list[7]))
elif net_style == "MaxPool2d":
self.layers.append(
nn.MaxPool2d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4], args_values_list[5]))
elif net_style == "Linear":
self.layers.append(
nn.Linear(args_values_list[0], args_values_list[1],
args_values_list[2]))
elif net_style == "reshape":
# 如果是特殊情况 reshape，就直接将目标向量尺寸传入
# print(type(args_values_list[0]))
self.layers.append(args_values_list[0])
elif net_style == "Conv1d":
self.layers.append(
nn.Conv1d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4], args_values_list[5],
args_values_list[6], args_values_list[7]))
elif net_style == "Conv3d":
self.layers.append(
nn.Conv3d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4], args_values_list[5],
args_values_list[6], args_values_list[7]))
elif net_style == "ConvTranspose1d":
self.layers.append(
nn.ConvTranspose1d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4], args_values_list[5],
args_values_list[6], args_values_list[7],
args_values_list[8]))
elif net_style == "ConvTranspose2d":
self.layers.append(
nn.ConvTranspose2d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4], args_values_list[5],
args_values_list[6], args_values_list[7],
args_values_list[8]))
elif net_style == "ConvTranspose3d":
self.layers.append(
nn.ConvTranspose3d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4], args_values_list[5],
args_values_list[6], args_values_list[7],
args_values_list[8]))
elif net_style == "Unfold":
self.layers.append(
nn.Unfold(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3]))
elif net_style == "Fold":
self.layers.append(
nn.Unfold(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "MaxPool1d":
self.layers.append(
nn.MaxPool1d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4], args_values_list[5]))
elif net_style == "MaxPool3d":
self.layers.append(
nn.MaxPool3d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4], args_values_list[5]))
elif net_style == "MaxUnpool1d":
self.layers.append(
nn.MaxUnpool1d(args_values_list[0], args_values_list[1],
args_values_list[2]))
elif net_style == "MaxUnpool2d":
self.layers.append(
nn.MaxUnpool2d(args_values_list[0], args_values_list[1],
args_values_list[2]))
elif net_style == "MaxUnpool3d":
self.layers.append(
nn.MaxUnpool3d(args_values_list[0], args_values_list[1],
args_values_list[2]))
elif net_style == "AvgPool1d":
self.layers.append(
nn.AvgPool1d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "AvgPool2d":
self.layers.append(
nn.AvgPool2d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "AvgPool3d":
self.layers.append(
nn.AvgPool3d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "FractionalMaxPool2d":
self.layers.append(
nn.FractionalMaxPool2d(args_values_list[0],
args_values_list[1],
args_values_list[2],
args_values_list[3],
args_values_list[4]))
elif net_style == "LPPool1d":
self.layers.append(
nn.LPPool1d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3]))
elif net_style == "LPPool2d":
self.layers.append(
nn.LPPool2d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3]))
elif net_style == "AdaptiveMaxPool1d":
self.layers.append(
nn.AdaptiveMaxPool1d(args_values_list[0], args_values_list[1]))
elif net_style == "AdaptiveMaxPool2d":
self.layers.append(
nn.AdaptiveMaxPool2d(args_values_list[0], args_values_list[1]))
elif net_style == "AdaptiveMaxPool3d":
self.layers.append(
nn.AdaptiveMaxPool3d(args_values_list[0], args_values_list[1]))
elif net_style == "AdaptiveAvgPool1d":
self.layers.append(nn.AdaptiveAvgPool1d(args_values_list[0]))
elif net_style == "AdaptiveAvgPool2d":
self.layers.append(nn.AdaptiveAvgPool2d(args_values_list[0]))
elif net_style == "AdaptiveAvgPool3d":
self.layers.append(nn.AdaptiveAvgPool3d(args_values_list[0]))
elif net_style == "ReflectionPad1d":
self.layers.append(nn.ReflectionPad1d(args_values_list[0]))
elif net_style == "ReflectionPad2d":
self.layers.append(nn.ReflectionPad2d(args_values_list[0]))
elif net_style == "ReplicationPad1d":
self.layers.append(nn.ReplicationPad1d(args_values_list[0]))
elif net_style == "ReplicationPad2d":
self.layers.append(nn.ReplicationPad2d(args_values_list[0]))
elif net_style == "ReplicationPad3d":
self.layers.append(nn.ReplicationPad3d(args_values_list[0]))
elif net_style == "ZeroPad2d":
self.layers.append(nn.ZeroPad2d(args_values_list[0]))
elif net_style == "ConstantPad1d":
self.layers.append(
nn.ConstantPad1d(args_values_list[0], args_values_list[1]))
elif net_style == "ConstantPad2d":
self.layers.append(
nn.ConstantPad2d(args_values_list[0], args_values_list[1]))
elif net_style == "ConstantPad3d":
self.layers.append(
nn.ConstantPad3d(args_values_list[0], args_values_list[1]))
elif net_style == "ELU":
self.layers.append(nn.ELU(args_values_list[0],
args_values_list[1]))
elif net_style == "Hardshrink":
self.layers.append(nn.Hardshrink(args_values_list[0]))
elif net_style == "Hardtanh":
self.layers.append(
nn.Hardtanh(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "LeakyReLU":
self.layers.append(
nn.LeakyReLU(args_values_list[0], args_values_list[1]))
elif net_style == "LogSigmoid":
self.layers.append(nn.LogSigmoid())
elif net_style == "PReLU":
self.layers.append(
nn.PReLU(args_values_list[0], args_values_list[1]))
elif net_style == "ReLU":
self.layers.append(nn.ReLU(args_values_list[0]))
elif net_style == "ReLU6":
self.layers.append(nn.ReLU6(args_values_list[0]))
elif net_style == "RReLU":
self.layers.append(
nn.RReLU(args_values_list[0], args_values_list[1],
args_values_list[2]))
elif net_style == "SELU":
self.layers.append(nn.SELU(args_values_list[0]))
elif net_style == "CELU":
self.layers.append(
nn.CELU(args_values_list[0], args_values_list[1]))
elif net_style == "Sigmoid":
self.layers.append(nn.Sigmoid())
elif net_style == "Softplus":
self.layers.append(
nn.Softplus(args_values_list[0], args_values_list[1]))
elif net_style == "Softshrink":
self.layers.append(nn.Softshrink(args_values_list[0]))
elif net_style == "Softsign":
self.layers.append(nn.Softsign())
elif net_style == "Tanh":
self.layers.append(nn.Tanh())
elif net_style == "Tanhshrink":
self.layers.append(nn.Tanhshrink())
elif net_style == "Threshold":
self.layers.append(
nn.Threshold(args_values_list[0], args_values_list[1],
args_values_list[2]))
elif net_style == "Softmin":
self.layers.append(nn.Softmin(args_values_list[0]))
elif net_style == "Softmax":
self.layers.append(nn.Softmax(args_values_list[0]))
elif net_style == "Softmax2d":
self.layers.append(nn.Softmax2d())
elif net_style == "LogSoftmax":
self.layers.append(nn.LogSoftmax(args_values_list[0]))
elif net_style == "AdaptiveLogSoftmaxWithLoss":
self.layers.append(
nn.AdaptiveLogSoftmaxWithLoss(args_values_list[0],
args_values_list[1],
args_values_list[2],
args_values_list[3],
args_values_list[4]))
elif net_style == "BatchNorm1d":
self.layers.append(
nn.BatchNorm1d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "BatchNorm2d":
self.layers.append(
nn.BatchNorm2d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "BatchNorm3d":
self.layers.append(
nn.BatchNorm3d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "GroupNorm":
self.layers.append(
nn.GroupNorm(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3]))
elif net_style == "InstanceNorm1d":
self.layers.append(
nn.InstanceNorm1d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "InstanceNorm2d":
self.layers.append(
nn.InstanceNorm2d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "InstanceNorm3d":
self.layers.append(
nn.InstanceNorm3d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "LayerNorm":
self.layers.append(
nn.LayerNorm(args_values_list[0], args_values_list[1],
args_values_list[2]))
elif net_style == "LocalResponseNorm":
self.layers.append(
nn.LocalResponseNorm(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3]))
elif net_style == "Linear":
self.layers.append(
nn.Linear(args_values_list[0], args_values_list[1],
args_values_list[2]))
elif net_style == "Dropout":
self.layers.append(
nn.Dropout(args_values_list[0], args_values_list[1]))
elif net_style == "Dropout2d":
self.layers.append(
nn.Dropout2d(args_values_list[0], args_values_list[1]))
elif net_style == "Dropout3d":
self.layers.append(
nn.Dropout3d(args_values_list[0], args_values_list[1]))
elif net_style == "AlphaDropout":
self.layers.append(
nn.AlphaDropout(args_values_list[0], args_values_list[1]))
from src.optimizer import RAdam, AdamW, PlainRAdam, \
AdaBound, AdaFactor, Lamb, Lookahead, Nadam, NovoGrad, Ralamb, RaLars, SGDW
activation = nn.ModuleDict([
['relu', nn.ReLU()],
['hardtanh', nn.Hardtanh()],
['relu6', nn.ReLU6()],
['sigmoid', nn.Sigmoid()],
['tanh', nn.Tanh()],
['softmax', nn.Softmax()],
['softmax2d', nn.Softmax2d()],
['logsoftmax', nn.LogSoftmax()],
['elu', nn.ELU()],
['selu', nn.SELU()],
['celu', nn.CELU()],
['hardshrink', nn.Hardshrink()],
['leakyrelu', nn.LeakyReLU()],
['logsigmoid', nn.LogSigmoid()],
['softplus', nn.Softplus()],
['softshrink', nn.Softshrink()],
['prelu', nn.PReLU()],
['softsign', nn.Softsign()],
['softmin', nn.Softmin()],
['tanhshrink', nn.Tanhshrink()],
['rrelu', nn.RReLU()],
['glu', nn.GLU()],
])
loss = nn.ModuleDict(
[['l1', nn.L1Loss()], ['nll', nn.NLLLoss()], ['kldiv',
nn.KLDivLoss()],
def __init__(self):
super(Model, self).__init__()
self.act_0 = nn.Hardshrink()
self.act_1 = nn.Hardshrink(lambd=0.3)
wt_efficiency_loader = DataLoader(wt_efficiency_set,
batch_size=batch_size,
shuffle=False)
eSpCas_loader = DataLoader(eSpCas_set, batch_size=batch_size, shuffle=False)
SpCas9_HF1_loader = DataLoader(SpCas9_HF1_set,
batch_size=batch_size,
shuffle=False)
# test_loader = DataLoader(test_set, batch_size=len(test_set), shuffle=False)
activation_functions = {
'Sigmoid': nn.Sigmoid(),
'Tanh': nn.Tanh(),
'ReLU': nn.ReLU(),
'LeakyReLU': nn.LeakyReLU(),
'ELU': nn.ELU(),
'Hardshrink': nn.Hardshrink(),
'Hardswish': nn.Hardswish(),
'ReLU6': nn.ReLU6(),
'PReLU': nn.PReLU(),
'None': nn.Identity()
}
pooling_funtion = {
'avg': nn.AvgPool1d,
'max': nn.MaxPool1d,
'none': nn.Identity
}
def hook_layer(name):
def hook(model, input, output):
def __init__(self):
# Control speed of learning
self.learning_rate = 0.05
# Control number of hidden )neurons
self.loss_function_key='BCELoss'
self.hidden_size1 =128
self.hidden_size2 =128
self.hidden_size3 =128
self.hidden_size4 =128
self.hidden_size5 =8
self.multiplier = 1.25
self.momentum = 0.9
self.lr_limit=5
self.best_correct=0
self.best_step=200
self.activation_hidden_key_1 = "hardshrink"
self.activation_hidden_key_2 = "prelu"
self.activation_hidden_key_3 = "hardshrink"
self.activation_hidden_key_4 = "prelu"#"prelu"
self.batch_norm=True
self.activations = {
'sigmoid': nn.Sigmoid(),
'relu': nn.ReLU(),
'relu6': nn.ReLU6(),
'htang1': nn.Hardtanh(-1, 1),
'tanh': nn.Tanh(),
'selu': nn.SELU(),
'hardshrink': nn.Hardshrink(),
'prelu': nn.PReLU(),
}
self.loss_functions = {
'BCELoss': nn.BCELoss(),
'MSELoss': nn.MSELoss(),
}
#self.batch_norm_grid=[True,False]
#self.activation_hidden_key_1_grid = list(self.activations.keys())
#self.activation_hidden_key_2_grid = list(self.activations.keys())
#self.loss_function_key_grid = list(self.loss_functions.keys())
# Grid search settings, see grid_search_tutorial
#self.learning_rate_grid =[0.015]
#self.activation_function=["relu_","selu_","sigmoid"]
#self.loss_function_grid=["BCELoss","MSE","L1Loss","CrossEntropyLoss","CTCLoss","BCELoss","BCEWithLogitsLoss",]
#self.loss_function_grid=["MSE","BCELoss"]
#self.learning_rate_grid =[0.009,0.011]
#self.activation_function1_grid=["relu_","selu_","sigmoid","tanh"]
#self.activation_function2_grid=["relu_","selu_","sigmoid","tanh"]
#self.activation_function1_grid=["relu_"]
#self.activation_function2_grid=["sigmoid"]
self.learning_rate_grid =[0.01,0.015,0.05]
#self.momentum_grid=np.linspace(0.88,0.98,5)
#self.hidden_size1_grid = [32,64,88]
#self.hidden_size3_grid = [256,512]
#self.hidden_size2_grid = [256,512]
# grid search will initialize this field
self.grid_search = None
# grid search will initialize this field
self.iter = 0
# This fields indicate how many times to run with same arguments
self.iter_number = 5
def __init__(self,
ksize=7,
sigma=30,
initializeDCT=True,
shrinkage='hard'):
super(__class__, self).__init__()
"""
Args:
- ksize: patch size for the DCT
- sigma: noise level (multiplies the threshold)
- initializeDCT: if True, initializes the convolutional
layers as the DCT and iDCT transforms; if false it
uses a random initialization.
- shrinkage: type of shrinkage used (hard thresholding,
soft shrinkage or tanh shrinkage)
Returns:
- model: initialized model
"""
from scipy.fftpack import dct, idct
import numpy as np
dtype = torch.FloatTensor
if torch.cuda.is_available(): dtype = torch.cuda.FloatTensor
self.sigma = sigma
self.dct = initializeDCT
ch = ksize**2
# pad by reflection: to have the output with the same size
# as the input we pad the image boundaries. Usually, zero
# padding is used for CNNs. However, since we want to
# reproduce the DCT denoising, we use reflection padding.
# Reflection padding is a differentiable layer.
self.padding = nn.ReflectionPad2d(2 * ksize // 2 - 1)
# first convolutional layer (e.g. DCT transform)
self.conv_in = nn.Conv2d(in_channels=1,
out_channels=ch,
kernel_size=ksize,
stride=1,
padding=0,
bias=not initializeDCT)
# threshold parameter (one variable per frequency)
self.thr = nn.Parameter(dtype(np.ones((1, ch, 1, 1))),
requires_grad=True)
# shrinkage function
if shrinkage == 'hard': self.shrinkage = nn.Hardshrink(1.)
elif shrinkage == 'soft': self.shrinkage = nn.Softshrink(1.)
elif shrinkage == 'tanh': self.shrinkage = nn.Tanhshrink()
else: print('DCTlike: unknown shrinkage option %s' % (shrinkage))
# output conv layer (e.g. inverse DCT transform)
self.conv_out = nn.Conv2d(in_channels=ch,
out_channels=1,
kernel_size=ksize,
stride=1,
padding=0,
bias=not initializeDCT)
# initialize the isometric DCT transforms
if initializeDCT:
# thresholding parameters (one per feature)
factor = 3.0 if shrinkage == 'hard' else 1.5
thr = np.ones((1, ch, 1, 1)) * sigma / 255. * factor
thr[0, 0] = 1e-3 # don't threshold DC component
self.thr.data = nn.Parameter(dtype(thr), requires_grad=True)
for i in range(ch):
# compute dct coefficients using scipy.fftpack
a = np.zeros((ksize, ksize))
a.flat[i] = 1
# first layer with direct dct transform
a1 = dct(dct(a.T, norm='ortho', type=3).T,
norm='ortho',
type=3)
self.conv_in.weight.data[i, 0, :, :] = nn.Parameter(dtype(a1))
# second layer, inverse transform rotated pi degrees
a2 = idct(idct(a.T, norm='ortho', type=2).T,
norm='ortho',
type=2)
a2 = np.flip(np.flip(a2, axis=0), axis=1) # pi-rotation
self.conv_out.weight.data[0,
i, :, :] = 1 / (ch) * nn.Parameter(
dtype(a2.copy()))
# random initialization
else:
# this comes from:
# 1) that the image data follows N(1/2, 1/4) (so 0.5 +- 2*sigma = [0,1])
# 2) imposing the output variance to be 0.5 (the default threshold for
# hardshrink)
std = 2. / np.sqrt(5.) / ksize
for i in range(ch):
self.conv_in.weight.data[i, 0, :, :] = dtype(
std * np.random.randn(ksize, ksize))
self.conv_out.weight.data[0, i, :, :] = dtype(
std * np.random.randn(ksize, ksize))
import numpy as np
from ..utils import solutionmanager as sm
from ..utils.gridsearch import GridSearch
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
activations = [
# nn.Sigmoid(),
# nn.LogSigmoid(),
nn.ReLU6(),
nn.LeakyReLU(negative_slope=0.01),
# nn.ELU(),
# nn.SELU(),
# nn.Hardtanh(),
# nn.Hardshrink(),
nn.Hardshrink(1),
]
class SolutionModel(nn.Module):
def __init__(self, input_size, output_size, params):
super(SolutionModel, self).__init__()
self.input_size = input_size
self.params = params
sizes = [input_size] + params.hidden_sizes + [output_size]
self.layers = nn.ModuleList(
nn.Linear(sizes[idx], sizes[idx+1]) for idx in range(len(sizes)-1)
).to(device)
