def __init__(self): self.device = torch.device("cuda") self.iterator = AudioDataset.NoisyMusicDataset( noisy_music_folder="Processed") super(Net, self).__init__() self.layer1 = nn.Sequential( nn.Conv1d(2, 32, kernel_size=3, stride=1, padding=0), nn.Tanhshrink(), nn.Conv1d(32, 64, kernel_size=3, stride=1, padding=0), nn.MaxPool1d(kernel_size=2, stride=1), nn.Dropout(0.25)) self.layer2 = nn.Sequential( nn.Conv1d(64, 64, kernel_size=3, stride=1, padding=0), nn.Tanhshrink(), nn.Conv1d(64, 128, kernel_size=3, stride=1, padding=0), nn.MaxPool1d(kernel_size=2, stride=1), nn.Dropout(0.5)) self.layer3 = nn.Sequential( nn.Conv1d(128, 128, kernel_size=3, stride=1, padding=0), nn.Tanhshrink(), nn.Conv1d(128, 512, kernel_size=3, stride=1, padding=0), nn.MaxPool1d(kernel_size=2, stride=1), nn.Dropout(0.5)) self.fc1 = nn.Sequential(nn.Linear(512, 10), nn.Tanhshrink(), nn.Dropout(0.5), nn.Linear(10, 2), nn.Softmax())
def __init__(self, matrix, n_modules, f_in=50, f_out=1): super(FLUX, self).__init__() # gene to flux self.inSize = f_in self.m_encoder = nn.ModuleList([ nn.Sequential( nn.Linear(self.inSize, 8, bias=False), nn.Tanhshrink(), nn.Linear(8, f_out), nn.Tanhshrink() # nn.SELU() ) for i in range(n_modules) ])
def __init__(self): super(GCLoss, self).__init__() sobel_x = torch.Tensor([[1, 0, -1], [2, 0, -2], [1, 0, -1]]).view( (1, 1, 3, 3)).repeat(1, 3, 1, 1) sobel_y = torch.Tensor([[1, 2, 1], [0, 0, 0], [-1, -2, -1]]).view( (1, 1, 3, 3)).repeat(1, 3, 1, 1) self.G_x_B = nn.Conv2d(3, 1, kernel_size=3, stride=1, padding=0, bias=False) self.G_x_B.weight = nn.Parameter(sobel_x) for param in self.G_x_B.parameters(): param.requires_grad = False self.G_y_B = nn.Conv2d(3, 1, kernel_size=3, stride=1, padding=0, bias=False) self.G_y_B.weight = nn.Parameter(sobel_y) for param in self.G_y_B.parameters(): param.requires_grad = False self.G_x_R = nn.Conv2d(3, 1, kernel_size=3, stride=1, padding=0, bias=False) self.G_x_R.weight = nn.Parameter(sobel_x) for param in self.G_x_R.parameters(): param.requires_grad = False self.G_y_R = nn.Conv2d(3, 1, kernel_size=3, stride=1, padding=0, bias=False) self.G_y_R.weight = nn.Parameter(sobel_y) for param in self.G_y_R.parameters(): param.requires_grad = False self.af_B = nn.Tanhshrink() self.af_R = nn.Tanhshrink()
def create_str_to_activations_converter(self): """Creates a dictionary which converts strings to activations""" str_to_activations_converter = { "elu": nn.ELU(), "hardshrink": nn.Hardshrink(), "hardtanh": nn.Hardtanh(), "leakyrelu": nn.LeakyReLU(), "logsigmoid": nn.LogSigmoid(), "prelu": nn.PReLU(), "relu": nn.ReLU(), "relu6": nn.ReLU6(), "rrelu": nn.RReLU(), "selu": nn.SELU(), "sigmoid": nn.Sigmoid(), "softplus": nn.Softplus(), "logsoftmax": nn.LogSoftmax(), "softshrink": nn.Softshrink(), "softsign": nn.Softsign(), "tanh": nn.Tanh(), "tanhshrink": nn.Tanhshrink(), "softmin": nn.Softmin(), "softmax": nn.Softmax(dim=1), "none": None } return str_to_activations_converter
def __init__(self, alpha=1.0): super().__init__() self.activations = [ nn.ELU(), nn.Hardshrink(), nn.Hardtanh(), nn.LeakyReLU(), nn.LogSigmoid(), nn.ReLU(), nn.PReLU(), nn.SELU(), nn.CELU(), nn.Sigmoid(), nn.Softplus(), nn.Softshrink(), nn.Softsign(), nn.Tanh(), nn.Tanhshrink() ] self.P = [ torch.nn.Parameter(torch.randn(1, requires_grad=True)) for _ in self.activations ] for activation, param in zip(self.activations, self.P): activation_name = str(activation).split("(")[0] self.add_module(name=activation_name, module=activation) self.register_parameter(name=activation_name + "p", param=param)
def __init__(self, num_inputs): super(Decoder_c, self).__init__() self.decoder = nn.Sequential(nn.Linear(num_inputs, num_inputs), nn.Dropout(0.5), nn.Tanhshrink(), nn.Linear(num_inputs, num_inputs)) self.decoder.apply(init_weights)
def __init__(self, in_feats=64, out_feats=64, activation='relu', skip=False): super(RGCN, self).__init__() self.in_feats = in_feats self.out_feats = out_feats self.activation = activation self.skip = skip # weight bases in equation (3) self.weight = nn.Parameter( torch.Tensor(3, self.in_feats, self.out_feats)) nn.init.kaiming_uniform_(self.weight, mode='fan_in', nonlinearity='relu') self.batchnorm = nn.BatchNorm1d(self.out_feats) if activation == 'relu': self.activation_ = nn.ReLU() elif activation == 'tanhshrink': self.activation_ = nn.Tanhshrink() else: print('Activation function not implemented.') sys.exit(-1)
def __init__(self, in_feats=64, out_feats=64, activation='relu', batchnorm=True): super(Embedding, self).__init__() self.in_feats = in_feats self.out_feats = out_feats self.activation = activation self.batchnorm = batchnorm layers = [] layer = nn.Linear(self.in_feats, self.out_feats) nn.init.kaiming_uniform_(layer.weight, mode='fan_in', nonlinearity='relu') layers.append(layer) if self.batchnorm: layer = nn.BatchNorm1d(self.out_feats) layers.append(layer) if self.activation == 'relu': layer = nn.ReLU() elif self.activation == 'tanhshrink': layer = nn.Tanhshrink() else: print('Activation function not implemented.') sys.exit(-1) layers.append(layer) self.fc = nn.Sequential(*layers)
def parse_activation(act): if act is None: return lambda x: x act, kwargs = parse_str(act) if act == 'sigmoid': return nn.Sigmoid(**kwargs) if act == 'tanh': return nn.Tanh(**kwargs) if act == 'relu': return nn.ReLU(**kwargs, inplace=True) if act == 'relu6': return nn.ReLU6(**kwargs, inplace=True) if act == 'elu': return nn.ELU(**kwargs, inplace=True) if act == 'selu': return nn.SELU(**kwargs, inplace=True) if act == 'prelu': return nn.PReLU(**kwargs) if act == 'leaky_relu': return nn.LeakyReLU(**kwargs, inplace=True) if act == 'threshold': return nn.Threshold(**kwargs, inplace=True) if act == 'hardtanh': return nn.Hardtanh(**kwargs, inplace=True) if act == 'log_sigmoid': return nn.LogSigmoid(**kwargs) if act == 'softplus': return nn.Softplus(**kwargs) if act == 'softshrink': return nn.Softshrink(**kwargs) if act == 'tanhshrink': return nn.Tanhshrink(**kwargs) if act == 'softmin': return nn.Softmin(**kwargs) if act == 'softmax': return nn.Softmax(**kwargs) if act == 'softmax2d': return nn.Softmax2d(**kwargs) if act == 'log_softmax': return nn.LogSoftmax(**kwargs) raise ValueError(f'unknown activation: {repr(act)}')
def initializePopulation(population_size, runs, featuresUsed): # create initial population and store in dictionary for competition populationDict = {} for i in range(0, population_size): # initialize learning rate - range [10^-1, 10^-6] learning_rate = 10**(-1*np.random.uniform(1, 6)) # initialize epochs - range [1, 100] epochs = int(np.random.uniform(1, 100)) # initialize momentum - range [0, 0.99] momentum = np.random.uniform(0, 0.99) # initialize layerSizes - range [2, 80] layerSizes = 2+np.random.randint(79, size=(1, 1))[0][0] # initialize activationFunction activationFunctionList = [nn.ReLU(), nn.Sigmoid(), nn.Tanh(), nn.Tanhshrink(), nn.Hardtanh()] activationFunction = random.choice(activationFunctionList) # initialize encoding of solution and find accuracy populationDict[i] = MyEncoding(i, learning_rate,epochs, momentum, featuresUsed, layerSizes, activationFunction) populationDict[i].accuracy = cross_train(populationDict[i], runs) return populationDict
def str2act(s): if s is 'none': return None elif s is 'hardtanh': return nn.Hardtanh() elif s is 'sigmoid': return nn.Sigmoid() elif s is 'relu6': return nn.ReLU6() elif s is 'tanh': return nn.Tanh() elif s is 'tanhshrink': return nn.Tanhshrink() elif s is 'hardshrink': return nn.Hardshrink() elif s is 'leakyrelu': return nn.LeakyReLU() elif s is 'softshrink': return nn.Softshrink() elif s is 'softsign': return nn.Softsign() elif s is 'relu': return nn.ReLU() elif s is 'prelu': return nn.PReLU() elif s is 'softplus': return nn.Softplus() elif s is 'elu': return nn.ELU() elif s is 'selu': return nn.SELU() else: raise ValueError("[!] Invalid activation function.")
def forward(self, data): act = nn.Tanhshrink() act = F.relu #act = nn.LeakyReLU(0.25) # first conv block data.x = act(self.conv1( data.x, data.edge_index, data.edge_attr)) cluster = get_preloaded_cluster(data.cluster0, data.batch) data = community_pooling(cluster, data) # second conv block data.x = act(self.conv2( data.x, data.edge_index, data.edge_attr)) cluster = get_preloaded_cluster(data.cluster1, data.batch) x, batch = max_pool_x(cluster, data.x, data.batch) # FC x = scatter_mean(x, batch, dim=0) x = act(self.fc1(x)) x = self.fc2(x) #x = F.dropout(x, training=self.training) return x
def __init__(self): super(energy_latency2_10, self).__init__() self.name = "energy_latency2_10" self.find = nn.Sequential(nn.Linear(10 * 14, 128), nn.Softplus(), nn.Linear(128, 128), nn.Softplus(), nn.Linear(128, 64), nn.Tanhshrink(), nn.Linear(64, 2), nn.Sigmoid())
def __init__(self): super(energy_50, self).__init__() self.name = "energy_50" self.find = nn.Sequential(nn.Linear(50 * 51, 128), nn.Softplus(), nn.Linear(128, 128), nn.Softplus(), nn.Linear(128, 64), nn.Tanhshrink(), nn.Linear(64, 1), nn.Sigmoid())
def skipadd_forward(self, x): """FORWARD CALCULATION - amplitude part. skip connection through adding Args: configuration (Tensor): tensor with the shape (1,batch, channel, ncell). the configuration is already reordered if pbc is true Returns: output(Tensor): wave function amplitude with the shape (groups, batch, n_skipth, ncell) . """ lrelu = nn.LeakyReLU(0.1) tanhsh = nn.Tanhshrink() x = self.causal_in(x) x = tanhsh(x) skip = 0 for i, dil_ly in enumerate(self.dil_act): x = dil_ly(x) x = lrelu(x) skip += self.skip_1x1[i]( x) / self.n_resch #(1, batch, self.ngroup * nskipth, ncell) skip = lrelu(skip) skip = skip.permute(0, -1, 1, 2).reshape(self.ncell, -1, self.ngroup * self.n_skipch) skip = self.local_linear(skip) skip = skip.reshape(self.ncell, -1, self.ngroup, self.n_skipch).permute(2, 1, 3, 0) return skip
def forward(self, x): """FORWARD CALCULATION - amplitude part. direct skip connection Args: configuration (Tensor): tensor with the shape (1,batch, channel, ncell). the configuration is already reordered if pbc is true Returns: output(Tensor): wave function amplitude with the shape (groups, batch, n_skipth, ncell) . """ nbatch = x.shape[1] lrelu = nn.LeakyReLU(0.1) tanhsh = nn.Tanhshrink() x = self.causal_in(x) x = tanhsh(x) skip = [] for i, dil_ly in enumerate(self.dil_act): x = dil_ly(x) x = lrelu(x) # #(1, batch, n_resch, ncell) skip.append(x.clone()) skip = th.cat(skip, 2) #(1, batch, n_resch*layers, ncell) skip = self.skip_conv(skip) #(1, batch, n_resch, ncell) skip = lrelu(skip) skip = skip.squeeze(0).permute(-1, 0, 1) #(ncell,batch, n_resch) skip = self.local_linear(skip) #(ncell,batch, ngroup*n_skipch*2) skip = lrelu(skip) skip = skip.reshape(self.ncell, nbatch, self.ngroup, -1).permute(0, 2, 1, 3).reshape(self.ncell * self.ngroup, nbatch, -1) skip = self.final_linear(skip) # (ncell*ngroup,batch, n_skipch) skip = skip.reshape(self.ncell, self.ngroup, -1, self.n_skipch).permute(1, 2, 3, 0) return skip
def mutateActivationFunction(self, id): # randomly re-select an activation function activationFunctionList = [nn.ReLU(), nn.Sigmoid(), nn.Tanh(), nn.Tanhshrink(), nn.Hardtanh()] new_activationFunction = random.choice(activationFunctionList) return MyEncoding(id, self.learningRate, self.epochs, self.momentum, self.featuresUsed, self.layerSizes, new_activationFunction)
def forward(self, data): act = nn.Tanhshrink() act = F.relu #act = nn.LeakyReLU(0.25) data.x = act(self.conv1(data.x, data.edge_index, data.edge_attr)) cluster = community_detection(data.internal_edge_index, data.num_nodes, edge_attr=None, batches=data.batches) data = community_pooling(cluster, data) data.x = act(self.conv2(data.x, data.edge_index, data.edge_attr)) cluster = community_detection(data.internal_edge_index, data.num_nodes, edge_attr=None) x, batch = max_pool_x(cluster, data.x, data.batch) x = scatter_mean(x, batch, dim=0) x = act(self.fc1(x)) x = self.fc2(x) #x = F.dropout(x, training=self.training) return x
def __init__(self, inner_size=1024, bil_size=32): super().__init__() self._layer_1 = nn.Linear(512 + 128, bil_size) self._layer_2 = nn.Bilinear(512 + 128, bil_size, inner_size) self._activation = nn.Tanhshrink() self._output = nn.Linear(inner_size, YOUTUBE8M_LABELS_N)
def get_activation(activation_type): if activation_type == "relu": return nn.ReLU() elif activation_type == "relu6": return nn.ReLU6() elif activation_type == "prelu": return nn.PReLU() elif activation_type == "selu": return nn.SELU() elif activation_type == "celu": return nn.CELU() elif activation_type == "gelu": return nn.GELU() elif activation_type == "sigmoid": return nn.Sigmoid() elif activation_type == "softplus": return nn.Softplus() elif activation_type == "softshrink": return nn.Softshrink() elif activation_type == "softsign": return nn.Softsign() elif activation_type == "tanh": return nn.Tanh() elif activation_type == "tanhshrink": return nn.Tanhshrink() else: raise ValueError("Unknown activation type {}".format(activation_type))
def __init__(self): super(NNActivationModule, self).__init__() self.activations = nn.ModuleList([ nn.ELU(), nn.Hardshrink(), nn.Hardsigmoid(), nn.Hardtanh(), nn.Hardswish(), nn.LeakyReLU(), nn.LogSigmoid(), # nn.MultiheadAttention(), nn.PReLU(), nn.ReLU(), nn.ReLU6(), nn.RReLU(), nn.SELU(), nn.CELU(), nn.GELU(), nn.Sigmoid(), nn.SiLU(), nn.Mish(), nn.Softplus(), nn.Softshrink(), nn.Softsign(), nn.Tanh(), nn.Tanhshrink(), # nn.Threshold(0.1, 20), nn.GLU(), nn.Softmin(), nn.Softmax(), nn.Softmax2d(), nn.LogSoftmax(), # nn.AdaptiveLogSoftmaxWithLoss(), ])
def __init__(self, num_inputs): super(Encoder, self).__init__() self.encoder = nn.Sequential(nn.BatchNorm1d(num_inputs), nn.Linear(num_inputs, num_inputs), nn.Dropout(0.5), nn.Tanhshrink(), nn.Linear(num_inputs, num_inputs)) self.encoder.apply(init_weights)
def mutateEverything(self, id): # re-initialize everything # activationFunction options activationFunctionList = [nn.ReLU(), nn.Sigmoid(), nn.Tanh(), nn.Tanhshrink(), nn.Hardtanh()] return MyEncoding(id, 10**(-1*np.random.uniform(1, 6)), int(np.random.uniform(1, 500)), np.random.uniform(0, 0.99), self.featuresUsed, 2+np.random.randint(79, size=(1, 1))[0][0], random.choice(activationFunctionList))
def __init__( self, in_manifold: RiemannianManifold, out_manifold: RiemannianManifold, in_dimension: int, out_dimension: int, non_linearity=None, num_poles=3, log_base_init: torch.Tensor = None, exp_base_init: torch.Tensor = None, ortho_init=False, ): super(ManifoldLayer, self).__init__() self.in_manifold = in_manifold self.out_manifold = out_manifold self.in_dimension = in_dimension self.out_dimension = out_dimension self.num_poles = num_poles if log_base_init is not None: self.log_base = ManifoldParameter(log_base_init, manifold=in_manifold, lr_scale=1) else: self.log_base = ManifoldParameter(torch.Tensor( num_poles, in_dimension), manifold=in_manifold, lr_scale=1) if exp_base_init is not None: self.exp_base = ManifoldParameter(exp_base_init, manifold=out_manifold, lr_scale=1) else: self.exp_base = ManifoldParameter(torch.Tensor(out_dimension), manifold=out_manifold, lr_scale=1) self.linear_layer = nn.Linear(in_dimension * num_poles, out_dimension, bias=False) if ortho_init: orthogonal_(self.linear_layer.weight) with torch.no_grad(): self.linear_layer.weight /= sqrt(in_dimension) self.non_linearity = None self.non_linearity_name = non_linearity if non_linearity is not None: if non_linearity == "relu": self.non_linearity = nn.ReLU() elif non_linearity == "tanh": self.non_linearity = nn.Tanh() elif non_linearity == "tanhshrink": self.non_linearity = nn.Tanhshrink() elif non_linearity == "leakyrelu": self.non_linearity = nn.LeakyReLU() elif non_linearity == "elu": self.non_linearity = nn.ELU()
def __init__(self, args, kgemb_dim, encoder: Seq2VecEncoder, vocab): super().__init__(vocab) self.args = args self.encoder = encoder self.first_and_last_emb = encoder.get_output_dim() self.loss = nn.MSELoss() self.projector = nn.Linear(kgemb_dim, encoder.get_output_dim()) self.projector2 = nn.Linear(kgemb_dim, encoder.get_output_dim()) self.projector3 = nn.Linear(kgemb_dim, encoder.get_output_dim()) self.nonlinear = nn.Tanhshrink()
def __init__(self): super(energy_50_RL, self).__init__() self.name = "energy_RL" self.feature = nn.Sequential(nn.Linear(50 * 51, 128), nn.Softplus(), nn.Linear(128, 128), nn.Softplus()) self.value = nn.Sequential(nn.Linear(128, 64), nn.Tanhshrink(), nn.Linear(64, 1), nn.Sigmoid()) self.action = nn.Sequential(nn.Linear(128, 256), nn.Softplus(), nn.Linear(256, 50 * 50)) self.softmax = nn.Softmax(dim=1)
def __init__(self): super(energy_latency_50_RL, self).__init__() self.name = "energy_latency_50_" + str(Coeff_Energy) + "_" + str( Coeff_Latency) + "_RL" self.feature = nn.Sequential(nn.Linear(50 * 52, 128), nn.Softplus(), nn.Linear(128, 128), nn.Softplus()) self.value = nn.Sequential(nn.Linear(128, 64), nn.Tanhshrink(), nn.Linear(64, 1), nn.Sigmoid()) self.action = nn.Sequential(nn.Linear(128, 256), nn.Softplus(), nn.Linear(256, 10 * 10)) self.softmax = nn.Softmax(dim=1)
def __init__(self, input_size, feature_size): super().__init__() self.input_layer = nn.Linear(input_size, feature_size, bias=False) self.hidden_layer_1 = nn.Linear(16, 16, bias=False) self.hidden_layer_2 = nn.Linear(128, 128, bias=False) #self.hidden_layer_3 = nn.Linear(32, 128, bias=False) self.output_layer = nn.Linear(16, feature_size, bias=False) self.prelu = nn.PReLU(1, 0.25) self.silu = nn.SiLU() self.elu = nn.ELU() self.tanshrink = nn.Tanhshrink()
def __init__(self, input_size, hidden_size, hidden_depth, output_size, device): super(NeuralNetSimple, self).__init__() self.fc_in = nn.Linear(input_size, hidden_size).to(device) self.fcs = nn.ModuleList() #collections.OrderedDict() self.hidden_depth = hidden_depth for i in range(self.hidden_depth): self.fcs.append(nn.Linear(hidden_size, hidden_size).to(device)) self.fc_out = nn.Linear(hidden_size, output_size).to(device) #activations: self.ths = nn.Tanhshrink().to(device) self.l_relu = nn.LeakyReLU().to(device)
def __init__(self, n1=2, n2=4, n3=8, n4=2): super(Autoencoder, self).__init__() self.nl = nn.Tanhshrink() # encoder self.enc1 = nn.Linear(n1, n2) self.enc2 = nn.Linear(n2, n3) self.enc3 = nn.Linear(n3, n4) # decoder self.dec1 = nn.Linear(n4, n3) self.dec2 = nn.Linear(n3, n2) self.dec3 = nn.Linear(n2, n1)