def forward(self, x):
new_features = super(_DenseLayer, self).forward(x)
if self.drop_rate > 0:
new_features = nn.dropout(new_features,
p=self.drop_rate,
training=self.training)
return torch.cat([x, new_features], 1)
def __init__(self, classes_num=10):
super(AlexNet2, self).__init__()
self.layer1 = nn.Sequential(
nn.Conv2d(in_channels=3,
out_channels=64,
kernel_size=3,
stride=2,
padding=1), nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2))
self.layer2 = nn.Sequential(
nn.Conv2d(in_channels=64,
out_channels=192,
kernel_size=3,
padding=1), nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2))
self.layer3 = nn.Sequential(
nn.Conv2d(in_channels=192,
out_channels=384,
kernel_size=3,
padding=1), nn.ReLU(inplace=True))
self.layer4 = nn.Sequential(
nn.Conv2d(in_channels=384,
out_channels=256,
kernel_size=3,
padding=1), nn.ReLU(inplace=True))
self.layer5 = nn.Sequential(
nn.Conv2d(in_channels=256,
out_channels=256,
kernel_size=3,
padding=1), nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2))
self.layer6 = nn.Sequential(
nn.dropout(), nn.Linear(in_features=256 * 2 * 2,
out_features=4096), nn.ReLU(inplace=True))
self.layer7 = nn.Sequential(
nn.dropout(), nn.Linear(in_features=4096, out_features=4096),
nn.ReLU(inplace=True))
self.layer8 = nn.Sequential(
nn.Linear(in_features=4096, out_features=classes_num))
def __init__(self):
super().__init__()
self.effnet = Efficientnet.from_pretrained(
'efficientnet-b3',
weights_path=wp3,
)
self.effnet._conv_stem.in_channels = 1
weight = self.effnet._conv_stem.weight.mean(1, keepdim=True)
self.effnet._conv_stem.weight = nn.Parameter(weight)
self.dropout = nn.dropout(0.1)
self.out = nn.Linear(1536, 24)
self.step_scheduler_after = 'epoch'
self.step_scheduler_metric = 'valid_label_rank_avg_prec_sc'
def __init__(self):
super(SensorToDryspotBoolModel, self).__init__()
self.dropout = nn.dropout(0.1)
self.maxpool = nn.maxpool2d(2, 2)
self.conv1 = nn.conv2d(1, 32, (7, 7))
self.conv2 = nn.conv2d(32, 64, (5, 5))
self.conv3 = nn.conv2d(64, 128, (3, 3))
self.conv4 = nn.conv2d(128, 256, (3, 3))
self.fc1 = nn.linear(256, 1024)
self.fc2 = nn.linear(1024, 512)
self.fc3 = nn.linear(512, 128)
self.fc_f = nn.linear(128, 1)
def __init__(self, h, d_model, dropout=0.1): """Initialize the class [Inputs] h : No of heads to be used d_model : No of model dimensions dropout : dropout rate""" super(MultiHeadedAttention, self).__init__() assert d_model%h==0 d_k = d_model//h self.h = h self.linears = clones(nn.Linear(d_model, d_model), 4) self.attn = None self.dropout = nn.dropout(p=dropout)
def __init__(self, params): #embedding_size, hidden_size, vocab_size, batch_size, layers, dropout, use_gpu):
super(LSTMLM, self).__init__()
self.use_gpu = params["use_gpu"]
self.hidden_size = params["rnn_size"]
self.vocab_size = params["vocab_size"]
self.batch_size = params["batch_size"]
self.embedding_size = params["embedding_size"]
self.dropout = nn.Dropout(params["dropout"])
self.layers = params["rnn_layers"]
self.word_embeddings = nn.Embedding(self.vocab_size, self.embedding_size)
# added separate layers to have control over each layer
self.lstms = nn.ModuleList([])
self.lstms.append(nn.LSTM(self.embedding_size, self.hidden_size))
for l in range(1,self.layers):
self.lstms.append(nn.dropout(dropout))
self.lstms.append(nn.LSTM(self.hidden_size, self.hidden_size))
self.hidden2output = nn.Linear(self.hidden_size, self.vocab_size)
self.hidden = self.init_hidden()
def __init__(self,
in_channels,
out_channels,
down=True,
act="relu",
use_dropout=False):
super().__init__()
self.conv = nn.Sequential(
nn.Conv2d(in_channels,
out_channels,
4,
2,
1,
bias=False,
padding_mode="reflect") if down else nn.ConvTranspose2d(
in_channels, out_channels, 4, 2, 1, bias=False),
nn.BatchNorm2d(out_channels),
nn.ReLU() if act == "relu" else nn.LeakyReLU(0.2),
)
self.use_dropout = use_dropout
self.dropout = nn.dropout(0.5)
def forward(self, inp, adj):
"""
inp: input_fea [N, in_features] in_features表示节点的输入特征向量元素个数
adj: 图的邻接矩阵 [N, N] 非零即一，数据结构基本知识"""
h = torch.mm(inp, self.W) # [N, out_features]
N = h.size()[0] # N 图的节点数
a_input = torch.cat([h.repeat(1, N).view(N * N, -1), h.repeat(N, 1)], dim=1).view(N, N, 2 * self.out_features)
# [N, N, 2*out_features]
e = self.leakyrelu(torch.matmul(a_input, self.a).squeeze(2))
# [N, N, 1] => [N, N] 图注意力的相关系数（未归一化）
zero_vec = -1e12 * torch.ones_like(e) # 将没有连接的边置为负无穷
attention = torch.where(adj > 0, e, zero_vec) # [N, N]
# 表示如果邻接矩阵元素大于0时，则两个节点有连接，该位置的注意力系数保留，否则需要mask并置为非常小的值，原因是softmax的时候这个最小值会不考虑。
attention = nn.softmax(attention, dim=1) # softmax形状保持不变 [N, N]，得到归一化的注意力权重！
attention = nn.dropout(attention, self.dropout, training=self.training) # dropout，防止过拟合
h_prime = torch.matmul(attention, h) # [N, N].[N, out_features] => [N, out_features]
# 得到由周围节点通过注意力权重进行更新的表示
if self.concat:
return self.elu(h_prime)
else:
return h_prime
def __init__(self, embedding_dim, hidden_dim, encoder_dim, dropout=0.1):
super(Encoder, self).__init__()
self.linear1 = nn.Linear(embedding_dim, hidden_dim)
self.linear2 = nn.Linear(hidden_dim, encoder_dim)
self.activation = nn.Tanh()
self.dropout = nn.dropout(dropout)
def forward(self, x):
x = nn.dropout(x, self.dropout, training=self.training) # dropout，防止过拟合
x = torch.cat([att(x) for att in self.attentions], dim=1) # 将每个head得到的表示进行拼接
x = nn.dropout(x, self.dropout, training=self.training) # dropout，防止过拟合
x = self.elu(self.out_att(x)) # 输出并激活
return self.log_softmax(x, dim=1) # log_softmax速度变快，保持数值稳定
def __init__(self, model_part, seq_len, feature_num, global_conv, conv_layers, layer_sizes, learned, embedding_dim, activation, output_activation, transfer, transfer_dim, dropout, posembed=True, batchnorm=True):
super().__init__()
self.learned = learned
if learned:
self.embed = nn.Embedding(AMINO_ACID, LEARNED_DIM, padding_idx=0)
self.conv2 = torch.nn.Conv1d(feature_num, global_conv["filters"], seq_len)
self.active2 = get_activation(global_conv["activation"])
# posembed
self.addpos = posembed
self.batchnorm = batchnorm
if posembed == True:
self.posembed = PositionEmbedding(seq_len, POS_DIM, "Random")
feature_num = feature_num + POS_DIM*2
self.clayer_num = len(conv_layers)
self.kmer = 0
# conv
dw = seq_len
kw = feature_num
if self.clayer_num > 0:
self.convs = nn.ModuleList()
self.cactive = nn.ModuleList()
if batchnorm:
self.bns1 = nn.ModuleList()
for i, conv_layer in enumerate(conv_layers):
self.kmer += conv_layer["kernel_size"]-1
self.convs.append(nn.Conv1d(kw, conv_layer["filters"], conv_layer["kernel_size"]))
dw = dw - conv_layer["kernel_size"] + 1
kw = conv_layer["filters"]
self.cactive.append(get_activation(conv_layer["activation"]))
if batchnorm:
self.bns1.append(nn.BatchNorm1d(conv_layer["filters"]))
else:
raise ValueError("The number of convolution layer must be larger than zero.")
if batchnorm:
self.bns1.append(nn.BatchNorm1d(global_conv["filters"]))
self.model_part = model_part
self.attn, output_size = aggregate_feature(model_part, kw, dw)
output_size = output_size + global_conv["filters"]
self.layer_num = len(layer_sizes)
self.transfer = transfer
if transfer:
self.trans_layer = nn.Linear(transfer_dim, int(output_size/2))
output_size = int(output_size*3/2)
if self.layer_num > 0:
if batchnorm:
self.bns2 = nn.ModuleList()
self.denses = nn.ModuleList()
self.dactive = nn.ModuleList()
for i, layer_size in enumerate(layer_sizes):
self.denses.append(nn.Linear(output_size, layer_size))
output_size = layer_size
self.dactive.append(get_activation(activation))
if batchnorm:
self.bns2.append(nn.BatchNorm1d(layer_size))
self.dropout = None
if dropout > 0:
self.dropout = nn.dropout(dropout)
self.output_layer = nn.Linear(output_size, embedding_dim)
self.output_active = get_activation(output_activation)
