def __init__(self,input_size,output_size, n_features, d_model=256,nhead=8, num_layers=3, dropout=0.1):
super(TransformerDecoderModel, self).__init__()
self.d_model = d_model
self.criterion = nn.L1Loss()
self.warmup_steps = 4000
self.output_size = output_size
self.n_features = n_features
self.encoder = nn.Linear(n_features, d_model)
self.pos_encoder = PositionalEncoding(d_model, dropout)
self.decoder = nn.Linear(n_features, d_model)
self.pos_decoder = PositionalEncoding(d_model, dropout)
decoder_layer = nn.TransformerDecoderLayer(d_model=d_model, nhead=nhead,dim_feedforward=d_model*4, dropout=dropout, activation='relu')
self.transformer_decoder = nn.TransformerDecoder(decoder_layer, num_layers=num_layers)
self.fc_out = nn.Linear(d_model, n_features)
self.src_mask = None
self.trg_mask = None
self.memory_mask = None
def __init__(self, config, src_vocab):
super(Transformer, self).__init__()
self.config = config
self.src_vocab = src_vocab
# 超参数
# h是多头数量， N是层数， dropout是比率
h, N, dropout = self.config.h, self.config.N, self.config.dropout
# 词向量维度，全连接维度
d_model, d_ff = self.config.d_model, self.config.d_ff
# 多头注意力层
attn = MultiHeadedAttention(h, d_model)
# 全连接层
ff = PositionwiseFeedForward(d_model, d_ff, dropout)
# 位置向量
position = PositionalEncoding(d_model, dropout)
self.encoder = Encoder(
EncoderLayer(config.d_model, deepcopy(attn), deepcopy(ff),
dropout), N)
self.src_embed = nn.Sequential(
Embedding(self.config.d_model, self.src_vocab),
deepcopy(position)) # embedding with position encoding
self.fc = nn.Linear(self.config.d_model, self.config.output_size)
self.softmax = nn.Softmax()
def make_model(cnn3d,
tgt_vocab,
N=6,
d_model=512,
d_ff=2048,
h=8,
dropout=0.1):
"Helper: Construct a model from hyperparameters."
c = copy.deepcopy
attn = MultiHeadedAttention(h, d_model)
ff = PositionwiseFeedForward(d_model, d_ff, dropout)
position = PositionalEncoding(d_model, dropout)
model = EncoderDecoder(
Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), N),
Decoder(DecoderLayer(d_model, c(attn), c(attn), c(ff), dropout), N),
nn.Sequential(c(position)),
nn.Sequential(Embeddings(d_model, tgt_vocab), c(position)),
Generator(d_model, tgt_vocab), cnn3d)
# This was important from their code.
# Initialize parameters with Glorot / fan_avg.
for p in model.named_parameters():
if not p[0].startswith(
"cnn3d") and p[1].requires_grad and p[1].dim() > 1:
nn.init.xavier_uniform_(p[1])
return model
def __init__(self, dm, dropout=0.1):
super(EncoderBlock, self).__init__()
self.pe = PositionalEncoding(dm, dropout)
self.self_attn = Attn()
self.ffn = PositionWiseFFN(dm, dm // 2)
self.dropout = dropout
self.highways = utils.clones(HighWay(dm, dropout), 2)
def __init__(self,dimbed,dimlat,dimout,nL):
super(SelfAttentionPE,self).__init__()
l = []
for i in range(nL):
l.append(AttentionLayer(dimbed,dimlat))
l.append(nn.ReLU())
l.append(nn.Dropout())
self.attention = nn.Sequential(*l)
self.classifier = nn.Linear(dimbed,dimout)
self.sgma = nn.Sigmoid()
self.pe = PositionalEncoding(d_model=dimbed,max_len=3000).to(device)
def __init__(self, outtoken, hidden, enc_layers=1, dec_layers=1, nhead=1, dropout=0.1, pretrained=True):
super(TransformerModel, self).__init__()
self.enc_layers = enc_layers
self.dec_layers = dec_layers
self.backbone_name = 'resnet50'
self.backbone = models.resnet50(pretrained=pretrained)
self.backbone.fc = nn.Conv2d(2048, int(hidden/2), 1)
self.pos_encoder = PositionalEncoding(hidden, dropout)
self.decoder = nn.Embedding(outtoken, hidden)
self.pos_decoder = PositionalEncoding(hidden, dropout)
self.transformer = nn.Transformer(d_model=hidden, nhead=nhead, num_encoder_layers=enc_layers,
num_decoder_layers=dec_layers, dim_feedforward=hidden * 4, dropout=dropout,
activation='relu')
self.fc_out = nn.Linear(hidden, outtoken)
self.src_mask = None
self.trg_mask = None
self.memory_mask = None
log_config(self)
def __init__(self, config, pre_train_weight, embedding_size):
super(Transformer, self).__init__()
self.config = config
self.pre_train_weight = pre_train_weight
self.embedding_size = embedding_size
# 超参数
# h是多头数量， N是层数， dropout是比率
h, N, dropout = self.config.h, self.config.N, self.config.dropout
# 词向量维度，全连接维度
d_model, d_ff = self.config.d_model, self.config.d_ff
# 多头注意力层
attn = MultiHeadedAttention(h, d_model)
# 全连接层
ff = PositionwiseFeedForward(d_model, d_ff, dropout)
# 位置向量
position = PositionalEncoding(d_model, dropout)
self.encoder = Encoder(EncoderLayer(config.d_model, deepcopy(attn), deepcopy(ff), dropout), N)
self.src_embed = nn.Sequential(Embedding(self.config.d_model, self.pre_train_weight, self.embedding_size), deepcopy(position)) # embedding with position encoding
def __init__(self,
outtoken,
hidden,
enc_layers=1,
dec_layers=1,
nhead=1,
dropout=0.1):
super(TransformerModel, self).__init__()
self.enc_layers = enc_layers
self.dec_layers = dec_layers
self.backbone_name = 'conv(64)->conv(64)->conv(128)->conv(256)->conv(256)->conv(512)->conv(512)'
self.conv0 = Conv2d(1,
64,
kernel_size=(3, 3),
stride=(1, 1),
padding=(1, 1))
self.conv1 = Conv2d(64,
128,
kernel_size=(3, 3),
stride=(1, 1),
padding=(1, 1))
self.conv2 = Conv2d(128,
256,
kernel_size=(3, 3),
stride=(2, 1),
padding=(1, 1))
self.conv3 = Conv2d(256,
256,
kernel_size=(3, 3),
stride=(1, 1),
padding=(1, 1))
self.conv4 = Conv2d(256,
512,
kernel_size=(3, 3),
stride=(2, 1),
padding=(1, 1))
self.conv5 = Conv2d(512,
512,
kernel_size=(3, 3),
stride=(1, 1),
padding=(1, 1))
self.conv6 = Conv2d(512, 512, kernel_size=(2, 1), stride=(1, 1))
self.pool1 = MaxPool2d(kernel_size=2,
stride=2,
padding=0,
dilation=1,
ceil_mode=False)
self.pool3 = MaxPool2d(kernel_size=2,
stride=2,
padding=0,
dilation=1,
ceil_mode=False)
self.pool5 = MaxPool2d(kernel_size=(2, 2),
stride=(2, 1),
padding=(0, 1),
dilation=1,
ceil_mode=False)
self.bn0 = BatchNorm2d(64,
eps=1e-05,
momentum=0.1,
affine=True,
track_running_stats=True)
self.bn1 = BatchNorm2d(128,
eps=1e-05,
momentum=0.1,
affine=True,
track_running_stats=True)
self.bn2 = BatchNorm2d(256,
eps=1e-05,
momentum=0.1,
affine=True,
track_running_stats=True)
self.bn3 = BatchNorm2d(256,
eps=1e-05,
momentum=0.1,
affine=True,
track_running_stats=True)
self.bn4 = BatchNorm2d(512,
eps=1e-05,
momentum=0.1,
affine=True,
track_running_stats=True)
self.bn5 = BatchNorm2d(512,
eps=1e-05,
momentum=0.1,
affine=True,
track_running_stats=True)
self.bn6 = BatchNorm2d(512,
eps=1e-05,
momentum=0.1,
affine=True,
track_running_stats=True)
self.activ = LeakyReLU()
self.pos_encoder = PositionalEncoding(hidden, dropout)
self.decoder = nn.Embedding(outtoken, hidden)
self.pos_decoder = PositionalEncoding(hidden, dropout)
self.transformer = nn.Transformer(d_model=hidden,
nhead=nhead,
num_encoder_layers=enc_layers,
num_decoder_layers=dec_layers,
dim_feedforward=hidden * 4,
dropout=dropout)
self.fc_out = nn.Linear(hidden, outtoken)
self.src_mask = None
self.trg_mask = None
self.memory_mask = None
log_config(self)
def _activate_tf(self, layer):
# Preparation before layer loop
self.pos_enc = PositionalEncoding(layer.embed_dim, layer.max_len)
self.norm = LayerNorm(layer.embed_dim)
self.dropout = Dropout(0.1)