def __init__(self,
ntoken,
ninp,
nhead,
nhid,
nlayers,
dropout=0.5):
super(TransformerModel, self).__init__()
from torch.nn import TransformerEncoder, TransformerEncoderLayer
self.model_type = 'Transformer'
self.pos_encoder = PositionalEncoding(ninp, dropout)
encoder_layers = TransformerEncoderLayer(
ninp, nhead, nhid, dropout)
self.transformer_encoder = TransformerEncoder(
encoder_layers, nlayers)
self.encoder = nn.Embedding(ntoken, ninp)
self.ninp = ninp
self.decoder = nn.Linear(ninp, ntoken)
self.init_weights()
def __init__(self, ntoken, emb_size, nhead, nhid, nlayers):
"""
emb_size: Embedding Size for the input
ntoken: Number of tokens Vocab Size
nhead: Number of transformer heads in the encoder
nhid: Number of hidden units in transformer encoder layer
nlayer: Number of layers in transformer encoder
"""
super(TransformerModel, self).__init__()
from torch.nn import TransformerEncoder, TransformerEncoderLayer
self.emb_size = emb_size
self.ntoken = ntoken
self.nhead = nhead
self.nhid = nhid
self.nlayer = nlayers
self.ninp = emb_size ### NEED TO CHECK THIS
"""
1. Initialize position input embedding, position encoding layers
2. Initialize transformer encoder with nlayers and each layer
having nhead heads and nhid hidden units.
3. Decoder can be implemented directly on top of the encoder as a linear layer.
To keep things simple, we are predicting one token for each of the input tokens.
We can pad the input to have the same length as target to ensure we can generate all target tokens.
You may experiment with a transformer decoder and use teacher forcing during training,
but it is not necessary to do so.
"""
# 1
# ninp == emb_size
self.pos_encoder = PositionalEncoding(emb_size)
encoder_layers = TransformerEncoderLayer(emb_size, nhead, nhid)
# 2
self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
self.encoder = nn.Embedding(ntoken, emb_size)
#3
self.decoder = nn.Linear(emb_size, ntoken)
self.init_weights()
def __init__(self,
input_vocab_size,
target_vocab_size,
d_model=512,
nhead=8,
dim_feedforward=2048,
num_encoder_layers=6,
num_decoder_layers=6,
dropout=0.1,
activation="relu"):
super(TransformerModel, self).__init__()
try:
from torch.nn import TransformerEncoderLayer, TransformerDecoderLayer
except:
raise ImportError(
'TransformerEncoder module does not exist in PyTorch 1.1 or lower.'
)
self.model_type = 'Transformer'
encoder_layers = TransformerEncoderLayer(d_model, nhead,
dim_feedforward, dropout,
activation)
encoder_norm = nn.LayerNorm(d_model)
self.encoder = TransformerEncoder(encoder_layers, num_encoder_layers,
d_model, input_vocab_size, dropout,
encoder_norm)
decoder_layers = TransformerDecoderLayer(d_model, nhead,
dim_feedforward, dropout,
activation)
decoder_norm = nn.LayerNorm(d_model)
self.decoder = TransformerDecoder(decoder_layers, num_decoder_layers,
d_model, target_vocab_size, dropout,
decoder_norm)
self.linear = nn.Linear(d_model, target_vocab_size)
self._reset_parameters()
self.d_model = d_model
self.nhead = nhead
def __init__(self):
super().__init__()
self.conv = nn.Sequential(
nn.Conv1d(24, 96, 7, stride=2, padding=3, groups=24),
nn.InstanceNorm1d(80),
nn.ELU(inplace=True),
nn.Conv1d(96, 128, 3, stride=1, padding=1),
nn.BatchNorm1d(128),
nn.ELU(inplace=True),
)
self.elayer1 = nn.Sequential(ECABasicBlock(128, 128),
ECABasicBlock(128, 128))
self.elayer2 = nn.Sequential(ECABasicBlock(128, 256),
ECABasicBlock(256, 256))
self.elayer3 = nn.Sequential(ECABasicBlock(256, 512, stride=2))
self.elayer4 = nn.Sequential(ECABasicBlock(512, 1024, stride=2),
nn.AvgPool1d(2))
encoder_layer = TransformerEncoderLayer(
d_model=1024,
nhead=8,
dim_feedforward=2048,
dropout=0.1,
activation="relu",
)
self.encoder = TransformerEncoder(encoder_layer, 8)
self.decoder1 = ECABasicBlock(1024, 1536, stride=2)
self.decoder2 = ECABasicBlock(1536, 2048)
self.decoder3 = ECABasicBlock(2048, 3070, stride=2)
self.decoder4 = ECABasicBlock(3070, 4094)
self.fc = nn.Sequential(
nn.AdaptiveAvgPool1d(1),
nn.Flatten(),
nn.Dropout(p=0.05),
nn.Linear(4094, 1024),
nn.ELU(inplace=True),
nn.Dropout(p=0.05),
nn.Linear(1024, 61),
)
def __init__(self,
ntoken,
ninp,
nhead,
nhid,
nlayers,
nclasses,
idropout=0.1,
hdropout=0.5,
layer_norm=0,
src_scale=0,
mlp=0):
super(TransformerModel, self).__init__()
self.model_type = 'Transformer'
if ninp > 0:
self.encoder = nn.Linear(ntoken, ninp)
else:
self.encoder = None
ninp = nhid = ntoken
encoder_layers = TransformerEncoderLayer(ninp,
nhead,
nhid,
hdropout,
activation='gelu')
self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
self.ninp = ninp
self.mlp = mlp
if mlp:
self.lins = nn.ModuleList()
self.lins.append(nn.Linear(nhid, nhid * 4))
self.lins.append(nn.Linear(nhid * 4, nclasses))
self.bns = nn.ModuleList()
self.bns.append(nn.BatchNorm1d(nhid * 4))
self.hdropout = hdropout
else:
self.decoder = nn.Linear(ninp, nclasses)
self.dropout = nn.Dropout(p=idropout)
self.src_scale = src_scale
self.layer_norm = layer_norm
if layer_norm:
self.input_layer_norm = nn.LayerNorm(ninp)
def __init__(self, args, use_pretrained=True):
super(ImageTransformer, self).__init__()
from torch.nn import TransformerEncoder, TransformerEncoderLayer
self.args = args
# Embedding
if args.encoder == "resnext101":
if use_pretrained:
self.embedder = torch.hub.load("facebookresearch/WSL-Images",
"resnext101_32x8d_wsl")
else:
self.embedder = tvmodels.resnext101_32x8d(pretrained=False)
if self.args.freeze_encoder:
self.freeze_encoder()
self.units = args.units
if self.units == 2048:
self.embedder.fc = nn.Identity()
else:
self.embedder.fc = nn.Linear(2048, self.units)
for m in self.embedder.fc.modules():
if isinstance(m, nn.Linear):
nn.init.kaiming_normal_(m.weight)
if hasattr(m, "bias") and m.bias is not None:
nn.init.constant_(m.bias, 0)
# self.src_mask = None
self.position_encoder = PositionalEncoding1D(
self.units) # (ninp, dropout)
encoder_layer = TransformerEncoderLayer(d_model=self.units,
nhead=8,
dim_feedforward=2048,
dropout=0.1)
self.transformer_encoder = TransformerEncoder(
encoder_layer=encoder_layer, num_layers=2)
self.transformer_decoder = nn.Linear(self.units, self.args.num_classes)
def __init__(self, vocab_size, embedding_dim, max_seq_len, num_heads, dim_feedforward, num_layers, dropout=0.5):
super(MyAttentionModelWithPooling, self).__init__()
try:
from torch.nn import TransformerEncoder, TransformerEncoderLayer
except:
raise ImportError('TransformerEncoder module does not exist in PyTorch 1.1 or lower.')
self.model_type = 'Transformer'
self.src_mask = None
self.pos_encoder = attention_models.PositionalEncoding(embedding_dim, dropout=dropout, max_len=max_seq_len)
encoder_layers = TransformerEncoderLayer(embedding_dim, num_heads, dim_feedforward, dropout)
# output shape (batch_size, max_seq_len, embedding_dim)
self.transformer_encoder = TransformerEncoder(encoder_layers, num_layers)
self.encoder = nn.Embedding(vocab_size, embedding_dim)
self.embedding_dim = embedding_dim
self.max_seq_len = max_seq_len
self.decoder = nn.Linear(self.embedding_dim, 1)
self.decoder_act = nn.Sigmoid()
self.pooler = nn.AvgPool1d(max_seq_len, stride=1)
self.init_weights()
def __init__(self, config, metagraph, dendrite, device):
super(Validator, self).__init__()
self.layers = TransformerEncoderLayer(bittensor.__network_dim__,
config.nucleus.nhead,
config.nucleus.nhid,
config.nucleus.dropout,
batch_first=True)
self.encoder = TransformerEncoder(self.layers, config.nucleus.nlayers)
self.decoder = torch.nn.Linear(bittensor.__network_dim__,
bittensor.__vocab_size__,
bias=False)
self.loss_fct = torch.nn.CrossEntropyLoss()
self.peer_weights = torch.nn.Parameter(
torch.ones([metagraph().n.item()],
requires_grad=True,
device=device))
self.noise_offset = 0.0000001
self.metagraph = metagraph
self.dendrite = dendrite
self.config = config
self.device = device
def __init__(self,
ntoken,
nout,
ninp,
nhead,
nhid,
nlayers,
max_len,
dropout=0.0,
layer_norm=False):
super(TransformerEncoderModel, self).__init__()
from torch.nn import TransformerEncoder, TransformerEncoderLayer
self.pos_encoder = PositionalEncoding(ninp, dropout, max_len=max_len)
encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers, norm= \
nn.LayerNorm(normalized_shape=ninp, eps=1e-6) if layer_norm else None)
self.encoder = nn.Linear(ntoken, ninp)
self.ninp = ninp
self.decoder = nn.Linear(ninp, nout)
self.init_weights()
def __init__(self,
num_encoder_layers: int,
num_decoder_layers: int,
emb_size: int,
vocab_size: int,
dim_feedforward: int = 512,
dropout: float = 0.1):
super(Seq2SeqTransformer, self).__init__()
encoder_layer = TransformerEncoderLayer(
d_model=emb_size, nhead=NHEAD, dim_feedforward=dim_feedforward)
self.transformer_encoder = TransformerEncoder(
encoder_layer, num_layers=num_encoder_layers)
decoder_layer = TransformerDecoderLayer(
d_model=emb_size, nhead=NHEAD, dim_feedforward=dim_feedforward)
self.transformer_decoder = TransformerDecoder(
decoder_layer, num_layers=num_decoder_layers)
self.generator = nn.Linear(emb_size, vocab_size)
self.tok_emb = TokenEmbedding(vocab_size, emb_size)
self.positional_encoding = PositionalEncoding(emb_size,
dropout=dropout)
def __init__(self, num_inputs, num_outputs, num_layers, nhid = 2 , dropout=0.1, nhead=29):
# def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5):
super(TransformerController, self).__init__()
# print(num_inputs, num_outputs, num_layers)
self.num_inputs = num_inputs
self.num_outputs = num_outputs
self.num_layers = num_layers
#self.model_type = 'Transformer'
self.src_mask = None
self.pos_encoder = PositionalEncoding(num_inputs, dropout)
#print(num_inputs)
encoder_layers = TransformerEncoderLayer(num_inputs, nhead, nhid, dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers, num_layers)
#self.encoder = nn.Embedding(ntoken, num_inputs)
# print('num inputs',num_inputs)
# print('num outputs', num_outputs)
#self.decoder = nn.Linear(num_inputs, num_outputs)
self.decoder = nn.Linear(num_inputs, num_outputs)
self.init_weights()
def __init__(self, model_path, itos, device, nhead, nhid, nlayers,
dropout):
self.model = load_model(model_path)
input_matrix = self.model.get_input_matrix()
vocab_size, emb_size = input_matrix.shape # (vocab_size, emb_size)
super().__init__(vocab_size, emb_size)
self.weight.data.copy_(torch.FloatTensor(input_matrix))
from torch.nn import TransformerEncoder, TransformerEncoderLayer
self.emb_size = emb_size
self.pos_encoder = PositionalEncoding(emb_size, dropout)
encoder_layers = TransformerEncoderLayer(emb_size, nhead, nhid,
dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
self.itos = itos
self.device = device
self.src_mask = None
self.subinds_cache = dict()
def __init__(self, ntokens, ninp, nhead, dim_mlp, nlayers, dropout=0.5):
super(TransformerModel, self).__init__()
try:
from torch.nn import TransformerEncoder, TransformerEncoderLayer
except:
raise ImportError(
'TransformerEncoder module does not exist in PyTorch 1.1 or lower.'
)
self.model_type = 'Transformer'
self.src_mask = None
self.pos_encoder = PositionalEncoding(ninp, dropout)
encoder_layers = TransformerEncoderLayer(d_model=ninp,
nhead=nhead,
dim_feedforward=dim_mlp,
dropout=dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
self.encoder = nn.Embedding(ntokens, ninp)
self.ninp = ninp
self.decoder = nn.Linear(ninp, ntokens)
self.ntokens = ntokens
self.init_weights()
def __init__(self,
M,
n_meds,
n_covs,
sequence_len,
emsize,
nhead,
nhid,
nlayers,
n_mc_smps,
dropout=0.5):
super(TransformerModel, self).__init__()
from torch.nn import TransformerEncoder, TransformerEncoderLayer
self.model_type = 'Transformer'
self.src_mask = None
self.encoder = nn.Linear(M + n_meds, emsize)
self.pos_encoder = PositionalEncoding(emsize, dropout)
encoder_layers = TransformerEncoderLayer(emsize, nhead, nhid, dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
self.emsize = emsize
self.final = torch.rand(size=(sequence_len * (emsize + n_covs), ))
# GP parameters
self.M = M
self.n_meds = n_meds
self.n_mc_smps = n_mc_smps
self.sequence_len = sequence_len
self.n_covs = n_covs
self.emsize = emsize
self.log_length = torch.normal(size=[1], mean=1, std=0.1)
self.log_noises = torch.normal(size=[self.M], mean=-2, std=0.1)
self.L_f_init = torch.eye(self.M)
self.log_length = torch.nn.Parameter(self.log_length)
self.log_noises = torch.nn.Parameter(self.log_noises)
self.L_f_init = torch.nn.Parameter(self.L_f_init)
self.final = torch.nn.Parameter(self.final)
self.init_weights()
def __init__(self,
kg_graph_repr: Dict[str, np.ndarray],
config: dict,
id2e: tuple = None):
if id2e is not None:
super(self.__class__, self).__init__(kg_graph_repr, config,
id2e[1])
else:
super(self.__class__, self).__init__(kg_graph_repr, config)
self.model_name = 'StarE_Transformer_Statement'
self.hid_drop2 = config['STAREARGS']['HID_DROP2']
self.feat_drop = config['STAREARGS']['FEAT_DROP']
self.num_transformer_layers = config['STAREARGS']['T_LAYERS']
self.num_heads = config['STAREARGS']['T_N_HEADS']
self.num_hidden = config['STAREARGS']['T_HIDDEN']
self.d_model = config['EMBEDDING_DIM']
self.positional = config['STAREARGS']['POSITIONAL']
self.p_option = config['STAREARGS']['POS_OPTION']
self.pooling = config['STAREARGS']['POOLING'] # min / avg / concat
self.hidden_drop = torch.nn.Dropout(self.hid_drop)
self.hidden_drop2 = torch.nn.Dropout(self.hid_drop2)
self.feature_drop = torch.nn.Dropout(self.feat_drop)
encoder_layers = TransformerEncoderLayer(
self.d_model, self.num_heads, self.num_hidden,
config['STAREARGS']['HID_DROP2'])
self.encoder = TransformerEncoder(encoder_layers,
config['STAREARGS']['T_LAYERS'])
self.position_embeddings = nn.Embedding(config['MAX_QPAIRS'] - 1,
self.d_model)
self.layer_norm = torch.nn.LayerNorm(self.emb_dim)
if self.pooling == "concat":
self.flat_sz = self.emb_dim * (config['MAX_QPAIRS'] - 1)
self.fc = torch.nn.Linear(self.flat_sz, self.emb_dim)
else:
self.fc = torch.nn.Linear(self.emb_dim, self.emb_dim)
def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5):
super(TransformerModel, self).__init__()
try:
from torch.nn import TransformerEncoder, TransformerEncoderLayer
except:
raise ImportError('TransformerEncoder module does not exist in PyTorch 1.1 or lower')
self.model_type = 'Transformer'
self.src_mask = None
self.pos_encoder = PositionalEncoding(ninp, dropout)
encoder_layer = TransformerEncoderLayer(ninp, nhead, nhid, dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
self.encoder = nn.Embedding(ntoken, ninp)
self.ninp = ninp
self.decoder = nn.Linear(ninp, ntoken)
self.init_weights()
def _generate_square_subsequent_mask(self, sz):
mask = torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)
mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))
return mask
def init_weights(self):
initrange = 0.1
self.encoder.weight.data.uniform_(-initrange, initrange)
self.decoder.bias.data.zero_()
self.decoder.weight.data.uniform_(-initrange, initrange)
def forward(self, src, has_mask=True):
if has_mask:
device = src.device
if self.src_mask is None or self.src_mask.size(0) != len(src):
mask = self._generate_square_subsequent_mask(len(src)).to(device)
self.src_mask = mask
else:
self.src_mask = None
src = self.encoder(src) * math.sqrt(self.ninp)
src = self.pos_encoder(src)
output = self.transformer_encoder(src, self.src_mask)
output = self.decoder(output)
return F.log_softmax(output, dim=-1)
def __init__(self, extractor, config):
super(TK_class, self).__init__()
self.embeddim = extractor.embeddings.shape[1]
self.p = config
self.mus = torch.tensor(
[-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9, 1.0],
dtype=torch.float)
self.mu_matrix = self.get_mu_matrix(extractor)
self.sigma = torch.tensor(0.1, requires_grad=False)
dropout = 0
non_trainable = not self.p["finetune"]
self.embedding = create_emb_layer(extractor.embeddings,
non_trainable=non_trainable)
self.cosine_module = StackedSimilarityMatrix(padding=extractor.pad)
self.position_encoder = PositionalEncoding(self.embeddim)
self.mixer = nn.Parameter(
torch.full([1, 1, 1], 0.9, dtype=torch.float32,
requires_grad=True))
encoder_layers = TransformerEncoderLayer(self.embeddim,
config["numattheads"],
config["ffdim"], dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers,
config["numlayers"])
self.s_log_fcc = nn.Linear(len(self.mus), 1, bias=False)
self.s_len_fcc = nn.Linear(len(self.mus), 1, bias=False)
self.comb_fcc = nn.Linear(2, 1, bias=False)
# init with small weights, otherwise the dense output is way to high for the tanh -> resulting in loss == 1 all the time
torch.nn.init.uniform_(self.s_log_fcc.weight, -0.014,
0.014) # inits taken from matchzoo
torch.nn.init.uniform_(self.s_len_fcc.weight, -0.014,
0.014) # inits taken from matchzoo
# init with small weights, otherwise the dense output is way to high for the tanh -> resulting in loss == 1 all the time
torch.nn.init.uniform_(self.comb_fcc.weight, -0.014,
0.014) # inits taken from matchzoo
def __init__(self, O_CONFIG, use_cls_token=False, requires_grad=False):
super(TransformerNet, self).__init__()
self.embed_dim = O_CONFIG["embed_dim"]
self.dropout = O_CONFIG["dropout"]
self.max_len = O_CONFIG["max_len"]
self.num_labels = O_CONFIG["num_labels"]
self.head_num = O_CONFIG["head_num"]
self.dim_feedforward = O_CONFIG["dim_feedforward"]
self.encoder_layter_num = O_CONFIG["encoder_layter_num"]
self.src_mask = None
self.use_cls_token = use_cls_token
self.embedding = nn.Embedding.from_pretrained(
O_CONFIG["embed_pretrained"],
freeze=(not O_CONFIG["update_embed"]))
# self.embedding.weight = nn.Parameter(weights, requires_grad=requires_grad) # Assigning the look-up table to the pre-trained GloVe word embedding.
self.pos_encoder = PositionalEncoding(self.embed_dim, self.dropout)
encoder_layers = TransformerEncoderLayer(self.embed_dim, self.head_num,
self.dim_feedforward,
self.dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers,
self.encoder_layter_num)
if (use_cls_token == True):
print('Enable CLS token for classification... ')
self.cls_token_vector = torch.empty(self.embed_dim).uniform_(
-0.1, 0.1)
else:
print('Enable weighted sum hidden states for classification...')
self.weighted_sum_layer = nn.Linear(self.max_len, 1, bias=False)
self.linear = nn.Linear(self.embed_dim, self.num_labels)
self.softmax = nn.Softmax(dim=1)
self.init_weights()
def __init__(self,
embedding_size,
hidden_size,
output_size,
num_layers,
dropout,
device,
nhead,
sequence_length,
latent_size=8):
super(Classical_Music_Transformer, self).__init__()
self.model_type = 'Transformer'
self.hidden_size = hidden_size
self.device = device
self.sequence_length = sequence_length
self.embedding_size = embedding_size
self.pos_encoder = PositionalEncoding(embedding_size, dropout)
encoder_layers = TransformerEncoderLayer(embedding_size, nhead,
hidden_size, dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers,
num_layers)
self.encoder = nn.Embedding(output_size, embedding_size)
self.output_size = output_size
self.sequence_length = sequence_length
self.decoder = nn.Linear(embedding_size, output_size)
self.latent_size = latent_size
self.next_node_projection = nn.Sequential(
nn.Linear(embedding_size, embedding_size // 2), nn.ReLU(),
nn.Linear(embedding_size // 2, latent_size * 2))
self.expand = nn.Sequential(
nn.Linear(latent_size, embedding_size // 2), nn.ReLU(),
nn.Linear(embedding_size // 2, embedding_size))
self.stop_symbol = '/'
self._start_symbols = [self.stop_symbol] * sequence_length
self.init_weights()
def __init__(self, input_size, emb_size, nhead, nhid, nlayers):
"""
emb_size: Embedding Size for the input
ntoken: Number of tokens Vocab Size
nhead: Number of transformer heads in the encoder
nhid: Number of hidden units in transformer encoder layer
nlayer: Number of layers in transformer encoder
"""
super(TransformerModel, self).__init__()
from torch.nn import TransformerEncoder, TransformerEncoderLayer
# Initialized position input embedding, position encoding layers
# self.encoder = nn.Embedding(ntoken, emb_size)
self.linear = nn.Linear(input_size, emb_size)
self.pos_encoding = PositionalEncoding(emb_size)
# Initialized transformer encoder with nlayers and each layer having nhead heads and nhid hidden units.
encoder_layers = TransformerEncoderLayer(emb_size,
nhead,
nhid,
dropout=0.1)
self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
def __init__(self,
vocab_size,
feature_dim_size,
ff_hidden_size,
sampled_num,
num_self_att_layers,
dropout,
device,
num_heads,
num_neighbors,
initialization=None):
super(SANNE, self).__init__()
self.feature_dim_size = feature_dim_size
self.ff_hidden_size = ff_hidden_size
self.num_self_att_layers = num_self_att_layers
self.vocab_size = vocab_size
self.sampled_num = sampled_num
self.device = device
self.num_heads = num_heads
self.num_neighbors = num_neighbors
if initialization == None:
self.input_feature = nn.Embedding(self.vocab_size,
self.feature_dim_size)
nn.init.xavier_uniform_(self.input_feature.weight.data)
else:
self.input_feature = nn.Embedding.from_pretrained(initialization)
#
encoder_layers = TransformerEncoderLayer(
d_model=self.feature_dim_size,
nhead=1,
dim_feedforward=self.ff_hidden_size,
dropout=0.5) # embed_dim must be divisible by num_heads
self.transformer_encoder = TransformerEncoder(encoder_layers,
self.num_self_att_layers)
# Linear function
self.dropouts = nn.Dropout(dropout)
self.ss = SampledSoftmax(self.vocab_size, self.sampled_num,
self.feature_dim_size, self.device)
def __init__(self,
max_num_usages,
embedding_dim,
num_heads,
hidden_dim,
num_layers=1,
dropout=0.1):
"""
:param max_num_usages: the max number of usages to use,
:param embedding_dim: the dimension of usage embedding,
:param num_heads: the number of heads in multiHeadAttention,
:param hidden_dim: the dimension of the feedforward network after multiHeadAttention,
:param num_layers: the number of sub-encoder-layers,
:param dropout: the dropout value.
"""
super(UsageEncoder, self).__init__()
self.max_num_usages = max_num_usages
self.positional_encoder = PositionalEncoding(embedding_dim, dropout)
encoder_layer = TransformerEncoderLayer(embedding_dim, num_heads,
hidden_dim, dropout)
self.encoder = TransformerEncoder(encoder_layer, num_layers)
def __init__(self,
ntoken=None,
emsize=None,
nhid=None,
nlayers=None,
nhead=None,
dropout=None):
super(Transformer, self).__init__()
# Move this elsewhere (to dataloader)
ntoken = len(TEXT.vocab.stoi)
#emsize == n_inputs
self.model_type = 'Transformer'
self.src_mask = None
self.pos_encoder = PositionalEncoding(emsize, dropout)
encoder_layers = TransformerEncoderLayer(emsize, nhead, nhid, dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
self.encoder = nn.Embedding(ntoken, emsize)
self.ninp = emsize
self.decoder = nn.Linear(emsize, ntoken)
self.init_weights()
def __init__(self,
chan_in,
emsize,
nhead,
nhid,
nlayers,
dropout=0.1,
chan_out=-1,
stencil=7):
super(TransformerModel, self).__init__()
from torch.nn import TransformerEncoder, TransformerEncoderLayer
self.model_type = 'Transformer'
self.src_mask = None
self.pos_encoder = PositionalEncoding(emsize, dropout)
encoder_layers = TransformerEncoderLayer(emsize, nhead, nhid, dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
self.encoder = CNN(chan_in, 2 * chan_in, 3 * chan_in, emsize, stencil)
self.ninp = emsize
if chan_out < 0:
chan_out = chan_in
self.decoder = CNN(emsize, 2 * emsize, 3 * emsize, chan_out, stencil)
def __init__(self,
vocab_size,
num_tags,
embed_dim,
num_heads,
hid_dim,
num_layers,
dropout=0.5):
super(SequenceTaggingTransformer, self).__init__()
self.padder = SequencePadding(padding_value=vocab_size)
self.pos_encoder = PositionalEncoding(embed_dim, dropout)
encoder_layers = TransformerEncoderLayer(embed_dim, num_heads, hid_dim,
dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers,
num_layers)
self.embedder = nn.Embedding(vocab_size + 1,
embed_dim,
padding_idx=vocab_size)
self.embed_dim = embed_dim
self.decoder = nn.Linear(embed_dim, num_tags + 1)
self.init_weights()
def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0):
super(TransformerModel, self).__init__()
try:
from torch.nn import TransformerEncoder, TransformerEncoderLayer
except:
raise ImportError(
'TransformerEncoder module does not exist in PyTorch 1.1 or lower.'
)
self.model_type = 'Transformer'
self.src_mask = None
self.pos_encoder = PositionalEncoding(ninp, dropout)
# self.list_layer = nn.ModuleList(
# [TransformerEncoderLayer(ninp, nhead, nhid, dropout) for i in range(nlayers)]
# )
encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
self.encoder = nn.Embedding(ntoken, ninp)
self.ninp = ninp
self.decoder = nn.Linear(ninp, ntoken)
self.init_weights()
def __init__(self,
ntoken=None,
ninp=128,
nhead=8,
nhid=2048,
nlayers=8,
dropout=0.5):
#ntoken: len(dictionary)
#ninp : embedding dimension
#nhead: # of multiheadattention
#nhid : dim(feedforward network model)
#nlayer: # of nn.TransofrmerEncoderLayer
super(TransformerModel, self).__init__()
from torch.nn import TransformerEncoder, TransformerEncoderLayer, Transformer, TransformerDecoder, TransformerDecoderLayer
if not ntoken:
self.ntoken = len(load_object("engDictAnn.pkl")) + 2
else:
self.ntoken = ntoken
self.model_type = "Transformer"
self.pos_encoder = PositionalEncoding(ninp, dropout)
encdoer_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)
self.transformer_encoder = TransformerEncoder(encdoer_layers, 2)
#self.encoder = nn.Embedding(self.ntoken, ninp)
self.encoder = nn.Linear(2048, ninp)
self.ninp = ninp
#self.transformer_decoder = Transformer(d_model=2048, nhead=nhead, )
self.embedded = nn.Embedding(self.ntoken, ninp)
self.decoder = nn.Linear(ninp, self.ntoken)
decoder_layers = TransformerDecoderLayer(
d_model=ninp,
nhead=8,
)
self.transformer_decoder = TransformerDecoder(decoder_layers,
num_layers=4)
self.init_weights()
def __init__(self,
ntoken,
ninp,
nhead,
nhid,
nlayers,
tie_layers=True,
tie_encoder_decoder=True,
dropout=0.5,
**kwargs):
super(TransformerModel, self).__init__()
self.model_type = 'Transformer'
self.src_mask = None
self.pos_encoder = PositionalEncoding(ninp, dropout)
encoder_layers = TransformerEncoderLayer(ninp,
nhead,
nhid,
dropout,
activation='gelu')
if tie_layers:
self.transformer_encoder = TiedTransformerEncoder(
encoder_layers, nlayers)
else:
self.transformer_encoder = TransformerEncoder(
encoder_layers, nlayers)
self.encoder = nn.Embedding(ntoken, ninp)
self.ninp = ninp
self.tie_encoder_decoder = tie_encoder_decoder
if self.tie_encoder_decoder:
self.decoder = nn.Linear(ninp, ntoken)
self.decoder.weight = self.encoder.weight
else:
self.decoder = nn.Linear(ninp, ntoken)
self.init_weights()
def __init__(self,
vocab,
n_emb,
n_head,
n_hid,
n_layers,
batch_size,
dropout=0.5,
n_actions=5):
super(TextOnly, self).__init__()
self.vocab = vocab
self.n_actions = n_actions
self.n_vocab = len(vocab)
self.n_emb = n_emb
self.n_head = n_head
self.n_hid = n_hid
self.n_layers = n_layers
# action & reward buffer
self.rewards = [[] for bb in range(batch_size)]
self.saved_actions = [[] for bb in range(batch_size)]
self.batch_size = batch_size
self.pos_encoder = PositionalEncoding(n_emb, dropout)
encoder_layers = TransformerEncoderLayer(n_emb, n_head, n_hid, dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers, n_layers)
self.encoder = nn.Embedding(self.n_vocab, n_emb)
# enc -> location prediction
self.enc2pred = nn.Linear(n_emb, 400 * 400)
# actor's layer
self.action_head0 = nn.Linear(n_emb, 512)
self.action_head1 = nn.Linear(512, 256)
self.action_head2 = nn.Linear(256, 128)
self.action_head3 = nn.Linear(128, self.n_actions)
# critic's layer
self.value_head = nn.Linear(n_emb, 1)
def __init__(self,
n_speakers,
in_size,
n_heads,
n_units,
n_layers,
dim_feedforward=2048,
dropout=0.5,
has_pos=False):
""" Self-attention-based diarization model.
Args:
n_speakers (int): Number of speakers in recording
in_size (int): Dimension of input feature vector
n_heads (int): Number of attention heads
n_units (int): Number of units in a self-attention block
n_layers (int): Number of transformer-encoder layers
dropout (float): dropout ratio
"""
super(TransformerModel, self).__init__()
self.n_speakers = n_speakers
self.in_size = in_size
self.n_heads = n_heads
self.n_units = n_units
self.n_layers = n_layers
self.has_pos = has_pos
self.src_mask = None
self.encoder = nn.Linear(in_size, n_units)
self.encoder_norm = nn.LayerNorm(n_units)
if self.has_pos:
self.pos_encoder = PositionalEncoding(n_units, dropout)
encoder_layers = TransformerEncoderLayer(n_units, n_heads,
dim_feedforward, dropout)
self.transformer_encoder = TransformerEncoder(encoder_layers, n_layers)
self.decoder = nn.Linear(n_units, n_speakers)
self.init_weights()
