def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5): super(TransformerModel, self).__init__() from torch.nn import TransformerEncoder, TransformerEncoderLayer self.model_type = 'Transformer' self.pos_encoder = PositionalEncoding(ninp, dropout) encoder_layers = TransformerEncoderLayer( ninp, nhead, nhid, dropout) self.transformer_encoder = TransformerEncoder( encoder_layers, nlayers) self.encoder = nn.Embedding(ntoken, ninp) self.ninp = ninp self.decoder = nn.Linear(ninp, ntoken) self.init_weights()
def __init__(self, ntoken, emb_size, nhead, nhid, nlayers): """ emb_size: Embedding Size for the input ntoken: Number of tokens Vocab Size nhead: Number of transformer heads in the encoder nhid: Number of hidden units in transformer encoder layer nlayer: Number of layers in transformer encoder """ super(TransformerModel, self).__init__() from torch.nn import TransformerEncoder, TransformerEncoderLayer self.emb_size = emb_size self.ntoken = ntoken self.nhead = nhead self.nhid = nhid self.nlayer = nlayers self.ninp = emb_size ### NEED TO CHECK THIS """ 1. Initialize position input embedding, position encoding layers 2. Initialize transformer encoder with nlayers and each layer having nhead heads and nhid hidden units. 3. Decoder can be implemented directly on top of the encoder as a linear layer. To keep things simple, we are predicting one token for each of the input tokens. We can pad the input to have the same length as target to ensure we can generate all target tokens. You may experiment with a transformer decoder and use teacher forcing during training, but it is not necessary to do so. """ # 1 # ninp == emb_size self.pos_encoder = PositionalEncoding(emb_size) encoder_layers = TransformerEncoderLayer(emb_size, nhead, nhid) # 2 self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers) self.encoder = nn.Embedding(ntoken, emb_size) #3 self.decoder = nn.Linear(emb_size, ntoken) self.init_weights()
def __init__(self, input_vocab_size, target_vocab_size, d_model=512, nhead=8, dim_feedforward=2048, num_encoder_layers=6, num_decoder_layers=6, dropout=0.1, activation="relu"): super(TransformerModel, self).__init__() try: from torch.nn import TransformerEncoderLayer, TransformerDecoderLayer except: raise ImportError( 'TransformerEncoder module does not exist in PyTorch 1.1 or lower.' ) self.model_type = 'Transformer' encoder_layers = TransformerEncoderLayer(d_model, nhead, dim_feedforward, dropout, activation) encoder_norm = nn.LayerNorm(d_model) self.encoder = TransformerEncoder(encoder_layers, num_encoder_layers, d_model, input_vocab_size, dropout, encoder_norm) decoder_layers = TransformerDecoderLayer(d_model, nhead, dim_feedforward, dropout, activation) decoder_norm = nn.LayerNorm(d_model) self.decoder = TransformerDecoder(decoder_layers, num_decoder_layers, d_model, target_vocab_size, dropout, decoder_norm) self.linear = nn.Linear(d_model, target_vocab_size) self._reset_parameters() self.d_model = d_model self.nhead = nhead
def __init__(self): super().__init__() self.conv = nn.Sequential( nn.Conv1d(24, 96, 7, stride=2, padding=3, groups=24), nn.InstanceNorm1d(80), nn.ELU(inplace=True), nn.Conv1d(96, 128, 3, stride=1, padding=1), nn.BatchNorm1d(128), nn.ELU(inplace=True), ) self.elayer1 = nn.Sequential(ECABasicBlock(128, 128), ECABasicBlock(128, 128)) self.elayer2 = nn.Sequential(ECABasicBlock(128, 256), ECABasicBlock(256, 256)) self.elayer3 = nn.Sequential(ECABasicBlock(256, 512, stride=2)) self.elayer4 = nn.Sequential(ECABasicBlock(512, 1024, stride=2), nn.AvgPool1d(2)) encoder_layer = TransformerEncoderLayer( d_model=1024, nhead=8, dim_feedforward=2048, dropout=0.1, activation="relu", ) self.encoder = TransformerEncoder(encoder_layer, 8) self.decoder1 = ECABasicBlock(1024, 1536, stride=2) self.decoder2 = ECABasicBlock(1536, 2048) self.decoder3 = ECABasicBlock(2048, 3070, stride=2) self.decoder4 = ECABasicBlock(3070, 4094) self.fc = nn.Sequential( nn.AdaptiveAvgPool1d(1), nn.Flatten(), nn.Dropout(p=0.05), nn.Linear(4094, 1024), nn.ELU(inplace=True), nn.Dropout(p=0.05), nn.Linear(1024, 61), )
def __init__(self, ntoken, ninp, nhead, nhid, nlayers, nclasses, idropout=0.1, hdropout=0.5, layer_norm=0, src_scale=0, mlp=0): super(TransformerModel, self).__init__() self.model_type = 'Transformer' if ninp > 0: self.encoder = nn.Linear(ntoken, ninp) else: self.encoder = None ninp = nhid = ntoken encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, hdropout, activation='gelu') self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers) self.ninp = ninp self.mlp = mlp if mlp: self.lins = nn.ModuleList() self.lins.append(nn.Linear(nhid, nhid * 4)) self.lins.append(nn.Linear(nhid * 4, nclasses)) self.bns = nn.ModuleList() self.bns.append(nn.BatchNorm1d(nhid * 4)) self.hdropout = hdropout else: self.decoder = nn.Linear(ninp, nclasses) self.dropout = nn.Dropout(p=idropout) self.src_scale = src_scale self.layer_norm = layer_norm if layer_norm: self.input_layer_norm = nn.LayerNorm(ninp)
def __init__(self, args, use_pretrained=True): super(ImageTransformer, self).__init__() from torch.nn import TransformerEncoder, TransformerEncoderLayer self.args = args # Embedding if args.encoder == "resnext101": if use_pretrained: self.embedder = torch.hub.load("facebookresearch/WSL-Images", "resnext101_32x8d_wsl") else: self.embedder = tvmodels.resnext101_32x8d(pretrained=False) if self.args.freeze_encoder: self.freeze_encoder() self.units = args.units if self.units == 2048: self.embedder.fc = nn.Identity() else: self.embedder.fc = nn.Linear(2048, self.units) for m in self.embedder.fc.modules(): if isinstance(m, nn.Linear): nn.init.kaiming_normal_(m.weight) if hasattr(m, "bias") and m.bias is not None: nn.init.constant_(m.bias, 0) # self.src_mask = None self.position_encoder = PositionalEncoding1D( self.units) # (ninp, dropout) encoder_layer = TransformerEncoderLayer(d_model=self.units, nhead=8, dim_feedforward=2048, dropout=0.1) self.transformer_encoder = TransformerEncoder( encoder_layer=encoder_layer, num_layers=2) self.transformer_decoder = nn.Linear(self.units, self.args.num_classes)
def __init__(self, vocab_size, embedding_dim, max_seq_len, num_heads, dim_feedforward, num_layers, dropout=0.5): super(MyAttentionModelWithPooling, self).__init__() try: from torch.nn import TransformerEncoder, TransformerEncoderLayer except: raise ImportError('TransformerEncoder module does not exist in PyTorch 1.1 or lower.') self.model_type = 'Transformer' self.src_mask = None self.pos_encoder = attention_models.PositionalEncoding(embedding_dim, dropout=dropout, max_len=max_seq_len) encoder_layers = TransformerEncoderLayer(embedding_dim, num_heads, dim_feedforward, dropout) # output shape (batch_size, max_seq_len, embedding_dim) self.transformer_encoder = TransformerEncoder(encoder_layers, num_layers) self.encoder = nn.Embedding(vocab_size, embedding_dim) self.embedding_dim = embedding_dim self.max_seq_len = max_seq_len self.decoder = nn.Linear(self.embedding_dim, 1) self.decoder_act = nn.Sigmoid() self.pooler = nn.AvgPool1d(max_seq_len, stride=1) self.init_weights()
def __init__(self, config, metagraph, dendrite, device): super(Validator, self).__init__() self.layers = TransformerEncoderLayer(bittensor.__network_dim__, config.nucleus.nhead, config.nucleus.nhid, config.nucleus.dropout, batch_first=True) self.encoder = TransformerEncoder(self.layers, config.nucleus.nlayers) self.decoder = torch.nn.Linear(bittensor.__network_dim__, bittensor.__vocab_size__, bias=False) self.loss_fct = torch.nn.CrossEntropyLoss() self.peer_weights = torch.nn.Parameter( torch.ones([metagraph().n.item()], requires_grad=True, device=device)) self.noise_offset = 0.0000001 self.metagraph = metagraph self.dendrite = dendrite self.config = config self.device = device
def __init__(self, ntoken, nout, ninp, nhead, nhid, nlayers, max_len, dropout=0.0, layer_norm=False): super(TransformerEncoderModel, self).__init__() from torch.nn import TransformerEncoder, TransformerEncoderLayer self.pos_encoder = PositionalEncoding(ninp, dropout, max_len=max_len) encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout) self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers, norm= \ nn.LayerNorm(normalized_shape=ninp, eps=1e-6) if layer_norm else None) self.encoder = nn.Linear(ntoken, ninp) self.ninp = ninp self.decoder = nn.Linear(ninp, nout) self.init_weights()
def __init__(self, num_encoder_layers: int, num_decoder_layers: int, emb_size: int, vocab_size: int, dim_feedforward: int = 512, dropout: float = 0.1): super(Seq2SeqTransformer, self).__init__() encoder_layer = TransformerEncoderLayer( d_model=emb_size, nhead=NHEAD, dim_feedforward=dim_feedforward) self.transformer_encoder = TransformerEncoder( encoder_layer, num_layers=num_encoder_layers) decoder_layer = TransformerDecoderLayer( d_model=emb_size, nhead=NHEAD, dim_feedforward=dim_feedforward) self.transformer_decoder = TransformerDecoder( decoder_layer, num_layers=num_decoder_layers) self.generator = nn.Linear(emb_size, vocab_size) self.tok_emb = TokenEmbedding(vocab_size, emb_size) self.positional_encoding = PositionalEncoding(emb_size, dropout=dropout)
def __init__(self, num_inputs, num_outputs, num_layers, nhid = 2 , dropout=0.1, nhead=29): # def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5): super(TransformerController, self).__init__() # print(num_inputs, num_outputs, num_layers) self.num_inputs = num_inputs self.num_outputs = num_outputs self.num_layers = num_layers #self.model_type = 'Transformer' self.src_mask = None self.pos_encoder = PositionalEncoding(num_inputs, dropout) #print(num_inputs) encoder_layers = TransformerEncoderLayer(num_inputs, nhead, nhid, dropout) self.transformer_encoder = TransformerEncoder(encoder_layers, num_layers) #self.encoder = nn.Embedding(ntoken, num_inputs) # print('num inputs',num_inputs) # print('num outputs', num_outputs) #self.decoder = nn.Linear(num_inputs, num_outputs) self.decoder = nn.Linear(num_inputs, num_outputs) self.init_weights()
def __init__(self, model_path, itos, device, nhead, nhid, nlayers, dropout): self.model = load_model(model_path) input_matrix = self.model.get_input_matrix() vocab_size, emb_size = input_matrix.shape # (vocab_size, emb_size) super().__init__(vocab_size, emb_size) self.weight.data.copy_(torch.FloatTensor(input_matrix)) from torch.nn import TransformerEncoder, TransformerEncoderLayer self.emb_size = emb_size self.pos_encoder = PositionalEncoding(emb_size, dropout) encoder_layers = TransformerEncoderLayer(emb_size, nhead, nhid, dropout) self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers) self.itos = itos self.device = device self.src_mask = None self.subinds_cache = dict()
def __init__(self, ntokens, ninp, nhead, dim_mlp, nlayers, dropout=0.5): super(TransformerModel, self).__init__() try: from torch.nn import TransformerEncoder, TransformerEncoderLayer except: raise ImportError( 'TransformerEncoder module does not exist in PyTorch 1.1 or lower.' ) self.model_type = 'Transformer' self.src_mask = None self.pos_encoder = PositionalEncoding(ninp, dropout) encoder_layers = TransformerEncoderLayer(d_model=ninp, nhead=nhead, dim_feedforward=dim_mlp, dropout=dropout) self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers) self.encoder = nn.Embedding(ntokens, ninp) self.ninp = ninp self.decoder = nn.Linear(ninp, ntokens) self.ntokens = ntokens self.init_weights()
def __init__(self, M, n_meds, n_covs, sequence_len, emsize, nhead, nhid, nlayers, n_mc_smps, dropout=0.5): super(TransformerModel, self).__init__() from torch.nn import TransformerEncoder, TransformerEncoderLayer self.model_type = 'Transformer' self.src_mask = None self.encoder = nn.Linear(M + n_meds, emsize) self.pos_encoder = PositionalEncoding(emsize, dropout) encoder_layers = TransformerEncoderLayer(emsize, nhead, nhid, dropout) self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers) self.emsize = emsize self.final = torch.rand(size=(sequence_len * (emsize + n_covs), )) # GP parameters self.M = M self.n_meds = n_meds self.n_mc_smps = n_mc_smps self.sequence_len = sequence_len self.n_covs = n_covs self.emsize = emsize self.log_length = torch.normal(size=[1], mean=1, std=0.1) self.log_noises = torch.normal(size=[self.M], mean=-2, std=0.1) self.L_f_init = torch.eye(self.M) self.log_length = torch.nn.Parameter(self.log_length) self.log_noises = torch.nn.Parameter(self.log_noises) self.L_f_init = torch.nn.Parameter(self.L_f_init) self.final = torch.nn.Parameter(self.final) self.init_weights()
def __init__(self, kg_graph_repr: Dict[str, np.ndarray], config: dict, id2e: tuple = None): if id2e is not None: super(self.__class__, self).__init__(kg_graph_repr, config, id2e[1]) else: super(self.__class__, self).__init__(kg_graph_repr, config) self.model_name = 'StarE_Transformer_Statement' self.hid_drop2 = config['STAREARGS']['HID_DROP2'] self.feat_drop = config['STAREARGS']['FEAT_DROP'] self.num_transformer_layers = config['STAREARGS']['T_LAYERS'] self.num_heads = config['STAREARGS']['T_N_HEADS'] self.num_hidden = config['STAREARGS']['T_HIDDEN'] self.d_model = config['EMBEDDING_DIM'] self.positional = config['STAREARGS']['POSITIONAL'] self.p_option = config['STAREARGS']['POS_OPTION'] self.pooling = config['STAREARGS']['POOLING'] # min / avg / concat self.hidden_drop = torch.nn.Dropout(self.hid_drop) self.hidden_drop2 = torch.nn.Dropout(self.hid_drop2) self.feature_drop = torch.nn.Dropout(self.feat_drop) encoder_layers = TransformerEncoderLayer( self.d_model, self.num_heads, self.num_hidden, config['STAREARGS']['HID_DROP2']) self.encoder = TransformerEncoder(encoder_layers, config['STAREARGS']['T_LAYERS']) self.position_embeddings = nn.Embedding(config['MAX_QPAIRS'] - 1, self.d_model) self.layer_norm = torch.nn.LayerNorm(self.emb_dim) if self.pooling == "concat": self.flat_sz = self.emb_dim * (config['MAX_QPAIRS'] - 1) self.fc = torch.nn.Linear(self.flat_sz, self.emb_dim) else: self.fc = torch.nn.Linear(self.emb_dim, self.emb_dim)
def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5): super(TransformerModel, self).__init__() try: from torch.nn import TransformerEncoder, TransformerEncoderLayer except: raise ImportError('TransformerEncoder module does not exist in PyTorch 1.1 or lower') self.model_type = 'Transformer' self.src_mask = None self.pos_encoder = PositionalEncoding(ninp, dropout) encoder_layer = TransformerEncoderLayer(ninp, nhead, nhid, dropout) self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers) self.encoder = nn.Embedding(ntoken, ninp) self.ninp = ninp self.decoder = nn.Linear(ninp, ntoken) self.init_weights() def _generate_square_subsequent_mask(self, sz): mask = torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1) mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0)) return mask def init_weights(self): initrange = 0.1 self.encoder.weight.data.uniform_(-initrange, initrange) self.decoder.bias.data.zero_() self.decoder.weight.data.uniform_(-initrange, initrange) def forward(self, src, has_mask=True): if has_mask: device = src.device if self.src_mask is None or self.src_mask.size(0) != len(src): mask = self._generate_square_subsequent_mask(len(src)).to(device) self.src_mask = mask else: self.src_mask = None src = self.encoder(src) * math.sqrt(self.ninp) src = self.pos_encoder(src) output = self.transformer_encoder(src, self.src_mask) output = self.decoder(output) return F.log_softmax(output, dim=-1)
def __init__(self, extractor, config): super(TK_class, self).__init__() self.embeddim = extractor.embeddings.shape[1] self.p = config self.mus = torch.tensor( [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9, 1.0], dtype=torch.float) self.mu_matrix = self.get_mu_matrix(extractor) self.sigma = torch.tensor(0.1, requires_grad=False) dropout = 0 non_trainable = not self.p["finetune"] self.embedding = create_emb_layer(extractor.embeddings, non_trainable=non_trainable) self.cosine_module = StackedSimilarityMatrix(padding=extractor.pad) self.position_encoder = PositionalEncoding(self.embeddim) self.mixer = nn.Parameter( torch.full([1, 1, 1], 0.9, dtype=torch.float32, requires_grad=True)) encoder_layers = TransformerEncoderLayer(self.embeddim, config["numattheads"], config["ffdim"], dropout) self.transformer_encoder = TransformerEncoder(encoder_layers, config["numlayers"]) self.s_log_fcc = nn.Linear(len(self.mus), 1, bias=False) self.s_len_fcc = nn.Linear(len(self.mus), 1, bias=False) self.comb_fcc = nn.Linear(2, 1, bias=False) # init with small weights, otherwise the dense output is way to high for the tanh -> resulting in loss == 1 all the time torch.nn.init.uniform_(self.s_log_fcc.weight, -0.014, 0.014) # inits taken from matchzoo torch.nn.init.uniform_(self.s_len_fcc.weight, -0.014, 0.014) # inits taken from matchzoo # init with small weights, otherwise the dense output is way to high for the tanh -> resulting in loss == 1 all the time torch.nn.init.uniform_(self.comb_fcc.weight, -0.014, 0.014) # inits taken from matchzoo
def __init__(self, O_CONFIG, use_cls_token=False, requires_grad=False): super(TransformerNet, self).__init__() self.embed_dim = O_CONFIG["embed_dim"] self.dropout = O_CONFIG["dropout"] self.max_len = O_CONFIG["max_len"] self.num_labels = O_CONFIG["num_labels"] self.head_num = O_CONFIG["head_num"] self.dim_feedforward = O_CONFIG["dim_feedforward"] self.encoder_layter_num = O_CONFIG["encoder_layter_num"] self.src_mask = None self.use_cls_token = use_cls_token self.embedding = nn.Embedding.from_pretrained( O_CONFIG["embed_pretrained"], freeze=(not O_CONFIG["update_embed"])) # self.embedding.weight = nn.Parameter(weights, requires_grad=requires_grad) # Assigning the look-up table to the pre-trained GloVe word embedding. self.pos_encoder = PositionalEncoding(self.embed_dim, self.dropout) encoder_layers = TransformerEncoderLayer(self.embed_dim, self.head_num, self.dim_feedforward, self.dropout) self.transformer_encoder = TransformerEncoder(encoder_layers, self.encoder_layter_num) if (use_cls_token == True): print('Enable CLS token for classification... ') self.cls_token_vector = torch.empty(self.embed_dim).uniform_( -0.1, 0.1) else: print('Enable weighted sum hidden states for classification...') self.weighted_sum_layer = nn.Linear(self.max_len, 1, bias=False) self.linear = nn.Linear(self.embed_dim, self.num_labels) self.softmax = nn.Softmax(dim=1) self.init_weights()
def __init__(self, embedding_size, hidden_size, output_size, num_layers, dropout, device, nhead, sequence_length, latent_size=8): super(Classical_Music_Transformer, self).__init__() self.model_type = 'Transformer' self.hidden_size = hidden_size self.device = device self.sequence_length = sequence_length self.embedding_size = embedding_size self.pos_encoder = PositionalEncoding(embedding_size, dropout) encoder_layers = TransformerEncoderLayer(embedding_size, nhead, hidden_size, dropout) self.transformer_encoder = TransformerEncoder(encoder_layers, num_layers) self.encoder = nn.Embedding(output_size, embedding_size) self.output_size = output_size self.sequence_length = sequence_length self.decoder = nn.Linear(embedding_size, output_size) self.latent_size = latent_size self.next_node_projection = nn.Sequential( nn.Linear(embedding_size, embedding_size // 2), nn.ReLU(), nn.Linear(embedding_size // 2, latent_size * 2)) self.expand = nn.Sequential( nn.Linear(latent_size, embedding_size // 2), nn.ReLU(), nn.Linear(embedding_size // 2, embedding_size)) self.stop_symbol = '/' self._start_symbols = [self.stop_symbol] * sequence_length self.init_weights()
def __init__(self, input_size, emb_size, nhead, nhid, nlayers): """ emb_size: Embedding Size for the input ntoken: Number of tokens Vocab Size nhead: Number of transformer heads in the encoder nhid: Number of hidden units in transformer encoder layer nlayer: Number of layers in transformer encoder """ super(TransformerModel, self).__init__() from torch.nn import TransformerEncoder, TransformerEncoderLayer # Initialized position input embedding, position encoding layers # self.encoder = nn.Embedding(ntoken, emb_size) self.linear = nn.Linear(input_size, emb_size) self.pos_encoding = PositionalEncoding(emb_size) # Initialized transformer encoder with nlayers and each layer having nhead heads and nhid hidden units. encoder_layers = TransformerEncoderLayer(emb_size, nhead, nhid, dropout=0.1) self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
def __init__(self, vocab_size, feature_dim_size, ff_hidden_size, sampled_num, num_self_att_layers, dropout, device, num_heads, num_neighbors, initialization=None): super(SANNE, self).__init__() self.feature_dim_size = feature_dim_size self.ff_hidden_size = ff_hidden_size self.num_self_att_layers = num_self_att_layers self.vocab_size = vocab_size self.sampled_num = sampled_num self.device = device self.num_heads = num_heads self.num_neighbors = num_neighbors if initialization == None: self.input_feature = nn.Embedding(self.vocab_size, self.feature_dim_size) nn.init.xavier_uniform_(self.input_feature.weight.data) else: self.input_feature = nn.Embedding.from_pretrained(initialization) # encoder_layers = TransformerEncoderLayer( d_model=self.feature_dim_size, nhead=1, dim_feedforward=self.ff_hidden_size, dropout=0.5) # embed_dim must be divisible by num_heads self.transformer_encoder = TransformerEncoder(encoder_layers, self.num_self_att_layers) # Linear function self.dropouts = nn.Dropout(dropout) self.ss = SampledSoftmax(self.vocab_size, self.sampled_num, self.feature_dim_size, self.device)
def __init__(self, max_num_usages, embedding_dim, num_heads, hidden_dim, num_layers=1, dropout=0.1): """ :param max_num_usages: the max number of usages to use, :param embedding_dim: the dimension of usage embedding, :param num_heads: the number of heads in multiHeadAttention, :param hidden_dim: the dimension of the feedforward network after multiHeadAttention, :param num_layers: the number of sub-encoder-layers, :param dropout: the dropout value. """ super(UsageEncoder, self).__init__() self.max_num_usages = max_num_usages self.positional_encoder = PositionalEncoding(embedding_dim, dropout) encoder_layer = TransformerEncoderLayer(embedding_dim, num_heads, hidden_dim, dropout) self.encoder = TransformerEncoder(encoder_layer, num_layers)
def __init__(self, ntoken=None, emsize=None, nhid=None, nlayers=None, nhead=None, dropout=None): super(Transformer, self).__init__() # Move this elsewhere (to dataloader) ntoken = len(TEXT.vocab.stoi) #emsize == n_inputs self.model_type = 'Transformer' self.src_mask = None self.pos_encoder = PositionalEncoding(emsize, dropout) encoder_layers = TransformerEncoderLayer(emsize, nhead, nhid, dropout) self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers) self.encoder = nn.Embedding(ntoken, emsize) self.ninp = emsize self.decoder = nn.Linear(emsize, ntoken) self.init_weights()
def __init__(self, chan_in, emsize, nhead, nhid, nlayers, dropout=0.1, chan_out=-1, stencil=7): super(TransformerModel, self).__init__() from torch.nn import TransformerEncoder, TransformerEncoderLayer self.model_type = 'Transformer' self.src_mask = None self.pos_encoder = PositionalEncoding(emsize, dropout) encoder_layers = TransformerEncoderLayer(emsize, nhead, nhid, dropout) self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers) self.encoder = CNN(chan_in, 2 * chan_in, 3 * chan_in, emsize, stencil) self.ninp = emsize if chan_out < 0: chan_out = chan_in self.decoder = CNN(emsize, 2 * emsize, 3 * emsize, chan_out, stencil)
def __init__(self, vocab_size, num_tags, embed_dim, num_heads, hid_dim, num_layers, dropout=0.5): super(SequenceTaggingTransformer, self).__init__() self.padder = SequencePadding(padding_value=vocab_size) self.pos_encoder = PositionalEncoding(embed_dim, dropout) encoder_layers = TransformerEncoderLayer(embed_dim, num_heads, hid_dim, dropout) self.transformer_encoder = TransformerEncoder(encoder_layers, num_layers) self.embedder = nn.Embedding(vocab_size + 1, embed_dim, padding_idx=vocab_size) self.embed_dim = embed_dim self.decoder = nn.Linear(embed_dim, num_tags + 1) self.init_weights()
def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0): super(TransformerModel, self).__init__() try: from torch.nn import TransformerEncoder, TransformerEncoderLayer except: raise ImportError( 'TransformerEncoder module does not exist in PyTorch 1.1 or lower.' ) self.model_type = 'Transformer' self.src_mask = None self.pos_encoder = PositionalEncoding(ninp, dropout) # self.list_layer = nn.ModuleList( # [TransformerEncoderLayer(ninp, nhead, nhid, dropout) for i in range(nlayers)] # ) encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout) self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers) self.encoder = nn.Embedding(ntoken, ninp) self.ninp = ninp self.decoder = nn.Linear(ninp, ntoken) self.init_weights()
def __init__(self, ntoken=None, ninp=128, nhead=8, nhid=2048, nlayers=8, dropout=0.5): #ntoken: len(dictionary) #ninp : embedding dimension #nhead: # of multiheadattention #nhid : dim(feedforward network model) #nlayer: # of nn.TransofrmerEncoderLayer super(TransformerModel, self).__init__() from torch.nn import TransformerEncoder, TransformerEncoderLayer, Transformer, TransformerDecoder, TransformerDecoderLayer if not ntoken: self.ntoken = len(load_object("engDictAnn.pkl")) + 2 else: self.ntoken = ntoken self.model_type = "Transformer" self.pos_encoder = PositionalEncoding(ninp, dropout) encdoer_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout) self.transformer_encoder = TransformerEncoder(encdoer_layers, 2) #self.encoder = nn.Embedding(self.ntoken, ninp) self.encoder = nn.Linear(2048, ninp) self.ninp = ninp #self.transformer_decoder = Transformer(d_model=2048, nhead=nhead, ) self.embedded = nn.Embedding(self.ntoken, ninp) self.decoder = nn.Linear(ninp, self.ntoken) decoder_layers = TransformerDecoderLayer( d_model=ninp, nhead=8, ) self.transformer_decoder = TransformerDecoder(decoder_layers, num_layers=4) self.init_weights()
def __init__(self, ntoken, ninp, nhead, nhid, nlayers, tie_layers=True, tie_encoder_decoder=True, dropout=0.5, **kwargs): super(TransformerModel, self).__init__() self.model_type = 'Transformer' self.src_mask = None self.pos_encoder = PositionalEncoding(ninp, dropout) encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout, activation='gelu') if tie_layers: self.transformer_encoder = TiedTransformerEncoder( encoder_layers, nlayers) else: self.transformer_encoder = TransformerEncoder( encoder_layers, nlayers) self.encoder = nn.Embedding(ntoken, ninp) self.ninp = ninp self.tie_encoder_decoder = tie_encoder_decoder if self.tie_encoder_decoder: self.decoder = nn.Linear(ninp, ntoken) self.decoder.weight = self.encoder.weight else: self.decoder = nn.Linear(ninp, ntoken) self.init_weights()
def __init__(self, vocab, n_emb, n_head, n_hid, n_layers, batch_size, dropout=0.5, n_actions=5): super(TextOnly, self).__init__() self.vocab = vocab self.n_actions = n_actions self.n_vocab = len(vocab) self.n_emb = n_emb self.n_head = n_head self.n_hid = n_hid self.n_layers = n_layers # action & reward buffer self.rewards = [[] for bb in range(batch_size)] self.saved_actions = [[] for bb in range(batch_size)] self.batch_size = batch_size self.pos_encoder = PositionalEncoding(n_emb, dropout) encoder_layers = TransformerEncoderLayer(n_emb, n_head, n_hid, dropout) self.transformer_encoder = TransformerEncoder(encoder_layers, n_layers) self.encoder = nn.Embedding(self.n_vocab, n_emb) # enc -> location prediction self.enc2pred = nn.Linear(n_emb, 400 * 400) # actor's layer self.action_head0 = nn.Linear(n_emb, 512) self.action_head1 = nn.Linear(512, 256) self.action_head2 = nn.Linear(256, 128) self.action_head3 = nn.Linear(128, self.n_actions) # critic's layer self.value_head = nn.Linear(n_emb, 1)
def __init__(self, n_speakers, in_size, n_heads, n_units, n_layers, dim_feedforward=2048, dropout=0.5, has_pos=False): """ Self-attention-based diarization model. Args: n_speakers (int): Number of speakers in recording in_size (int): Dimension of input feature vector n_heads (int): Number of attention heads n_units (int): Number of units in a self-attention block n_layers (int): Number of transformer-encoder layers dropout (float): dropout ratio """ super(TransformerModel, self).__init__() self.n_speakers = n_speakers self.in_size = in_size self.n_heads = n_heads self.n_units = n_units self.n_layers = n_layers self.has_pos = has_pos self.src_mask = None self.encoder = nn.Linear(in_size, n_units) self.encoder_norm = nn.LayerNorm(n_units) if self.has_pos: self.pos_encoder = PositionalEncoding(n_units, dropout) encoder_layers = TransformerEncoderLayer(n_units, n_heads, dim_feedforward, dropout) self.transformer_encoder = TransformerEncoder(encoder_layers, n_layers) self.decoder = nn.Linear(n_units, n_speakers) self.init_weights()