def __init__(self, input_size, hidden_size, num_layers): super(TorchRNN, self).__init__() self.rnn = nn.RNN(input_size=input_size, hidden_size=hidden_size, num_layers=num_layers) self.softmax = nn.LogSoftmax(dim=1)
def __init__(self,input_dim, embedding_dim, hidden_dim, output_dim): super().__init__() self.embedding = nn.Embedding(input_dim, embedding_dim) self.rnn = nn.RNN(embedding_dim, hidden_dim) self.fc = nn.Linear(hidden_dim,output_dim)
import torch import torch.nn as nn from torch.autograd import Variable # One hot encoding for each char in 'hello' h = [1, 0, 0, 0] e = [0, 1, 0, 0] l = [0, 0, 1, 0] o = [0, 0, 0, 1] # One cell RNN input_dim (4) -> output_dim (2). sequence: 5 cell = nn.RNN(input_size=4, hidden_size=2, batch_first=True) # (num_layers * num_directions, batch, hidden_size) # (batch, num_layers * num_directions, hidden_size) for batch_first=True hidden = (Variable(torch.randn(1, 1, 2))) # Propagate input through RNN # Input: (batch, seq_len, input_size) when batch_first=True inputs = Variable(torch.Tensor([h, e, l, l, o])) for one in inputs: one = one.view(1, 1, -1) # Input: (batch, seq_len, input_size) when batch_first=True out, hidden = cell(one, hidden) print("one input size", one.size(), "out size", out.size()) # We can do the whole at once # Propagate input through RNN # Input: (batch, seq_len, input_size) when batch_first=True inputs = inputs.view(1, 5, -1) out, hidden = cell(inputs, hidden)
def __init__(self, hidden_size): super(RNN, self).__init__() self.rnn = nn.RNN(300, hidden_size, 1) self.linear = nn.Linear(hidden_size,1) self.sigmoid = nn.Sigmoid()
import time import math import numpy as np import torch from torch import nn, optim import torch.functional as F import d2lzh_pytorch as d2l device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') (corpus_indices, char_to_idx, idx_to_char, vocab_size) = d2l.load_data_jay_lyrics() num_hiddens = 256 rnn_layer = nn.RNN(input_size = vocab_size, hidden_size = num_hiddens) # num_steps = 35 # batch_size = 2 # state = None # X = torch.rand(num_steps, batch_size, vocab_size) # Y, state_new = rnn_layer(X, state) # print(Y.shape, len(state_new), state_new[0].shape) class RNNModel(nn.Module): def __init__(self, rnn_layer, vocab_size): super(RNNModel, self).__init__() self.rnn = rnn_layer self.hidden_size = rnn_layer.hidden_size * (2 if rnn_layer.bidirectional else 1) self.vocab_size = vocab_size self.dense = nn.Linear(self.hidden_size, vocab_size) self.state = None def forward(self, inputs, state): # inputs:(batch,seq_len)
def __init__(self, input_size, hidden_size, output_size, rnn_type='lstm', num_layers=1, num_hidden_layers=2, bias=True, batch_first=True, batch_size=32, dropout=0, bidirectional=False, nr_cells=5, read_heads=2, cell_size=10, nonlinearity='tanh', gpu_id=-1, independent_linears=False, share_memory=True, debug=False, define_layers=True, clip=20): super(DNC, self).__init__() # todo: separate weights and RNNs for the interface and output vectors self.input_size = input_size self.real_output_size = output_size self.hidden_size = hidden_size self.rnn_type = rnn_type self.num_layers = num_layers self.num_hidden_layers = num_hidden_layers self.bias = bias self.batch_first = batch_first self.batch_size = batch_size self.dropout = dropout self.bidirectional = bidirectional self.nr_cells = nr_cells self.read_heads = read_heads self.cell_size = cell_size self.nonlinearity = nonlinearity self.gpu_id = gpu_id self.independent_linears = independent_linears self.share_memory = share_memory self.debug = debug self.clip = clip self.define_layers = define_layers self.w = self.cell_size self.r = self.read_heads self.read_vectors_size = self.r * self.w self.output_size = self.hidden_size self.nn_input_size = self.input_size + self.read_vectors_size self.nn_output_size = self.output_size + self.read_vectors_size self.rnns = [] self.memories = [] if self.define_layers: for layer in range(self.num_layers): if self.rnn_type.lower() == 'rnn': self.rnns.append( nn.RNN((self.input_size if layer == 0 else self.nn_output_size), self.output_size, bias=self.bias, nonlinearity=self.nonlinearity, batch_first=self.batch_first, dropout=self.dropout, num_layers=self.num_hidden_layers)) elif self.rnn_type.lower() == 'gru': self.rnns.append( nn.GRU((self.input_size if layer == 0 else self.nn_output_size), self.output_size, bias=self.bias, batch_first=self.batch_first, dropout=self.dropout, num_layers=self.num_hidden_layers)) if self.rnn_type.lower() == 'lstm': self.rnns.append( nn.LSTM((self.input_size if layer == 0 else self.nn_output_size), self.output_size, bias=self.bias, batch_first=self.batch_first, dropout=self.dropout, num_layers=self.num_hidden_layers)) setattr(self, self.rnn_type.lower() + '_layer_' + str(layer), self.rnns[layer]) # memories for each layer if not self.share_memory: self.memories.append( Memory(input_size=self.output_size, mem_size=self.nr_cells, cell_size=self.w, read_heads=self.r, gpu_id=self.gpu_id, independent_linears=self.independent_linears)) setattr(self, 'rnn_layer_memory_' + str(layer), self.memories[layer]) # only one memory shared by all layers if self.share_memory: self.memories.append( Memory(input_size=self.output_size, mem_size=self.nr_cells, cell_size=self.w, read_heads=self.r, gpu_id=self.gpu_id, independent_linears=self.independent_linears)) setattr(self, 'rnn_layer_memory_shared', self.memories[0]) # final output layer # self.output = nn.Linear(self.nn_output_size, self.real_output_size) # orthogonal(self.output.weight) if self.define_layers: self.output = nn.Sequential( nn.Linear(self.nn_output_size, self.hidden_size), nn.Tanh(), nn.Linear(self.hidden_size, self.real_output_size)) if self.gpu_id != -1: [x.cuda(self.gpu_id) for x in self.rnns] [x.cuda(self.gpu_id) for x in self.memories] self.output.cuda()
def __init__(self, eye_size, eyebrow_size, nose_size, mouth_size, h1_size, h2_size, h3_size, h4_size, h5_size, h6_size, total_length=30, bidirectional=True, bias=False, num_classes=6): super(PHRNN, self).__init__() self.length = total_length self.rnn1_1 = nn.RNN(input_size=eye_size, hidden_size=h1_size, bias=bias, batch_first=True, bidirectional=True) self.rnn1_2 = nn.RNN(input_size=eyebrow_size, hidden_size=h1_size, bias=bias, batch_first=True, bidirectional=True) self.rnn1_3 = nn.RNN(input_size=nose_size, hidden_size=h1_size, bias=bias, batch_first=True, bidirectional=True) self.rnn1_4 = nn.RNN(input_size=mouth_size, hidden_size=h1_size, bias=bias, batch_first=True, bidirectional=True) if bidirectional: h1_size = h1_size * 2 self.rnn2_1 = nn.RNN(input_size=h1_size * 2, hidden_size=h2_size, bias=bias, batch_first=True, bidirectional=True) self.rnn2_2 = nn.RNN(input_size=h1_size, hidden_size=h2_size, bias=bias, batch_first=True, bidirectional=True) self.rnn2_3 = nn.RNN(input_size=h1_size, hidden_size=h2_size, bias=bias, batch_first=True, bidirectional=True) if bidirectional: h2_size = h2_size * 2 self.rnn3_1 = nn.RNN(input_size=h2_size, hidden_size=h3_size, bias=bias, batch_first=True, bidirectional=True) self.rnn3_2 = nn.RNN(input_size=h2_size, hidden_size=h3_size, bias=bias, batch_first=True, bidirectional=True) self.rnn3_3 = nn.RNN(input_size=h2_size, hidden_size=h3_size, bias=bias, batch_first=True, bidirectional=True) if bidirectional: h3_size = h3_size * 2 self.rnn4_1 = nn.RNN(input_size=h3_size * 2, hidden_size=h4_size, bias=bias, batch_first=True, bidirectional=True) self.rnn4_2 = nn.RNN(input_size=h3_size * 2, hidden_size=h4_size, bias=bias, batch_first=True, bidirectional=True) if bidirectional: h4_size = h4_size * 2 self.rnn5_1 = nn.RNN(input_size=h4_size, hidden_size=h5_size, bias=bias, batch_first=True, bidirectional=True) self.rnn5_2 = nn.RNN(input_size=h4_size, hidden_size=h5_size, bias=bias, batch_first=True, bidirectional=True) if bidirectional: h5_size = h5_size * 2 self.lstm = nn.LSTM(input_size=h5_size * 2, hidden_size=h6_size, bias=bias, batch_first=True, bidirectional=True) if bidirectional: h6_size = h6_size * 2 self.h6_size = h6_size self.fc1 = nn.Linear(h6_size * total_length, 1024) self.fc2 = nn.Linear(1024, 256) self.fc3 = nn.Linear(256, num_classes)
def test_rnn_single_layer(self): rnn = nn.RNN(10, 20, 1, nonlinearity='relu') input = Variable(torch.randn(5, 3, 10)) h0 = Variable(torch.randn(1, 3, 20)) self.assertONNX(rnn, input, h0)
def test_rnn(self): rnn = nn.RNN(10, 20, 2) input = Variable(torch.randn(5, 3, 10)) h0 = Variable(torch.randn(2, 3, 20)) self.assertONNX(rnn, input, h0)
def __init__(self): super().__init__() self.embed_layer = nn.Embedding(10, 32) self.dense1 = nn.Linear(64, 64) self.rnn = nn.RNN(64, 64, 2, batch_first=True, nonlinearity='relu') self.dense = nn.Linear(64, 10)
def __init__(self, rnn_type, vocab_size, embedding_size, hidden_size, num_layers, batch_first, dropout, bidirectional=False, tied=False): """ Parameters ---------- rnn_type: str The type of RNN to use. Available options are "LSTM" for a LSTM, "GRU" for a GRU, "RNN_TANH" for a Elman RNN with tanh activation, and "RNN_RELU" for a Elman RNN with ReLU activation. vocab_size: int The number of types in the vocabulary. embedding_size: int The dimension of the embeddings. hidden_size: int The dimension of the hidden state vector. num_layers: int The number of layers to use in the RNN. batch_first: boolean If True, then the input and output tensors are provided as (batch, seq, feature). dropout: float The dropout proportion applied to RNN layers, the embedding, and the RNN output. bidirectional: boolean, optional (default=False) Whether to use a bidirectional RNN. Only valid if the class is a FixedContextRNNLM tied: boolean, optional (default=False) Whether or not to tie the embedding and the output embedding weights. """ super(EmbeddingRNN, self).__init__() self.rnn_type = rnn_type self.hidden_size = hidden_size self.num_layers = num_layers self.batch_first = batch_first self.dropout = nn.Dropout(dropout) self.bidirectional = bidirectional self.vocab_size = vocab_size self.embedding_size = embedding_size self.embedding = nn.Embedding(self.vocab_size, self.embedding_size) self.bidirectional = bidirectional if rnn_type in ["LSTM", "GRU"]: # Create a LSTM or GRU self.rnn = getattr(nn, rnn_type)(input_size=self.embedding_size, hidden_size=self.hidden_size, num_layers=self.num_layers, batch_first=self.batch_first, dropout=dropout, bidirectional=self.bidirectional) elif rnn_type in ["RNN_TANH", "RNN_RELU"]: # Create an Elman RNN with the specified activation function. nonlinearity = {"RNN_TANH": "tanh", "RNN_RELU": "relu"}[rnn_type] self.rnn = nn.RNN(input_size=self.embedding_size, hidden_size=hidden_size, num_layers=num_layers, nonlinearity=nonlinearity, batch_first=self.batch_first, dropout=dropout, bidirectional=self.bidirectional) else: raise ValueError("An invalid rnn_type {} was specified. " "Options are \"LSTM\", \"GRU\", \"RNN_TANH\", " "or \"RNN_RELU\".".format(rnn_type)) if self.bidirectional: self.rnn_output_size = self.hidden_size * 2 else: self.rnn_output_size = self.hidden_size self.decoder = nn.Linear(self.rnn_output_size, self.vocab_size) self.tied = tied if self.tied: if self.hidden_size != self.embedding_size: raise ValueError("When using the tied flag, hidden " "size must equal embedding size.") self.decoder.weight = self.embedding.weight
def __init__(self, n_items, hidden_size=64, dim=128): super(RNN_rec, self).__init__() # 随机初始化所有物品向量 self.items = nn.Embedding(n_items, dim, max_norm=1) self.rnn = nn.RNN(dim, hidden_size, batch_first=True) self.dense = self.dense_layer(hidden_size, 1)
from d2l import torch as d2l import torch from torch import nn from torch.nn import functional as F batch_size, num_steps = 32, 35 train_iter, vocab = d2l.load_data_time_machine(batch_size, num_steps) num_hiddens = 256 rnn_layer = nn.RNN(len(vocab), num_hiddens) state = torch.zeros( (1, batch_size, num_hiddens )) # (number of hidden layers, batch size, number of hidden units). state.shape class RNNModel(nn.Module): """The RNN model.""" def __init__(self, rnn_layer, vocab_size, **kwargs): super(RNNModel, self).__init__(**kwargs) self.rnn = rnn_layer self.vocab_size = vocab_size self.num_hiddens = self.rnn.hidden_size # If the RNN is bidirectional (to be introduced later), # `num_directions` should be 2, else it should be 1. if not self.rnn.bidirectional: self.num_directions = 1 self.linear = nn.Linear(self.num_hiddens, self.vocab_size) else: self.num_directions = 2
def __init__(self, input_size, embedding_size, hidden_size, output_size): super(RNNClassifier, self).__init__() self.hidden_size = hidden_size self.embedding = nn.Embedding(input_size, embedding_size) self.rnn = nn.RNN(embedding_size, hidden_size, batch_first=True) self.o2o = nn.Linear(hidden_size, output_size)
def __init__(self, nhidden, dropout): super(BrnnStructure, self).__init__() self.rnn = nn.RNN(nhidden, nhidden, 1, bidirectional=True) self.compress = FC(3 * nhidden, nhidden, relu=True) self.compress_r = FC(3 * nhidden, nhidden, relu=True) self.dropout = dropout
def __init__(self): super(Decoder, self).__init__() self.dec_cell = nn.RNN(input_size=len_n, hidden_size=n_hidden, dropout=0.5)
def __init__(self, config): super(CoVeEncoder, self).__init__() print('---build batched CoVeEncoder---') self.gpu = config.gpu self.bidirectional = config.bid_flag self.batch_size = config.batch_size self.char_hidden_dim = 0 self.use_char = config.use_char if self.use_char: self.char_hidden_dim = config.char_hidden_dim self.char_embedding_dim = config.char_emb_dim self.char = Char(config.char_features, config.char_alphabet.size(), self.char_embedding_dim, self.char_hidden_dim, config.dropout, self.gpu) self.rnn = nn.LSTM(300, 300, num_layers=2, bidirectional=True, batch_first=True) self.rnn.load_state_dict( model_zoo.load_url(model_urls['wmt-lstm'], model_dir=model_cache)) self.embedding_dim = config.word_emb_dim # catnlp self.hidden_dim = config.hidden_dim self.hyper_hidden_dim = config.hyper_hidden_dim self.hyper_embedding_dim = config.hyper_embedding_dim self.layers = config.layers self.drop = nn.Dropout(config.dropout) self.word_embeddings = nn.Embedding(config.word_alphabet.size(), self.embedding_dim) if config.pretrain_word_embedding is not None: self.word_embeddings.weight.data.copy_( torch.from_numpy(config.pretrain_word_embedding)) else: self.word_embeddings.weight.data.copy_( torch.from_numpy( self.random_embedding(config.word_alphabet.size(), self.embedding_dim))) self.mode = config.word_features self.embedding_dim = 600 if self.mode == 'BaseRNN': self.encoder = nn.RNN(self.char_hidden_dim + self.embedding_dim, self.hidden_dim, num_layers=self.layers, batch_first=True) elif self.mode == 'RNN': self.encoder = RNN(self.char_hidden_dim + self.embedding_dim, self.hidden_dim, num_layers=self.layers, gpu=self.gpu) elif self.mode == 'MetaRNN': self.encoder = MetaRNN(self.char_hidden_dim + self.embedding_dim, self.hidden_dim, self.hyper_hidden_dim, self.hyper_embedding_dim, num_layers=self.layers, gpu=self.gpu) elif self.mode == 'BaseLSTM': self.encoder = nn.LSTM(self.char_hidden_dim + self.embedding_dim, self.hidden_dim // 2, num_layers=self.layers, batch_first=True, bidirectional=True) elif self.mode == 'LSTM': self.encoder = LSTM(self.char_hidden_dim + self.embedding_dim, self.hidden_dim // 2, num_layers=self.layers, gpu=self.gpu, bidirectional=self.bidirectional) elif self.mode == 'MetaLSTM': self.encoder = MetaLSTM(self.char_hidden_dim + self.embedding_dim, self.hidden_dim // 2, self.hyper_hidden_dim, self.hyper_embedding_dim, num_layers=self.layers, gpu=self.gpu, bidirectional=self.bidirectional) else: print( 'Error word feature selection, please check config.word_features.' ) exit(0) if config.gpu: self.rnn = self.rnn.cuda() self.encoder = self.encoder.cuda() self.hidden2tag = nn.Linear(self.hidden_dim, config.label_alphabet_size)
def __init__(self, embed_size: int, field_size: int, padding_idx: int = 0, rnn_method: str = "lstm", output_method: str = "avg_pooling", nn_embedding: nn.Parameter = None, **kwargs): r"""Initialize SequenceIndicesEmbedding. Args: embed_size (int): Size of embedding tensor field_size (int): Size of inputs field padding_idx (int, optional): Padding index. Defaults to 0. rnn_method (str, optional): Method of RNN. Allow: ["gru", "lstm", "rnn"]. Defaults to "lstm". output_method (str, optional): Method of aggregation. Allow: ["avg_pooling", "max_pooling", "mean", "none", "sum"]. Defaults to "avg_pooling". nn_embedding (nn.Parameter, optional): Pretrained embedding values. Defaults to None. Kwargs: num_layers (int): Number of layers of RNN. Default to 1. bias (bool): Whether bias is added to RNN or not. Default to True. dropout (float): Probability of Dropout in RNN. Default to 0.0. bidirectional (bool): Whether bidirectional is used in RNN or not. Default to False. Attributes: length (int): Size of embedding tensor. embedding (torch.nn.Module): Embedding layer. rnn_layers (torch.nn.Module): RNN layers. aggregation (Union[torch.nn.Module, callable]): Pooling layer or aggregation function. output_method (string): Type of output_method. Raises: ValueError: when rnn_method is not in ["gru", "lstm", "rnn"]. ValueError: when output_method is not in ["avg_pooling", "max_pooling", "mean", "sum"]. """ # refer to parent class super(SequenceIndicesEmbedding, self).__init__() # bind embedding to pre-trained embedding module if nn_embedding is not None if nn_embedding is not None: self.length = nn_embedding.size("E") self.embedding = nn.Embedding.from_pretrained(nn_embedding) # else, create a embedding module with the given arguments else: self.length = embed_size self.embedding = nn.Embedding(field_size, embed_size, padding_idx=padding_idx, **kwargs) # parse bidirectional from kwargs bidirectional = kwargs.get("bidirectional", False) if bidirectional: # set hidden_size to embed_size // 2 if bidirectional is True hidden_size = embed_size // 2 else: # else, set hidden_size to embed_size hidden_size = embed_size # parse arguments of RNN layers rnn_args = dict(input_size=embed_size, hidden_size=hidden_size, num_layers=kwargs.get("num_layers", 1), bias=kwargs.get("bias", True), batch_first=True, dropout=kwargs.get("dropout", 0.0), bidirectional=bidirectional) # initialize RNN layers if rnn_method == "rnn": self.rnn_layers = nn.RNN(**rnn_args) elif rnn_method == "lstm": self.rnn_layers = nn.LSTM(**rnn_args) elif rnn_method == "gru": self.rnn_layers = nn.GRU(**rnn_args) else: raise ValueError('rnn_method only allows ["rnn", "lstm", "gru"].') # initialize aggregation layer for outputs and bind output_method to output_method if output_method == "avg_pooling": self.aggregation = nn.AdaptiveAvgPool1d(1) elif output_method == "max_pooling": self.aggregation = nn.AdaptiveMaxPool1d(1) elif output_method == "mean": self.aggregation = partial(torch.mean, dim="N", keepdim=True) elif output_method == "none": self.aggregation = torch.Tensor elif output_method == "sum": self.aggregation = partial(torch.sum, dim="N", keepdim=True) else: raise ValueError( 'output_method only allows ["avg_pooling", "max_pooling", "mean", "none", "sum"].' ) self.output_method = output_method
def __init__(self, rnn_type, ntoken, ninp, nhid, nlayers, dropout=0.5, tie_weights=False): super(RNNModel, self).__init__() self.drop = nn.Dropout(dropout) self.encoder = nn.Embedding(ntoken, ninp) if rnn_type in ['LSTM', 'GRU']: self.rnn = getattr(nn, rnn_type)(ninp, nhid, nlayers, dropout=dropout) elif rnn_type in ['python_LSTM']: self.rnn = NaiveLSTM(ninp, nhid) elif rnn_type in ['new_LSTM']: self.rnn = NewLSTM(ninp, nhid) else: try: nonlinearity = { 'RNN_TANH': 'tanh', 'RNN_RELU': 'relu' }[rnn_type] except KeyError: raise ValueError( """An invalid option for `--model` was supplied, options are ['LSTM', 'GRU', 'RNN_TANH' or 'RNN_RELU']""" ) self.rnn = nn.RNN(ninp, nhid, nlayers, nonlinearity=nonlinearity, dropout=dropout) if rnn_type in ['new_LSTM']: self.decoder = nn.Linear(nhid + nhid, ntoken) else: self.decoder = nn.Linear(nhid, ntoken) # Optionally tie weights as in: # "Using the Output Embedding to Improve Language Models" (Press & Wolf 2016) # https://arxiv.org/abs/1608.05859 # and # "Tying Word Vectors and Word Classifiers: A Loss Framework for Language Modeling" (Inan et al. 2016) # https://arxiv.org/abs/1611.01462 if tie_weights: if nhid != ninp: raise ValueError( 'When using the tied flag, nhid must be equal to emsize') self.decoder.weight = self.encoder.weight # ??? self.init_weights() self.rnn_type = rnn_type self.nhid = nhid self.nlayers = nlayers
def __init__(self, in_dim, n_hidden, n_layer, out_dim): super(RNN, self).__init__() self.layer1 = nn.RNN(in_dim, n_hidden, n_layer, nonlinearity='tanh',batch_first=True) self.layer2 = nn.Linear(n_hidden, out_dim)
def __init__(self): super(Model,self).__init__() self.rnn=nn.RNN(input_size=input_size,hidden_size=hidden_size,batch_first=True)
def __init__(self): super(Seq2Seq, self).__init__() self.enc_cell = nn.RNN(input_size=n_class, hidden_size=n_hidden, dropout=0.5) self.dec_cell = nn.RNN(input_size=n_class, hidden_size=n_hidden, dropout=0.5) self.fc = nn.Linear(n_hidden, n_class)
# -*- coding: utf-8 -*- """ Created on 11:42 AM 12/3/18 @author: ngunhuconchocon @brief: """ from __future__ import print_function import torch import torch.nn as nn import torch.nn.functional as F from torchsummary.torchsummary import summary rnn = nn.RNN(10, 20, 2) input = torch.randn(5, 3, 10) h0 = torch.randn(2, 3, 20) output, hn = rnn(input, h0) print(output) # #In this example i have 2 different nets. The input is fed to the first net. # #The output of the first net is fed to the second net. # #I only want to train the second net in this example # # net1_input = get_data() # net1_output = net1(net1_input) # #This creates a computation graph from input to output (i.e. all operations performed on the input to acquire the output). This is used when we want to get gradients for the weights we want to improve. # #As we only want to train the second net we dont need the computation graph from the first net. Therefore we detach the output from the previous computation graph # net2_input = net1_output.detach() # net2_output = net2(net2_input)
import torch import torch.nn as nn rnn = nn.RNN(5, 6, 1) input = torch.randn(1, 3, 5) h0 = torch.randn(1, 3, 6) output, hn = rnn(input, h0) print(output)
def __init__(self, vocab_size, embedding_dim, rnn_hidden_size, rnn_num_layers=1, rnn_type="LSTM", tie_weights=False): super(PoetryGenerator, self).__init__() self.rnn_type = rnn_type self.rnn_hidden_size = rnn_hidden_size self.rnn_num_layers = rnn_num_layers self.encoder = self.init_embedding( embedding_dim) # use embedding layer as first layer # TODO: better CSM # ============================================================================================== # Convolutional Sentence Model (CSM) layers, compresses a line (sequence of vectors) to a vector # use full convolutional layers without pooling # ============================================================================================== self.csm_l1 = nn.Sequential( nn.Conv3d(1, 1, kernel_size=(1, 2, 1), stride=(1, 1, 1)), nn.Dropout2d()) self.csm_l2 = nn.Sequential( nn.Conv3d(1, 1, kernel_size=(1, 2, 1), stride=(1, 1, 1)), nn.Dropout2d()) self.csm_l3 = nn.Sequential( nn.Conv3d(1, 1, kernel_size=(1, 3, 1), stride=(1, 1, 1)), nn.Dropout2d()) self.csm_l4 = nn.Sequential( nn.Conv3d(1, 1, kernel_size=(1, 3, 1), stride=(1, 1, 1)), nn.Dropout2d()) # TODO: better context # ==================================================================================================== # Context Model (CM) layers, compresses vectors of lines to one vector # for convenience, define 2 selectable layers for the training data is QUATRAIN which contains 4 lines # ==================================================================================================== # Compress 2 lines context into 1 vector self.cm_21 = nn.Sequential( nn.Conv2d(1, 1, kernel_size=(2, 1), stride=(1, 1)), nn.Dropout2d()) # Compress 3 lines context into 1 vector self.cm_31 = nn.Sequential( nn.Conv2d(1, 1, kernel_size=(3, 1), stride=(1, 1)), nn.Dropout2d()) # ============================================================================================== # Recurrent Generation Model (RGM) layers, # generates one word according to the previous words in the current line and the previous lines # ============================================================================================== # the inputs is concatenation of word embedding and lines vector (the same dimension as word embedding just now) if self.rnn_type == "LSTM": self.rnn = nn.LSTM(self.rnn_hidden_size * 2, self.rnn_hidden_size, self.rnn_num_layers, batch_first=True, dropout=0.5) elif self.rnn_type == "GRU": self.rnn = nn.GRU(self.rnn_hidden_size * 2, self.rnn_hidden_size, self.rnn_num_layers, batch_first=True, dropout=0.5) else: self.rnn = nn.RNN(self.rnn_hidden_size * 2, self.rnn_hidden_size, self.rnn_num_layers, batch_first=True, dropout=0.5) self.decoder = nn.Linear(self.rnn_hidden_size, vocab_size) # tie weights, e.i. use same weight as encoder for decoder, (I learned this trick from PyTorch Example). if tie_weights: self.decoder.weight = self.encoder.weight
def __init__( self, *, d_hid: int, n_hid_lyr: int, p_hid: float, **kwargs: Optional[Dict], ): super().__init__() # Create vanilla RNN layers and put in module list. # RNN in `self.recur` are treated as sequential RNN. # Input tensor : Output of `SAttnRNNModel.pre_hid`. # Input shape : `(B, S, H)`. # Input dtype : `torch.float32`. # Output tensor: Batch of recurrent token hidden states. # Output shape : `(B, S, H)`. # Output dtype : `torch.float32`. self.recur = nn.ModuleList([ nn.RNN(input_size=d_hid, hidden_size=d_hid, batch_first=True) for _ in range(n_hid_lyr) ]) # Create self attention query, key and value transformation layers and # put in module list. # Input tensor : Output of `self.recur`. # Input shape : `(B, S, H)`. # Input dtype : `torch.float32`. # Output tensor: Query, key and value token features. # Output shape : `(B, S, H)`. # Output dtype : `torch.float32`. self.query = nn.ModuleList([ nn.Linear(in_features=d_hid, out_features=d_hid) for _ in range(n_hid_lyr) ]) self.key = nn.ModuleList([ nn.Linear(in_features=d_hid, out_features=d_hid) for _ in range(n_hid_lyr) ]) self.value = nn.ModuleList([ nn.Linear(in_features=d_hid, out_features=d_hid) for _ in range(n_hid_lyr) ]) # Create self attention final output transformation layers and put in # module list. # Input tensor : Output of self attention weighted sum. # Input shape : `(B, S, H)`. # Input dtype : `torch.float32`. # Output tensor: Linear transformation on self attention weighted sum. # Output shape : `(B, S, H)`. # Output dtype : `torch.float32`. self.out = nn.ModuleList([ nn.Linear(in_features=d_hid, out_features=d_hid) for _ in range(n_hid_lyr) ]) # Create dropout layers. # Only need to create `n_hid_lyr - 1` since `SAttnRNNModel.post_hid` # drop output of `SAttnRNNModel.hid`. # Input tensor : Output of `self.out`. # Input shape : `(B, S, H)`. # Input dtype : `torch.float32`. # Output tensor: Sparse output of `self.out`. # Output shape : `(B, S, H)`. # Output dtype : `torch.float32`. dp: List[nn.Module] = [ nn.Dropout(p=p_hid) for _ in range(n_hid_lyr - 1) ] dp.append(nn.Identity()) self.dp = nn.ModuleList(dp)
def __init__(self, device, cnn_type='new', cnn_freeze=False, rnn_type='lstm', bidirection='False', regression='last_only', input_sequence_length=2, hidden_size=100, num_layers=2, learning_rate=0.0001): super().__init__() ### Deep Neural Network Setup ### self.cnn_type = cnn_type if cnn_type == 'new': self.CNN = nn.Sequential( nn.Conv2d(in_channels=3, out_channels=64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False), nn.BatchNorm2d(64), nn.LeakyReLU(0.1), nn.Conv2d(in_channels=64, out_channels=128, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), bias=False), nn.BatchNorm2d(128), nn.LeakyReLU(0.1), nn.Conv2d(in_channels=128, out_channels=256, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), bias=False), nn.BatchNorm2d(256), nn.LeakyReLU(0.1), nn.Conv2d(in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False), nn.BatchNorm2d(256), nn.LeakyReLU(0.1), nn.Conv2d(in_channels=256, out_channels=512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False), nn.BatchNorm2d(512), nn.LeakyReLU(0.1), nn.MaxPool2d(kernel_size=3, stride=1), nn.Conv2d(in_channels=512, out_channels=512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False), nn.BatchNorm2d(512), nn.LeakyReLU(0.1), nn.Conv2d(in_channels=512, out_channels=512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False), nn.BatchNorm2d(512), nn.LeakyReLU(0.1), nn.Conv2d(in_channels=512, out_channels=512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False), nn.BatchNorm2d(512), nn.LeakyReLU(0.1), # nn.AvgPool2d(kernel_size=3, stride=1), nn.Conv2d(in_channels=512, out_channels=1024, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False), nn.BatchNorm2d(1024), nn.LeakyReLU(0.1), nn.Conv2d(in_channels=1024, out_channels=1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False), nn.BatchNorm2d(1024), nn.LeakyReLU(0.1), nn.AvgPool2d(kernel_size=3, stride=1), nn.Conv2d(in_channels=1024, out_channels=2048, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False), nn.BatchNorm2d(2048), nn.LeakyReLU(0.1), ) CNN_flat_output_size = 16384 self.CNN.to(device) elif cnn_type == 'mobilenetV3_large': self.CNN = models.mobilenet_v3_large(pretrained=True) CNN_flat_output_size = 1000 if cnn_freeze == True: print('Freeze ' + cnn_type) for name, module in self.CNN.named_children(): for layer in module.children(): for param in layer.parameters(): param.requires_grad = False # print(param) elif cnn_type == 'mobilenetV3_small': self.CNN = models.mobilenet_v3_small(pretrained=True) CNN_flat_output_size = 1000 if cnn_freeze == True: print('Freeze ' + cnn_type) for name, module in self.CNN.named_children(): for layer in module.children(): for param in layer.parameters(): param.requires_grad = False # print(param) elif cnn_type == 'vgg16': self.CNN = models.vgg16(pretrained=True) CNN_flat_output_size = 1000 if cnn_freeze == True: print('Freeze ' + cnn_type) for name, module in self.CNN.named_children(): for layer in module.children(): for param in layer.parameters(): param.requires_grad = False # print(param) self.hidden_size = hidden_size self.num_layers = num_layers self.rnn_type = rnn_type self.regression = regression if rnn_type == 'rnn': if num_layers > 1: self.RNN = nn.RNN(input_size=CNN_flat_output_size, hidden_size=hidden_size, num_layers=num_layers, batch_first=True, dropout=0.5) else: self.RNN = nn.RNN(input_size=CNN_flat_output_size, hidden_size=hidden_size, num_layers=num_layers, batch_first=True) elif rnn_type == 'lstm': if num_layers > 1: self.RNN = nn.LSTM(input_size=CNN_flat_output_size, hidden_size=hidden_size, num_layers=num_layers, batch_first=True, dropout=0.5) else: self.RNN = nn.LSTM(input_size=CNN_flat_output_size, hidden_size=hidden_size, num_layers=num_layers, batch_first=True) # self.linear = nn.Linear(in_features=hidden_size, out_features=6) if regression == 'last_only': self.linear = nn.Linear( in_features=hidden_size, out_features=3) # For last sequence only case elif regression == 'full_sequence': in_features_num = input_sequence_length * hidden_size out_features_num = (num_layers * hidden_size) // 2 self.linear1 = nn.Linear( in_features=in_features_num, out_features=out_features_num) # For full sequence self.batchnorm_linear1 = nn.BatchNorm1d(out_features_num) self.leakyrelu_linear1 = nn.LeakyReLU(0.1) self.dropout_linear1 = nn.Dropout(p=0.5) in_features_num = out_features_num out_features_num = out_features_num // 2 self.linear2 = nn.Linear( in_features=in_features_num, out_features=out_features_num) # For full sequence self.batchnorm_linear2 = nn.BatchNorm1d(out_features_num) self.leakyrelu_linear2 = nn.LeakyReLU(0.1) self.dropout_linear2 = nn.Dropout(p=0.5) in_features_num = out_features_num out_features_num = 3 self.linear3 = nn.Linear( in_features=in_features_num, out_features=out_features_num) # For full sequence ### Training Setup ### self.device = device self.to(self.device) # self.optimizer = optim.RMSprop(self.parameters(), lr=learning_rate) self.optimizer = optim.Adam(self.parameters(), lr=learning_rate) self.translation_loss = nn.MSELoss()
def build_rnn(self): self.rnn = nn.RNN(input_size=512, hidden_size=self.num_class, num_layers=1, batch_first=True)
def __init__(self, n_features): super(SequentialLayer, self).__init__() self.rnn = nn.RNN(n_features, 3, 1) signal = torch.tensor([[-1.], [0.], [1.]]) self.register_buffer('signal', signal)
def __init__(self, embedding_matrix=None, hidden_dim=None, num_layers=None, dropout=None, bidirectional=None, useGlove=None, trainable=None, typeOfPadding="no_padding", typeOfRNN="simple", typeOfAttention="multiplicative"): super().__init__() # Create an embedding layer of dimension vocabulary size * 100 self.embeddingLayer = nn.Embedding(len(embedding_matrix), 100) # Set embedding to GloVe representation if flag is True if useGlove: self.embeddingLayer.weight.data.copy_(embedding_matrix) # Set weights of the matrix to non-updatable if flag is False if not trainable: self.embeddingLayer.weight.requires_grad = False # Initialize the RNN structure self.typeOfRNN = typeOfRNN if typeOfRNN == "simple": self.rnnLayer = nn.RNN(input_size=100, hidden_size=hidden_dim, batch_first=True, num_layers=num_layers, dropout=dropout, bidirectional=bidirectional) elif typeOfRNN == "GRU": self.rnnLayer = nn.GRU(input_size=100, hidden_size=hidden_dim, batch_first=True, num_layers=num_layers, dropout=dropout, bidirectional=bidirectional) else: self.rnnLayer = nn.LSTM(input_size=100, hidden_size=hidden_dim, batch_first=True, num_layers=num_layers, dropout=dropout, bidirectional=bidirectional) # Initialize variables to define the fully-connected layer self.num_directions = 2 if bidirectional else 1 self.typeOfPadding = typeOfPadding self.typeOfAttention = typeOfAttention self.num_layers = num_layers self.hidden_size = hidden_dim # Initialize the learnable attention weights if self.typeOfAttention == "multiplicative": self.attn_weight = nn.Linear(hidden_dim * self.num_directions, hidden_dim * self.num_directions) else: self.Wh = nn.Linear(self.hidden_size * self.num_directions, self.hidden_size * self.num_directions) self.Ws = nn.Linear(self.hidden_size * self.num_directions, self.hidden_size * self.num_directions) # Initialize a fully-connected layer self.fc = nn.Linear(hidden_dim * self.num_directions, 1) # Sigmoid activation function squashes the output between 0 and 1 self.sigmoid = nn.Sigmoid()