def __init__(self, rnn_hiddens, hiddens, embed_dim, prefix): super(GRU, self).__init__(prefix=prefix) with self.name_scope(): # gated recurrent units self.grus = [] for i, hidden in enumerate(rnn_hiddens): cell = rnn.GRUCell(hidden, prefix='gru%d_' % i) self.register_child(cell) self.grus += [cell] # dense layers (mf dense layers) self.denses = nn.Sequential() in_dims = [rnn_hiddens[-1]] + hiddens out_dims = hiddens + [FLOW_OUTPUT_DIM * FLOW_OUTPUT_LEN] for i, (in_dim, out_dim) in enumerate(zip(in_dims, out_dims)): activation = None if i == len(in_dims) - 1 else 'relu' if embed_dim == 0: self.denses.add( nn.Dense(out_dim, activation, flatten=False, prefix='dense%d_' % i)) else: self.denses.add( MFDense(N_LOC, embed_dim, in_dim, out_dim, activation, prefix='mf_dense%d_' % i))
def _get_rnn_cell(mode, num_layers, input_size, hidden_size, dropout, weight_dropout, var_drop_in, var_drop_state, var_drop_out): """create rnn cell given specs""" rnn_cell = rnn.SequentialRNNCell() with rnn_cell.name_scope(): for i in range(num_layers): if mode == 'rnn_relu': cell = rnn.RNNCell(hidden_size, 'relu', input_size=input_size) elif mode == 'rnn_tanh': cell = rnn.RNNCell(hidden_size, 'tanh', input_size=input_size) elif mode == 'lstm': cell = rnn.LSTMCell(hidden_size, input_size=input_size) elif mode == 'gru': cell = rnn.GRUCell(hidden_size, input_size=input_size) if var_drop_in + var_drop_state + var_drop_out != 0: cell = contrib.rnn.VariationalDropoutCell( cell, var_drop_in, var_drop_state, var_drop_out) rnn_cell.add(cell) if i != num_layers - 1 and dropout != 0: rnn_cell.add(rnn.DropoutCell(dropout)) if weight_dropout: apply_weight_drop(rnn_cell, 'h2h_weight', rate=weight_dropout) return rnn_cell
def __init__(self, n_hidden, vocab_size, embed_dim, max_seq_length, **kwargs): super(korean_autospacing, self).__init__(**kwargs) # 입력 시퀀스 길이 self.in_seq_len = max_seq_length # 출력 시퀀스 길이 self.out_seq_len = max_seq_length # GRU의 hidden 개수 self.n_hidden = n_hidden # 고유문자개수 self.vocab_size = vocab_size # max_seq_length self.max_seq_length = max_seq_length # 임베딩 차원수 self.embed_dim = embed_dim with self.name_scope(): self.embedding = nn.Embedding(input_dim=self.vocab_size, output_dim=self.embed_dim) self.conv_unigram = nn.Conv2D(channels=128, kernel_size=(1, self.embed_dim)) self.conv_bigram = nn.Conv2D(channels=256, kernel_size=(2, self.embed_dim), padding=(1, 0)) self.conv_trigram = nn.Conv2D(channels=128, kernel_size=(3, self.embed_dim), padding=(1, 0)) self.conv_forthgram = nn.Conv2D(channels=64, kernel_size=(3, self.embed_dim), padding=(2, 0)) self.conv_fifthgram = nn.Conv2D(channels=32, kernel_size=(3, self.embed_dim), padding=(2, 0)) self.bi_gru = rnn.BidirectionalCell( rnn.GRUCell(hidden_size=self.n_hidden), rnn.GRUCell(hidden_size=self.n_hidden)) self.dense_sh = nn.Dense(100, activation='relu', flatten=False) self.dense = nn.Dense(1, activation='sigmoid', flatten=False)
def __init__(self, samples_per_step: int = 5, **kwargs): """ Args: samples_per_step: How many samples to generate at each decoding step. **kwargs: """ super(SinDecoder, self).__init__(**kwargs) self._samples_per_step = samples_per_step with self.name_scope(): self._decoder_rnn = rnn.GRUCell(hidden_size=20) self._projection_layer = nn.Dense(samples_per_step, flatten=False)
def __init__(self, n_hidden, vocab_size, embed_dim, max_seq_length, end_idx, attention=False, **kwargs): super(korean_english_translator, self).__init__(**kwargs) self.end_idx = end_idx #입력 시퀀스 길이 self.in_seq_len = max_seq_length #출력 시퀀스 길이 self.out_seq_len = max_seq_length # GRU의 hidden 개수 self.n_hidden = n_hidden #고유문자개수 self.vocab_size = vocab_size #max_seq_length self.max_seq_length = max_seq_length #임베딩 차원수 self.embed_dim = embed_dim self.attention = attention with self.name_scope(): self.embedding = nn.Embedding(input_dim=vocab_size, output_dim=embed_dim, dtype="float16") self.encoder = rnn.GRUCell(hidden_size=n_hidden) self.decoder = rnn.GRUCell(hidden_size=n_hidden) self.batchnorm = nn.BatchNorm(axis=2) #flatten을 false로 할 경우 마지막 차원에 fully connected가 적용된다. self.dense = nn.Dense(self.vocab_size, flatten=False) if self.attention: self.dropout = nn.Dropout(0.3) self.attdense = nn.Dense(self.max_seq_length, flatten=False) self.attn_combine = nn.Dense(self.n_hidden, flatten=False)
def _get_rnn_cell(mode, num_layers, input_size, hidden_size, dropout, weight_dropout, var_drop_in, var_drop_state, var_drop_out, skip_connection, proj_size=None, cell_clip=None, proj_clip=None): """create rnn cell given specs Parameters ---------- mode : str The type of RNN cell to use. Options are 'lstmpc', 'rnn_tanh', 'rnn_relu', 'lstm', 'gru'. num_layers : int The number of RNN cells in the encoder. input_size : int The initial input size of in the RNN cell. hidden_size : int The hidden size of the RNN cell. dropout : float The dropout rate to use for encoder output. weight_dropout: float The dropout rate to the hidden to hidden connections. var_drop_in: float The variational dropout rate for inputs. Won’t apply dropout if it equals 0. var_drop_state: float The variational dropout rate for state inputs on the first state channel. Won’t apply dropout if it equals 0. var_drop_out: float The variational dropout rate for outputs. Won’t apply dropout if it equals 0. skip_connection : bool Whether to add skip connections (add RNN cell input to output) proj_size : int The projection size of each LSTMPCellWithClip cell. Only available when the mode=lstmpc. cell_clip : float Clip cell state between [-cellclip, cell_clip] in LSTMPCellWithClip cell. Only available when the mode=lstmpc. proj_clip : float Clip projection between [-projclip, projclip] in LSTMPCellWithClip cell Only available when the mode=lstmpc. """ assert mode == 'lstmpc' or proj_size is None, \ 'proj_size takes effect only when mode is lstmpc' assert mode == 'lstmpc' or cell_clip is None, \ 'cell_clip takes effect only when mode is lstmpc' assert mode == 'lstmpc' or proj_clip is None, \ 'proj_clip takes effect only when mode is lstmpc' rnn_cell = rnn.HybridSequentialRNNCell() with rnn_cell.name_scope(): for i in range(num_layers): if mode == 'rnn_relu': cell = rnn.RNNCell(hidden_size, 'relu', input_size=input_size) elif mode == 'rnn_tanh': cell = rnn.RNNCell(hidden_size, 'tanh', input_size=input_size) elif mode == 'lstm': cell = rnn.LSTMCell(hidden_size, input_size=input_size) elif mode == 'gru': cell = rnn.GRUCell(hidden_size, input_size=input_size) elif mode == 'lstmpc': cell = LSTMPCellWithClip(hidden_size, proj_size, cell_clip=cell_clip, projection_clip=proj_clip, input_size=input_size) if var_drop_in + var_drop_state + var_drop_out != 0: cell = contrib.rnn.VariationalDropoutCell( cell, var_drop_in, var_drop_state, var_drop_out) if skip_connection: cell = rnn.ResidualCell(cell) rnn_cell.add(cell) if i != num_layers - 1 and dropout != 0: rnn_cell.add(rnn.DropoutCell(dropout)) if weight_dropout: apply_weight_drop(rnn_cell, 'h2h_weight', rate=weight_dropout) return rnn_cell
def _get_rnn_cell(mode, num_layers, input_size, hidden_size, dropout, var_drop_in, var_drop_state, var_drop_out, skip_connection, proj_size=None): """create rnn cell given specs Parameters ---------- mode : str The type of RNN cell to use. Options are 'rnn_tanh', 'rnn_relu', 'lstm', 'lstmp', 'gru'. num_layers : int The number of RNN cells in the encoder. input_size : int The initial input size of in the RNN cell. hidden_size : int The hidden size of the RNN cell. dropout : float The dropout rate to use for encoder output. var_drop_in: float The variational dropout rate for inputs. Won’t apply dropout if it equals 0. var_drop_state: float The variational dropout rate for state inputs on the first state channel. Won’t apply dropout if it equals 0. var_drop_out: float The variational dropout rate for outputs. Won’t apply dropout if it equals 0. skip_connection : bool Whether to add skip connections (add RNN cell input to output) proj_size : int The projection size of each LSTMPCell cell. Only available when the mode=lstmpc. """ if mode == 'lstmps': assert proj_size is not None, \ 'proj_size takes effect only when mode is lstmp' rnn_cell = rnn.HybridSequentialRNNCell() with rnn_cell.name_scope(): for i in range(num_layers): if mode == 'rnn_relu': cell = rnn.RNNCell(hidden_size, 'relu', input_size=input_size) elif mode == 'rnn_tanh': cell = rnn.RNNCell(hidden_size, 'tanh', input_size=input_size) elif mode == 'lstm': cell = rnn.LSTMCell(hidden_size, input_size=input_size) elif mode == 'lstmp': cell = gluon.contrib.rnn.LSTMPCell(hidden_size, input_size, proj_size) elif mode == 'gru': cell = rnn.GRUCell(hidden_size, input_size=input_size) if var_drop_in + var_drop_state + var_drop_out != 0: cell = gluon.contrib.rnn.VariationalDropoutCell( cell, var_drop_in, var_drop_state, var_drop_out) if skip_connection: cell = rnn.ResidualCell(cell) rnn_cell.add(cell) if i != num_layers - 1 and dropout != 0: rnn_cell.add(rnn.DropoutCell(dropout)) return rnn_cell
def __init__(self, hidden_size, prefix=None): super(MyGRUCell, self).__init__(prefix=prefix) self.hidden_size = hidden_size with self.name_scope(): self.cell = rnn.GRUCell(self.hidden_size)