def __init__(self, n_neurons=128, batch_size=4, user=6):
"""
Actor Encoder class
Args:
n_neurons: int
Hidden layer of LSTM
user: int
Num of user
Outputs:
enc_outputs: 3D tensor [batch, user, n_neurons]
Whole sequence outputs
enc_state: 1D list [tensor, tensor]
enc_state[0]: 2D tensor [batch, n_neurons]
Final memory state
enc_state[1]: 2D tensor [batch, n_neurons]
Final carry state
"""
self.n_neurons = n_neurons
self.batch_size = batch_size
self.user = user
# Define Recursive cell
self.enc_rec_cell = LSTMCell(self.n_neurons)
def __init__(self, n_neurons=128, batch_size=4, seq_length=10):
# パラメタ設定
self.n_neurons = n_neurons
self.batch_size = batch_size
self.seq_length = seq_length
# 再帰セル定義
self.enc_rec_cell = LSTMCell(self.n_neurons)
def __init__(self):
super(CharRNN, self).__init__()
self.dense_layer = Dense(65)
self.cells = [
LSTMCell(units=512),
LSTMCell(units=512),
LSTMCell(units=512)
]
self.lstm = StackedRNNCells(self.cells)
def __init__(self):
super(MyModel, self).__init__()
# define all the layers and input
self.conv2d = Conv2D(input_shape=(MyModel.img_width,
MyModel.img_height, 1),
padding='same',
strides=(1, 1),
filters=32,
kernel_size=(5, 5),
activation='relu')
self.max_pool = MaxPool2D(pool_size=(2, 2), strides=(2, 2))
self.batch_norm = BatchNormalization()
self.conv2d_1 = Conv2D(padding='same',
strides=(1, 1),
filters=64,
kernel_size=(5, 5),
activation='relu')
self.max_pool_1 = MaxPool2D(pool_size=(2, 2), strides=(2, 2))
self.batch_norm_1 = BatchNormalization()
self.conv2d_2 = Conv2D(padding='same',
strides=(1, 1),
filters=128,
kernel_size=(3, 3),
activation='relu')
self.max_pool_2 = MaxPool2D(pool_size=(1, 5), strides=(1, 1))
self.batch_norm_2 = BatchNormalization()
self.conv2d_3 = Conv2D(padding='same',
strides=(1, 1),
filters=128,
kernel_size=(3, 3),
activation='relu')
self.max_pool_3 = MaxPool2D(pool_size=(1, 3), strides=(1, 1))
self.batch_norm_3 = BatchNormalization()
self.conv2d_4 = Conv2D(padding='same',
strides=(1, 1),
filters=256,
kernel_size=(3, 3),
activation='relu')
self.max_pool_4 = MaxPool2D(pool_size=(1, 2), strides=(1, 1))
self.batch_norm_4 = BatchNormalization()
self.rnn = Bidirectional(
RNN([
LSTMCell(256),
LSTMCell(256),
], return_sequences=True))
self.atrous_conv2d = Conv2D(padding='same',
kernel_size=(1, 1),
filters=MyModel.num_chars)
def __init__(self,
input_seq_len=60,
output_seq_len=14,
layers=[128],
n_in_features=1,
n_out_features=1,
bidirectional=False,
loss='mean_absolute_error'):
self.input_seq_len = input_seq_len
self.output_seq_len = output_seq_len
self.n_in_features = n_in_features
self.n_out_features = n_out_features
n_layers = len(layers)
## Encoder
encoder_inputs = Input(shape=(None, n_in_features))
lstm_cells = [LSTMCell(hidden_dim) for hidden_dim in layers]
# lstm_cells = LSTMCell(layers)
if bidirectional:
encoder = Bidirectional(RNN(lstm_cells, return_state=True))
encoder_outputs_and_states = encoder(encoder_inputs)
bi_encoder_states = encoder_outputs_and_states[1:]
encoder_states = []
for i in range(int(len(bi_encoder_states) / 2)):
temp = concatenate([
bi_encoder_states[i], bi_encoder_states[2 * n_layers + i]
],
axis=-1)
encoder_states.append(temp)
else:
encoder = RNN(lstm_cells, return_state=True)
encoder_outputs_and_states = encoder(encoder_inputs)
encoder_states = encoder_outputs_and_states[1:]
## Decoder
decoder_inputs = Input(shape=(None, n_out_features))
if bidirectional:
decoder_cells = [LSTMCell(hidden_dim * 2) for hidden_dim in layers]
else:
decoder_cells = [LSTMCell(hidden_dim) for hidden_dim in layers]
decoder_lstm = RNN(decoder_cells,
return_sequences=True,
return_state=True)
decoder_outputs_and_states = decoder_lstm(decoder_inputs,
initial_state=encoder_states)
decoder_outputs = decoder_outputs_and_states[0]
decoder_dense = Dense(n_out_features)
decoder_outputs = decoder_dense(decoder_outputs)
self.model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
self.model.compile(Adam(), loss=loss)
def create_model(layers, bidirectional=False):
n_layers = len(layers)
## Encoder
encoder_inputs = Input(shape=(None, n_in_features))
lstm_cells = [LSTMCell(hidden_dim) for hidden_dim in layers]
if bidirectional:
encoder = Bidirectional(RNN(lstm_cells, return_state=True))
encoder_outputs_and_states = encoder(encoder_inputs)
bi_encoder_states = encoder_outputs_and_states[1:]
encoder_states = []
for i in range(int(len(bi_encoder_states) / 2)):
temp = []
for j in range(2):
temp.append(
concatenate([
bi_encoder_states[i][j],
bi_encoder_states[n_layers + i][j]
],
axis=-1))
encoder_states.append(temp)
else:
encoder = RNN(lstm_cells, return_state=True)
encoder_outputs_and_states = encoder(encoder_inputs)
encoder_states = encoder_outputs_and_states[1:]
## Decoder
decoder_inputs = Input(shape=(None, n_out_features))
if bidirectional:
decoder_cells = [LSTMCell(hidden_dim * 2) for hidden_dim in layers]
else:
decoder_cells = [LSTMCell(hidden_dim) for hidden_dim in layers]
decoder_lstm = RNN(decoder_cells, return_sequences=True, return_state=True)
decoder_outputs_and_states = decoder_lstm(decoder_inputs,
initial_state=encoder_states)
decoder_outputs = decoder_outputs_and_states[0]
decoder_dense = Dense(n_out_features)
decoder_outputs = decoder_dense(decoder_outputs)
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
return model
def __init__(self,
encoder=None,
vocab_size=1,
embedding_size=32,
hidden_size=64):
"""Initialization method.
Args:
encoder (IntegerEncoder): An index to vocabulary encoder.
vocab_size (int): The size of the vocabulary.
embedding_size (int): The size of the embedding layer.
hidden_size (int): The amount of hidden neurons.
"""
logger.info('Overriding class: Generator -> BiLSTMGenerator.')
# Overrides its parent class with any custom arguments if needed
super(BiLSTMGenerator, self).__init__(name='G_bi_lstm')
# Creates a property for holding the used encoder
self.encoder = encoder
# Creates an embedding layer
self.embedding = Embedding(vocab_size,
embedding_size,
name='embedding')
# Creates a forward LSTM cell
cell_f = LSTMCell(hidden_size, name='lstm_cell_f')
# And a orward RNN layer
self.forward = RNN(cell_f,
name='forward_rnn',
return_sequences=True,
stateful=True)
# Creates a backward LSTM cell
cell_b = LSTMCell(hidden_size, name='lstm_cell_b')
# And a backward RNN layer
self.backward = RNN(cell_b,
name='backward_rnn',
return_sequences=True,
stateful=True,
go_backwards=True)
# Creates the linear (Dense) layer
self.linear = Dense(vocab_size, name='out')
logger.info('Class overrided.')
def __init__(self, units_1, units_2, dropout_1, dropout_2, output_steps,
output_size):
super().__init__()
self.output_steps = output_steps
self.units_1 = units_1
self.units_2 = units_2
self.lstm_cells = [
LSTMCell(units_1, dropout=dropout_1),
LSTMCell(units_2, dropout=dropout_2),
]
self.lstm_stack = StackedRNNCells(self.lstm_cells)
self.lstm_rnn = RNN(self.lstm_stack, return_state=True)
self.dense = Dense(output_size, activation="softmax")
def __init__(self, units_1, units_2, dropout_1, dropout_2, output_steps, output_size):
super().__init__()
self.output_steps = output_steps
self.units_1 = units_1
self.units_2 = units_2
self.dropout_1 = dropout_1
self.dropout_2 = dropout_2
self.lstm_cell_1 = LSTMCell(units_1, dropout=dropout_1)
self.lstm_cell_2 = LSTMCell(units_2, dropout=dropout_2)
self.lstm_rnn_1 = RNN(self.lstm_cell_1, return_state=True, return_sequences=True)
self.lstm_rnn_2 = RNN(self.lstm_cell_2, return_state=True)
self.dense = Dense(output_size, activation="softmax")
def __init__(self, opts):
super(LSTM, self).__init__()
self.cell = LSTMCell(opts.lstm_units)
# Output layer.
self.__out_layer = Dense(opts.output_size,
activation=None,
use_bias=True)
# Interface layer.
self.__interface = Dense(opts.interface_size,
activation=None,
use_bias=True)
def init_layers(self,
layers=(20, 20),
hardness=0.0,
learning_rate=0.01,
epsilon=1e-10,
pool=[],
use_meta_features=False,
time_scale=1000.,
lr_multiplier_scale=0.0,
warmup_lstm_update=False,
**kwargs):
"""Initialize layers."""
self.choices = [
getattr(analytical, p["class_name"] + "Optimizer")(**p["config"])
for p in pool
]
self.hardness = hardness
self.epsilon = epsilon
self.use_meta_features = use_meta_features
self.time_scale = time_scale
self.lr_multiplier_scale = lr_multiplier_scale
self.warmup_lstm_update = warmup_lstm_update
self.learning_rate = learning_rate
self.recurrent = [LSTMCell(hsize, **kwargs) for hsize in layers]
self.choice = Dense(len(pool), input_shape=(layers[-1], ))
if self.lr_multiplier_scale > 0.0:
self.lr_multiplier = Dense(len(pool),
input_shape=(layers[-1], ),
kernel_initializer='zeros',
bias_initializer='zeros')
def __init__(self, lr=1e-4, dropout_rate=0.2, units=300, beam_width=12, vocab_size=9000):
super().__init__()
self.embedding = tf.Variable(np.load('../data/embedding.npy'),
dtype=tf.float32,
name='pretrained_embedding',
trainable=False)
self.encoder = Encoder(units=units, dropout_rate=dropout_rate)
self.attention_mechanism = BahdanauAttention(units=units)
self.decoder_cell = AttentionWrapper(
LSTMCell(units),
self.attention_mechanism,
attention_layer_size=units)
self.projected_layer = ProjectedLayer(self.embed.embedding)
self.sampler = tfa.seq2seq.sampler.TrainingSampler()
self.decoder = BasicDecoder(
self.decoder_cell,
self.sampler,
output_layer=self.projected_layer)
self.beam_search = BeamSearchDecoder(
self.decoder_cell,
beam_width=beam_width,
embedding_fn=lambda x: tf.nn.embedding_lookup(self.embedding, x),
output_layer=self.projected_layer)
self.vocab_size = vocab_size
self.optimizer = Adam(lr)
self.accuracy = tf.keras.metrics.Accuracy()
self.mean = tf.keras.metrics.Mean()
self.decay_lr = tf.optimizers.schedules.ExponentialDecay(lr, 1000, 0.95)
self.logger = logging.getLogger('tensorflow')
self.logger.setLevel(logging.INFO)
def setUp(self):
seq = Input((None, ), dtype="int32")
x = Embedding(8000, 128)(seq)
self.embedder = Model(seq, x)
x = RNN(LSTMCell(128))(x)
self.lstmer = Model(seq, x)
def __init__(self, n_neurons=128, batch_size=4, seq_length=10):
# パラメタ設定
self.n_neurons = n_neurons
self.batch_size = batch_size
self.seq_length = seq_length
# 地点マスク用罰則
self.infty = 1.0E+08
# 地点マスクbit（テンソル）
self.mask = 0
# サンプリングのシード
self.seed = None
# 初期入力値のパラメータ変数(Encoderセルの出力次元、
# [batch_size, n_neuron])
first_input = tf.get_variable('GO', [1, self.n_neurons])
self.dec_first_input = tf.tile(first_input, [self.batch_size, 1])
# Pointing機構のパラメータ変数
self.W_ref = tf.get_variable('W_ref',
[1, self.n_neurons, self.n_neurons])
self.W_out = tf.get_variable('W_out', [self.n_neurons, self.n_neurons])
self.v = tf.get_variable('v', [self.n_neurons])
# 再帰セル定義
self.dec_rec_cell = LSTMCell(self.n_neurons)
def __init__(self, Shape, Episode_Length, Input_Dim, Output_Dim, Alpha = 0, Activation = 'tanh'):
Activation_Method = self._Activation(Activation)
Regularizer = tf.keras.regularizers.l2(Alpha) if Alpha != 0 else None
self.Episode_Length = Episode_Length
self.State = tf.placeholder(shape = [None, self.Episode_Length, Input_Dim], dtype = tf.float32, name = 'A_States')
self.Action = tf.placeholder(shape = [None, Output_Dim], dtype = tf.float32, name = 'Action_True')
self.Advantage = tf.placeholder(shape = [None, 1], dtype = tf.float32, name = 'Advantage')
self.Sigma = tf.placeholder(shape = [None, Output_Dim], dtype = tf.float32, name = 'Sigma')
Input = self.State
cells = []
for Neuron_units in Shape:
cells.append(LSTMCell(Neuron_units,activation = Activation_Method, kernel_regularizer = Regularizer))
LSTM_Layers = RNN(cells, return_sequences = False)
Last_Layer = LSTM_Layers(Input)
self.Action_Pred = tf.layers.dense(Last_Layer, Output_Dim, activation = None, activity_regularizer = Regularizer, name = 'Action_Pred')
self.Learning_Rate = tf.placeholder(tf.float32, shape = (), name = 'A_Learning_Rate')
self.Optimizer = tf.train.AdamOptimizer(learning_rate = self.Learning_Rate)
Loglik_Loss = tf.log(tf.sqrt(2 * np.pi * (self.Sigma ** 2))) + ((self.Action - self.Action_Pred) ** 2) / (2 * (self.Sigma ** 2))
Loglik_Loss = tf.reduce_sum(Loglik_Loss, axis = 1, keepdims = True) * self.Advantage
self.Fit = self.Optimizer.minimize(tf.reduce_mean(Loglik_Loss))
def __init__(self, Shape, Episode_Length, Input_Dim, Output_Dim, Alpha = 0, Activation = "Sigmoid"):
Activation_Method = self._Activation(Activation)
Regularizer = tf.keras.regularizers.l2(Alpha) if Alpha != 0 else None
self.Episode_Length = Episode_Length
self.State = tf.placeholder(shape = [None, self.Episode_Length, Input_Dim], dtype = tf.float32, name = 'A_States')
self.Value = tf.placeholder(shape = [None, Output_Dim], dtype = tf.float32, name = 'Value_True')
Input = self.State
cells = [] # To create a multiple layers stacked RNN
for Neuron_units in Shape:
cells.append(LSTMCell(Neuron_units,activation = Activation_Method, kernel_regularizer = Regularizer))
LSTM_Layers = RNN(cells, return_sequences = False) # We abandon all the intermediate processings and only train with final state
Last_Layer = LSTM_Layers(Input)
self.Value_Pred = tf.layers.dense(Last_Layer, Output_Dim, activation = None, activity_regularizer = Regularizer, name = 'Value_Pred')
self.Learning_Rate = tf.placeholder(tf.float32, shape = (), name = 'C_Learning_Rate')
self.Optimizer = tf.train.AdamOptimizer(learning_rate = self.Learning_Rate)
self.Fit = self.Optimizer.minimize(tf.losses.mean_squared_error(self.Value, self.Value_Pred))
class LSTM(Model):
def __init__(self, opts):
super(LSTM, self).__init__()
self.cell = LSTMCell(opts.lstm_units)
# Output layer.
self.__out_layer = Dense(opts.output_size,
activation=None,
use_bias=True)
# Interface layer.
self.__interface = Dense(opts.interface_size,
activation=None,
use_bias=True)
def __call__(self, inputs, state):
hidden, state = self.cell(inputs, state)
ctrl_output = {
'output': self.__out_layer(hidden),
'interface': self.__interface(hidden)
}
return ctrl_output, state
def initialize(self, batch_size):
return self.cell.get_initial_state(batch_size=batch_size,
dtype=tf.float32)
def __init__(self, embedding_size=32, hidden_size=64):
"""Initialization method.
Args:
embedding_size (int): The size of the embedding layer.
hidden_size (int): The amount of hidden neurons.
"""
logger.info('Overriding class: Discriminator -> LSTMDiscriminator.')
# Overrides its parent class with any custom arguments if needed
super(LSTMDiscriminator, self).__init__(name='D_lstm')
# Creates an embedding layer
self.embedding = Dense(embedding_size, name='embedding')
# Creates a LSTM cell
self.cell = LSTMCell(hidden_size, name='lstm_cell')
# Creates the RNN loop itself
self.rnn = RNN(self.cell,
name='rnn_layer',
return_sequences=True,
stateful=True)
# And finally, defining the output layer
self.out = Dense(1, name='out')
logger.info('Class overrided.')
def init_layers(self,
layers=(20, 20),
beta_1=0.9,
beta_2=0.999,
learning_rate=None,
epsilon=1e-10,
time_scale=2000.,
warmup_lstm_update=False,
**kwargs):
"""Initialize layers."""
self.beta_1 = beta_1
self.beta_2 = beta_2
self.epsilon = epsilon
self.time_scale = time_scale
self.warmup_lstm_update = warmup_lstm_update
if learning_rate is None:
learning_rate = {
"sgd": 0.2,
"momentum": 0.5,
"rmsprop": 0.005,
"adam": 0.005,
"powersign": 0.1,
"addsign": 0.1
}
self.learning_rate = learning_rate
self.recurrent = [LSTMCell(hsize, **kwargs) for hsize in layers]
self.choice = Dense(6, input_shape=(layers[-1], ))
def _build_model(self):
# Neural Net for Deep-Q learning Model
observation_input = Input(shape=(self.state_size, ))
UNITS = 32
cell_0_units = UNITS
cell_1_units = UNITS
cell_2_units = UNITS
cell_0_state_in = [Input(shape=(cell_0_units, )) for _ in range(2)]
cell_1_state_in = [Input(shape=(cell_1_units, )) for _ in range(2)]
cell_2_state_in = [Input(shape=(cell_2_units, )) for _ in range(2)]
output, cell_0_output, cell_0_cell_state = LSTMCell(
cell_0_units,
input_shape=(self.state_size, ))(inputs=observation_input,
states=cell_0_state_in)
output = Dropout(rate=0.3)(output)
output, cell_1_output, cell_1_cell_state = LSTMCell(
cell_1_units,
input_shape=(self.state_size, ))(inputs=observation_input,
states=cell_1_state_in)
output = Dropout(rate=0.3)(output)
#output, cell_2_state = LSTMCell(cell_2_units, input_shape=(self.state_size,))([observation_input, cell_2_state_in])
#output = Dropout(rate=0.3)(output)
value_estimator = Dense(1)(output)
value_estimator = LeakyReLU()(value_estimator)
advantage_estimator = Dense(self.action_size)(output)
advantage_estimator = LeakyReLU()(advantage_estimator)
quality_estimator = Lambda(lambda tensors: tensors[0] + tensors[
1] - keras_backend.mean(tensors[1], axis=1, keepdims=True),
output_shape=(self.action_size, ))(
[value_estimator, advantage_estimator])
model = Model(inputs=[observation_input] + cell_0_state_in +
cell_1_state_in + cell_2_state_in,
outputs=[
quality_estimator, cell_0_output, cell_0_cell_state,
cell_1_output, cell_1_cell_state
])
model.compile(loss=mean_squared_error,
optimizer=Adam(lr=self.learning_rate))
return model
def __init__(self,
model,
lstm_size=64,
lstm_num_layers=1,
tanh_constant=1.5,
cell_exit_extra_step=False,
skip_target=0.4,
temperature=None,
branch_bias=0.25,
entropy_reduction='sum'):
super().__init__(model)
self.tanh_constant = tanh_constant
self.temperature = temperature
self.cell_exit_extra_step = cell_exit_extra_step
cells = [
LSTMCell(units=lstm_size, use_bias=False)
for _ in range(lstm_num_layers)
]
self.lstm = RNN(cells, stateful=True)
self.g_emb = tf.random.normal((1, 1, lstm_size)) * 0.1
self.skip_targets = tf.constant([1.0 - skip_target, skip_target])
self.max_layer_choice = 0
self.bias_dict = {}
for mutable in self.mutables:
if isinstance(mutable, LayerChoice):
if self.max_layer_choice == 0:
self.max_layer_choice = len(mutable)
assert self.max_layer_choice == len(mutable), \
"ENAS mutator requires all layer choice have the same number of candidates."
if 'reduce' in mutable.key:
bias = []
for choice in mutable.choices:
if 'conv' in str(type(choice)).lower():
bias.append(branch_bias)
else:
bias.append(-branch_bias)
self.bias_dict[mutable.key] = tf.constant(bias)
# exposed for trainer
self.sample_log_prob = 0
self.sample_entropy = 0
self.sample_skip_penalty = 0
# internal nn layers
self.embedding = Embedding(self.max_layer_choice + 1, lstm_size)
self.soft = Dense(self.max_layer_choice, use_bias=False)
self.attn_anchor = Dense(lstm_size, use_bias=False)
self.attn_query = Dense(lstm_size, use_bias=False)
self.v_attn = Dense(1, use_bias=False)
assert entropy_reduction in [
'sum', 'mean'
], 'Entropy reduction must be one of sum and mean.'
self.entropy_reduction = tf.reduce_sum if entropy_reduction == 'sum' else tf.reduce_mean
self.cross_entropy_loss = SparseCategoricalCrossentropy(
from_logits=True, reduction=Reduction.NONE)
self._first_sample = True
def __init__(self, units, dropout, output_steps, output_size):
super().__init__()
self.output_steps = output_steps
self.units = units
self.lstm_cell = LSTMCell(units, dropout=dropout)
self.lstm_rnn = RNN(self.lstm_cell, return_state=True)
self.dense = Dense(output_size, activation="softmax")
def __init__(self):
self.layers = hps.DecoderRNN_layers
self.size = hps.DecoderRNN_size
self.lstm_list = [LSTMCell(self.size) for _ in range(self.layers)]
self.lstm_cell = RNN(self.lstm_list,
return_state=True,
return_sequences=True)
def __init__(self, num_class, name='rnn-classifier', **kwargs):
super(BiLSTMClassifier, self).__init__(name=name, **kwargs)
self.hidden_size = 128
self.cells = [
LSTMCell(self.hidden_size, kernel_regularizer=l2(),
recurrent_regularizer=l2(), dropout=0.2,
recurrent_dropout=0.2),
LSTMCell(self.hidden_size, kernel_regularizer=l2(),
recurrent_regularizer=l2(), dropout=0.2,
recurrent_dropout=0.2)
]
self.rnn1 = Bidirectional(
RNN(self.cells, return_sequences=True)
)
self.rnn2 = Bidirectional(
LSTM(num_class, return_sequences=True, activation='softmax', kernel_regularizer=l2(),
recurrent_regularizer=l2(), dropout=0.2,
recurrent_dropout=0.2), merge_mode='sum')
self.crf = CRF(num_class)
def __init__(self, n_units, n_acts, conv_fn="impala_cnn", **conv_kwarg):
super(RLconv, self).__init__()
self.n_units = n_units
self.n_acts = n_acts
self.cnn_model = cnn_dict[conv_fn](**conv_kwarg)
self.lstmcell = LSTMCell(self.n_units)
self.act_dense = Dense(self.n_acts,
kernel_initializer=normalized_col_init(0.01))
self.critic_dense = Dense(1,
kernel_initializer=normalized_col_init(1.0))
def create_model(data, config, is_training=True):
with tf.varaiable_scope("tacotron2", reuse=tf.AUTO_REUSE):
inputs,input_sequences_length,target,target_sequences_length, \
target_inputs = data['inputs'],data['input_sequences_length'],data['target'], \
data['target_sequences_length'],data['target_inputs']
batch_size = tf.shape(inputs)[0]
# Start of Encoder layers
embedding_table = tf.Variable(
name='embedding_table',
shape=[VOCABULARY_SIZE, 128],
dtype=tf.float32,
initializer=tf.truncated_normal_initializer(stddev=0.5))
embedding_layer = tf.nn.embedding_lookup(embedding_table, inputs)
conv_outputs = n_layer_1d_convolution(embedding_layer,
n=3,
filter_width=5,
channels=512,
name="convolution_encoder")
cell_fw = ZoneoutWrapper(LSTMCell(256, name="encoder_lstm_forward"),
0.1,
is_training=is_training)
cell_bw = ZoneoutWrapper(LSTMCell(256, name="encoder_lstm_backward"),
0.1,
is_training=is_training)
outputs = tf.keras.layers.Bidirectional(cell_fw,
backward_layer=cell_bw)
# Stacking rnn starts
model = tf.keras.Sequential([
Bidirectional(layer=cell_fw,
merge_mode="concat",
backward_layer=cell_bw),
])
inputs = conv_outputs
def __init__(self):
tf.reset_default_graph()
self.encoder_sege_file = "./tf_data_new/enc.segement"
self.decoder_sege_file = "./tf_data_new/dec.segement"
self.encoder_vocabulary = "./tf_data_new/enc.vocab"
self.decoder_vocabulary = "./tf_data_new/dec.vocab"
self.eval_enc = "./tf_data_new/eval_enc"
self.eval_dec = "./tf_data_new/eval_dec"
self.vocab_file = "./tf_data_new/en_de_vocabs"
self.batch_size = 20
self.max_batches = 15000
self.show_epoch = 10
self.model_path = './model_2/'
self.transform_model = Transformer(
embedding_size=128,
num_layers=6,
keep_prob_rate=0.2,
learning_rate=0.0001,
learning_decay_rate=0.99,
clip_gradient=True,
is_embedding_scale=True,
multihead_num=8,
max_gradient_norm=5,
vocab_size=40020,
max_encoder_len=200,
max_decoder_len=200,
share_embedding=True,
pad_index=0,
learning_decay_steps=500,
dimension_feedforword=2048,
dimension_model=512,
)
self.LSTMmodel = dynamicSeq2seq(encoder_cell=LSTMCell(500),
decoder_cell=LSTMCell(500),
encoder_vocab_size=70824,
decoder_vocab_size=70833,
embedding_size=128,
attention=False,
bidirectional=False,
debug=False,
time_major=True)
def __init__(self, rnn_cell_sizes, label_dim, keep_prob):
"""
Construct RNNColorBot
rnn_cell_sizes: list of integers denoting the size of each LSTM cell in the RNN; rnn_cell_sizes[i] is the size of the i-th layer cell
label_dimension: the length of the labels
"""
super()__init__(name='')
self.rnn_cell_sizes = rnn_cell_sizes
self.label_dimension = label_dim
self.keep_prob = keep_prob
self.cells = [LSTMCell(size) for size in self.rnn_cell_sizes]
self.relu = Dense(self.label_dimension, activation=tf.nn.relu)
def test_lstm_cell():
n_inputs = 3
n_units = 4
batch_size = 1
inputs = tx.Input(n_units=n_inputs)
lstm0 = tx.LSTMCell(
inputs,
n_units,
activation=tf.tanh,
gate_activation=tf.sigmoid,
forget_bias_init=tf.initializers.ones(),
)
lstm1 = LSTMCell(n_units,
activation='tanh',
recurrent_activation='sigmoid',
unit_forget_bias=True,
implementation=2)
state0 = [s() for s in lstm0.previous_state]
# get_initial_state from keras returns either a tuple or a single
# state see `test_rnn_cell`, but the __call__ API requires an iterable
state1 = lstm1.get_initial_state(inputs, batch_size=1)
assert tx.tensor_equal(state1, state0)
inputs.value = tf.ones([batch_size, n_inputs])
res1 = lstm1(inputs, state0)
res1_ = lstm1(inputs, state0)
for r1, r2 in zip(res1, res1_):
assert tx.tensor_equal(r1, r2)
# the only difference is that keras kernels are fused together
kernel = tf.concat([w.weights.value() for w in lstm0.layer_state.w],
axis=-1)
w_i, _, _, _ = tf.split(kernel, 4, axis=1)
assert tx.tensor_equal(w_i, lstm0.w[0].weights.value())
recurrent_kernel = tf.concat([u.weights for u in lstm0.layer_state.u],
axis=-1)
bias = tf.concat([w.bias for w in lstm0.layer_state.w], axis=-1)
assert tx.tensor_equal(tf.shape(kernel), tf.shape(lstm1.kernel))
assert tx.tensor_equal(tf.shape(recurrent_kernel),
tf.shape(lstm1.recurrent_kernel))
assert tx.tensor_equal(tf.shape(bias), tf.shape(lstm1.bias))
lstm1.kernel = kernel
lstm1.recurrent_kernel = recurrent_kernel
lstm1.bias = bias
res2 = lstm1(inputs, state0)
for i in range(len(res1)):
assert not tx.tensor_equal(res1[i], res2[i])
res0 = lstm0()
assert tx.tensor_equal(res0, res2[0])
def add_lstm(self, inputs):
if 'lstm_conf' in self.nn_config:
lstm_conf = self.nn_config['lstm_conf']
activation = None if lstm_conf['batch_norm'] else lstm_conf[
'lstm_activation']
return_seq = True if '1dCNN' in self.nn_config else False
cell = LSTMCell(units=lstm_conf['lstm_units'],
activation=activation)
if lstm_conf['method'] == 'dynamic_rnn':
rnn_outputs1, states = dynamic_rnn(cell,
inputs,
dtype=tf.float32)
lstm_outputs = tf.reshape(rnn_outputs1[:, -1, :],
[-1, lstm_conf['lstm_units']])
elif lstm_conf['method'] == 'keras_lstm_layer':
lstm_outputs = LSTM(lstm_conf['lstm_units'],
name="first_lstm",
activation=activation,
input_shape=(self.data_config['lookback'],
self.ins),
return_sequences=return_seq)(inputs)
else:
rnn_layer = RNN(cell, return_sequences=return_seq)
lstm_outputs = rnn_layer(inputs) # [batch_size, neurons]
if self.verbose > 0:
print(lstm_outputs.shape, 'before reshaping',
K.eval(tf.rank(lstm_outputs)))
lstm_outputs = tf.reshape(lstm_outputs[:, :],
[-1, lstm_conf['lstm_units']])
if self.verbose > 0:
print(lstm_outputs.shape, 'after reshaping',
K.eval(tf.rank(lstm_outputs)))
if lstm_conf['batch_norm']:
rnn_outputs3 = BatchNormalization()(lstm_outputs)
lstm_outputs = Activation('relu')(rnn_outputs3)
if lstm_conf['dropout'] is not None:
lstm_outputs = Dropout(lstm_conf['dropout'])(lstm_outputs)
else:
lstm_outputs = inputs
return lstm_outputs
