def lstm_reshape(self,
inputs,
name_prefix,
index,
reshaped_inputs=None,
initial=False):
name_prefix = "{0}_{1}_{2}".format(self.controller_network_name,
name_prefix, index)
cell = LSTMCell(
self.lstm_cell_units,
kernel_initializer=get_weight_initializer(initializer="lstm"),
recurrent_initializer=get_weight_initializer(initializer="lstm"))
if initial:
x = RNN(cell,
return_state=True,
name="{0}_{1}".format(name_prefix, "lstm"))(inputs)
else:
x = RNN(cell,
return_state=True,
name="{0}_{1}".format(name_prefix,
"lstm"))(reshaped_inputs,
initial_state=inputs[1:])
rx = Reshape((-1, self.lstm_cell_units),
name="{0}_{1}".format(name_prefix, "reshape"))(x[0])
return x, rx
def build(self, input_shape):
cell = self.Cell(self.units)
self.rnn = RNN(cell, return_sequences=self.return_sequences, go_backwards=self.go_backwards)
self.rnn.build(input_shape=input_shape)
self._trainable_weights = self.rnn.trainable_weights
super(Complex_LSTM, self).build(input_shape)
def predict(self, x_train, y_train, x_test, y_test, embeddings, sequence_length, class_count):
req_type = type(np.array([]))
assert type(x_train) == req_type and type(x_test) == req_type
assert type(y_train) == req_type and type(y_test) == req_type
from keras.models import Model
from keras.layers import Input, Dense, Dropout, Flatten, Embedding, LSTM, Bidirectional, LSTMCell, StackedRNNCells, RNN
from keras.layers.merge import Concatenate
from keras.optimizers import Adam, Adagrad
from keras.regularizers import l2
input_shape = (sequence_length,)
model_input = Input(shape=input_shape)
print("Input tensor shape: ", int_shape(model_input))
model_embedding = Embedding(embeddings.shape[0], 100, input_length=sequence_length, name="embedding")(model_input)
# model_embedding = Embedding(embeddings.shape[0], embeddings.shape[1], weights=[embeddings], name="embedding")(model_input)
print("Embeddings tensor shape: ", int_shape(model_embedding))
# model_recurrent = Bidirectional(LSTM(embeddings.shape[1], activation='relu', dropout=0.2))(model_embedding)
#####################################################################################################################################
# cells_forward = [LSTMCell(units=self.output_size), LSTMCell(units=self.output_size), LSTMCell(units=self.output_size)]
# cells_backward = [LSTMCell(units=self.output_size), LSTMCell(units=self.output_size), LSTMCell(units=self.output_size)]
cells_forward = [LSTMCell(units=self.output_size)] * 3
cells_backward = [LSTMCell(units=self.output_size)] * 3
# LSTM_forward = RNN(cells_forward, go_backwards=False)(model_embedding)
# LSTM_backward = RNN(cells_backward, go_backwards=True)(model_embedding)
cells_forward_stacked = StackedRNNCells(cells_forward)
cells_backward_stacked = StackedRNNCells(cells_backward)
LSTM_forward = RNN(cells_forward_stacked, go_backwards=False)(model_embedding)
LSTM_backward = RNN(cells_backward_stacked, go_backwards=True)(model_embedding)
model_recurrent = Concatenate(axis=-1)([LSTM_forward, LSTM_backward])
# model_recurrent = Bidirectional(cells_forward)(model_embedding)
######################################################################################################################################
model_hidden = Dropout(0.5)(model_recurrent)
model_output = Dense(class_count, activation="softmax", kernel_regularizer=l2(0.1), bias_regularizer=l2(0.1))(model_hidden)
model = Model(model_input, model_output)
optimizer = Adam(lr=self.learning_rate)
# optimizer = Adagrad(lr=self.learning_rate)
model.compile(loss="categorical_crossentropy", optimizer=optimizer, metrics=["accuracy"])
# model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
# model.fit(x_train, y_train, batch_size=self.batch_size, epochs=self.num_epochs,
# validation_data=(x_test, y_test), verbose=2, shuffle=True)
model.fit(x_train, y_train, batch_size=self.batch_size, epochs=self.num_epochs,
validation_split=0.2, verbose=2, shuffle=True)
score, acc = model.evaluate(x_test, y_test,
batch_size=self.batch_size)
print('\nTest score:', score)
print('Test accuracy:', acc, '\n')
test_model(model, x_test, y_test)
return 0
def __init__(self,
n_dims,
units,
activation='tanh',
recurrent_activation='hard_sigmoid',
use_bias=True,
kernel_initializer='glorot_uniform',
recurrent_initializer='orthogonal',
bias_initializer='zeros',
kernel_regularizer=None,
recurrent_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
recurrent_constraint=None,
bias_constraint=None,
dropout=0.,
recurrent_dropout=0.,
implementation=1,
return_sequences=False,
return_state=False,
go_backwards=False,
stateful=False,
unroll=False,
reset_after=False,
**kwargs):
cell = GRUCellKeepDim(n_dims,
units,
activation=activation,
recurrent_activation=recurrent_activation,
use_bias=use_bias,
kernel_initializer=kernel_initializer,
recurrent_initializer=recurrent_initializer,
bias_initializer=bias_initializer,
kernel_regularizer=kernel_regularizer,
recurrent_regularizer=recurrent_regularizer,
bias_regularizer=bias_regularizer,
kernel_constraint=kernel_constraint,
recurrent_constraint=recurrent_constraint,
bias_constraint=bias_constraint,
dropout=dropout,
recurrent_dropout=recurrent_dropout,
implementation=implementation,
reset_after=reset_after)
RNN.__init__(self,
cell,
return_sequences=return_sequences,
return_state=return_state,
go_backwards=go_backwards,
stateful=stateful,
unroll=unroll,
**kwargs)
self.activity_regularizer = regularizers.get(activity_regularizer)
def rnn(month, features, true_price, d):
train_length = int(0.8 * len(month))
x_train = features[:train_length]
x_test = features[train_length:]
label_train = month[:train_length].reshape(-1, 1)
label_test = month[train_length:].reshape(-1, 1)
true = true_price[train_length:]
scaler = StandardScaler()
scaler = scaler.fit(x_train)
x_train = scaler.transform(x_train)
x_test = scaler.transform(x_test)
# scaler2 = StandardScaler()
# scaler2 = scaler2.fit(label_train)
# label_train = scaler2.transform(label_train)
# label_test = scaler2.transform(label_test)
x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], 1))
x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1))
K.clear_session()
model_rnn = Sequential()
model_rnn.add(
RNN(20,
input_shape=(x_train.shape[1], 1),
activation='relu',
return_sequences=True))
model_rnn.add(Dropout(0.2))
# model_rnn.add(RNN(64, activation='relu', return_sequences=True))
# model_rnn.add(Dropout(0.2))
model_rnn.add(RNN(10, activation='relu', return_sequences=False))
model_rnn.add(Dropout(0.2))
model_rnn.add(Dense(1))
model_rnn.compile(optimizer='adam', loss='mean_squared_error')
early_stop = EarlyStopping(monitor='loss', patience=5, verbose=1)
model_rnn.fit(x_train, label_train, epochs=1000, batch_size=32, verbose=0)
# evaluate the keras model
predicted_y = model_rnn.predict(x_test)
predicted_y = predicted_y.reshape(len(predicted_y), )
return true, predicted_y
number = 0
def build(self, input_shape):
# input to the kernelized LSTM is a 4D tensor:
nbatch, nfram, kernels, n_in = input_shape
cell = self.Cell(kernels, self.units, self.activation, self.recurrent_activation)
self.rnn = RNN(cell, return_sequences=self.return_sequences, go_backwards=self.go_backwards)
# the Keras RNN implementation does only work with 3D tensors, hence we flatten the last two dimensions of the input:
self.rnn.build(input_shape=(nbatch, nfram, kernels*n_in))
self._trainable_weights = self.rnn.trainable_weights
super(Kernelized_LSTM, self).build(input_shape)
def __call__(self, input):
rnn_cell_1 = GRUCell(units=self.rnn_units_1, dropout=self.dropout, recurrent_dropout=self.recurrent_dropout, name=self.name + '_rnn_cell_1' if self.name else None)
rnn_cell_2 = GRUCell(units=self.rnn_units_2, dropout=self.dropout, recurrent_dropout=self.recurrent_dropout, name=self.name + '_rnn_cell_2' if self.name else None)
# gru_cell_3 = GRUCell(units= rnn_units_3, dropout= rnn_dropout, recurrent_dropout= rnn_recurrent_dropout, reset_after= False)
rnn_stack_cell = StackedRNNCells(cells=[rnn_cell_1, rnn_cell_2], name=self.name + '_stacked_rnn_cell' if self.name else None)
rnn = RNN(cell=rnn_stack_cell, return_state=self.return_state, return_sequences=self.return_sequence, unroll=self.unroll, name=self.name)(input)
return rnn
def gen_RNN():
#lstm = CuDNNLSTM(6, stateful=True, return_sequences=True)(inp)
#lstm = LeakyReLU()(lstm)
#lstm = CuDNNLSTM(3, stateful=True, return_sequences=True)(lstm)
#lstm = Activation('sigmoid')(lstm)
input = Input(batch_shape=(1, 1, 64 * 64 * 3 + num_classes))
lstm = Concatenate()(
[Dense(32, activation='elu', name='zoinks')(input), input])
cells = []
for i in range(2):
cells.append(
LSTMCell(256,
recurrent_dropout=0.05,
implementation=2,
recurrent_initializer='glorot_normal'))
lstm = RNN(cells, stateful=True, name="RNN_yoooo")(lstm)
lstm = Dense(128, activation='tanh', name='jinkies')(lstm)
lstm = Dense(3, activation='tanh', name='yikes')(lstm)
lstm = Reshape((
1,
3,
))(lstm)
model = Model(inputs=input, outputs=lstm)
return model
def load_skip_thought_model(
encode_size,
embedding_words_per_sentence,
lookup_table
):
word_embedding_size = lookup_table.shape[1]
def __probabilities_calc(x):
result = K.dot(lookup_table_k, K.permute_dimensions(x, [0,2,1]))
result = K.permute_dimensions(result, [1, 2, 0])
result = K.exp(result)
return result
lookup_table_k = K.variable(value=lookup_table)
inputs = Input(shape=(embedding_words_per_sentence, word_embedding_size), name='sentences')
encoder = GRU(encode_size, use_bias=True, return_sequences=False, name='encoded_sentences')
previous_decoder_cell = DecoderGRUCell(word_embedding_size, encoder_size=encode_size)
next_decoder_cell = DecoderGRUCell(word_embedding_size, encoder_size=encode_size)
repeater = RepeatVector(embedding_words_per_sentence, name='repeat_encode_sentences')
joiner = Concatenate(name='concat_encode_w_inputs')
previous_decoder = RNN(previous_decoder_cell, return_sequences=True, name='decoded_previous_sentences')
next_decoder = RNN(next_decoder_cell, return_sequences=True, name='decoded_next_sentences')
prev_prob_calculator = Lambda(__probabilities_calc, name='previous_probabilities')
next_prob_calculator = Lambda(__probabilities_calc, name='next_probabilities')
#prev_prob_calculator = Lambda(__probabilities_calc)
#next_prob_calculator = Lambda(__probabilities_calc)
encoded_sentence = encoder(inputs)
repeated_encoded_sentence = repeater(encoded_sentence)
concatenated_inputs = joiner([repeated_encoded_sentence, inputs])
decoded_previous_words = previous_decoder(concatenated_inputs)
decoded_next_words = next_decoder(concatenated_inputs)
previous_probs = prev_prob_calculator(decoded_previous_words)
next_probs = next_prob_calculator(decoded_next_words)
model = Model(inputs=[inputs], outputs=[previous_probs, next_probs], name='skip_thought')
return model
def rnn(in_array, target_array):
# create and fit the model
model = Sequential()
model.add(RNN(512, input_shape=(in_array.shape[1], 1)))
model.add(Dense(target_array.shape[1], activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def create_stacked_lstms(self, hidden_size, num_layers, return_sequences,
return_state):
# Create a list of RNN Cells, these are then concatenated into a single layer
# with the RNN layer.
cells = []
for i in range(num_layers):
cells.append(LSTMCell(hidden_size)) # TODO: try regularizers
return RNN(cells,
return_sequences=return_sequences,
return_state=return_state)
def get_model_soft_sharing_lstm_singleoutput(emb_matrix,
time_steps,
learning_rate=0.001,
n_classes=1,
decay=0.1,
layers=3):
cells_1 = [LSTMCell(128, dropout=0.2) for i in range(layers)]
cells_2 = [LSTMCell(128, dropout=0.2) for i in range(layers)]
input = Input(shape=(time_steps, ), dtype='int32')
embedding = Embedding(input_dim=emb_matrix.shape[0],
output_dim=emb_matrix.shape[1],
weights=[emb_matrix],
input_length=time_steps,
trainable=True)
sequence_input = embedding(input)
x = Bidirectional(RNN(cells_1, return_sequences=True))(sequence_input)
x = Bidirectional(RNN(cells_2, return_sequences=False))(x)
x = Dropout(0.2)(x)
x = Dense(256, activation='relu')(x)
x = Dropout(0.2)(x)
x = Dense(128, activation='relu')(x)
x = Dropout(0.2)(x)
x = Dense(128, activation='relu')(x)
x = Dropout(0.2)(x)
x = Dense(64, activation='relu')(x)
x = Dropout(0.2)(x)
x = Dense(64, activation='relu')(x)
preds = Dense(n_classes, activation='sigmoid')(x)
model = Model(input, preds)
adam = optimizers.Adam(lr=learning_rate, decay=DECAY)
model.compile(loss='binary_crossentropy',
optimizer=adam,
metrics=['accuracy'])
print(model.summary())
return model
def get_model(self):
# Input text
encoder_inputs = Input(shape=(None, ))
# Input summary
decoder_inputs = Input(shape=(None, ))
# word embedding layer for text
encoder_inputs_emb = Embedding(input_dim=self.num_encoder_tokens + 1,
output_dim=self.embedding_dim,
mask_zero=True)(encoder_inputs)
# word embedding layer for summary
decoder_inputs_emb = Embedding(input_dim=self.num_decoder_tokens + 1,
output_dim=self.embedding_dim,
mask_zero=True)(decoder_inputs)
# Bidirectional LSTM encoder
encoder_out = Bidirectional(LSTM(self.hidden_dim // 2,
return_sequences=True,
return_state=True),
merge_mode='concat')(encoder_inputs_emb)
encoder_o = encoder_out[0]
initial_h_lstm = concatenate([encoder_out[1], encoder_out[2]])
initial_c_lstm = concatenate([encoder_out[3], encoder_out[4]])
initial_decoder_state = Dense(self.hidden_dim, activation='tanh')(
concatenate([initial_h_lstm, initial_c_lstm]))
# LSTM decoder + attention
initial_attention_h = Lambda(lambda x: K.zeros_like(x)[:, 0, :])(
encoder_o)
initial_state = [initial_decoder_state, initial_attention_h]
cell = DenseAnnotationAttention(cell=GRUCell(self.hidden_dim),
units=self.hidden_dim,
input_mode="concatenate",
output_mode="cell_output")
# TODO output_mode="concatenate", see TODO(3)/A
decoder_o, decoder_h, decoder_c = RNN(cell=cell,
return_sequences=True,
return_state=True)(
decoder_inputs_emb,
initial_state=initial_state,
constants=encoder_o)
decoder_o = Dense(self.hidden_dim * 2)(concatenate(
[decoder_o, decoder_inputs_emb]))
y_pred = TimeDistributed(
Dense(self.num_decoder_tokens + 1,
activation='softmax'))(decoder_o)
model = Model([encoder_inputs, decoder_inputs], y_pred)
return model
def _build_decoder(self):
decoder_cells = []
for n_hidden_neurons in self.decoder_layers:
decoder_cells.append(
self.cell(units=n_hidden_neurons,
dropout=self.dropout,
kernel_regularizer=l2(self.l2),
recurrent_regularizer=l2(self.l2)))
# return output for EACH timestamp
self.decoder = RNN(decoder_cells,
return_sequences=True,
return_state=True,
name='decoder')
def create_embed_model(obs_timesteps=10, pred_timesteps=3, nb_nodes=208, k=1, dgc_mode='hybrid', inner_act=None):
encoder = RNN(DGCRNNCell(k,dgc_mode=dgc_mode), return_state=True)
decoder = RNN(DGCRNNCell(k,dgc_mode=dgc_mode), return_sequences=True, return_state=True)
unstack_k = Lambda(unstack)
choice = Scheduled()
input_obs = Input(shape=(obs_timesteps, nb_nodes, 1))
input_gt = Input(shape=(pred_timesteps, nb_nodes, 1)) #(None, T, N, 1)
encoder_inputs = Lambda(lambda x: K.squeeze(x, axis = -1))(input_obs) # (None, T, N)
encoder_outputs, state_h = encoder(encoder_inputs)
unstacked = unstack_k(input_gt) #[(None, N, 1) x T] list
initial = unstacked[0] #(None, N, 1)
decoder_inputs = Lambda(lambda x: K.permute_dimensions(x, (0,2,1)))(initial) #(None, 1, N)
decoder_outputs_new, state_h_new = decoder(decoder_inputs, initial_state=state_h)
state_h = state_h_new
prediction = []
decoded_results = decoder_outputs_new
prediction.append(decoded_results)
if pred_timesteps > 1:
for i in range(1,pred_timesteps):
decoder_inputs = choice([prediction[-1], unstacked[i]])#(None, 208, 1)
decoder_inputs = Lambda(lambda x: K.permute_dimensions(x, (0,2,1)))(decoder_inputs)#(None, 1, 208)
decoder_outputs_new, state_h_new = decoder(decoder_inputs, initial_state=state_h)
state_h = state_h_new
decoded_results = decoder_outputs_new
prediction.append(decoded_results)
outputs = Lambda(stack)(prediction)
model = Model([input_obs, input_gt], outputs)
return model
def Emojify_RNN_model(input_shape, word_to_vec_map, word_to_index):
sentence_indices = Input(input_shape, dtype='int32')
embedding_layer = pretrained_embedding_layer(word_to_vec_map,
word_to_index)
embeddings = embedding_layer(sentence_indices)
# https://stackoverflow.com/questions/45989610/setting-up-the-input-on-an-rnn-in-keras
# Desired result is a sequence with same length, we will use return_sequences=True. (Else, you'd get only one result).
# keras.layers.RNN(cell, return_sequences=False, return_state=False, go_backwards=False, stateful=False, unroll=False)
cells = [MinimalRNNCell(32), MinimalRNNCell(64)]
X = RNN(cells, return_sequences=True)(embeddings)
X = Dropout(0.5)(X)
X = Dense(5)(X)
X = Activation('sigmoid')(X)
model = Model(inputs=sentence_indices, outputs=X)
return model
def _build_encoder(self):
"""
Build the encoder multilayer RNN (stacked RNN)
"""
# Create a list of RNN Cells, these get stacked one after the other in the RNN,
# implementing an efficient stacked RNN
encoder_cells = []
for n_hidden_neurons in self.encoder_layers:
encoder_cells.append(
self.cell(units=n_hidden_neurons,
dropout=self.dropout,
kernel_regularizer=l2(self.l2),
recurrent_regularizer=l2(self.l2)))
self.encoder = RNN(encoder_cells, return_state=True, name='encoder')
def build_model(input_shape):
batch_shape = (BATCH_SIZE, ) + input_shape
inputs = []
for _ in range(FRAMES):
input = Input(batch_shape=batch_shape)
inputs.append(input)
layers = 1
conv_outputs = []
for input in inputs:
filters = 8
conv_input = input
for _ in range(layers):
conv = Conv2D(filters,
kernel_size=3,
activation='relu',
padding='same')(conv_input)
bnrm = BatchNormalization()(conv)
pool = MaxPooling2D(8)(bnrm)
conv_input = pool
filters *= 2
flattened = Flatten()(conv_input)
conv_outputs.append(flattened)
stacked = Stack()(conv_outputs)
units = FRAMES
cell = LSTMCell(units)
lstm = RNN(cell, unroll=True)(stacked)
drpt1 = Dropout(0.2)(lstm)
dense = Dense(units // 2, activation='relu')(drpt1)
drpt2 = Dropout(0.2)(dense)
output = Dense(1)(drpt2)
model = km.Model(inputs=inputs, outputs=output)
adam = optmzrs.Adam(learning_rate=0.0001, amsgrad=True)
model.compile(loss='mse', optimizer=adam, metrics=['mae'])
model.summary()
return model
def build_model(allow_cudnn_kernel=True): # CuDNN is only available at the layer level, and not at the cell level. # This means `LSTM(units)` will use the CuDNN kernel, # while RNN(LSTMCell(units)) will run on non-CuDNN kernel. if allow_cudnn_kernel: # The LSTM layer with default options uses CuDNN. lstm_layer = LSTM(units, input_shape=(None, input_dim)) else: # Wrapping a LSTMCell in a RNN layer will not use CuDNN. lstm_layer = RNN( LSTMCell(units), input_shape=(None, input_dim) ) model = keras.models.Sequential([ lstm_layer, BatchNormalization(), Dense(output_size), ]) return model
def __init__(self, gpus=1, batch_size=50, segment_size=12, output_size=12, window_size=15, cnn_filters=[2,3,4], hidden_sizes=[10,10], learning_rate=0.0001, learning_rate_decay=0, create_tensorboard=False): self.segment_size = segment_size self.output_size = output_size self.gpus = gpus # Define an input sequence. # 1 refers to a single channel of the input inputs = Input(shape=(segment_size, window_size, window_size, 1), name="input") # cnns out = TimeDistributed(Conv2D(cnn_filters[0], kernel_size=5, activation='relu', padding='same'), name="cnn_1")(inputs) out = TimeDistributed(MaxPooling2D(), name=f"max_pool")(out) out = TimeDistributed(Conv2D(cnn_filters[1], kernel_size=5, activation='relu', padding='same'), name="cnn_2")(out) out = TimeDistributed(AveragePooling2D(), name=f"avg_pool_1")(out) out = TimeDistributed(Conv2D(cnn_filters[2], kernel_size=5, activation='relu', padding='same'), name="cnn_3")(out) out = TimeDistributed(AveragePooling2D(), name=f"avg_pool_2")(out) out = TimeDistributed(Flatten(), name="flatten_before_lstm")(out) cells = [LSTMCell(hidden_sizes[0]), LSTMCell(hidden_sizes[1])] out = RNN(cells)(out) # out = Flatten(name="flatten_after_lstm")(out) out = Dense(100, activation='relu', name=f"mlp_relu")(out) out = Dense(output_size, activation='linear', name=f"mlp_linear")(out) self.model = Model(inputs=inputs, outputs=out) self.model = model_device_adapter.get_device_specific_model(self.model, gpus) optimizer = Adam(lr=learning_rate, decay=learning_rate_decay) self.model.compile(loss='mse', optimizer=optimizer) print(self.model.summary()) super(CnnLSTM, self).__init__(batch_size=batch_size, create_tensorboard=create_tensorboard)
def __init__(self,
rnn,
rnn_dim,
input_dim,
dropout_W=0.0,
dropout_U=0.0,
cnn_border_mode='same'):
if rnn == 'lstm':
from keras.layers import CuDNNLSTM as RNN
elif rnn == 'sru':
from nea.cell import SRU as RNN
elif rnn == 'nlstm':
from nea.cell import NestedLSTM as RNN
elif rnn == 'gru':
from keras.layers import CuDNNGRU as RNN
elif rnn == 'simple':
from keras.layers.recurrent import SimpleRNN as RNN
elif rnn == 'indrnn':
from nea.cell import IndRNN as RNN
self.model = Sequential()
self.model.add(
Conv1D(filters=100,
kernel_size=3,
padding=cnn_border_mode,
strides=1,
input_shape=input_dim))
for i in range(MC.DEPTH):
self.model.add(
Bidirectional(
RNN(
rnn_dim,
# dropout=dropout_W,
#recurrent_dropout=dropout_U,
return_sequences=True), ))
if MC.HIGHWAY:
self.model.add(TimeDistributed(Highway(activation='tanh')))
#self.model.add(TimeDistributed(Dense(MC.DENSE_DIM,activation='relu')))
self.model.add(Dropout(MC.DROPOUT))
self.model.add(Attention())
def cell_unit_rnn():
import keras
from keras.layers import RNN
import keras.backend.tensorflow_backend as K
class MinimalRNNCell(keras.layers.Layer):
def __init__(self, units, **kwargs):
self.units = units
self.state_size = units
super(MinimalRNNCell, self).__init__(**kwargs)
def build(self, input_shape):
self.kernel = self.add_weight(shape=(input_shape[-1], self.units),
initializer='uniform',
name='kernel')
self.recurrent_kernel = self.add_weight(
shape=(self.units, self.units),
initializer='uniform',
name='recurrent_kernel')
self.built = True
def call(self, inputs, states):
prev_output = states[0]
h = K.dot(inputs, self.kernel)
print('hidden', h)
output = h + K.dot(prev_output, self.recurrent_kernel)
return output, [output]
# Let's use this cell in a RNN layer:
cell = MinimalRNNCell(100)
x = keras.Input(batch_shape=(3000, 28, 28))
# When return_state is True,
# [<tf.Tensor 'rnn_1/TensorArrayReadV3:0' shape=(3000, 100) dtype=float64>,
# <tf.Tensor 'rnn_1/while/Exit_2:0' shape=(3000, 100) dtype=float64>]
# When return_state is False,
# Tensor("rnn_1/TensorArrayReadV3:0", shape=(3000, 100), dtype=float64)
layer = RNN(cell, return_state=True)
y = layer(x)
def __build_prednet_layer(self, model_type):
if model_type == ModelType.PREDNET or model_type == ModelType.SINGLE_PIXEL_ACTIVATION:
prednet = PredNet.build_from_params(self.params)
elif model_type == ModelType.CONV_PREDNET:
prednet = ConvPredNet.build_from_params(self.params)
if self.autoencoder is None and not self.trainable_autoencoder:
raise ValueError(
"Pretrained autoencoder not set and autoencoder is not trainable!"
)
elif model_type == ModelType.AMPLIF_ERROR:
prednet = AmplifiedErrorPredNet.build_from_params(self.params)
elif model_type == ModelType.STATE_VECTOR:
cell = StateVectorPredNetCell(self.balls * 4, self.nt,
self.nb_layers,
self.params.extrap_start_time,
self.params.extrap_end_time,
self.params.output_mode)
prednet = RNN(cell, return_sequences=True)
elif model_type == ModelType.CONCAT_PREDNET:
prednet = ConcatPredNet.build_from_params(self.params)
return prednet
def make_model(vocabulary_size,
hidden_size,
num_steps,
use_dropout=True,
lstm=False):
model = Sequential()
model.add(Embedding(vocabulary_size, hidden_size, input_length=num_steps))
if lstm:
model.add(LSTM(hidden_size, return_sequences=True))
model.add(LSTM(hidden_size, return_sequences=True))
else:
model.add(
RNN(SimpleRNNCell(hidden_size, hidden_size, hidden_size),
return_sequences=True))
if use_dropout:
model.add(Dropout(0.5))
model.add(TimeDistributed(Dense(vocabulary_size)))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['categorical_accuracy'])
return model
def test_Bidirectional_with_constants_layer_passing_initial_state():
class RNNCellWithConstants(Layer):
def __init__(self, units, **kwargs):
self.units = units
self.state_size = units
super(RNNCellWithConstants, self).__init__(**kwargs)
def build(self, input_shape):
if not isinstance(input_shape, list):
raise TypeError('expects constants shape')
[input_shape, constant_shape] = input_shape
# will (and should) raise if more than one constant passed
self.input_kernel = self.add_weight(shape=(input_shape[-1],
self.units),
initializer='uniform',
name='kernel')
self.recurrent_kernel = self.add_weight(shape=(self.units,
self.units),
initializer='uniform',
name='recurrent_kernel')
self.constant_kernel = self.add_weight(shape=(constant_shape[-1],
self.units),
initializer='uniform',
name='constant_kernel')
self.built = True
def call(self, inputs, states, constants):
[prev_output] = states
[constant] = constants
h_input = K.dot(inputs, self.input_kernel)
h_state = K.dot(prev_output, self.recurrent_kernel)
h_const = K.dot(constant, self.constant_kernel)
output = h_input + h_state + h_const
return output, [output]
def get_config(self):
config = {'units': self.units}
base_config = super(RNNCellWithConstants, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
# Test basic case.
x = Input((5, 5))
c = Input((3, ))
s_for = Input((32, ))
s_bac = Input((32, ))
cell = RNNCellWithConstants(32)
custom_objects = {'RNNCellWithConstants': RNNCellWithConstants}
with CustomObjectScope(custom_objects):
layer = wrappers.Bidirectional(RNN(cell))
y = layer(x, initial_state=[s_for, s_bac], constants=c)
model = Model([x, s_for, s_bac, c], y)
model.compile(optimizer='rmsprop', loss='mse')
model.train_on_batch([
np.zeros((6, 5, 5)),
np.zeros((6, 32)),
np.zeros((6, 32)),
np.zeros((6, 3))
], np.zeros((6, 64)))
# Test basic case serialization.
x_np = np.random.random((6, 5, 5))
s_fw_np = np.random.random((6, 32))
s_bk_np = np.random.random((6, 32))
c_np = np.random.random((6, 3))
y_np = model.predict([x_np, s_fw_np, s_bk_np, c_np])
weights = model.get_weights()
config = layer.get_config()
with CustomObjectScope(custom_objects):
layer = wrappers.Bidirectional.from_config(copy.deepcopy(config))
y = layer(x, initial_state=[s_for, s_bac], constants=c)
model = Model([x, s_for, s_bac, c], y)
model.set_weights(weights)
y_np_2 = model.predict([x_np, s_fw_np, s_bk_np, c_np])
assert_allclose(y_np, y_np_2, atol=1e-4)
# verify that state is used
y_np_2_different_s = model.predict(
[x_np, s_fw_np + 10., s_bk_np + 10., c_np])
with pytest.raises(AssertionError):
assert_allclose(y_np, y_np_2_different_s, atol=1e-4)
# test flat list inputs
with CustomObjectScope(custom_objects):
layer = wrappers.Bidirectional.from_config(copy.deepcopy(config))
y = layer([x, s_for, s_bac, c])
model = Model([x, s_for, s_bac, c], y)
model.set_weights(weights)
y_np_3 = model.predict([x_np, s_fw_np, s_bk_np, c_np])
assert_allclose(y_np, y_np_3, atol=1e-4)
data = np.array(newdata)
X = data[:, 0:101]
Y = data[:, 5]
train_x = X[:int(len(X) * 0.7)]
test_x = X[int(len(X) * 0.7) + 1:]
train_y = Y[:int(len(Y) * 0.7)]
test_y = Y[int(len(Y) * 0.7) + 1:]
train_x = train_x.reshape((train_x.shape[0], 1, train_x.shape[1]))
test_x = test_x.reshape((test_x.shape[0], 1, test_x.shape[1]))
print(train_x.shape)
print(test_y.shape)
# design network
model = Sequential()
model.add(RNN(200))
model.add(RNN(200))
model.add(Dense(1, activation='relu'))
model.compile(loss='mae', optimizer='adam', metrics=['accuracy'])
# fit network
history = model.fit(train_x,
train_y,
epochs=20,
batch_size=100,
validation_data=(test_x, test_y),
verbose=1,
shuffle=True)
# plot history
pyplot.plot(history.history['loss'], label='train')
pyplot.plot(history.history['val_loss'], label='test')
os.makedirs(config.log_dir) tf.set_random_seed(config.seed) np.random.seed(config.seed) num_steps_ahead = 1 n_layers = 1 layers = [config.m] * n_layers # Encoder # driving series with shape (T, n) encoder_inputs = Input(shape=(None, config.n)) # add endogenous series encoder = RNN([LSTMCell(units) for units in layers], return_state=True) encoder_out_and_states = encoder(encoder_inputs) # Decoder decoder_inputs = Input(shape=(config.T - num_steps_ahead, )) z_i = Dense(64)(decoder_inputs) encoder_out = concatenate(encoder_out_and_states + [z_i]) z = Dense(256)(encoder_out) decoder_outs = Dense(len(config.target_cols), activation="linear")(z) model = Model(inputs=[encoder_inputs, decoder_inputs], outputs=decoder_outs) model.compile(optimizer="adam", loss="mse")
def __construct_rnn_model__(self):
model = Sequential()
n_rnn_layers = len(self.n_units)
if n_rnn_layers > 1:
is_return_sequences = True
else:
is_return_sequences = False
if self.is_self_activation:
is_return_sequences_last_recurrent_layer = True
else:
is_return_sequences_last_recurrent_layer = False
print(is_return_sequences_last_recurrent_layer)
assert self.method is not None
if self.method == 'rnn':
print(self.max_sent_len)
model.add(
RNN(
self.n_units[0],
input_shape=(self.max_sent_len, self.word_emb_dim),
return_sequences=is_return_sequences,
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg,
dropout=self.dropout,
recurrent_dropout=self.recurrent_dropout,
))
for l_idx, n_unit in enumerate(self.n_units[1:]):
if l_idx < (n_rnn_layers - 2):
model.add(
RNN(
n_unit,
return_sequences=True,
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg,
dropout=self.dropout,
recurrent_dropout=self.recurrent_dropout,
))
else:
model.add(
RNN(
n_unit,
return_sequences=
is_return_sequences_last_recurrent_layer,
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg,
dropout=self.dropout,
recurrent_dropout=self.recurrent_dropout,
))
elif self.method == 'lstm':
print(self.max_sent_len)
model.add(
LSTM(
self.n_units[0],
input_shape=(self.max_sent_len, self.word_emb_dim),
return_sequences=is_return_sequences,
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg,
dropout=self.dropout,
recurrent_dropout=self.recurrent_dropout,
))
for l_idx, n_unit in enumerate(self.n_units[1:]):
if l_idx < n_rnn_layers - 2:
model.add(
LSTM(
n_unit,
return_sequences=True,
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg,
dropout=self.dropout,
recurrent_dropout=self.recurrent_dropout,
))
else:
model.add(
LSTM(
n_unit,
return_sequences=
is_return_sequences_last_recurrent_layer,
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg,
dropout=self.dropout,
recurrent_dropout=self.recurrent_dropout,
))
elif self.method == 'gru':
model.add(
GRU(self.n_units[0],
input_shape=(self.max_sent_len, self.word_emb_dim),
return_sequences=is_return_sequences,
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg))
for l_idx, n_unit in enumerate(self.n_units[1:]):
if l_idx < n_rnn_layers - 2:
model.add(
GRU(n_unit,
return_sequences=True,
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg))
else:
model.add(
GRU(n_unit,
return_sequences=
is_return_sequences_last_recurrent_layer,
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg))
elif self.method == 'bi-lstm':
print(self.max_sent_len)
model.add(
Bidirectional(
LSTM(
self.n_units[0],
return_sequences=(
is_return_sequences
or is_return_sequences_last_recurrent_layer),
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg,
dropout=self.dropout,
recurrent_dropout=self.recurrent_dropout,
),
input_shape=(self.max_sent_len, self.word_emb_dim),
))
for l_idx, n_unit in enumerate(self.n_units[1:]):
if l_idx < (n_rnn_layers - 2):
model.add(
Bidirectional(
LSTM(
n_unit,
return_sequences=True,
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg,
dropout=self.dropout,
recurrent_dropout=self.recurrent_dropout,
)))
else:
model.add(
Bidirectional(
LSTM(
n_unit,
return_sequences=
is_return_sequences_last_recurrent_layer,
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg,
dropout=self.dropout,
recurrent_dropout=self.recurrent_dropout,
)))
elif self.method == 'bi-gru':
model.add(
Bidirectional(
GRU(self.n_units[0],
return_sequences=is_return_sequences,
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg),
input_shape=(self.max_sent_len, self.word_emb_dim),
))
for l_idx, n_unit in enumerate(self.n_units[1:]):
if l_idx < (n_rnn_layers - 2):
model.add(
Bidirectional(
GRU(n_unit,
return_sequences=True,
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg)))
else:
model.add(
Bidirectional(
GRU(n_unit,
return_sequences=
is_return_sequences_last_recurrent_layer,
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg)))
elif self.method == 'lstm-cnn':
print(self.max_sent_len)
model.add(
Conv1D(self.filters,
self.kernel_size,
input_shape=(self.max_sent_len, self.word_emb_dim),
padding='valid',
activation='relu',
strides=1))
model.add(MaxPooling1D(pool_size=self.pool_size))
model.add(
LSTM(
self.n_units[0],
return_sequences=is_return_sequences,
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg,
dropout=self.dropout,
recurrent_dropout=self.recurrent_dropout,
))
for l_idx, n_unit in enumerate(self.n_units[1:]):
if l_idx < n_rnn_layers - 2:
model.add(
LSTM(
n_unit,
return_sequences=True,
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg,
dropout=self.dropout,
recurrent_dropout=self.recurrent_dropout,
))
else:
model.add(
LSTM(
n_unit,
return_sequences=
is_return_sequences_last_recurrent_layer,
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg,
dropout=self.dropout,
recurrent_dropout=self.recurrent_dropout,
))
elif self.method == 'gru-cnn':
print(self.max_sent_len)
model.add(
Conv1D(self.filters,
self.kernel_size,
input_shape=(self.max_sent_len, self.word_emb_dim),
padding='valid',
activation='relu',
strides=1))
model.add(MaxPooling1D(pool_size=self.pool_size))
model.add(
GRU(
self.n_units[0],
return_sequences=is_return_sequences,
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg,
dropout=self.dropout,
recurrent_dropout=self.recurrent_dropout,
))
for l_idx, n_unit in enumerate(self.n_units[1:]):
if l_idx < n_rnn_layers - 2:
model.add(
GRU(
n_unit,
return_sequences=True,
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg,
dropout=self.dropout,
recurrent_dropout=self.recurrent_dropout,
))
else:
model.add(
GRU(
n_unit,
return_sequences=
is_return_sequences_last_recurrent_layer,
kernel_regularizer=self.input_reg,
recurrent_regularizer=self.recurr_reg,
dropout=self.dropout,
recurrent_dropout=self.recurrent_dropout,
))
elif self.method == 'cnn':
print(self.max_sent_len)
model.add(
Conv1D(self.filters,
self.kernel_size,
input_shape=(self.max_sent_len, self.word_emb_dim),
padding='valid',
activation='relu',
strides=1))
model.add(GlobalMaxPooling1D())
if self.is_self_activation:
model.add(SeqSelfAttention(attention_activation='sigmoid'))
model.add(Flatten())
for dense_n_unit in self.dense_units:
model.add(Dense(dense_n_unit, activation='relu'))
model.add(Dense(self.label_dim, activation='sigmoid'))
if self.learn_algo == 'sg':
optimizer = SGD(lr=self.learning_rate,
momentum=0.0,
decay=0.0,
nesterov=False)
elif self.learn_algo == 'adam':
optimizer = Adam(
lr=self.learning_rate,
# beta_1=0.9,
# beta_2=0.999,
# epsilon=1e-08,
amsgrad=True,
)
else:
raise NotImplementedError
if self.is_cross_entropy_loss:
model.compile(
optimizer=optimizer,
loss='binary_crossentropy',
metrics=['binary_accuracy'],
)
else:
model.compile(
loss='mean_squared_error',
optimizer=optimizer,
)
return model
def build(self, input_shape):
self.kernel = self.add_weight(shape=(input_shape[-1], self.units),
initializer='uniform',
name='kernel')
self.recurrent_kernel = self.add_weight(shape=(self.units, self.units),
initializer='uniform',
name='recurrent_kernel')
self.built = True
def call(self, inputs, states):
prev_output = states[0]
h = K.dot(inputs, self.kernel)
output = h + K.dot(prev_output, self.recurrent_kernel)
return output, [output]
# Let's use this cell in a RNN layer:
cell = MinimalRNNCell(32)
x = keras.Input((None, 5))
layer = RNN(cell)
y = layer(x)
# Here's how to use the cell to build a stacked RNN:
cells = [MinimalRNNCell(32), MinimalRNNCell(64)]
x = keras.Input((None, 5))
layer = RNN(cells)
y = layer(x)
print(y)
earlyStopping = EarlyStopping(monitor='val_loss', patience=1, verbose=0)
saveBestModel = ModelCheckpoint(save_best_model_file,
monitor='val_loss',
verbose=0,
save_best_only=True,
save_weights_only=True)
embeddings_layer = Embedding(len(word_index) + 1,
embedding_dim,
input_length=max_sequence_length,
trainable=True)
SENT_HIDDEN_SIZE = 100
inputs = Input(shape=(max_sequence_length, ), dtype='int32', name='input')
embeddings_sequences = embeddings_layer(inputs)
output = RNN(SENT_HIDDEN_SIZE)(embeddings_sequences)
output = Dense(64, activation='relu', name='dense1')(output)
print(output)
output = Dense(1, activation='sigmoid')(output)
model = Model(inputs=inputs, outputs=[output])
model.summary()
model.compile(loss='binary_crossentropy',
optimizer=Adam(0.0001),
metrics=['accuracy'])
checkpoint_filepath = 'E:/DeepLearning/bully_code/diyu/indrnn.h5'
checkpoint = ModelCheckpoint(checkpoint_filepath,
monitor='acc',
verbose=0,
save_best_only=True,
