def _init_model_32(self, inputs): # 3 to 2 network x = inputs x = CuDNNLSTM(96, kernel_regularizer=L1L2(l1=0.01, l2=0.01), \ recurrent_regularizer=L1L2(l1=0.01, l2=0.01), return_sequences = True)(x) x = CuDNNLSTM(32, kernel_regularizer=L1L2(l1=0.01, l2=0.01), \ recurrent_regularizer=L1L2(l1=0.01, l2=0.01), return_sequences = False)(x) x = Dense(2, activation='softmax')(x) return x
def build_model(self, X_train, X_val, y_train, y_val):
model = Sequential()
model.add(
CuDNNLSTM(128,
input_shape=(X_train.shape[1:]),
return_sequences=True))
model.add(Dropout(0.2))
model.add(BatchNormalization())
model.add(
CuDNNLSTM(128,
input_shape=(X_train.shape[1:]),
return_sequences=True))
model.add(Dropout(0.1))
model.add(BatchNormalization())
model.add(CuDNNLSTM(128, input_shape=(X_train.shape[1:])))
model.add(Dropout(0.2))
model.add(BatchNormalization())
model.add(Dense(32, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(2, activation='softmax'))
opt = tf.keras.optimizers.Adam(lr=0.001, decay=1e-6)
model.compile(loss='sparse_categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
tensorboard = TensorBoard(
log_dir='logs/{}'.format(PriceClassification.NAME))
filepath = "RNN_Final-{epoch:02d}-{val_acc:.3f}"
checkpoint = ModelCheckpoint("../models/{}.model".format(
filepath,
monitor='val_acc',
verbose=2,
save_best_only=True,
mode='max'))
history = model.fit(X_train,
y_train,
batch_size=PriceClassification.BATCH_SIZE,
epochs=PriceClassification.EPOCHS,
validation_data=(X_val, y_val),
shuffle=True,
callbacks=[tensorboard, checkpoint])
# Score model
score = model.evaluate(X_val, y_val, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
# Save model
model.save("models/{}".format(PriceClassification.NAME))
def __init__(self, input_shape):
self.model = Sequential()
self.model.add(
CuDNNLSTM(2, input_shape=input_shape[1:], return_sequences=True))
self.model.add(CuDNNLSTM(1, return_sequences=False))
#self.model.add(CuDNNLSTM(1,return_sequences=False))
#self.model.add(BatchNormalization(axis=-1, momentum=0.99, epsilon=0.001))
self.model.compile(loss="mse", optimizer="adam", metrics=["accuracy"])
def build(self):
input_tensor = Input(name='inputs',
shape=(None, *self._feat_shape, 1),
dtype=tf.float32)
x = input_tensor
x = Conv2D(32, (11, 5),
use_bias=True,
activation='relu',
padding='same',
kernel_initializer='he_normal',
name="conv1")(x)
x = Conv2D(32, (11, 5),
use_bias=True,
activation='relu',
padding='same',
kernel_initializer='he_normal',
name="conv2")(x)
_, _, dim, channels = x.get_shape().as_list()
output_dim = dim * channels
x = Reshape((-1, output_dim))(x)
x = TimeDistributed(Dropout(0.8))(x)
x = CuDNNLSTM(512,
kernel_initializer='glorot_uniform',
bias_initializer='random_normal',
return_sequences=True,
name='lstm')(x)
x = Activation('tanh', name='activation')(x)
x = TimeDistributed(Dropout(0.8))(x)
x = CuDNNLSTM(512,
kernel_initializer='glorot_uniform',
bias_initializer='random_normal',
return_sequences=True,
name='lstm1')(x)
x = Activation('tanh', name='activation1')(x)
x = TimeDistributed(Dense(1024, activation='relu'))(x)
x = TimeDistributed(Dropout(0.5))(x)
# Output layer with softmax
x = TimeDistributed(Dense(self._vocab_size))(x)
input_length = Input(name='input_length', shape=[1], dtype='int64')
labels = Input(name='targets', shape=[None], dtype='int32')
label_length = Input(name='target_length', shape=[1], dtype='int64')
loss_out = Lambda(self.ctc_lambda_func, output_shape=(1, ),
name='ctc')([x, input_length, labels, label_length])
self._model = tf.keras.Model(
inputs=[input_tensor, labels, input_length, label_length],
outputs=[loss_out])
def get_model():
model = Sequential()
# IF you are running with a GPU, try out the CuDNNLSTM layer type instead (don't pass an activation, tanh is required)
model = Sequential()
model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(64, 300, 1)))
model.add(Conv2D(32, (3, 3)))
# model.add(Dropout(0.4))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(Conv2D(64, (3, 3)))
model.add(MaxPooling2D(pool_size=(2, 2)))
# model.add(Dropout(0.4))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(Conv2D(128, (3, 3)))
# model.add(Dropout(0.4))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(256, (3, 3), activation='relu'))
model.add(Conv2D(256, (3, 3)))
# model.add(Dropout(0.4))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(512, (3, 3), activation='relu'))
model.add(Conv2D(1100, (3, 3), activation='relu'))
model.add(GlobalAveragePooling2D())
model.add(Reshape((11, 100)))
model.add(CuDNNLSTM(128, return_sequences=True))
# model.add(Dropout(0.4))
model.add(CuDNNLSTM(128, return_sequences=True))
# model.add(Dropout(0.4))
model.add(CuDNNLSTM(256, return_sequences=True))
# model.add(Dropout(0.4))
model.add(CuDNNLSTM(256, return_sequences=True))
# model.add(Dropout(0.4))
model.add(TimeDistributed(Dense(17)))
model.add(Activation('softmax'))
return model
def rnn_train(X, Y, h1=64, h2=32, n_epochs=100, s_batch=100, Lr=0.001):
model = tf.keras.models.Sequential()
model.add(CuDNNLSTM(h1, input_shape=(X.shape[1:]), return_sequences=True))
model.add(CuDNNLSTM(h2))
model.add(Dense(1))
optimizer = tf.keras.optimizers.Adam(lr=Lr)
model.compile(optimizer=optimizer, loss='mse', metrics=['mae', 'mse'])
hist = model.fit(X, Y, epochs=n_epochs, batch_size=s_batch)
return hist
def define_model(self, look_back_steps, batch_size, state_ful=False):
# The base LM model
# batch_input_shape=(batch_size, timesteps, data_dim)
if state_ful:
word_input = Input((look_back_steps, self.vocab_size), batch_size=batch_size)
else:
word_input = Input((look_back_steps, self.vocab_size))
# The output vector from the LSTM layer is 200d and return_sequences=True must be set in order to use
lstm_layer1 = CuDNNLSTM(100, return_sequences=True, stateful=state_ful)(word_input)
lstm_layer2 = CuDNNLSTM(100, stateful=state_ful)(lstm_layer1)
#lstm_layer1 = LSTM(100, return_sequences=True, stateful=state_ful)(word_input)
#lstm_layer2 = LSTM(100, stateful=state_ful)(lstm_layer1)
# Set the weights to the weights corresponding to the embedding
dense_word_decoder = Dense(self.vocab_size, trainable=False, name='embedding_layer')(lstm_layer2)
#print('Embedding matrix shape: {}'.format(self.embedding.shape))
# The energy term for affect words
# The input is the LIWC feature extraction which has 5 categories (5 index input vector)
affect_input = Input((5,))
dense_1 = Dense(50, activation=sigmoid)(affect_input)
dense_2 = Dense(100, activation=sigmoid)(dense_1)
# The report does not say anything about the activation function for the second layer
dense_affect_decoder_1 = Dense(self.vocab_size)(dense_2)
# multiply the affect vector with the beta energy term
beta = 1.75
dense_affect_decoder = Lambda(lambda x: x*beta)(dense_affect_decoder_1)
# the bias term for unigram occurrences has not been added to the model
final_layer = Add()([dense_word_decoder, dense_affect_decoder])
prediction_layer = Softmax()(final_layer)
model = models.Model(inputs=[word_input, affect_input], outputs=[prediction_layer])
weights = model.get_layer('embedding_layer').get_weights()
#print(weights[1].shape)
weights[0] = self.embedding
weights[1] = self.bias_vector
#print(weights)
model.get_layer('embedding_layer').set_weights(weights)
self.start_weights.append(weights[0])
self.start_weights.append(weights[1])
model.summary()
return model
def __init__(self,
units,
kernel_initializer='glorot_uniform',
recurrent_initializer='orthogonal',
bias_initializer='zeros',
unit_forget_bias=True,
kernel_regularizer=None,
recurrent_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
recurrent_constraint=None,
bias_constraint=None,
return_sequences=False,
**kwargs) -> None:
super().__init__(**kwargs)
self.return_sequences = return_sequences
if not return_sequences:
raise NotImplementedError("Must set return_sequences=True")
self.forward_lstm = CuDNNLSTM(units,
kernel_initializer,
recurrent_constraint,
bias_initializer,
unit_forget_bias,
kernel_regularizer,
recurrent_regularizer,
bias_regularizer,
activity_regularizer,
kernel_constraint,
recurrent_constraint,
bias_constraint,
return_sequences=True)
self.reverse_lstm = CuDNNLSTM(units,
kernel_initializer,
recurrent_constraint,
bias_initializer,
unit_forget_bias,
kernel_regularizer,
recurrent_regularizer,
bias_regularizer,
activity_regularizer,
kernel_constraint,
recurrent_constraint,
bias_constraint,
return_sequences=True)
def get_cudnncnnlstm(shape, dropout):
model = Sequential()
with tf.variable_scope("Conv1D1", reuse=tf.AUTO_REUSE) as scope:
model.add(
Conv1D(128,
input_shape=(train_X.shape[1:]),
kernel_size=3,
activation='relu'))
model.add(BatchNormalization())
model.add(Dropout(0.5))
with tf.variable_scope("Conv1D2", reuse=tf.AUTO_REUSE) as scope:
model.add(Conv1D(128, kernel_size=3, activation='relu'))
model.add(BatchNormalization())
model.add(Dropout(0.5))
with tf.variable_scope("CuDNNLSTM1", reuse=tf.AUTO_REUSE) as scope:
model.add(CuDNNLSTM(64, return_sequences=True))
model.add(BatchNormalization())
model.add(Dropout(dropout))
with tf.variable_scope("CuDNNLSTM2", reuse=tf.AUTO_REUSE) as scope:
model.add(CuDNNLSTM(64, return_sequences=True))
model.add(BatchNormalization())
model.add(Dropout(dropout))
with tf.variable_scope("CuDNNLSTM3", reuse=tf.AUTO_REUSE) as scope:
model.add(CuDNNLSTM(64, return_sequences=True))
model.add(BatchNormalization())
model.add(Dropout(dropout))
with tf.variable_scope("CuDNNLSTM4", reuse=tf.AUTO_REUSE) as scope:
model.add(CuDNNLSTM(64, return_sequences=False))
model.add(BatchNormalization())
model.add(Dropout(dropout))
with tf.variable_scope("DENSE1", reuse=tf.AUTO_REUSE) as scope:
model.add(Dense(128, activation='relu'))
model.add(BatchNormalization())
model.add(Dropout(dropout))
with tf.variable_scope("DENSE2", reuse=tf.AUTO_REUSE) as scope:
model.add(Dense(1, activation='sigmoid'))
opt = tf.keras.optimizers.Adam(lr=1e-2, decay=1e-3)
model.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=['accuracy'])
model.summary()
return model
def dynamicLayerCreator(self, lastLstm=False):
if self.typeNet == "LSTM":
if not lastLstm:
if self.gpu:
self.model.add(
CuDNNLSTM(self.neurons,
return_sequences=True,
batch_size=self.batchSize,
stateful=True))
else:
self.model.add(
LSTM(self.neurons,
return_sequences=True,
batch_size=self.batchSize,
stateful=True))
self.model.add(BatchNormalization())
else:
if self.gpu:
self.model.add(
CuDNNLSTM(self.neurons,
batch_size=self.batchSize,
stateful=True))
else:
self.model.add(
LSTM(self.neurons,
batch_size=self.batchSize,
stateful=True))
self.model.add(BatchNormalization())
self.model.add(Dropout(0.2))
elif self.typeNet == "Conv1D":
paddingType = ["same", "valid"]
self.model.add(
Conv1D(self.neurons,
self.filterSize,
activation="relu",
padding=paddingType[self.padding],
batch_size=self.batchSize))
self.model.add(BatchNormalization())
self.model.add(Dropout(0.2))
elif self.typeNet == "Dense":
self.model.add(
Dense(self.neurons,
activation='relu',
batch_size=self.batchSize))
self.model.add(BatchNormalization())
self.model.add(Dropout(0.2))
else:
logging.error("No valid NN type selected")
def model_lstm_atten(embedding_matrix):
inp = Input(shape=(maxlen, ))
x = Embedding(max_features,
embed_size,
weights=[embedding_matrix],
trainable=False)(inp)
x = SpatialDropout1D(0.1)(x)
x = Bidirectional(CuDNNLSTM(40, return_sequences=True))(x)
y = Bidirectional(CuDNNGRU(40, return_sequences=True))(x)
atten_1 = Attention(maxlen)(x) # skip connect
atten_2 = Attention(maxlen)(y)
avg_pool = GlobalAveragePooling1D()(y)
max_pool = GlobalMaxPooling1D()(y)
conc = concatenate([atten_1, atten_2, avg_pool, max_pool])
conc = Dense(16, activation="relu")(conc)
conc = Dropout(0.1)(conc)
outp = Dense(1, activation="sigmoid")(conc)
model = Model(inputs=inp, outputs=outp)
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[f1])
return model
def create_model(self):
"""
Creates the model
"""
self.mod = Sequential()
# Add the RNN layer
# Note 1: For RNN, we set unroll=True to enable fast GPU usage
# Note 2: You need to set the CuDNN version of LSTM
unroll = tf.test.is_gpu_available()
if (self.mod_type == 'lstm'):
# LSTM model
if tf.test.is_gpu_available():
self.mod.add(CuDNNLSTM(self.nh, input_shape=(self.nt, self.nin),\
return_sequences=False, name='RNN'))
else:
self.mod.add(LSTM(self.nh, input_shape=(self.nt, self.nin),\
return_sequences=False, name='RNN',unroll=unroll))
elif self.is_complex:
# Complex RNN
cell = ComplexRNNCell(nh=self.nh)
self.mod.add(RNN(cell, input_shape=(self.nt, self.nin),\
return_sequences=False, name='RNN',unroll=True))
else:
# Real RNN model
self.mod.add(SimpleRNN(self.nh, input_shape=(self.nt, self.nin),\
return_sequences=False, name='RNN',activation='relu',unroll=unroll))
self.mod.add(Dense(nout, activation='softmax', name='Output'))
self.mod.summary()
def get_model():
speed = Input(shape=(4, 2), name='speed_input')
img = Input(shape=(64, 64, 12), name='img_input')
conv_0 = Conv2D(32, 3, strides=(1, 1), padding='same',
activation='relu')(img)
conv_1 = Conv2D(64, 3, strides=(2, 2), padding='same',
activation='relu')(conv_0)
maxpool_1 = MaxPool2D(padding='same')(conv_1)
conv_2 = Conv2D(64, 3, padding='same', activation='relu')(maxpool_1)
maxpool_2 = MaxPool2D(padding='same')(conv_2)
conv_3 = Conv2D(128, 3, padding='same', activation='relu')(maxpool_2)
flatten_0 = Flatten()(conv_3)
fc_0 = Dense(64)(flatten_0)
fc_1 = CuDNNLSTM(64)(speed)
con_0 = Concatenate()([fc_0, fc_1])
steer = Dense(1, activation='tanh', name='steer_output')(con_0)
acc = Dense(1, activation='tanh', name='acc_output')(con_0)
model = tf.keras.models.Model([img, speed], [steer, acc])
model.compile(optimizer='adam', loss='mse')
return model
def model_lstm(input_shape):
inp = Input(shape=(
input_shape[1],
input_shape[2],
))
x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(inp)
x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
x = Attention(input_shape[1])(x)
x = Dense(64, activation="relu")(x)
x = Dense(1, activation="sigmoid")(x)
model = Model(inputs=inp, outputs=x)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=[matthews_correlation])
return model
def training_run(X_train, y_train, X_val, y_val, MAX_LEN, bias_reg, kernel_reg,
recur_reg, word_index, class_assoc, X_test, y_test,
num_epochs, embedding_layer):
print('Training model.')
# Train an LSTM - the model will only contain one layer
# to add additional layers, simply add another CuDNNLSTM layer with option return_sequences = True
# to use a different number of neurons change the first argument of the CuDNNLSTM layer
model = Sequential()
model.add(embedding_layer)
model.add(
CuDNNLSTM(100,
bias_regularizer=bias_reg,
kernel_regularizer=kernel_reg,
recurrent_regularizer=recur_reg))
if len(class_assoc) > 2: #multiclass classification
model.add(Dense(len(class_assoc), activation='softmax'))
model.compile(loss="categorical_crossentropy",
optimizer='adam',
metrics=['accuracy'])
else: #binary classification
model.add(Dense(1, activation='sigmoid'))
model.compile(loss="binary_crossentropy",
optimizer='adam',
metrics=['accuracy'])
hist = model.fit(X_train,
y_train,
epochs=num_epochs,
batch_size=1000,
validation_data=(X_val, y_val))
hist = hist.history
predictions = model.predict(X_test, batch_size=1000)
return hist, predictions
def train_lstm():
# Create symbolic vars
x = Input(shape=(None, in_dim), dtype='float32', name='input')
# Create network
# fw_cell = LSTM(hidden_units_size, return_sequences=False,
# implementation=2)(x)
fw_cell = CuDNNLSTM(hidden_units_size, return_sequences=False)(x)
h3 = Dense(classes, activation='softmax', use_bias=False)(fw_cell)
model = Model(inputs=x, outputs=h3)
validate_lstm_in_out(model)
start = timer.perf_counter()
model.compile(optimizer='Adam', loss='categorical_crossentropy')
end = timer.perf_counter()
print('>>> Model compilation took {:.1f} seconds'.format(end - start))
# Print parameter count
params = model.count_params()
print('# network parameters: ' + str(params))
# Start training
batch_time = []
batch_loss = []
train_start = timer.perf_counter()
for i in range(nb_batches):
batch_start = timer.perf_counter()
loss = model.train_on_batch(x=bX,
y=to_categorical(bY, num_classes=classes))
batch_end = timer.perf_counter()
batch_time.append(batch_end - batch_start)
batch_loss.append(loss)
train_end = timer.perf_counter()
print_results(batch_loss, batch_time, train_start, train_end)
def initialise_model():
global model, bottleneck_mdl
from tensorflow.keras.layers import CuDNNGRU, Dropout, Dense, Embedding, CuDNNLSTM, Input, add
#===========================================================#
#=========================[MODEL]===========================#
#===========================================================#
bottleneck_input = Input(shape=(2048, ))
fe1 = Dropout(0.5)(bottleneck_input)
fe2 = Dense(256, activation=tf.nn.selu)(fe1)
#Partial Caption Input
cap_inputs = Input(shape=(72, ))
#se1 is already pretrained on GLOVE model
se1 = Embedding(5411, 200)(cap_inputs)
se2 = Dropout(0.5)(se1)
se3 = CuDNNGRU(256, return_sequences=True)(se2)
se4 = CuDNNLSTM(256)(se3)
decoder1 = add([fe2, se4])
decoder2 = Dense(256, activation=tf.nn.selu)(decoder1)
outputs = Dense(5411, activation=tf.nn.softmax)(decoder2)
model = Model(inputs=[bottleneck_input, cap_inputs], outputs=outputs)
model.load_weights('model_xeon.h5')
#===========================================================#
#=========================[MODEL]===========================#
#===========================================================#
incep_mdl = InceptionV3(weights="imagenet")
bottleneck_mdl = Model(incep_mdl.input, incep_mdl.layers[-2].output)
def create_network(network_input, n_vocab):
""" create the structure of the neural network """
model = Sequential()
model.add(CuDNNLSTM(256, input_shape=(network_input.shape[1], network_input.shape[2]), return_sequences=True))
model.add(CuDNNLSTM(128, return_sequences=True))
model.add(CuDNNLSTM(64))
model.add(Dense(n_vocab))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
# Load the weights to each node
model.load_weights(weight)
return model
def _build_model(self):
for i in range(self.num_of_lstms):
input_vec = Input(shape=(self.maxlen, self.inner_maxlen))
bidi1 = Bidirectional(
CuDNNLSTM(self.units[i], return_sequences=True))(input_vec)
do1 = Dropout(self.dropouts[i])(bidi1)
bidi2 = Bidirectional(
CuDNNLSTM(self.units[i], return_sequences=True))(do1)
do2 = Dropout(self.dropouts[i])(bidi2)
output = TimeDistributed(
Dense(len(ANSWER_REVERSE_MAPPER) + 1,
activation='softmax'))(do2)
self.models.append(Model(inputs=[input_vec], outputs=[output]))
self.models[-1].compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['acc'])
def biLSTM(X_1, X_2, y, X_1_val, X_2_val, y_val, maxlen, vocab_size):
# best so far:
# lr-0.001-epochs-10-batch_size-64-lstm_dim-32-dense_dim-64-emb_dim-32-num_dnn-1
# lr-0.001-epochs-20-batch_size-32-lstm_dim-64-dense_dim-64-emb_dim-64-num_dnn-2
tf.keras.backend.clear_session()
with tf.device('/GPU:0'):
lr = 1e-3 # Learning rate
epochs = 20 # Number of epochs
batch_size = 32 # Batch size
lstm_dim = 64 # LSTM dimension
emb_dim = 64 # Embedding dimension
num_dnn = 2 # Number of outer DNN layers
input_aux = Input(shape=(maxlen, ))
x = Embedding(input_dim=vocab_size,
output_dim=emb_dim,
input_length=maxlen)(input_aux)
#output_aux_1 = CuDNNLSTM(lstm_dim, go_backwards=False)(x)
#output_aux_2 = CuDNNLSTM(lstm_dim)(x)
#output_aux = concatenate([output_aux_1, output_aux_2])
output_aux = Bidirectional(CuDNNLSTM(lstm_dim))(x)
model_aux = Model(inputs=[input_aux], outputs=[output_aux])
# Creating outer model
input_1 = Input(shape=(maxlen, ))
input_2 = Input(shape=(maxlen, ))
x_1 = model_aux(input_1)
x_2 = model_aux(input_2)
x = concatenate([x_1, x_2])
for i in range(num_dnn):
dense_dim = {{choice([32, 64])}}
x = Dense(dense_dim, activation='relu')(x)
x = Flatten()(x)
output = Dense(1, activation='sigmoid')(x)
model = Model(inputs=[input_1, input_2], outputs=[output])
#tensorboard = TensorBoard(log_dir=param)
model.compile(optimizer=tf.keras.optimizers.Adam(lr=lr),
loss='binary_crossentropy',
metrics=['accuracy'])
result = model.fit(
[X_1, X_2],
y,
batch_size=batch_size,
epochs=epochs,
#callbacks=[tensorboard],
verbose=2,
validation_data=([X_1_val, X_2_val], y_val))
validation_acc = np.amax(result.history['val_acc'])
print(param)
print('Best validation acc of epoch:', validation_acc)
return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}
def lstm_cnn_chars(input_shape: Tuple[int, ...],
output_shape: Tuple[int, ...],
dropout: float = 0.2) -> Model:
num_classes = output_shape[0]
inputs = Input(input_shape)
#8700 as it is 29 words x 300 the dimension of the embeddings
# charsInputs = Lambda(lambda x: x[:,:240])(inputs)
charsEmbeddings = Embedding(193, output_dim=4)(inputs)
chars1 = Conv1D(3,
kernel_size=2,
activation='tanh',
input_shape=input_shape,
kernel_initializer='glorot_normal')(charsEmbeddings)
chars1 = Flatten()(chars1)
chars2 = Conv1D(4,
kernel_size=3,
activation='tanh',
input_shape=input_shape,
kernel_initializer='glorot_normal')(charsEmbeddings)
chars2 = Flatten()(chars2)
chars3 = Conv1D(5,
kernel_size=4,
activation='tanh',
input_shape=input_shape,
kernel_initializer='glorot_normal')(charsEmbeddings)
chars3 = Flatten()(chars3)
chars = Concatenate(axis=-1)([chars1, chars2, chars3])
chars = Lambda(lambda x: tf.expand_dims(x, -1))(chars)
chars = MaxPooling1D()(chars)
chars = BatchNormalization()(chars)
chars = Bidirectional(
CuDNNLSTM(8, return_sequences=True,
kernel_initializer='glorot_normal'))(chars)
chars = Bidirectional(
CuDNNLSTM(8, return_sequences=True,
kernel_initializer='glorot_normal'))(chars)
chars = Bidirectional(
CuDNNLSTM(8, return_sequences=True,
kernel_initializer='glorot_normal'))(chars)
chars = Flatten()(chars)
X = Dropout(dropout)(chars)
output = Dense(num_classes,
activation='sigmoid',
kernel_initializer='random_normal')(chars)
return Model(inputs=inputs, outputs=output)
def model(feat_seq, max_len, embed_shape, emb_dim, env):
inputs = []
models = []
input_ = Input(shape=(max_len, len(feat_seq)),
dtype='float32',
name='input_1')
lstm_ = CuDNNLSTM(30,
name='cudnnlstm_1')(input_) if env == 'cloud' else LSTM(
30, name='lstm_1')(input_)
inputs.append(input_)
models.append(lstm_)
for key, value in embed_shape.items():
input_ = Input(shape=(max_len, ),
dtype='float32',
name='input_%s' % key)
inputs.append(input_)
embedding_ = Embedding(value,
emb_dim,
input_length=max_len,
name='embedding_%s' % key)(input_)
lstm_ = CuDNNLSTM(30, name='cudnnlstm_%s' %
key)(embedding_) if env == 'cloud' else LSTM(
30, name='lstm_%s' % key)(embedding_)
models.append(lstm_)
models_merged = Concatenate(axis=1)(models)
output = Dense(1, use_bias=False)(models_merged)
output = BatchNormalization()(output)
output = Activation("sigmoid")(output)
model_params = {
'loss': 'binary_crossentropy',
'optimizer': 'adam',
'metrics': ['acc']
}
model = Model(inputs=inputs, outputs=output)
model.compile(**model_params)
logger.info(model.summary())
return model
def GRUModel(max_words, embedding_size, sequence_length, optimizer, num_classes):
model = Sequential()
model.add(keras.layers.Embedding(max_words+1, embedding_size, input_length=sequence_length, trainable=True))
model.add(CuDNNLSTM(512))
model.add(keras.layers.Dropout(0.3))
model.add(keras.layers.Dense(num_classes, activation='sigmoid'))
model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['categorical_accuracy'])
return model
def newModel():
model = Sequential()
model.add(CuDNNLSTM(128, input_shape=(maxInputLength, vocabSize)))
model.add(Dense(vocabSize, activation="softmax"))
optimizer = tf.keras.optimizers.RMSprop(lr=0.01)
model.compile(optimizer=optimizer, loss="categorical_crossentropy")
return model
def __init__(self, num_units, backwards, batch_size, embedding_dim,
src_vocab_size):
super(tf.keras.Model, self).__init__()
self.embd = Embedding(src_vocab_size,
embedding_dim,
batch_input_shape=[batch_size, None])
self.lstm_layers = Sequential([
# Dropout(0.5),
CuDNNLSTM(num_units,
return_sequences=True,
return_state=False,
go_backwards=backwards),
# Dropout(0.5),
CuDNNLSTM(num_units, return_sequences=True, return_state=False),
# Dropout(0.5),
CuDNNLSTM(num_units, return_sequences=False, return_state=True)
])
self.dropout = Dropout(0.5)
def makeCuDNNLSTM(cells=25):
inputs = Input(shape=(sequence_length,1))
x = CuDNNLSTM(cells,return_sequences=False)(inputs)
x = Dropout(0.1)(x)
outputs = Dense(2,activation='softmax')(x)
model = Model(inputs=inputs, outputs=outputs)
model.compile(loss='categorical_crossentropy',optimizer=optimizers.RMSprop(),
metrics=['accuracy'])
model.summary()
return model
def lstm_chars(input_shape: Tuple[int, ...],
output_shape: Tuple[int, ...],
dropout: float = 0.0) -> Model:
num_classes = output_shape[0]
inputs = Input(input_shape)
#8700 as it is 29 words x 300 the dimension of the embeddings
#chars = Lambda(lambda x: x[:,:240])(inputs)
chars = Embedding(193, output_dim=4)(inputs)
chars = Bidirectional(CuDNNLSTM(121, return_sequences=False))(chars)
chars = Lambda(lambda x: tf.expand_dims(x, -1))(chars)
#chars = BatchNormalization()(chars)
chars = Bidirectional(CuDNNLSTM(8, return_sequences=True))(chars)
chars = Bidirectional(CuDNNLSTM(8, return_sequences=True))(chars)
chars = Flatten()(chars)
X = Dropout(dropout)(chars)
output = Dense(num_classes, activation='sigmoid')(X)
return Model(inputs=inputs, outputs=output)
def modelLSTM(num_units1, num_units2, time_steps, n_features, _batch_size,
n_classes, _patience, sub_db, n_epochs):
(train_images, train_labels_encoded, test_images,
test_labels_encoded) = sub_db
modelLSTM = ks.Sequential()
#Batch size should be (at most) the same number of hidden cells
#no activation selection
modelLSTM.add(
Bidirectional(CuDNNLSTM(num_units1,
unit_forget_bias='true',
return_sequences='true'),
input_shape=(time_steps, n_features)))
modelLSTM.add(Bidirectional(CuDNNLSTM(num_units2,
unit_forget_bias='true')))
modelLSTM.add(Dense(n_classes, activation='softmax'))
modelLSTM.compile(loss='sparse_categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
#Definizione callback
es = ks.callbacks.EarlyStopping(monitor='val_loss',
patience=_patience,
mode='auto',
restore_best_weights=True,
verbose=0)
time_callback = TimeHistory()
#Validation_data è usato al termine di ogni epoch;
hist = modelLSTM.fit(train_images,
train_labels_encoded,
batch_size=_batch_size,
validation_data=(test_images, test_labels_encoded),
epochs=n_epochs,
shuffle='true',
callbacks=[time_callback, es],
verbose=0)
return modelLSTM, hist, time_callback
def AttRNNSpeechModelWave(input_shape, n_classes):
X_input = Input(input_shape)
X = Lambda(lambda q: expand_dims(q, -1), name='expand_dims')(X_input)
X = Conv1D(16, 9, activation=relu, padding='valid')(X)
X = MaxPool1D(8)(X)
X = Conv1D(32, 9, activation=relu, padding='valid')(X)
X = MaxPool1D(8)(X)
X = Conv1D(32, 9, activation=relu, padding='valid')(X)
X = MaxPool1D(6)(X)
#X = Lambda(lambda q: squeeze(q, -1), name='squeeze_last_dim') (X)
X = Bidirectional(CuDNNLSTM(64, return_sequences=True))(X)
X = Dropout(0.5)(X)
X = Bidirectional(CuDNNLSTM(64, return_sequences=True))(X)
X = Dropout(0.5)(X)
xFirst = Lambda(lambda q: q[:, 16])(X)
query = Dense(128)(xFirst)
query = Dropout(0.5)(query)
attScores = Dot(axes=[1, 2])([query, X])
attScores = Softmax(name='attSoftmax')(attScores)
attVector = Dot(axes=[1, 1])([attScores, X])
X = Dense(64, activation='relu')(attVector)
X = Dropout(0.5)(X)
X = Dense(32)(X)
X = Dropout(0.5)(X)
X = Dense(n_classes, activation='softmax', name='output')(X)
model = Model(inputs=X_input, outputs=X)
return model
def get_model():
model = Sequential()
model.add(
CuDNNLSTM(36,
stateful=True,
return_sequences=True,
batch_input_shape=(BATCH_SIZE, train_x.shape[1],
train_x.shape[2])))
model.add(Activation('relu'))
model.add(CuDNNLSTM(36, stateful=True, return_sequences=False))
model.add(Activation('relu'))
model.add(Dense(1))
""".....compile model....."""
model.compile(loss='mae', optimizer='adam', metrics=['mse', 'mae'])
return model
