#
DATA_PATH = './donquijote.txt'
LONG_SEC = 100 # Número de caracteres en cada ejemplo
# Lectura de datos y creación del set de entrenamiento
x_train, y_train, TAM_VOCAB, ix_to_char, char_to_ix = cargar_datos(
DATA_PATH, LONG_SEC)
#
# 1. MODELO: red LSTM con 1024 neuronas
#
modelo = Sequential()
modelo.add(LSTM(1024, input_shape=(None, 92), return_sequences=True))
modelo.add(TimeDistributed(Dense(92)))
modelo.add(Activation('softmax'))
#
# 2/3. COMPILACIÓN Y ENTRENAMIENTO DEL MODELO
#
modelo.compile(loss="categorical_crossentropy", optimizer="adam")
modelo.fit(x_train, y_train, batch_size=64, epochs=20, verbose=1)
#
# 4. PREDICCIÓN
#
print(generar_texto(modelo, 500, 92, ix_to_char))
# Secuencia de 500 caracteres generada
# ver lo que de mi padre le había de ser con la muerte de la
def five_images_to_LSF_model1():
input_lips = Input(shape=(5, 64, 64, 1))
# lips parts
lips_conv1_1 = TimeDistributed(
Conv2D(16, (5, 5), padding="same", activation="relu"))(input_lips)
lips_conv1_2 = TimeDistributed(
Conv2D(16, (5, 5), padding="same", activation="relu"))(lips_conv1_1)
lips_pooling1 = TimeDistributed(MaxPooling2D(pool_size=(2,
2)))(lips_conv1_2)
lips_conv2_1 = TimeDistributed(
Conv2D(32, (3, 3), padding="same", activation="relu"))(lips_pooling1)
lips_conv2_2 = TimeDistributed(
Conv2D(32, (3, 3), padding="same", activation="relu"))(lips_conv2_1)
lips_pooling2 = TimeDistributed(MaxPooling2D(pool_size=(2,
2)))(lips_conv2_2)
lips_conv3_1 = TimeDistributed(
Conv2D(64, (3, 3), padding="same", activation="relu"))(lips_pooling2)
lips_conv3_2 = TimeDistributed(
Conv2D(64, (3, 3), padding="same", activation="relu"))(lips_conv3_1)
lips_pooling3 = TimeDistributed(MaxPooling2D(pool_size=(2,
2)))(lips_conv3_2)
# tongues parts
input_tongues = Input(shape=(5, 64, 64, 1))
tongues_conv1_1 = TimeDistributed(
Conv2D(16, (5, 5), padding="same", activation="relu"))(input_tongues)
tongues_conv1_2 = TimeDistributed(
Conv2D(16, (5, 5), padding="same", activation="relu"))(tongues_conv1_1)
tongues_pooling1 = TimeDistributed(
MaxPooling2D(pool_size=(2, 2)))(tongues_conv1_2)
tongues_conv2_1 = TimeDistributed(
Conv2D(32, (3, 3), padding="same",
activation="relu"))(tongues_pooling1)
tongues_conv2_2 = TimeDistributed(
Conv2D(32, (3, 3), padding="same", activation="relu"))(tongues_conv2_1)
tongues_pooling2 = TimeDistributed(
MaxPooling2D(pool_size=(2, 2)))(tongues_conv2_2)
lips_flat_layer = TimeDistributed(Flatten())(lips_pooling3)
tongues_flat_layer = TimeDistributed(Flatten())(tongues_pooling2)
cc = concatenate([lips_flat_layer, tongues_flat_layer])
lstm1 = LSTM(1024, return_sequences=False)(cc)
lstm_dr1 = Dropout(0.2)(lstm1)
fc = Dense(13, activation="linear")(lstm_dr1)
mymodel = Model([input_lips, input_tongues], fc)
return mymodel
def simple_cnn_1D(args):
#
# Model parameters
#
model_type = args.model_type
input_shape = args.input_shape
n_convs = args.n_layers_convs
n_dense = args.n_layers_dense
max_str_len = args.max_str_len
num_classes = args.num_classes
activation = args.act
advanced_act = args.aact
padding = args.padding
start_filter = args.sf_dim
kernel_size = args.kernel_size
kernel_init = args.kernel_init
l2_reg = args.l2_reg
shared_axes = args.shared_axes
opt = args.opt
#
# Layer parameters
#
# Conv layers parameters
convArgs = {
"activation": activation,
"data_format": "channels_last",
"padding": padding,
"bias_initializer": "zeros",
"kernel_regularizer": l2(l2_reg),
"kernel_initializer": kernel_init,
"use_bias": True
}
# Dense layers parameters
denseArgs = {
"activation": activation,
"kernel_regularizer": l2(l2_reg),
"kernel_initializer": kernel_init, # "glorot_normal"
"bias_initializer": "zeros",
"use_bias": True
}
#
# Model building
#
# Input Stage
if model_type == "quaternion":
quat_init = 'quaternion'
convArgs.update({"kernel_initializer": quat_init})
input_data = Input(name='the_input', shape=input_shape, dtype='float64')
# Convolutional Stage
if model_type == 'real':
# batch-normalization
O = BatchNormalization(axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,
scale=True)(input_data)
for idx in range(n_convs):
conv_filters = start_filter * (2**idx)
O = Conv1D(conv_filters,
kernel_size,
name='conv_{}'.format(idx),
**convArgs)(O)
if advanced_act == 'prelu':
O = PReLU()(O)
# batch-normalization
O = BatchNormalization(axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,
scale=True)(O)
else:
# batch-normalization
O = QuaternionBatchNormalization(axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,
scale=True)(input_data)
for idx in range(n_convs):
conv_filters = start_filter * (2**idx)
O = QuaternionConv1D(conv_filters,
kernel_size,
name='conv_{}'.format(idx),
**convArgs)(O)
if advanced_act == 'prelu':
O = PReLU()(O)
# batch-normalization
O = QuaternionBatchNormalization(axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,
scale=True)(O)
# conv_shape = O.shape
# reshape
# O = Reshape(target_shape=[K.int_shape(O)[1] * K.int_shape(O)[2]])(O)
# Fully-Connected Stage
if model_type == 'real':
for idx in range(n_dense):
O = TimeDistributed(
Dense(1024, **denseArgs, name='dense_{}'.format(idx)))(O)
if advanced_act == 'prelu':
O = PReLU()(O)
# batch-normalization
O = BatchNormalization(axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,
scale=True)(O)
else:
for idx in range(n_dense):
O = TimeDistributed(
QuaternionDense(256, **denseArgs,
name='dense_{}'.format(idx)))(O)
if advanced_act == 'prelu':
O = PReLU()(O)
# batch-normalization
O = QuaternionBatchNormalization(axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,
scale=True)(O)
#
# Prediction (Output) Stage
#
inner = Dense(num_classes,
kernel_regularizer=l2(l2_reg),
use_bias=True,
bias_initializer="zeros",
kernel_initializer=kernel_init)(O)
y_pred = Activation('softmax', name='softmax')(inner)
# Print summary of neural model
Model(inputs=input_data, outputs=y_pred).summary()
labels = Input(name='the_labels', shape=[max_str_len], dtype='float32')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
# Keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
loss_out = Lambda(ctc_lambda_func, output_shape=(1, ),
name='ctc')([y_pred, labels, input_length, label_length])
# clipnorm seems to speeds up convergence
# sgd = SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5)
opt_dict = {
'adam': Adam(lr=0.02, clipnorm=5),
'sgd': SGD(lr=0.02,
decay=1e-6,
momentum=0.9,
nesterov=True,
clipnorm=5)
}
optimizer = opt_dict[opt]
model = Model(inputs=[input_data, labels, input_length, label_length],
outputs=loss_out)
# the loss calc occurs elsewhere, so use a dummy lambda func for the loss
model.compile(loss={
'ctc': lambda y_true, y_pred: y_pred
},
optimizer=optimizer)
return model
def build_model(X_Train, Y_Train, n_input, X_Test, Y_Test):
# mini-bacht gradient descent als Trainingsalgorithmen ausgewählt
#(Sequenzgroße=14 <Batch-Size < Große des Trainingsdatensatzes )
# So wurde die Batch-Size als vielfacheit der Eingangs- und Ausgangssequenzlänge gewählt
verbose, epochs, batch_size = 1, 3000, 28
n_daysteps, n_features, n_outputs = X_Train.shape[1], X_Train.shape[
2], Y_Train.shape[1]
Y_Train = Y_Train.reshape((Y_Train.shape[0], Y_Train.shape[1], 1))
#Neues leeres Modell anlegen
model = Sequential()
model.add(
LSTM(256, activation='relu', input_shape=(n_daysteps, n_features)))
model.add(BatchNormalization())
model.add(RepeatVector(n_outputs))
model.add(LSTM(256, activation='relu', return_sequences=True))
model.add(BatchNormalization())
model.add(TimeDistributed(Dense(128, activation='relu')))
model.add(TimeDistributed(Dense(1)))
model.compile(loss='mae', optimizer='adam', metrics=['acc', 'mae'])
# Modell trainieren
#X_Val, Y_Val = X_Train[-val_split:], Y_Train[-val_split:]
history = model.fit(X_Train,
Y_Train,
epochs=epochs,
batch_size=batch_size,
validation_data=(X_Test, Y_Test),
verbose=verbose)
# Model Architektur
print(model.summary())
acc = history.history['acc']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(acc) + 1)
#plotting training and validation loss
plt.plot(epochs, loss, label='Training loss')
plt.plot(epochs, val_loss, label='validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
plt.show()
#plotting training and validation accuracy
plt.clf()
#val_acc = history.history['val_acc']
#plt.plot(epochs, acc, label='Training acc')
#plt.plot(epochs, val_acc, label='Validation acc')
# plt.title('Training and validation accuracy')
# plt.xlabel('Epochs')
# plt.ylabel('acc')
# plt.legend()
# plt.show()
model.save('LSTM_Model_14_2to1.h5')
return model
def _build_model(self, x, y):
"""Construct ASAC model using feature and label statistics.
Args:
- x: temporal feature
- y: labels
Returns:
- model: asac model
"""
# Parameters
h_dim = self.h_dim
n_layer = self.n_layer
dim = len(x[0, 0, :])
max_seq_len = len(x[0, :, 0])
# Build one input, two outputs model
main_input = Input(shape=(max_seq_len, dim), dtype='float32')
mask_layer = Masking(mask_value=-1.)(main_input)
previous_input = Input(shape=(max_seq_len, dim), dtype='float32')
previous_mask_layer = Masking(mask_value=-1.)(previous_input)
select_layer = rnn_layer(previous_mask_layer,
self.model_type,
h_dim,
return_seq=True)
for _ in range(n_layer):
select_layer = rnn_layer(select_layer,
self.model_type,
h_dim,
return_seq=True)
select_layer = TimeDistributed(Dense(
dim, activation='sigmoid'))(select_layer)
# Sampling the selection
select_layer = Lambda(lambda x: x - 0.5)(select_layer)
select_layer = Activation('relu')(select_layer)
select_out = Lambda(lambda x: x * 2, name='select')(select_layer)
# Second output
pred_layer = Multiply()([mask_layer, select_out])
for _ in range(n_layer - 1):
pred_layer = rnn_layer(pred_layer,
self.model_type,
h_dim,
return_seq=True)
return_seq_bool = len(y.shape) == 3
pred_layer = rnn_layer(pred_layer, self.model_type, h_dim,
return_seq_bool)
if self.task == 'classification': act_fn = 'sigmoid'
elif self.task == 'regression': act_fn = 'linear'
if len(y.shape) == 3:
pred_out = TimeDistributed(Dense(y.shape[-1], activation=act_fn),
name='predict')(pred_layer)
elif len(y.shape) == 2:
pred_out = Dense(y.shape[-1], activation=act_fn,
name='predict')(pred_layer)
model = Model(inputs=[main_input, previous_input],
outputs=[select_out, pred_out])
# Optimizer
adam = tf.keras.optimizers.Adam(learning_rate=self.learning_rate,
beta_1=0.9,
beta_2=0.999,
amsgrad=False)
# Model compile
if self.task == 'classification':
model.compile(loss={
'select': select_loss,
'predict': binary_cross_entropy_loss
},
optimizer=adam,
loss_weights={
'select': 0.01,
'predict': 1
})
elif self.task == 'regression':
model.compile(loss={
'select': select_loss,
'predict': rmse_loss
},
optimizer=adam,
loss_weights={
'select': 0.01,
'predict': 1
})
return model
def PanopticNet(backbone,
input_shape,
inputs=None,
backbone_levels=['C3', 'C4', 'C5'],
pyramid_levels=['P3', 'P4', 'P5', 'P6', 'P7'],
create_pyramid_features=__create_pyramid_features,
create_semantic_head=__create_semantic_head,
frames_per_batch=1,
temporal_mode=None,
num_semantic_classes=[3],
required_channels=3,
norm_method=None,
pooling=None,
location=True,
use_imagenet=True,
lite=False,
upsample_type='upsampling2d',
interpolation='bilinear',
name='panopticnet',
z_axis_convolutions=False,
**kwargs):
"""Constructs a Mask-RCNN model using a backbone from
``keras-applications`` with optional semantic segmentation transforms.
Args:
backbone (str): Name of backbone to use.
input_shape (tuple): The shape of the input data.
backbone_levels (list): The backbone levels to be used.
to create the feature pyramid.
pyramid_levels (list): Pyramid levels to use.
create_pyramid_features (function): Function to get the pyramid
features from the backbone.
create_semantic_head (function): Function to build a semantic head
submodel.
frames_per_batch (int): Size of z axis in generated batches.
If equal to 1, assumes 2D data.
temporal_mode: Mode of temporal convolution. Choose from
``{'conv','lstm','gru', None}``.
num_semantic_classes (list or dict): Number of semantic classes
for each semantic head. If a ``dict``, keys will be used as
head names and values will be the number of classes.
norm_method (str): Normalization method to use with the
:mod:`deepcell.layers.normalization.ImageNormalization2D` layer.
location (bool): Whether to include a
:mod:`deepcell.layers.location.Location2D` layer.
use_imagenet (bool): Whether to load imagenet-based pretrained weights.
lite (bool): Whether to use a depthwise conv in the feature pyramid
rather than regular conv.
upsample_type (str): Choice of upsampling layer to use from
``['upsamplelike', 'upsampling2d', 'upsampling3d']``.
interpolation (str): Choice of interpolation mode for upsampling
layers from ``['bilinear', 'nearest']``.
pooling (str): optional pooling mode for feature extraction
when ``include_top`` is ``False``.
- None means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- 'avg' means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- 'max' means that global max pooling will
be applied.
z_axis_convolutions (bool): Whether or not to do convolutions on
3D data across the z axis.
required_channels (int): The required number of channels of the
backbone. 3 is the default for all current backbones.
kwargs (dict): Other standard inputs for ``retinanet_mask``.
Raises:
ValueError: ``temporal_mode`` not 'conv', 'lstm', 'gru' or ``None``
Returns:
tensorflow.keras.Model: Panoptic model with a backbone.
"""
channel_axis = 1 if K.image_data_format() == 'channels_first' else -1
conv = Conv3D if frames_per_batch > 1 else Conv2D
conv_kernel = (1, 1, 1) if frames_per_batch > 1 else (1, 1)
# Check input to __merge_temporal_features
acceptable_modes = {'conv', 'lstm', 'gru', None}
if temporal_mode is not None:
temporal_mode = str(temporal_mode).lower()
if temporal_mode not in acceptable_modes:
raise ValueError('temporal_mode {} not supported. Please choose '
'from {}.'.format(temporal_mode, acceptable_modes))
# TODO only works for 2D: do we check for 3D as well?
# What are the requirements for 3D data?
img_shape = input_shape[1:] if channel_axis == 1 else input_shape[:-1]
if img_shape[0] != img_shape[1]:
raise ValueError('Input data must be square, got dimensions {}'.format(
img_shape))
if not math.log(img_shape[0], 2).is_integer():
raise ValueError('Input data dimensions must be a power of 2, '
'got {}'.format(img_shape[0]))
# Check input to interpolation
acceptable_interpolation = {'bilinear', 'nearest'}
if interpolation not in acceptable_interpolation:
raise ValueError('Interpolation mode "{}" not supported. '
'Choose from {}.'.format(
interpolation, list(acceptable_interpolation)))
if inputs is None:
if frames_per_batch > 1:
if channel_axis == 1:
input_shape_with_time = tuple(
[input_shape[0], frames_per_batch] + list(input_shape)[1:])
else:
input_shape_with_time = tuple(
[frames_per_batch] + list(input_shape))
inputs = Input(shape=input_shape_with_time, name='input_0')
else:
inputs = Input(shape=input_shape, name='input_0')
# Normalize input images
if norm_method is None:
norm = inputs
else:
if frames_per_batch > 1:
norm = TimeDistributed(ImageNormalization2D(
norm_method=norm_method, name='norm'), name='td_norm')(inputs)
else:
norm = ImageNormalization2D(norm_method=norm_method,
name='norm')(inputs)
# Add location layer
if location:
if frames_per_batch > 1:
# TODO: TimeDistributed is incompatible with channels_first
loc = TimeDistributed(Location2D(name='location'),
name='td_location')(norm)
else:
loc = Location2D(name='location')(norm)
concat = Concatenate(axis=channel_axis,
name='concatenate_location')([norm, loc])
else:
concat = norm
# Force the channel size for backbone input to be `required_channels`
fixed_inputs = conv(required_channels, conv_kernel, strides=1,
padding='same', name='conv_channels')(concat)
# Force the input shape
axis = 0 if K.image_data_format() == 'channels_first' else -1
fixed_input_shape = list(input_shape)
fixed_input_shape[axis] = required_channels
fixed_input_shape = tuple(fixed_input_shape)
model_kwargs = {
'include_top': False,
'weights': None,
'input_shape': fixed_input_shape,
'pooling': pooling
}
_, backbone_dict = get_backbone(backbone, fixed_inputs,
use_imagenet=use_imagenet,
frames_per_batch=frames_per_batch,
return_dict=True,
**model_kwargs)
backbone_dict_reduced = {k: backbone_dict[k] for k in backbone_dict
if k in backbone_levels}
ndim = 2 if frames_per_batch == 1 else 3
pyramid_dict = create_pyramid_features(backbone_dict_reduced,
ndim=ndim,
lite=lite,
interpolation=interpolation,
upsample_type=upsample_type,
z_axis_convolutions=z_axis_convolutions)
features = [pyramid_dict[key] for key in pyramid_levels]
if frames_per_batch > 1:
temporal_features = [__merge_temporal_features(f, mode=temporal_mode,
frames_per_batch=frames_per_batch)
for f in features]
for f, k in zip(temporal_features, pyramid_levels):
pyramid_dict[k] = f
semantic_levels = [int(re.findall(r'\d+', k)[0]) for k in pyramid_dict]
target_level = min(semantic_levels)
semantic_head_list = []
if not isinstance(num_semantic_classes, dict):
num_semantic_classes = {
k: v for k, v in enumerate(num_semantic_classes)
}
for k, v in num_semantic_classes.items():
semantic_head_list.append(create_semantic_head(
pyramid_dict, n_classes=v,
input_target=inputs, target_level=target_level,
semantic_id=k, ndim=ndim, upsample_type=upsample_type,
interpolation=interpolation, **kwargs))
outputs = semantic_head_list
model = Model(inputs=inputs, outputs=outputs, name=name)
return model
def get_test_model_recurrent():
"""Returns a minimalistic test model for recurrent layers."""
input_shapes = [(17, 4), (1, 10), (20, 40), (6, 7, 10, 3)]
outputs = []
inputs = [Input(shape=s) for s in input_shapes]
inp = PReLU()(inputs[0])
lstm = Bidirectional(
LSTM(
units=4,
return_sequences=True,
bias_initializer=
'random_uniform', # default is zero use random to test computation
activation='tanh',
recurrent_activation='relu'),
merge_mode='concat')(inp)
lstm2 = Bidirectional(LSTM(units=6,
return_sequences=True,
bias_initializer='random_uniform',
activation='elu',
recurrent_activation='hard_sigmoid'),
merge_mode='sum')(lstm)
lstm3 = LSTM(units=10,
return_sequences=False,
bias_initializer='random_uniform',
activation='selu',
recurrent_activation='sigmoid')(lstm2)
outputs.append(lstm3)
conv1 = Conv1D(2, 1, activation='sigmoid')(inputs[1])
lstm4 = LSTM(units=15,
return_sequences=False,
bias_initializer='random_uniform',
activation='tanh',
recurrent_activation='elu')(conv1)
dense = (Dense(23, activation='sigmoid'))(lstm4)
outputs.append(dense)
time_dist_1 = TimeDistributed(Conv2D(2, (3, 3), use_bias=True))(inputs[3])
flatten_1 = TimeDistributed(Flatten())(time_dist_1)
outputs.append(
Bidirectional(LSTM(units=6,
return_sequences=True,
bias_initializer='random_uniform',
activation='tanh',
recurrent_activation='sigmoid'),
merge_mode='ave')(flatten_1))
outputs.append(TimeDistributed(MaxPooling2D(2, 2))(inputs[3]))
outputs.append(TimeDistributed(AveragePooling2D(2, 2))(inputs[3]))
outputs.append(TimeDistributed(BatchNormalization())(inputs[3]))
nested_inputs = Input(shape=input_shapes[0][1:])
nested_x = Dense(5, activation='relu')(nested_inputs)
nested_predictions = Dense(3, activation='softmax')(nested_x)
nested_model = Model(inputs=nested_inputs, outputs=nested_predictions)
nested_model.compile(loss='categorical_crossentropy', optimizer='nadam')
outputs.append(TimeDistributed(nested_model)(inputs[0]))
nested_sequential_model = Sequential()
nested_sequential_model.add(Flatten(input_shape=input_shapes[0][1:]))
nested_sequential_model.compile(optimizer='rmsprop',
loss='categorical_crossentropy')
outputs.append(TimeDistributed(nested_sequential_model)(inputs[0]))
model = Model(inputs=inputs, outputs=outputs, name='test_model_recurrent')
model.compile(loss='mse', optimizer='nadam')
# fit to dummy data
training_data_size = 2
data_in = generate_input_data(training_data_size, input_shapes)
initial_data_out = model.predict(data_in)
data_out = generate_output_data(training_data_size, initial_data_out)
model.fit(data_in, data_out, epochs=10)
return model
def build_transformer(source_vocabulary_size,
target_vocabulary_size,
max_length,
share_word_embedding=False,
n=6,
h=8,
d_k=64,
d_v=64,
d_model=512,
optimizer="adam",
null_token_value=0):
source_input = Input(shape=(None, ), name="source_input")
target_input = Input(shape=(None, ), name="target_input")
enc_input = Lambda(lambda x: x[:, 1:])(source_input)
dec_input = Lambda(lambda x: x[:, :-1])(target_input)
dec_target_output = Lambda(lambda x: x[:, 1:])(target_input)
# create embedding
source_word_embedding = Embedding(
source_vocabulary_size,
d_model,
name="source_embedding" if share_word_embedding else "source_embedding"
) # weights=[_get_positional_encoding_matrix(max_length, d_model)]
if share_word_embedding:
target_word_embedding = source_word_embedding
else:
target_word_embedding = Embedding(target_vocabulary_size,
d_model,
name="target_embedding")
# embedding for the position encoding
position_encoding = Embedding(
max_length,
d_model,
trainable=False,
weights=[_get_positional_encoding_matrix(max_length, d_model)],
name="position_embedding")
enc = Encoder(source_word_embedding,
position_encoding,
n=n,
h=h,
d_k=d_k,
d_v=d_v,
d_model=d_model,
d_inner_hid=512)
dec = Decoder(target_word_embedding,
position_encoding,
n=n,
h=h,
d_k=d_k,
d_v=d_v,
d_model=d_model,
d_inner_hid=512)
enc_output = enc(enc_input)
dec_output = dec(dec_input, enc_output)
# lin_dense = TimeDistributed(Dense(d_model))
fin_output = TimeDistributed(Dense(target_vocabulary_size,
activation=None,
use_bias=False),
name="output") # "softmax"
# lin_dense_out = lin_dense(dec_output)
fin_output_out = fin_output(dec_output) # lin_dense_out)
accuracy = Lambda(_get_accuracy,
arguments={"null_token_value": null_token_value
})([fin_output_out, dec_target_output])
loss = Lambda(_get_loss, arguments={"null_token_value": null_token_value
})([fin_output_out, dec_target_output])
train_model = Model(inputs=[source_input, target_input], outputs=loss)
train_model.add_loss([loss])
train_model.compile(optimizer, None)
train_model.metrics_names.append('accuracy')
# when using tf.keras
#train_model.metrics_tensors.append(accuracy)
train_model.metrics.append(accuracy)
inference_model = Model([source_input, target_input], fin_output_out)
return train_model, inference_model
signature="tokens",
as_dict=True)["elmo"]
from tensorflow.keras.layers import Dense, Flatten, Embedding, LSTM, TimeDistributed, Dropout, Bidirectional, Lambda
from tensorflow.keras.models import Model
from tensorflow.keras.layers import concatenate
from tensorflow.keras import Input
input = Input(shape=(max_len,), dtype=tf.string)
embeddings = Lambda(ElmoEmbedding, output_shape=(max_len, 1024))(input)
x = Bidirectional(LSTM(units=512, return_sequences=True, recurrent_dropout=0.2, dropout=0.2))(embeddings)
x_rnn = Bidirectional(LSTM(units=512, return_sequences=True, recurrent_dropout=0.2, dropout=0.2))(x)
x = concatenate([x, x_rnn])
out = TimeDistributed(Dense(n_tags, activation='softmax'))(x)
model = Model(input, out)
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
X_tr, X_val = X_train[:1213*batch_size], X_train[-135*batch_size:]
y_tr, y_val = y_train[:1213*batch_size], y_train[-135*batch_size:]
y_tr = y_tr.reshape(y_tr.shape[0], y_tr.shape[1], 1)
y_val = y_val.reshape(y_val.shape[0], y_val.shape[1], 1)
m = np.array(X_tr)
print(m.shape)
n = np.array(X_val)
print(n.shape)
history = model.fit(np.array(X_tr), y_tr, validation_data=(np.array(X_val), y_val),
def __init__(self, window, hop=None):
super(MOSNet, self).__init__(name='MOSNet', window=window, hop=hop)
# constants
self.fixed_rate = 16000
self.mono = True
self.absolute = True
self.FFT_SIZE = 512
self.SGRAM_DIM = self.FFT_SIZE // 2 + 1
self.HOP_LENGTH = 256
self.WIN_LENGTH = 512
self.model = dotdict({'predict': None})
pre_trained_dir = os.path.dirname(__file__)
try:
tflite_model = open(os.path.join(pre_trained_dir, 'mosnet.tflite'),
"rb").read()
#converter = tf.lite.TFLiteConverter.from_keras_model(self.model)
##converter.optimizations = [tf.lite.Optimize.DEFAULT]
#tflite_model = converter.convert()
#open(os.path.join(pre_trained_dir, 'mosnet.tflite'), "wb").write(tflite_model)
self.interpreter = tf.lite.Interpreter(model_content=tflite_model)
self.interpreter.allocate_tensors()
self.input_details = self.interpreter.get_input_details()
self.output_details = self.interpreter.get_output_details()
self.input_shape = self.input_details[0]['shape']
def predict(mag, verbose, batch_size):
self.interpreter.resize_tensor_input(
self.input_details[0]['index'], mag.shape)
self.interpreter.allocate_tensors()
self.interpreter.set_tensor(self.input_details[0]['index'],
mag)
self.interpreter.invoke()
return self.interpreter.get_tensor(
self.output_details[0]['index'])
self.model.predict = predict
return
except:
raise
_input = keras.Input(shape=(None, 257)) #shape=(0, 257))
re_input = layers.Reshape((-1, 257, 1),
input_shape=(None, 257))(_input)
# CNN
conv1 = (Conv2D(16, (3, 3),
strides=(1, 1),
activation='relu',
padding='same'))(re_input)
conv1 = (Conv2D(16, (3, 3),
strides=(1, 1),
activation='relu',
padding='same'))(conv1)
conv1 = (Conv2D(16, (3, 3),
strides=(1, 3),
activation='relu',
padding='same'))(conv1)
conv2 = (Conv2D(32, (3, 3),
strides=(1, 1),
activation='relu',
padding='same'))(conv1)
conv2 = (Conv2D(32, (3, 3),
strides=(1, 1),
activation='relu',
padding='same'))(conv2)
conv2 = (Conv2D(32, (3, 3),
strides=(1, 3),
activation='relu',
padding='same'))(conv2)
conv3 = (Conv2D(64, (3, 3),
strides=(1, 1),
activation='relu',
padding='same'))(conv2)
conv3 = (Conv2D(64, (3, 3),
strides=(1, 1),
activation='relu',
padding='same'))(conv3)
conv3 = (Conv2D(64, (3, 3),
strides=(1, 3),
activation='relu',
padding='same'))(conv3)
conv4 = (Conv2D(128, (3, 3),
strides=(1, 1),
activation='relu',
padding='same'))(conv3)
conv4 = (Conv2D(128, (3, 3),
strides=(1, 1),
activation='relu',
padding='same'))(conv4)
conv4 = (Conv2D(128, (3, 3),
strides=(1, 3),
activation='relu',
padding='same'))(conv4)
re_shape = layers.Reshape((-1, 4 * 128),
input_shape=(-1, 4, 128))(conv4)
# BLSTM
blstm1 = Bidirectional(LSTM(128,
return_sequences=True,
dropout=0.3,
recurrent_dropout=0.3,
recurrent_constraint=max_norm(0.00001)),
merge_mode='concat')(re_shape)
# DNN
flatten = TimeDistributed(layers.Flatten())(blstm1)
dense1 = TimeDistributed(Dense(128, activation='relu'))(flatten)
dense1 = Dropout(0.3)(dense1)
frame_score = TimeDistributed(Dense(1), name='frame')(dense1)
import warnings
average_score = layers.GlobalAveragePooling1D(name='avg')(frame_score)
self.model = Model(outputs=[average_score, frame_score], inputs=_input)
# weights are in the directory of this file
# load pre-trained weights. CNN_BLSTM is reported as best
self.model.load_weights(os.path.join(pre_trained_dir, 'cnn_blstm.h5'))
input_dim = x_train_loaded.shape[2]
encoding_dim = n_steps * n_features
# Encoder
encoder = Sequential(name='Encoder')
encoder.add(
LSTM(encoding_dim,
activation='relu',
input_shape=(n_steps, input_dim),
name='encoderlayer1'))
# Decoder
decoder = Sequential(name='Decoder')
decoder.add(RepeatVector(n_steps))
decoder.add(LSTM(encoding_dim, return_sequences=True, name='decoderlayer1'))
decoder.add(TimeDistributed(Dense(input_dim)))
# Compile and fit autoencoder
SAE = Sequential([encoder, decoder], name='SAE')
SAE.compile(optimizer='adam', loss='mse', metrics=['accuracy'])
encoder.compile(optimizer='adam', loss='mse', metrics=['accuracy'])
SAE_history = SAE.fit(x_train_loaded,
x_train_loaded,
epochs=epochs,
batch_size=batch_size)
# Save model
SAE.save('Models/SAE_train_5timestep_%s.h5' % today)
encoder.save('Models/Encoder_train_5timestep_%s.h5' % today)
# Note, the following assumes that you have the graphviz graph library and the Python interface installed
def create_lstm_vae_model(batch_size, time_steps, number_of_features, int_dim,
latent_dim):
def vae_sampling(args):
z_mean, z_log_sigma = args
batch_size = shape(z_mean)[0]
latent_dim = shape(z_mean)[1]
epsilon = K.random_normal(shape=(batch_size, latent_dim),
mean=0,
stddev=1)
return z_mean + K.exp(z_log_sigma / 2) * epsilon
# Encoder
input_x = Input(shape=(time_steps, number_of_features))
encoder_LSTM_int = LSTM(int_dim, return_sequences=True)(input_x)
encoder_LSTM_latent = LSTM(latent_dim,
return_sequences=False)(encoder_LSTM_int)
z_mean = Dense(latent_dim)(encoder_LSTM_latent)
z_log_sigma = Dense(latent_dim)(encoder_LSTM_latent)
z_encoder_output = Lambda(
vae_sampling, output_shape=(latent_dim, ))([z_mean, z_log_sigma])
encoder = Model(input_x, [z_mean, z_log_sigma, z_encoder_output])
# Decoder
decoder_input = Input(shape=(latent_dim))
decoder_repeated = RepeatVector(time_steps)(decoder_input)
decoder_LSTM_int = LSTM(int_dim, return_sequences=True)(decoder_repeated)
decoder_LSTM = LSTM(number_of_features,
return_sequences=True)(decoder_LSTM_int)
decoder_dense1 = TimeDistributed(Dense(number_of_features *
2))(decoder_LSTM)
decoder_output = TimeDistributed(Dense(number_of_features))(decoder_LSTM)
decoder = Model(decoder_input, decoder_output)
# VAE
output = decoder(
encoder(input_x)[2]
) # this is the part encoder and decoder are connected together. Decoder takes the encoder output's[2] as input
lstm_vae = keras.Model(input_x, output, name='lstm_vae')
# Loss
reconstruction_loss = mse(input_x, output)
reconstruction_loss *= number_of_features
kl_loss = 1 + z_log_sigma - K.square(z_mean) - K.exp(z_log_sigma)
kl_loss = K.sum(kl_loss)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss)
lstm_vae.add_loss(vae_loss)
return encoder, decoder, lstm_vae #
def all_metrics_together(y, y_hat):
accuracy = metrics.accuracy_score(y, y_hat)
recall = metrics.recall_score(y, y_hat)
precision = metrics.precision_score(y, y_hat)
f1 = metrics.f1_score(y, y_hat)
df = pd.DataFrame(
{
'Accuracy': accuracy,
'Recall': recall,
'Precision': precision,
'F1': f1
},
index=['metric_value'])
return df
def call(self, inputs, training=None, mask=None):
conv1 = TimeDistributed(MaxPool2D(
(2, 2)))(TimeDistributed(self.conv1)(inputs))
conv2 = TimeDistributed(MaxPool2D(
(2, 2)))(TimeDistributed(self.conv2)(conv1))
return TimeDistributed(self.flat)(conv2)
def output_layer(self):
return TimeDistributed(Dense(self.token_count, activation='softmax'))
def __init__(self, dim, batch_norm, dropout, rec_dropout, partition,
ihm_pos, target_repl=False, depth=1, input_dim=76, **kwargs):
print("==> not used params in network class:", kwargs.keys())
self.dim = dim
self.batch_norm = batch_norm
self.dropout = dropout
self.rec_dropout = rec_dropout
self.depth = depth
# Input layers and masking
X = Input(shape=(None, input_dim), name='X')
mX = Masking()(X)
# Masks
ihm_M = Input(shape=(1,), name='ihm_M')
decomp_M = Input(shape=(None,), name='decomp_M')
los_M = Input(shape=(None,), name='los_M')
inputs = [X, ihm_M, decomp_M, los_M]
# Main part of the network
for i in range(depth):
mX = LSTM(units=dim,
activation='tanh',
return_sequences=True,
recurrent_dropout=rec_dropout,
dropout=dropout)(mX)
L = mX
if dropout > 0:
L = Dropout(dropout)(L)
# Output modules
outputs = []
# ihm output
# NOTE: masking for ihm prediction works this way:
# if ihm_M = 1 then we will calculate an error term
# if ihm_M = 0, our prediction will be 0 and as the label
# will also be 0 then error_term will be 0.
if target_repl > 0:
ihm_seq = TimeDistributed(Dense(1, activation='sigmoid'), name='ihm_seq')(L)
ihm_y = GetTimestep(ihm_pos)(ihm_seq)
ihm_y = Multiply(name='ihm_single')([ihm_y, ihm_M])
outputs += [ihm_y, ihm_seq]
else:
ihm_seq = TimeDistributed(Dense(1, activation='sigmoid'))(L)
ihm_y = GetTimestep(ihm_pos)(ihm_seq)
ihm_y = Multiply(name='ihm')([ihm_y, ihm_M])
outputs += [ihm_y]
# decomp output
decomp_y = TimeDistributed(Dense(1, activation='sigmoid'))(L)
decomp_y = ExtendMask(name='decomp', add_epsilon=True)([decomp_y, decomp_M])
outputs += [decomp_y]
# los output
if partition == 'none':
los_y = TimeDistributed(Dense(1, activation='relu'))(L)
else:
los_y = TimeDistributed(Dense(10, activation='softmax'))(L)
los_y = ExtendMask(name='los', add_epsilon=True)([los_y, los_M])
outputs += [los_y]
# pheno output
if target_repl:
pheno_seq = TimeDistributed(Dense(25, activation='sigmoid'), name='pheno_seq')(L)
pheno_y = LastTimestep(name='pheno_single')(pheno_seq)
outputs += [pheno_y, pheno_seq]
else:
pheno_seq = TimeDistributed(Dense(25, activation='sigmoid'))(L)
pheno_y = LastTimestep(name='pheno')(pheno_seq)
outputs += [pheno_y]
super(Network, self).__init__(inputs=inputs, outputs=outputs)
def interspeech_TIMIT_Model(d):
n = d.num_layers
sf = d.start_filter
activation = d.act
advanced_act = d.aact
drop_prob = d.dropout
inputShape = (778, 3, 40) # (3,41,None)
filsize = (3, 5)
Axis = 1
if advanced_act != "none":
activation = 'linear'
convArgs = {
"activation": activation,
"data_format": "channels_first",
"padding": "same",
"bias_initializer": "zeros",
"kernel_regularizer": l2(d.l2),
"kernel_initializer": "random_uniform",
}
denseArgs = {
"activation": d.act,
"kernel_regularizer": l2(d.l2),
"kernel_initializer": "random_uniform",
"bias_initializer": "zeros",
"use_bias": True
}
#### Check kernel_initializer for quaternion model ####
if d.model == "quaternion":
convArgs.update({"kernel_initializer": d.quat_init})
#
# Input Layer & CTC Parameters for TIMIT
#
if d.model == "quaternion":
I = Input(shape=(778, 4, 40)) # Input(shape=(4,41,None))
else:
I = Input(shape=inputShape)
labels = Input(name='the_labels', shape=[None], dtype='float32')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
#
# Input stage:
#
if d.model == "real":
O = Conv2D(sf, filsize, name='conv', use_bias=True, **convArgs)(I)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
else:
O = QuaternionConv2D(sf,
filsize,
name='conv',
use_bias=True,
**convArgs)(I)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
#
# Pooling
#
O = MaxPooling2D(pool_size=(1, 3), padding='same')(O)
#
# Stage 1
#
for i in range(0, n // 2):
if d.model == "real":
O = Conv2D(sf,
filsize,
name='conv' + str(i),
use_bias=True,
**convArgs)(O)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(drop_prob)(O)
else:
O = QuaternionConv2D(sf,
filsize,
name='conv' + str(i),
use_bias=True,
**convArgs)(O)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(drop_prob)(O)
#
# Stage 2
#
for i in range(0, n // 2):
if d.model == "real":
O = Conv2D(sf * 2,
filsize,
name='conv' + str(i + n / 2),
use_bias=True,
**convArgs)(O)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(drop_prob)(O)
else:
O = QuaternionConv2D(sf * 2,
filsize,
name='conv' + str(i + n / 2),
use_bias=True,
**convArgs)(O)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(drop_prob)(O)
#
# Permutation for CTC
#
print("Last Q-Conv2D Layer (output): ", K.int_shape(O))
print("Shape tuple: ",
K.int_shape(O)[0],
K.int_shape(O)[1],
K.int_shape(O)[2],
K.int_shape(O)[3])
#### O = Permute((3,1,2))(O)
#### print("Last Q-Conv2D Layer (Permute): ", O.shape)
# O = Lambda(lambda x: K.reshape(x, (K.shape(x)[0], K.shape(x)[1],
# K.shape(x)[2] * K.shape(x)[3])),
# output_shape=lambda x: (None, None, x[2] * x[3]))(O)
# O = Lambda(lambda x: K.reshape(x, (K.int_shape(x)[0], K.int_shape(x)[1],
# K.int_shape(x)[2] * K.int_shape(x)[3])),
# output_shape=lambda x: (None, None, x[2] * x[3]))(O)
O = tf.keras.layers.Reshape(
target_shape=[-1, K.int_shape(O)[2] * K.int_shape(O)[3]])(O)
#
# Dense
#
print("Q-Dense input: ", K.int_shape(O))
if d.model == "quaternion":
print("first Q-dense layer: ", O.shape)
O = TimeDistributed(QuaternionDense(256, **denseArgs))(O)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(drop_prob)(O)
O = TimeDistributed(QuaternionDense(256, **denseArgs))(O)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(drop_prob)(O)
O = TimeDistributed(QuaternionDense(256, **denseArgs))(O)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
else:
O = TimeDistributed(Dense(1024, **denseArgs))(O)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(drop_prob)(O)
O = TimeDistributed(Dense(1024, **denseArgs))(O)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(drop_prob)(O)
O = TimeDistributed(Dense(1024, **denseArgs))(O)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
# pred = TimeDistributed( Dense(61, activation='softmax', kernel_regularizer=l2(d.l2), use_bias=True, bias_initializer="zeros", kernel_initializer='random_uniform' ))(O)
pred = TimeDistributed(
Dense(61,
kernel_regularizer=l2(d.l2),
use_bias=True,
bias_initializer="zeros",
kernel_initializer='random_uniform'))(O)
output = Activation('softmax', name='softmax')(pred)
network = CTCModel([I], [output])
# network.compile(Adam(lr=0.0001))
""" CTC Loss - Implemented differently
if d.ctc:
# CTC For sequence labelling
O = Lambda(ctc_lambda_func, output_shape=(1,), name='ctc')([pred, labels,input_length,label_length])
# Creating a function for testing and validation purpose
val_function = K.function([I],[pred])
# Return the model
return Model(inputs=[I, input_length, labels, label_length], outputs=O), val_function
"""
# return Model(inputs=I, outputs=pred)
return network
def line_lstm_ctc(input_shape, output_shape, window_width=28, window_stride=14):
image_height, image_width = input_shape
output_length, num_classes = output_shape
num_windows = int((image_width - window_width) / window_stride) + 1
if num_windows < output_length:
raise ValueError(f'Window width/stride need to generate at least {output_length} windows (currently {num_windows})')
image_input = Input(shape=input_shape, name='image')
y_true = Input(shape=(output_length,), name='y_true')
input_length = Input(shape=(1,), name='input_length')
label_length = Input(shape=(1,), name='label_length')
gpu_present = len(device_lib.list_local_devices()) > 1
lstm_fn = CuDNNLSTM if gpu_present else LSTM
# Your code should use slide_window and extract image patches from image_input.
# Pass a convolutional model over each image patch to generate a feature vector per window.
# Pass these features through one or more LSTM layers.
# Convert the lstm outputs to softmax outputs.
# Note that lstms expect a input of shape (num_batch_size, num_timesteps, feature_length).
##### Your code below (Lab 3)
image_reshaped = Reshape((image_height, image_width, 1))(image_input)
# (image_height, image_width, 1)
image_patches = Lambda(
slide_window,
arguments={'window_width': window_width, 'window_stride': window_stride}
)(image_reshaped)
# (num_windows, image_height, window_width, 1)
# Make a LeNet and get rid of the last two layers (softmax and dropout)
convnet = lenet((image_height, window_width, 1), (num_classes,))
convnet = KerasModel(inputs=convnet.inputs, outputs=convnet.layers[-2].output)
convnet_outputs = TimeDistributed(convnet)(image_patches)
# (num_windows, 128)
lstm_output1 = lstm_fn(128, return_sequences=True)(convnet_outputs)
lstm_output = lstm_fn(64, return_sequences=True)(lstm_output1)
# (num_windows, 128)
softmax_output = Dense(num_classes, activation='softmax', name='softmax_output')(lstm_output)
# (num_windows, num_classes)
##### Your code above (Lab 3)
input_length_processed = Lambda(
lambda x, num_windows=None: x * num_windows,
arguments={'num_windows': num_windows}
)(input_length)
ctc_loss_output = Lambda(
lambda x: K.ctc_batch_cost(x[0], x[1], x[2], x[3]),
name='ctc_loss'
)([y_true, softmax_output, input_length_processed, label_length])
ctc_decoded_output = Lambda(
lambda x: ctc_decode(x[0], x[1], output_length),
name='ctc_decoded'
)([softmax_output, input_length_processed])
model = KerasModel(
inputs=[image_input, y_true, input_length, label_length],
outputs=[ctc_loss_output, ctc_decoded_output]
)
return model
# One-hot encoding
x = utils.to_categorical(x, num_classes=num_classes)
# reshape X to be [samples, time steps, features]
x = np.reshape(x, (-1, len(x), data_dim))
print(x.shape)
# One-hot encoding
y = utils.to_categorical(y, num_classes=num_classes)
# time steps
y = np.reshape(y, (-1, len(y), data_dim))
print(y.shape)
model = Sequential()
model.add(LSTM(128, input_shape=(
timesteps, data_dim), return_sequences=True))
model.add(TimeDistributed(Dense(num_classes,activation='softmax')))
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop', metrics=['accuracy'])
history=model.fit(x, y, epochs=100, verbose=2)
predictions = model.predict(x, verbose=2)
for i, prediction in enumerate(predictions):
print(prediction)
x_index = np.argmax(x[i], axis=1)
x_str = [char_set[j] for j in x_index]
print(x_index, ''.join(x_str))
index = np.argmax(prediction, axis=1)
def build_model(self, predict, custom_batch_size=None):
conf = self.conf
model_conf = conf['model']
rnn_size = model_conf['rnn_size']
rnn_type = model_conf['rnn_type']
regularization = model_conf['regularization']
dense_regularization = model_conf['dense_regularization']
use_batch_norm = False
if 'use_batch_norm' in model_conf:
use_batch_norm = model_conf['use_batch_norm']
dropout_prob = model_conf['dropout_prob']
length = model_conf['length']
pred_length = model_conf['pred_length']
# skip = model_conf['skip']
stateful = model_conf['stateful']
return_sequences = model_conf['return_sequences']
# model_conf['output_activation']
output_activation = conf['data']['target'].activation
use_signals = conf['paths']['use_signals']
num_signals = sum([sig.num_channels for sig in use_signals])
num_conv_filters = model_conf['num_conv_filters']
# num_conv_layers = model_conf['num_conv_layers']
size_conv_filters = model_conf['size_conv_filters']
pool_size = model_conf['pool_size']
dense_size = model_conf['dense_size']
batch_size = self.conf['training']['batch_size']
if predict:
batch_size = self.conf['model']['pred_batch_size']
# so we can predict with one time point at a time!
if return_sequences:
length = pred_length
else:
length = 1
if custom_batch_size is not None:
batch_size = custom_batch_size
if rnn_type == 'LSTM':
rnn_model = LSTM
elif rnn_type == 'CuDNNLSTM':
rnn_model = CuDNNLSTM
elif rnn_type == 'SimpleRNN':
rnn_model = SimpleRNN
else:
print('Unkown Model Type, exiting.')
exit(1)
batch_input_shape = (batch_size, length, num_signals)
# batch_shape_non_temporal = (batch_size, num_signals)
indices_0d, indices_1d, num_0D, num_1D = self.get_0D_1D_indices()
def slicer(x, indices):
return x[:, indices]
def slicer_output_shape(input_shape, indices):
shape_curr = list(input_shape)
assert len(shape_curr) == 2 # only valid for 3D tensors
shape_curr[-1] = len(indices)
return tuple(shape_curr)
pre_rnn_input = Input(shape=(num_signals, ))
if num_1D > 0:
pre_rnn_1D = Lambda(
lambda x: x[:, len(indices_0d):],
output_shape=(len(indices_1d), ))(pre_rnn_input)
pre_rnn_0D = Lambda(
lambda x: x[:, :len(indices_0d)],
output_shape=(len(indices_0d), ))(pre_rnn_input)
# slicer(x,indices_0d),lambda s:
# slicer_output_shape(s,indices_0d))(pre_rnn_input)
pre_rnn_1D = Reshape(
(num_1D, len(indices_1d) // num_1D))(pre_rnn_1D)
pre_rnn_1D = Permute((2, 1))(pre_rnn_1D)
if ('simple_conv' in model_conf.keys()
and model_conf['simple_conv'] is True):
for i in range(model_conf['num_conv_layers']):
pre_rnn_1D = Convolution1D(num_conv_filters,
size_conv_filters,
padding='valid',
activation='relu')(pre_rnn_1D)
pre_rnn_1D = MaxPooling1D(pool_size)(pre_rnn_1D)
else:
for i in range(model_conf['num_conv_layers']):
div_fac = 2**i
'''The first conv layer learns `num_conv_filters//div_fac`
filters (aka kernels), each of size
`(size_conv_filters, num1D)`. Its output will have shape
(None, len(indices_1d)//num_1D - size_conv_filters + 1,
num_conv_filters//div_fac), i.e., for
each position in the input spatial series (direction along
radius), the activation of each filter at that position.
'''
'''For i=1 first conv layer would get:
(None, (len(indices_1d)//num_1D - size_conv_filters
+ 1)/pool_size-size_conv_filters + 1,
num_conv_filters//div_fac)
'''
pre_rnn_1D = Convolution1D(num_conv_filters // div_fac,
size_conv_filters,
padding='valid')(pre_rnn_1D)
if use_batch_norm:
pre_rnn_1D = BatchNormalization()(pre_rnn_1D)
pre_rnn_1D = Activation('relu')(pre_rnn_1D)
'''The output of the second conv layer will have shape
(None, len(indices_1d)//num_1D - size_conv_filters + 1,
num_conv_filters//div_fac),
i.e., for each position in the input spatial series
(direction along radius), the activation of each filter
at that position.
For i=1, the second layer would output
(None, (len(indices_1d)//num_1D - size_conv_filters + 1)/
pool_size-size_conv_filters + 1,num_conv_filters//div_fac)
'''
pre_rnn_1D = Convolution1D(num_conv_filters // div_fac,
1,
padding='valid')(pre_rnn_1D)
if use_batch_norm:
pre_rnn_1D = BatchNormalization()(pre_rnn_1D)
pre_rnn_1D = Activation('relu')(pre_rnn_1D)
'''Outputs (None, (len(indices_1d)//num_1D - size_conv_filters
+ 1)/pool_size, num_conv_filters//div_fac)
For i=1, the pooling layer would output:
(None,((len(indices_1d)//num_1D- size_conv_filters
+ 1)/pool_size-size_conv_filters+1)/pool_size,
num_conv_filters//div_fac)
'''
pre_rnn_1D = MaxPooling1D(pool_size)(pre_rnn_1D)
pre_rnn_1D = Flatten()(pre_rnn_1D)
pre_rnn_1D = Dense(
dense_size,
kernel_regularizer=l2(dense_regularization),
bias_regularizer=l2(dense_regularization),
activity_regularizer=l2(dense_regularization))(pre_rnn_1D)
if use_batch_norm:
pre_rnn_1D = BatchNormalization()(pre_rnn_1D)
pre_rnn_1D = Activation('relu')(pre_rnn_1D)
pre_rnn_1D = Dense(
dense_size // 4,
kernel_regularizer=l2(dense_regularization),
bias_regularizer=l2(dense_regularization),
activity_regularizer=l2(dense_regularization))(pre_rnn_1D)
if use_batch_norm:
pre_rnn_1D = BatchNormalization()(pre_rnn_1D)
pre_rnn_1D = Activation('relu')(pre_rnn_1D)
pre_rnn = Concatenate()([pre_rnn_0D, pre_rnn_1D])
else:
pre_rnn = pre_rnn_input
if model_conf['rnn_layers'] == 0 or (
'extra_dense_input' in model_conf.keys()
and model_conf['extra_dense_input']):
pre_rnn = Dense(
dense_size,
activation='relu',
kernel_regularizer=l2(dense_regularization),
bias_regularizer=l2(dense_regularization),
activity_regularizer=l2(dense_regularization))(pre_rnn)
pre_rnn = Dense(
dense_size // 2,
activation='relu',
kernel_regularizer=l2(dense_regularization),
bias_regularizer=l2(dense_regularization),
activity_regularizer=l2(dense_regularization))(pre_rnn)
pre_rnn = Dense(
dense_size // 4,
activation='relu',
kernel_regularizer=l2(dense_regularization),
bias_regularizer=l2(dense_regularization),
activity_regularizer=l2(dense_regularization))(pre_rnn)
pre_rnn_model = tf.keras.Model(inputs=pre_rnn_input, outputs=pre_rnn)
# TODO(KGF): uncomment following lines to get summary of pre-RNN model
# from mpi4py import MPI
# comm = MPI.COMM_WORLD
# task_index = comm.Get_rank()
# if not predict and task_index == 0:
# print('Printing out pre_rnn model...')
# fr = open('model_architecture.log', 'w')
# ori = sys.stdout
# sys.stdout = fr
# pre_rnn_model.summary()
# sys.stdout = ori
# fr.close()
# pre_rnn_model.summary()
x_input = Input(batch_shape=batch_input_shape)
# TODO(KGF): Ge moved this inside a new conditional in Dec 2019. check
# x_in = TimeDistributed(pre_rnn_model)(x_input)
if (num_1D > 0 or ('extra_dense_input' in model_conf.keys()
and model_conf['extra_dense_input'])):
x_in = TimeDistributed(pre_rnn_model)(x_input)
else:
x_in = x_input
# ==========
# TCN MODEL
# ==========
if ('keras_tcn' in model_conf.keys()
and model_conf['keras_tcn'] is True):
print('Building TCN model....')
tcn_layers = model_conf['tcn_layers']
tcn_dropout = model_conf['tcn_dropout']
nb_filters = model_conf['tcn_hidden']
kernel_size = model_conf['kernel_size_temporal']
nb_stacks = model_conf['tcn_nbstacks']
use_skip_connections = model_conf['tcn_skip_connect']
activation = model_conf['tcn_activation']
use_batch_norm = model_conf['tcn_batch_norm']
for _ in range(model_conf['tcn_pack_layers']):
x_in = TCN(use_batch_norm=use_batch_norm,
activation=activation,
use_skip_connections=use_skip_connections,
nb_stacks=nb_stacks,
kernel_size=kernel_size,
nb_filters=nb_filters,
num_layers=tcn_layers,
dropout_rate=tcn_dropout)(x_in)
x_in = Dropout(dropout_prob)(x_in)
else: # end TCN model
# ==========
# RNN MODEL
# ==========
# LSTM in ONNX: "The maximum opset needed by this model is only 9."
model_kwargs = dict(
return_sequences=return_sequences,
# batch_input_shape=batch_input_shape,
stateful=stateful,
kernel_regularizer=l2(regularization),
recurrent_regularizer=l2(regularization),
bias_regularizer=l2(regularization),
)
if rnn_type != 'CuDNNLSTM':
# Dropout is unsupported in CuDNN library
model_kwargs['dropout'] = dropout_prob
model_kwargs['recurrent_dropout'] = dropout_prob
for _ in range(model_conf['rnn_layers']):
x_in = rnn_model(rnn_size, **model_kwargs)(x_in)
x_in = Dropout(dropout_prob)(x_in)
if return_sequences:
# x_out = TimeDistributed(Dense(100,activation='tanh')) (x_in)
x_out = TimeDistributed(Dense(
1, activation=output_activation))(x_in)
model = tf.keras.Model(inputs=x_input, outputs=x_out)
# bug with tensorflow/Keras
# TODO(KGF): what is this bug? this is the only direct "tensorflow"
# import outside of mpi_runner.py and runner.py
# if (conf['model']['backend'] == 'tf'
# or conf['model']['backend'] == 'tensorflow'):
# first_time = "tensorflow" not in sys.modules
# import tensorflow as tf
# if first_time:
# tf.compat.v1.keras.backend.get_session().run(
# tf.global_variables_initializer())
model.reset_states()
return model
state_h = Concatenate()([forward_h, backward_h])
state_c = Concatenate()([forward_c, backward_c])
# Set up the decoder, using `encoder_states` as initial state
encoder_states = [state_h, state_c]
decoder_inputs = Input(shape=(None, ), name="Decoder_Input")
dec_emb = Embedding(num_decoder_tokens, latent_dim)(decoder_inputs)
decoder_lstm = LSTM(latent_dim * 2,
return_sequences=True,
return_state=True,
name="Decoder_LSTM")
decoder_outputs, _, _ = decoder_lstm(dec_emb, initial_state=encoder_states)
decoder_dense = Dense(num_decoder_tokens, activation='softmax')
decoder_outputs = TimeDistributed(decoder_dense)(decoder_outputs)
# Define the model that will turn
# `encoder_input_data` & `decoder_input_data` into `decoder_target_data`
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
train_samples = len(X_train)
val_samples = len(X_test)
batch_size = 128
epochs = 5
model.fit(generate_batch(X_train, y_train, batch_size=batch_size),
emb = Embedding( input_dim=VOCAB_SIZE, output_dim=50, weights=[embedding_matrix],
trainable=False, input_length=MAX_SEQ_LEN)(inp)
embedding = Model( inp, emb )
save_model( embedding, "embedding_model.h5" )
#Model for identifying tags of each word
inp = Input( shape=(MAX_SEQ_LEN, 50) )
drop = Dropout(0.1)(inp)
#two bidirectinal LSTM layers
lstm1 = LSTM( 30, return_sequences=True, recurrent_dropout=0.1)
seq1 = Bidirectional(lstm1)( drop )
lstm2 = LSTM( 30, return_sequences=True, recurrent_dropout=0.1)
seq2 = Bidirectional(lstm2)( seq1 )
# TIME_DISTRIBUTED -> ( MAX_SEQ_LEN, 50 ) -> (MAX_SEQ_LEN, NER_SIZE)
tags = TimeDistributed( Dense(NER_SIZE, activation="relu") )(seq2)
model = Model( inp, tags )
model.compile( optimizer="rmsprop", loss="categorical_crossentropy", metrics=[ "accuracy" ] )
#batch generator for model training
def getBatch(sentences, targets, batch_size=128):
n = len(sentences)//batch_size
for i in range( n+1 ):
x = sentences[ i*batch_size : (i+1)*batch_size ]
x = embedding.predict(x)
y = targets[ i*batch_size : (i+1)*batch_size ]
yield x,y
#do training
all_metrics = []
for epoch in range(1,51):
# Import Dense and TimeDistributed layers
from tensorflow.keras.layers import Dense, TimeDistributed
# Define a softmax dense layer that has fr_vocab outputs
de_dense = Dense(fr_vocab, activation='softmax')
# Wrap the dense layer in a TimeDistributed layer
de_dense_time = TimeDistributed(de_dense)
# Get the final prediction of the model
de_pred = de_dense_time(de_out)
print("Prediction shape: ", de_pred.shape)
def create_model_lstm(data_type,
cnn=False,
layer_nb=5,
classes=len(labels),
lstm_nodes=64,
learning_rate=0.0001):
if data_type == 'mfcc':
input_shape = (98, 13) if not cnn else (98, 13, 1)
if data_type == 'ssc':
input_shape = (98, 26) if not cnn else (98, 26, 1)
model = Sequential(name='lstm_{}'.format('' if not cnn else 'cnn_') +
data_type)
model.add(Input(shape=input_shape))
if cnn:
cnn = Sequential(name='cnn_entry_' + data_type)
cnn.add(Conv1D(22, 3, padding='same', name='conv1'))
cnn.add(BatchNormalization(name='batch_norm1'))
cnn.add(tf.keras.layers.Activation('relu'))
cnn.add(Conv1D(44, 3, padding='same', name='conv2'))
cnn.add(BatchNormalization(name='batch_norm2'))
cnn.add(tf.keras.layers.Activation('relu'))
cnn.add(Conv1D(22, 3, padding='same', name='conv3'))
cnn.add(BatchNormalization(name='batch_norm3'))
cnn.add(tf.keras.layers.Activation('relu'))
cnn.add(AveragePooling1D(2, strides=2, name='pooling'))
model.add(TimeDistributed(cnn))
model.add(Reshape((98, -1)))
if layer_nb == 1:
model.add(LSTM(lstm_nodes, name='lstm_entry'))
model.add(Lambda(lambda x: K.l2_normalize(x, axis=1)))
model.add(BatchNormalization(name='block1_batchnorm'))
else:
model.add(LSTM(lstm_nodes, return_sequences=True, name='lstm_entry'))
model.add(Lambda(lambda x: K.l2_normalize(x, axis=1)))
model.add(BatchNormalization(name='block1_batchnorm'))
for i in range(2, layer_nb):
model.add(
LSTM(lstm_nodes,
return_sequences=True,
name='lstm_{}'.format(i)))
model.add(Lambda(lambda x: K.l2_normalize(x, axis=1)))
model.add(
BatchNormalization(name='block{}_batchnorm'.format(str(i))))
model.add(LSTM(lstm_nodes, name='lstm_out'))
model.add(Lambda(lambda x: K.l2_normalize(x, axis=1)))
model.add(BatchNormalization(name='blockout_batchnorm'))
model.add(Flatten())
model.add(Dense(classes, activation='softmax', name='predictions'))
optimizer = RMSprop(lr=learning_rate)
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
model.summary()
return model
# LSTM 입력을 위해 3차원 텐서 형태로 변경
X = X.reshape(1, n_timesteps, 1)
y = y.reshape(1, n_timesteps, 1)
return X, y
# 하이퍼파라미터 정의
n_units = 20
n_timesteps = 4
# 양방향 LSTM 모델 정의
model = Sequential()
model.add(
Bidirectional(
LSTM(n_units, return_sequences=True, input_shape=(n_timesteps, 1))))
model.add(TimeDistributed(Dense(1, activation='sigmoid')))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# 모델 학습
# 에포크마다 학습 데이터를 생성해서 학습
for epoch in range(1000):
X, y = get_sequence(n_timesteps)
model.fit(X, y, epochs=1, batch_size=1, verbose=2)
# 모델 평가
X, y = get_sequence(n_timesteps)
yhat = model.predict(X, verbose=0)
yhat2 = np.argmax(yhat, axis=1)
print(yhat2)
state_c = Concatenate()([forward_c3,backward_c3]) decoder_inputs = Input(shape=(None,)) dec_emb_layer = Embedding(y_voc_size, latent_dim,trainable=True) dec_emb = dec_emb_layer(decoder_inputs) decoder_lstm = LSTM(600, return_sequences=True, return_state=True) decoder_outputs,decoder_fwd_state, decoder_back_state = decoder_lstm(dec_emb,initial_state=[state_h, state_c]) attn_layer = AttentionLayer(name='attention_layer') attn_out, attn_states = attn_layer([encoder_outputs, decoder_outputs]) decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([decoder_outputs, attn_out]) decoder_dense = TimeDistributed(Dense(y_voc_size, activation='softmax')) decoder_out = decoder_dense(decoder_concat_input) model = Model([encoder_inputs, decoder_inputs], decoder_out) model.summary() model.compile(optimizer='adam', loss='sparse_categorical_crossentropy',metrics=['accuracy']) es = EarlyStopping(monitor='val_loss', mode='min') history=model.fit([x_tr,y_tr[:,:-1]], y_tr.reshape(y_tr.shape[0],y_tr.shape[1], 1)[:,1:] ,epochs=50,callbacks=[es],batch_size=128, validation_data=([x_val,y_val[:,:-1]], y_val.reshape(y_val.shape[0],y_val.shape[1], 1)[:,1:])) from matplotlib import pyplot pyplot.plot(history.history['loss'], label='train') pyplot.plot(history.history['val_loss'], label='test') pyplot.legend() pyplot.show()
def init_nslr_model(preprocess_size, lrs_size, lrs_levels, preprocess='conv', preprocess_time=False, lrs_diff=True, lrs_time=True, recursive_tensors=True, name_only=False, input_shape=None, num_classes=None, lrs_norm='BN'):
preprocess = preprocess.lower()
if preprocess != 'conv' and preprocess != 'dense':
preprocess_name = ''
else:
preprocess_name = 'Conv' if preprocess == 'conv' else 'Dense'
preprocess_name = 'T{}'.format(preprocess_name) if preprocess_time else preprocess_name
lrs_time_tag = 'T' if lrs_time else ''
lrs_diff_tag = 'D' if lrs_diff else ''
lrs_recurrent_tag = 'R' if recursive_tensors else ''
lrs_name = '{}{}{}LRS{}'.format(lrs_time_tag, lrs_diff_tag, lrs_recurrent_tag, lrs_norm)
model_name = '{}{}_H{}_W{}'.format(preprocess_name, lrs_name, preprocess_size, lrs_size)
if name_only: return model_name
num_sig_layers = 3
model = Sequential()
model.add(InputLayer(input_shape=input_shape))
if preprocess_name != '':
if preprocess_time:
model.add(Time())
if preprocess == 'conv':
model.add(Conv1D(preprocess_size, 8, padding='same', kernel_initializer='he_uniform'))
else:
model.add(TimeDistributed(Dense(preprocess_size, kernel_initializer='he_uniform')))
model.add(BatchNormalization(axis=-1))
model.add(Activation('relu'))
if preprocess_time:
model.add(Time())
if preprocess == 'conv':
model.add(Conv1D(preprocess_size, 5, padding='same' , kernel_initializer='he_uniform'))
else:
model.add(TimeDistributed(Dense(preprocess_size, kernel_initializer='he_uniform')))
model.add(BatchNormalization(axis=-1))
model.add(Activation('relu'))
if preprocess_time:
model.add(Time())
if preprocess == 'conv':
model.add(Conv1D(preprocess_size, 3, padding='same', kernel_initializer='he_uniform'))
else:
model.add(Dense(preprocess_size, kernel_initializer='he_uniform'))
model.add(BatchNormalization(axis=-1))
model.add(Activation('relu'))
for i in range(num_sig_layers-1):
# model.add(Conv1D(conv_size, 3, padding='same', kernel_initializer='he_uniform'))
# model.add(BatchNormalization(axis=-1))
if lrs_time:
model.add(Time())
if lrs_diff:
model.add(Difference())
model.add(LRS(lrs_size, lrs_levels, return_sequences=True, recursive_tensors=recursive_tensors))
model.add(Reshape((input_shape[0]-i-1, lrs_levels, lrs_size,)))
if lrs_norm == 'BN':
model.add(BatchNormalization(axis=[-2]))
elif lrs_norm == 'TBN':
model.add(BatchNormalization(axis=[-2, -1]))
elif lrs_norm == 'LN':
model.add(LayerNormalization(axis=[-3, -1]))
# model.add(LayerNormalization(axis=[1, 3]))
#model.add(LayerNormalization(axis=[1, 3]))
model.add(Reshape((input_shape[0]-i-1, lrs_levels * lrs_size,)))
# model.add(Activation('relu'))
# if renorm:
# model.add(BatchNormalization(axis=[1, 2], renorm=True, center=False, scale=False))
# else:
#
# model.add(Conv1D(conv_size, 3, padding='same', kernel_initializer='he_uniform'))
# model.add(BatchNormalization(axis=-1))
if lrs_time:
model.add(Time())
if lrs_diff:
model.add(Difference())
model.add(LRS(lrs_size, lrs_levels, return_sequences=False, recursive_tensors=recursive_tensors))
# model.add(BatchNormalization(axis=1, center=False, scale=False))
model.add(Reshape((lrs_levels, lrs_size,)))
# model.add(LayerNormalization(axis=[2]))
model.add(BatchNormalization(axis=1))
model.add(Reshape((lrs_levels * lrs_size,)))
# model.add(BatchNormalization(axis=-1, renorm=renorm, center=False, scale=False))
# model.add(tf.keras.layers.Dropout(0.5))
# if gap:
# model.add(GlobalAveragePooling1D())
# else:
# model.add(Lambda(lambda X: X[:, -1]))
model.add(Dense(num_classes, activation='softmax'))
model._name = model_name
return model
def sequential_cnn_2D(args):
#
# Model parameters
#
model_type = args.model_type
# input_shape = args.input_shape
n_convs = args.n_layers_convs
n_dense = args.n_layers_dense
# max_str_len = args.max_str_len
num_classes = args.num_classes
activation = args.act
advanced_act = args.aact
padding = args.padding
start_filter = args.sf_dim
kernel_size = args.kernel_size
kernel_init = args.kernel_init
l2_reg = args.l2_reg
shared_axes = args.shared_axes
opt = args.opt
#
# Layer parameters
#
# Conv layers parameters
convArgs = {
"activation": activation,
"data_format": "channels_last",
"padding": padding,
"bias_initializer": "zeros",
"kernel_regularizer": l2(l2_reg),
"kernel_initializer": kernel_init,
"use_bias": True
}
# Dense layers parameters
denseArgs = {
"activation": activation,
"kernel_regularizer": l2(l2_reg),
"kernel_initializer": kernel_init, # "glorot_normal"
"bias_initializer": "zeros",
"use_bias": True
}
#
# Model building
#
if model_type == "quaternion":
quat_init = 'quaternion'
convArgs.update({"kernel_initializer": quat_init})
# input_data = Input(name='the_input', shape=input_shape, dtype='float64')
model = Sequential()
# Convolutional Stage
if model_type == 'real':
# batch-normalization
# O = BatchNormalization(axis=-1, momentum=0.99, epsilon=1e-3, center=True, scale=True)
for idx in range(n_convs):
conv_filters = start_filter * (2**idx)
O = Conv2D(conv_filters,
kernel_size,
name='conv_{}'.format(idx),
**convArgs)
model.add(O)
if advanced_act == 'prelu':
O = PReLU()
model.add(O)
# batch-normalization
O = BatchNormalization(axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,
scale=True)
model.add(O)
else:
# batch-normalization
# O = QuaternionBatchNormalization(axis=-1, momentum=0.99, epsilon=1e-3, center=True, scale=True)
for idx in range(n_convs):
conv_filters = start_filter * (2**idx)
O = QuaternionConv2D(conv_filters,
kernel_size,
name='conv_{}'.format(idx),
**convArgs)(O)
if advanced_act == 'prelu':
O = PReLU()
model.add(O)
# batch-normalization
O = QuaternionBatchNormalization(axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,
scale=True)(O)
# conv_shape = O.shape
#
# Reshaping Stage
#
O = Reshape(target_shape=[
K.int_shape(O)[1],
K.int_shape(O)[2] * K.int_shape(O)[3]
])(O)
# Fully-Connected Stage
if model_type == 'real':
for idx in range(n_dense):
O = TimeDistributed(Dense(1024, **denseArgs))
model.add(O)
if advanced_act == 'prelu':
O = PReLU()
model.add(O)
# batch-normalization
O = BatchNormalization(axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,
scale=True)
model.add(O)
else:
for idx in range(n_dense):
O = TimeDistributed(QuaternionDense(1024, **denseArgs))
model.add(O)
if advanced_act == 'prelu':
O = PReLU()
model.add(O)
# batch-normalization
O = QuaternionBatchNormalization(axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,
scale=True)(O)
#
# Prediction (Output) Stage
#
y_pred = Dense(num_classes,
activation='softmax',
kernel_regularizer=l2(l2_reg),
use_bias=True,
bias_initializer="zeros",
kernel_initializer=kernel_init)
model.add(y_pred)
# clipnorm seems to speeds up convergence
opt_dict = {
'sgd': SGD(lr=0.02,
decay=1e-6,
momentum=0.9,
nesterov=True,
clipnorm=5),
'adam': Adam(lr=0.02, clipnorm=5)
}
optimizer = opt_dict[opt]
# the loss calc occurs elsewhere, so use a dummy lambda func for the loss
model.compile(loss='categorical_crossentropy', optimizer=optimizer)
return model
x_train, x_test, y_train, y_test = train_test_split(X,
y,
test_size=0.2,
random_state=1)
from tensorflow.keras import Model, Input
from tensorflow.keras.layers import LSTM, Embedding, Dense
from tensorflow.keras.layers import TimeDistributed, SpatialDropout1D, Bidirectional
input_word = Input(shape=(max_len, ))
model = Embedding(input_dim=num_words, output_dim=50,
input_length=max_len)(input_word)
model = SpatialDropout1D(0.1)(model)
model = Bidirectional(
LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(model)
out = TimeDistributed(Dense(num_tags, activation="softmax"))(model)
model = Model(input_word, out)
model.summary()
model.compile(optimizer="adam",
loss="sparse_categorical_crossentropy",
metrics=["accuracy"])
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping
from livelossplot.tf_keras import PlotLossesCallback
chkpt = ModelCheckpoint("model_weights.h5",
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=True,
batch_size=384)
y_in = Input(shape=(512, 1))
y_err = Input(shape=(512, 1))
# h_enc = Conv1D(32, 2, activation='relu')(y_in)
# h_enc = Conv1D(32, 8, activation='relu')(h_enc)
# h_enc = CuDNNGRU(512, return_sequences=True)(y_in)
# h_enc = CuDNNGRU(256, return_sequences=True)(h_enc)
h_enc = CuDNNGRU(64, return_sequences=False)(y_in)
# h_enc = BatchNormalization()(h_enc)
h_enc = Dense(32, activation=None, name='bottleneck')(h_enc)
# h_enc = BatchNormalization()(h_enc)
h_dec = RepeatVector(512)(h_enc)
h_dec = CuDNNGRU(64, return_sequences=True)(h_dec)
# h_dec = CuDNNGRU(256, return_sequences=True)(h_dec)
h_dec = TimeDistributed(Dense(1))(h_dec)
model = Model(inputs=[y_in, y_err], outputs=h_dec)
model.compile(optimizer=Adam(clipvalue=0.5), loss=chi2(y_err))
# model.load_weights("model_weights/lstm_autoencoder_2020-01-09_18-20-21.h5")
training_time_stamp = datetime.datetime.now(
tz=pytz.timezone('Europe/London')).strftime("%Y-%m-%d_%H-%M-%S")
CB = EarlyStopping(monitor='val_loss',
min_delta=1e-5,
patience=100,
verbose=1,
mode='auto')
MC = ModelCheckpoint('model_weights/model_{}.h5'.format(training_time_stamp),
monitor='val_loss',
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x_padded, y_padded, test_size = 0.1)
"""**BUILD AND TRAIN THE MODEL**"""
# Sequential Model
model = Sequential()
# embedding layer
model.add(Embedding(english_vocab_size, 256, input_length = maxlen_english, mask_zero = True))
# encoder
model.add(LSTM(256))
# decoder
model.add(RepeatVector(maxlen_french))
model.add(LSTM(256, return_sequences= True ))
model.add(TimeDistributed(Dense(french_vocab_size, activation ='softmax')))
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
model.summary()
# change the shape of target from 2D to 3D
y_train = np.expand_dims(y_train, axis = 2)
y_train.shape
model.fit(x_train, y_train, batch_size=1024, validation_split= 0.1, epochs=20)
"""**Got 92.5 % accuracy for 20 epoch :)**"""
# save the model
model.save("weights.h5")
"""**ASSESS TRAINED MODEL PERFORMANCE**"""