def __init__(self,
vocab_len=50000,
embed_dims=128,
hidden_dims=256,
rnn_type="LSTM",
batch_size=128):
super(Encoder, self).__init__()
self.batch_size = batch_size
self.hidden_dims = hidden_dims
self.embedding = layers.Embedding(input_dim=vocab_len,
output_dim=embed_dims,
input_length=400)
if rnn_type == "LSTM":
self.recurrent_layer = layers.Bidirectional(
layers.LSTM(units=hidden_dims,
activation='tanh',
return_sequences=True,
return_state=True,
recurrent_initializer='glorot_uniform'))
if rnn_type == "GRU":
self.recurrent_layer = layers.Bidirectional(
layers.GRU(units=hidden_dims,
activation='tanh',
return_sequences=True,
return_state=True,
recurrent_initializer='glorot_uniform'))
if rnn_type == "RUM":
raise NotImplementedError
# Layer used to reduce output from bidirectional RNN layer.
# Referenced https://github.com/abisee/pointer-generator to
# examine their approach to this process, and adapted the
# approach here.
self.reduce_layer = layers.Dense(units=hidden_dims, activation='relu')
def __init__(self, config):
super(PredictorDetRNN, self).__init__()
self.rnn_type = config.rnn_type
# Layers
self.embedding = layers.Dense(config.emb_size,
activation=config.activation_func)
if self.rnn_type == "lstm":
logging.info("Using LSTM RNNs")
self.rnn = layers.LSTM(config.enc_hidden_size,
return_sequences=False,
return_state=True,
activation='tanh',
dropout=config.dropout_rate)
else:
logging.info("Using GRU RNNs")
self.rnn = layers.GRU(config.enc_hidden_size,
return_sequences=False,
return_state=True,
activation='tanh',
dropout=config.dropout_rate)
self.dropout = layers.Dropout(config.dropout_rate,
name="dropout_decode")
self.decoder = layers.Dense(config.P, activation=tf.identity)
# Loss function
self.loss_fun = losses.LogCosh()
def __init__(self, model_type: str, vocab_size: int, embed_dim: int, hidden_size: int = 128, training: bool = False):
super(MyAdvancedModel, self).__init__()
self.hidden_size = hidden_size
self.num_classes = len(ID_TO_CLASS)
self.model_type = model_type
self.embeddings = tf.Variable(tf.random.normal((vocab_size, embed_dim)))
if model_type == 'over':
self.decoder = layers.Dense(units=self.num_classes)
self.omegas1 = tf.Variable(tf.random.normal((hidden_size*2, hidden_size)))
self.omegas2 = tf.Variable(tf.random.normal((hidden_size, (int)(hidden_size/2))))
self.omegas3 = tf.Variable(tf.random.normal(((int)(hidden_size/2), 1)))
self.biDirection = layers.Bidirectional(tf.keras.layers.GRU(units=hidden_size, return_sequences=True, activation='tanh', recurrent_activation="tanh"))
self.batch_normed = layers.BatchNormalization(trainable=True)
elif model_type == 'sattn':
self.biDirection = layers.Bidirectional(tf.keras.layers.GRU(units=hidden_size, return_sequences=True, activation='tanh', recurrent_activation="tanh"))
self.decoder = layers.GRU(units=self.num_classes, return_sequences=True, activation='tanh')
else:
print ("CONV")
self.decoder = layers.Dense(units=self.num_classes)
self.biDirection = layers.Bidirectional(tf.keras.layers.GRU(units=hidden_size, return_sequences=True, activation='tanh', recurrent_activation="tanh"))
self.conv = layers.Conv1D(filters = self.hidden_size, kernel_size = 3)
self.omegas = tf.Variable(tf.random.normal((hidden_size, (int)(hidden_size/2))))
def __init__(self, input_dim: int, num_layers: int):
super(GruSequenceToVector, self).__init__(input_dim)
# TODO(students): start
self.num_layers = num_layers
self.hidden_layer = layers.GRU(
self._input_dim, activation='tanh', return_sequences=True
) # Using tanh activation for each layer , and using return_sequences to capture current representation at each step.
def regression_model(input_dim = 31,tcn = False):
if not tcn:
from tensorflow.keras import layers, models
import tensorflow as tf
model = models.Sequential()
model.add(layers.GRU(128, return_sequences=True, input_shape=(None, input_dim)))
model.add(layers.TimeDistributed(layers.Dense(13, activation='linear')))
model.summary()
model.compile(loss='mse',
optimizer=tf.keras.optimizers.Adam(lr=0.01))
else:
import tcn
import keras
from keras.layers import Dense,TimeDistributed
from keras.models import Input, Model
from tcn import TCN
i = Input(batch_shape=(None, None, input_dim))
o = TCN(return_sequences=True)(i) # The TCN layers are here.
o = TimeDistributed(Dense(13, activation='linear'))(o)
model = Model(inputs=[i], outputs=[o])
model.summary()
model.compile(loss='mse',
optimizer= keras.optimizers.Adam(lr=0.01))
return model
def build_transactions_rnn(transactions_cat_features,
embedding_projections,
product_col_name='product',
rnn_units=128,
classifier_units=32,
optimizer=None):
if not optimizer:
optimizer = keras.optimizers.Adam(lr=1e-3)
inputs = []
cat_embeds = []
for feature_name in transactions_cat_features:
inp = L.Input(shape=(None, ),
dtype='uint32',
name=f'input_{feature_name}')
inputs.append(inp)
source_size, projection = embedding_projections[feature_name]
emb = L.Embedding(source_size + 1,
projection,
trainable=True,
mask_zero=False,
name=f'embedding_{feature_name}')(inp)
cat_embeds.append(emb)
# product feature
inp = L.Input(shape=(1, ), dtype='uint32', name=f'input_product')
inputs.append(inp)
source_size, projection = embedding_projections['product']
product_emb = L.Embedding(source_size + 1,
projection,
trainable=True,
mask_zero=False,
name=f'embedding_product')(inp)
product_emb_reshape = L.Reshape((projection, ))(product_emb)
concated_cat_embeds = L.concatenate(cat_embeds)
dropout_embeds = L.SpatialDropout1D(0.05)(concated_cat_embeds)
sequences = L.Bidirectional(L.GRU(units=rnn_units,
return_sequences=True))(dropout_embeds)
pooled_avg_sequences = L.GlobalAveragePooling1D()(sequences)
pooled_max_sequences = L.GlobalMaxPooling1D()(sequences)
#add dropout=0.5
concated = L.concatenate(
[pooled_avg_sequences, pooled_max_sequences, product_emb_reshape])
dense_intermediate = L.Dense(classifier_units,
activation='relu',
kernel_regularizer=keras.regularizers.L1L2(
1e-7, 1e-5))(concated)
proba = L.Dense(1, activation='sigmoid')(dense_intermediate)
model = Model(inputs=inputs, outputs=proba)
model.compile(loss='binary_crossentropy', optimizer=optimizer)
return model
def get_gru(dim, act=None, is_bidirectional=False, merge_mode="sum"):
layer = layers.GRU(dim, activation=act, return_sequences=True,
kernel_initializer=kernel_initializer, kernel_regularizer=kernel_regularizer)
if is_bidirectional:
layer = layers.Bidirectional(layer, merge_mode=merge_mode)
return layer
def define_recurrent_encoder(timesteps, features, latent_dim):
''' Define recurrent encoder
args:
dimensions timesteps, feautres.
latent dim: hyper-parameter
return:
recurrent encoder model
'''
recurrent_encoder = tfkm.Sequential([
tfkl.Masking(mask_value=-1.,
input_shape=(timesteps, features)),
tfkl.Dropout(0.2),
tfkl.GRU(42, return_sequences = True),
tfkl.GRU(latent_dim)
])
return recurrent_encoder
def __init__(self,
K,
conv_channels,
pool_size,
projections,
highway_units,
highway_nums,
rnn_units,
name='cbhg'):
super(CBHG, self).__init__(name=name)
self.K = K
self.conv_channels = conv_channels
self.pool_size = pool_size
self.projections = projections
self.highway_units = highway_units
self.highway_nums = highway_nums
self.rnn_units = rnn_units
self.conv_banks = [ConvBlock('cab', '1D', conv_channels, k, name='conv_banks')
for k in range(1, K + 1)]
self.pool = layers.MaxPool1D(pool_size, strides=1, padding='same', name='pool')
acts = ['relu'] * (len(projections) - 1) + [None]
self.conv_projections = [ConvBlock('cab', '1D', c, 3, None, a, name='conv_projs')
for c, a in zip(projections, acts)]
self.highway_nets = [HighwayNet(highway_units, name='highway')
for _ in range(highway_nums)]
self.gru_layer = layers.Bidirectional(layers.GRU(rnn_units, return_sequences=True),
name='bigru')
self.supports_masking = True
def build_rnn_model(embedding_dim,
vocabulary_size,
recurrent_type='SimpleRNN',
n_rnn_units=64,
n_rnn_layers=1,
bidirect=True):
tf.random.set_seed(123)
model = tf.keras.Sequential()
model.add(
tkl.Embedding(input_dim=vocabulary_size,
output_dim=embedding_dim,
name='embedding_layer'))
for i in range(n_rnn_layers):
return_sequences = (i < n_rnn_layers - 1)
if recurrent_type == 'SimpleRNN':
recurrent_layer = tkl.SimpleRNN(
units=n_rnn_units,
return_sequences=return_sequences,
name='simple_rnn_layer{}'.format(i))
elif recurrent_type == 'LSTM':
recurrent_layer = tkl.LSTM(units=n_rnn_units,
return_sequences=return_sequences,
name='lstm_layer{}'.format(i))
elif recurrent_type == 'GRU':
recurrent_layer = tkl.GRU(units=n_rnn_units,
return_seq=return_sequences,
name='gru_layer{}'.format(i))
if bidirect:
recurrent_layer = tkl.Bidirectional(recurrent_layer,
name='bidirect_' +
recurrent_layer.name)
model.add(recurrent_layer)
model.add(tkl.Dense(64, activation='relu'))
model.add(tkl.Dense(1, activation='sigmoid'))
return model
def __init__(self,vocab_size,emb_size,hid_size,dropout=0.1,recurrent_dropout=0.2):
super(WordRNN,self).__init__()
self.embed = L.Embedding(vocab_size,emb_size)
self.gru = L.GRU(hid_size,dropout=dropout,recurrent_dropout=recurrent_dropout)
self.bi_gru = L.Bidirectional(self.gru)
self.fc = L.Dense(1)
def __init__(self,
vocab_size,
max_message_length,
embed_dim,
hidden_dim,
vision_module,
flexible_message_length=False,
activation='linear',
n_distractors=1,
image_dim=32):
super(Receiver, self).__init__(vocab_size,
max_message_length,
embed_dim,
hidden_dim,
vision_module,
flexible_message_length,
VtoH_activation=activation)
self.n_distractors = n_distractors
self.image_dim = image_dim
# if message length fixed, 0 counts as a standard symbol and no masking is applied
self.language_module = Sequential([
layers.Embedding(vocab_size, embed_dim, name='embedding'),
layers.GRU(hidden_dim, name='GRU_layer')
]) # GRU linear by default
self.__build()
def build_model_c_rnn(model_input):
x_input = layers.Input(model_input)
# -------Convolutional blocks------- #
# First convolutional block
x = layers.Conv2D(68, (3, 3),
strides=(1, 1),
padding='same',
name='conv_1')(x_input)
x = layers.BatchNormalization()(x)
x = layers.Activation('relu')(x)
x = layers.MaxPooling2D((2, 2), strides=(1, 1))(x)
x = layers.Dropout(0.1)(x)
# Second convolutional block
x = layers.Conv2D(137, (3, 3),
strides=(1, 1),
padding='same',
name='conv_2')(x)
x = layers.BatchNormalization()(x)
x = layers.Activation('relu')(x)
x = layers.MaxPooling2D((3, 3), strides=(1, 1))(x)
x = layers.Dropout(0.1)(x)
# Third convolutional block
x = layers.Conv2D(137, (3, 3),
strides=(1, 1),
padding='same',
name='conv_3')(x)
x = layers.BatchNormalization()(x)
x = layers.Activation('relu')(x)
x = layers.MaxPooling2D((4, 4), strides=(1, 1))(x)
x = layers.Dropout(0.1)(x)
# Fourth convolutional block
x = layers.Conv2D(137, (3, 3),
strides=(1, 1),
padding='same',
name='conv_4')(x)
x = layers.BatchNormalization()(x)
x = layers.Activation('relu')(x)
x = layers.MaxPooling2D((4, 4), strides=(1, 1))(x)
x = layers.Dropout(0.1)(x)
# -------Recurrent Block------- #
# GRU layer
lstm = layers.GRU(68)(x[:, :, :, 0])
# Softmax Output
output = layers.Dense(8, activation='softmax', name='preds')(lstm)
model_output = output
model = tf.keras.models.Model(x_input, model_output)
opt = Adam(lr=0.001)
# opt = tf.keras.optimizers.RMSprop(lr=0.0005) # Optimizer
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
print(model.summary())
return model
def Build_RNN_layer(Parametre_layer):
if Parametre_layer["type_cell"] == "LSTM":
cell = L.LSTM(units=Parametre_layer["units"],
dropout=Parametre_layer["dropout"],
recurrent_dropout=Parametre_layer["recurrent_dropout"],
return_sequences=True,
return_state=True,
stateful=Parametre_layer["stateful"],
unroll=Parametre_layer["unroll"])
elif Parametre_layer["type_cell"] == "CuDNNGRU":
cell = CuDNNGRU(units=Parametre_layer["units"],
dropout=Parametre_layer["dropout"],
recurrent_dropout=Parametre_layer["recurrent_dropout"],
return_sequences=True,
return_state=True,
stateful=Parametre_layer["stateful"],
unroll=Parametre_layer["unroll"])
elif Parametre_layer["type_cell"] == "GRU":
cell = L.GRU(units=Parametre_layer["units"],
return_sequences=True,
return_state=True,
stateful=Parametre_layer["stateful"])
else: #by default CuDNNLSTM
cell = CuDNNLSTM(units=Parametre_layer["units"],
return_sequences=True,
return_state=True,
stateful=Parametre_layer["stateful"])
return cell
def __init__(self):
super(LSTM, self).__init__()
# Define a Masking Layer with -1 as mask.
self.norm = x = layers.BatchNormalization()
self.CNN = tf.keras.applications.EfficientNetB0(
include_top=False, weights='imagenet', input_tensor=None,
input_shape=None, pooling=None)
self.CNN.trainable = train_CNN
self.dense = layers.Dense(LSTM_input_units)
# Define a LSTM layer to be applied over the Masking layer.
# Dynamic computation will automatically be performed to ignore -1 values.
self.lstm1 = layers.Bidirectional(layers.GRU(units=LSTM1_output_units, return_sequences=True,dropout=dropout))
self.lstm2 = layers.Bidirectional(layers.GRU(units=LSTM2_output_units, return_sequences=True,dropout=dropout))
self.lstm3 = layers.Bidirectional(layers.GRU(units=LSTM3_output_units, return_sequences=True,dropout=dropout))
# Output fully connected layer (5 classes).
self.out = layers.Dense(num_classes+1)
def build_classifier_model(embedding_matrix):
input = layers.Input(shape=(MAX_SEQ_LEN, ), dtype=np.int32)
embedding_layer = layers.Embedding(VOCAB_SIZE + 1,
EMBEDDING_DIMS,
weights=[embedding_matrix],
trainable=False)
embedded_input = embedding_layer(input)
gru_output = layers.Bidirectional(
layers.GRU(HIDDEN_UNITS, return_sequences=True))(embedded_input)
gru_output = layers.Bidirectional(layers.GRU(HIDDEN_UNITS))(gru_output)
prob = layers.Dense(NUM_OUTPUTS, activation='sigmoid')(gru_output)
bigru_model = Model(input, prob)
print('generated bigru model...')
return bigru_model
def main():
num_samples = 1000
num_features = 7
X = np.random.rand(num_samples, 2, 2, 3, 5)
y = np.random.rand(num_samples, 2, 2)
layer = GRU(512)
batch_size = 30
timesteps = 20
features = 15
out = layer(np.random.rand([batch_size, timesteps, features]))
model = Sequential([
keras.Input(shape=(None, num_features)),
layers.GRU(512),
layers.BatchNormalization()
])
model = Sequential()
model.add(Flatten())
model.add(Dense(12, input_dim=num_features, activation='relu'))
model.add(Dense(8, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# fit the keras model on the dataset
model.fit(X, y, epochs=15, batch_size=10)
# evaluate the keras model
_, accuracy = model.evaluate(X, y)
print('Accuracy: %.2f' % (accuracy))
def __init__(self, task):
self.task = task
options = self.build_options()
self.options = options
self.outputs = {}
self.my_losses = {}
with tf.compat.v1.variable_scope('vqa_quality'):
self.q_embed = layers.Embedding(
options['vocab_size'],
options['embed_size'],
input_length=options['question_len'])
self.q_gru = layers.GRU(options['q_encoded_size'],
dropout=options['drop_prob'])
self.topdown_tanh = layers.Conv2D(options['topdown_gated_size'],
1,
activation='tanh')
self.topdown_sig = layers.Conv2D(options['topdown_gated_size'],
1,
activation='sigmoid')
self.topdown_multiply = layers.Multiply()
self.topdown_conv = layers.Conv2D(1, 1)
self.q_gated_tanh = GatedTanh(options['q_encoded_size'],
options['q_encoded_size'])
self.img_gated_tanh = GatedTanh(options['img_feat_dim'],
options['q_encoded_size'])
self.VQ_joint = layers.Multiply()
self.ans_gated_tanh = GatedTanh(options['q_encoded_size'],
options['ans_fc_dim'])
self.ans_dense = layers.Dense(
1, input_shape=(options['ans_fc_dim'], ), activation='sigmoid')
def main():
mirrored_strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy()
with mirrored_strategy.scope():
seq_data = keras.Input(shape=(None, 1), name="seq_data")
seq_lengths = keras.Input(shape=(), name="seq_lengths", dtype=tf.int32)
mask = keras.layers.Lambda(lambda x: tf.sequence_mask(x))(seq_lengths)
conv = layers.Conv1D(32, 3, strides=1, padding='same', activation='relu')(seq_data)
rnn = layers.Bidirectional(layers.GRU(32, return_sequences=True))(conv,mask=mask)
#rnn = layers.Bidirectional(layers.GRU(32, return_sequences=True))(conv)
dense = layers.Dense(5, name="signal_mask")(rnn)
model = keras.Model(inputs=[seq_data, seq_lengths], outputs=[dense])
model.compile(optimizer=tf.keras.optimizers.Adam(
learning_rate=0.001),
loss=tf.keras.backend.sparse_categorical_crossentropy
)
batch_size = 1
time_step = 1000
signal_length = 1000
sequences = np.random.rand(batch_size, time_step, 1)
seq_lengths = np.array([signal_length])
signal_mask = np.random.randint(5, size=(1, time_step))
dataset = tf.data.Dataset.from_tensor_slices(
(sequences, seq_lengths, signal_mask)).repeat(1000).batch(1)\
.map(lambda a,b,c: ({"seq_data":a,"seq_lengths":b}, {"signal_mask":c}))
model.fit(dataset, epochs=1)
def create_stitchGAN_model(self,shape_in,shape_out,mode='stitch',
model_path='',stateful=False,batch_size=1,
optimizer=tf.keras.optimizers.Adam()):
if stateful:
print('\n Creating Stateful LSTM in inference Mode. \n')
inputs = tf.keras.Input(batch_shape=[batch_size]+
list(shape_in[0][1:]))
else:
inputs = tf.keras.Input(shape=shape_in[0][1:])
lstm_out=layers.GRU(64,return_sequences=True,
stateful=stateful)(inputs)
#out= layers.Conv1D(8,1)(lstm_out)
#out=layers.BatchNormalization()(lstm_out)
#out=tf.keras.activations.relu(out)
out=layers.Conv1D(1,1)(lstm_out)
#inputs = tf.keras.Input(shape=shape_in[0][1:])
#gru_out=layers.GRU(64,return_sequences=True)(inputs)
#gru_out=layers.Dropout(0.3,noise_shape=(None, 1, 64))(gru_out)
#out= layers.Conv1D(1,1)(gru_out)
model= tf.keras.Model(inputs=inputs,outputs=out)
loss= tf.keras.losses.MeanSquaredError()
metrics= ['mse']
model.compile(loss=loss,
optimizer=optimizer,
metrics=metrics)
model.summary()
print(model.optimizer)
#tf.keras.utils.plot_model(model, "simple_stitcher.png")
return model
def __init__(self,
vocab_size: int,
embedding_dim: int,
filters: List[int],
out_channels: int,
drop_prob: float,
nn_hidden_dim: int,
gru_hidden_dim: 128,
num_classes: int = 2):
super(CNNandAttentiveBiGRUmodel, self).__init__()
self._baseCNN = baseCNNmodel(embedding_dim, filters, out_channels)
self.embedding_dim = embedding_dim
self._embeddings = tf.Variable(tf.random.normal(
(vocab_size, embedding_dim)),
trainable=True)
self.omegas = tf.Variable(tf.random.normal((gru_hidden_dim * 2, 1)))
gru = layers.GRU(units=gru_hidden_dim, return_sequences=True)
self.bi_gru_layer = layers.Bidirectional(layer=gru,
merge_mode='concat')
self.mlp1_layer = layers.Dense(units=nn_hidden_dim, activation='tanh')
self.mlp2_layer = layers.Dense(units=50, activation='tanh')
self._classification_layer = layers.Dense(units=num_classes)
self.dropout = layers.Dropout(drop_prob)
def __init__(self, n_actions, input_shape):
self.n_actions = n_actions
self.input_shape = input_shape
self._layers = []
if len(input_shape) > 1:
self._layers.append(
layers.Conv2D(filters=16,
kernel_size=8,
strides=4,
activation=tf.nn.tanh))
self._layers.append(
layers.Conv2D(filters=32,
kernel_size=4,
strides=2,
activation=tf.nn.tanh))
self._layers.append(layers.Reshape((1, -1)))
self._layers.append(layers.GRU(256, return_state=True))
else:
self._layers.append(layers.Dense(units=32, activation=tf.nn.tanh))
self._layers.append(layers.Dense(units=32, activation=tf.nn.tanh))
# self._layers.append(layers.Reshape((1, -1)))
# self._layers.append(layers.GRU(256, return_state = True))
# self._layers.append(layers.GRU(512, return_state = True, return_sequences = True))
self._actor = layers.Dense(self.n_actions, activation=tf.nn.softmax)
self._critic = layers.Dense(1, activation=None)
def recurrent_model(units=32, celltype='GRU', Tx=None, Trej=0, print_summary=False, learning_rate=.001, name='recurrent_model', loss='mse', metrics='mse', initializer = keras.initializers.GlorotNormal()):
'''
Recurrent neural network model.
'''
# specify recurrent cell type
if celltype=='GRU':
Rcell = layers.GRU(units=units, return_sequences=True, kernel_initializer=initializer, name='Xrec')
elif celltype=='LSTM':
Rcell = layers.LSTM(units=units, return_sequences=True, kernel_initializer=initializer, name='Xrec')
elif celltype=='RNN':
Rcell = layers.SimpleRNN(units=units, return_sequences=True, kernel_initializer=initializer, name='Xrec')
# Model architecture
X_input = layers.Input(shape=(Tx,1), name='X0')
X = Rcell(X_input)
X = layers.Dense(1, activation='linear', kernel_initializer=initializer, name='Xdense')(X)
Y = layers.Add(name='Y')([X,X_input])
# Create model
model = keras.Model(inputs=[X_input], outputs=Y, name=name)
if print_summary:
model.summary()
# Compile model
opt = tf.keras.optimizers.Adam(learning_rate,clipvalue=10)
model.compile(optimizer=opt, loss=loss, metrics=metrics)
return model
def __init__(self,
filters,
kernel_size,
num_residual_blocks,
l2=None,
**kwargs):
super(StateEncoder, self).__init__(**kwargs)
self.residual_blocks = [
ResidualBlock(filters=filters,
kernel_size=3,
l2=l2,
name="residual_block_{:d}".format(i))
for i in range(num_residual_blocks)
]
self.recurrency = layers.Bidirectional(
layers.GRU(
units=filters,
stateful=False,
return_sequences=True,
return_state=True,
implementation=2, # GPU compatible
kernel_regularizer=regularizers.l2(l2)),
name="recurrency")
self.concatenate = layers.Concatenate()
self.attention = BahdanauAttention(filters)
def build_model(self):
""" builds and returns encoder decoder tensorflow model"""
inputs = tf.keras.Input(shape=(None, ))
x = layers.Embedding(self.num_encoder_tokens, self.latent_dim)(inputs)
x = layers.Dropout(0.4)(x)
if self.args.use_GRU:
x = layers.GRU(64, return_sequences=False, return_state=False)(x)
else:
x = layers.LSTM(64, return_sequences=False, return_state=False)(x)
x = layers.BatchNormalization()(x)
x = layers.Dropout(0.4)(x)
if self.args.use_document_context:
verfahrensart_input = tf.keras.Input(shape=(1, ))
dokumentart_input = tf.keras.Input(shape=(1, ))
concat_list = [x, verfahrensart_input, dokumentart_input]
x = tf.keras.layers.Concatenate()(concat_list)
x = tf.keras.layers.Dense(255, activation="relu")(x)
x = layers.Dropout(0.4)(x)
outputs = layers.Dense(self.num_decoder_tokens,
activation="softmax")(x)
inputs_list = [inputs]
if self.args.use_document_context:
inputs_list.append(verfahrensart_input)
inputs_list.append(dokumentart_input)
model = Model(inputs_list, outputs)
tf.keras.utils.plot_model(model,
"RNN_model.png",
show_shapes=True,
show_layer_names=False)
return model
def __init__(
self,
hidden_units,
dropout_rate=0.2,
aggregation_type="mean",
combination_type="concat",
normalize=False,
*args,
**kwargs,
):
super(GraphConvLayer, self).__init__(*args, **kwargs)
self.aggregation_type = aggregation_type
self.combination_type = combination_type
self.normalize = normalize
self.ffn_prepare = create_ffn(hidden_units, dropout_rate)
if self.combination_type == "gated":
self.update_fn = layers.GRU(
units=hidden_units,
activation="tanh",
recurrent_activation="sigmoid",
dropout=dropout_rate,
return_state=True,
recurrent_dropout=dropout_rate,
)
else:
self.update_fn = create_ffn(hidden_units, dropout_rate)
def build_model(wav_size=16384, chunk_size=256, chunk_hop=128, emb_size=32):
N_WIN = (wav_size - chunk_size) // chunk_hop + 1
hann_win = tf.signal.hann_window(chunk_size)
wav_input = tf.keras.Input(shape=(wav_size, ))
encoder = tf.signal.frame(wav_input, chunk_size, chunk_hop, axis=1)
encoder = tf.multiply(encoder, hann_win)
encoder = tf.expand_dims(encoder, axis=-1)
encoder = tfkl.TimeDistributed(tfkl.Conv1D(64, 8, padding='SAME'))(encoder)
encoder = tfkl.TimeDistributed(
tfkl.Conv1D(64, 8, padding='SAME', strides=8))(encoder)
encoder = tfkl.TimeDistributed(
tfkl.Conv1D(64, 8, padding='SAME', strides=4))(encoder)
encoder = tfkl.TimeDistributed(
tfkl.Conv1D(64, 8, padding='SAME', strides=2))(encoder)
encoder = tfkl.TimeDistributed(tfkl.Flatten())(encoder)
encoder = tfkl.GRU(emb_size, return_sequences=True)(encoder) #Global LSTM
encoder_model = tf.keras.Model(wav_input, encoder)
decoder_input = tf.keras.Input(shape=(N_WIN, emb_size), name='encoded_img')
decoder = tfkl.GRU(256, return_sequences=True)(decoder_input)
decoder = tfkl.TimeDistributed(tfkl.Reshape(target_shape=(4, 64)))(decoder)
decoder = tfkl.TimeDistributed(tfkl.Conv1D(64, 8, padding='SAME'))(decoder)
decoder = tfkl.TimeDistributed(tfkl.UpSampling1D(2))(decoder)
decoder = tfkl.TimeDistributed(tfkl.Conv1D(64, 8, padding='SAME'))(decoder)
decoder = tfkl.TimeDistributed(tfkl.UpSampling1D(4))(decoder)
decoder = tfkl.TimeDistributed(tfkl.Conv1D(64, 8, padding='SAME'))(decoder)
decoder = tfkl.TimeDistributed(tfkl.UpSampling1D(8))(decoder)
decoder = tfkl.TimeDistributed(
tfkl.Conv1D(1, 8, padding='SAME', activation='tanh',
dtype=tf.float32))(decoder)
decoder = tfkl.Reshape(target_shape=(N_WIN, chunk_size))(decoder)
decoder = tf.multiply(decoder, hann_win)
decoder = tf.signal.overlap_and_add(decoder, chunk_hop)
decoder_model = tf.keras.Model(decoder_input, decoder)
ae_input = tf.keras.Input(shape=(wav_size, ))
encoder_out = encoder_model(ae_input)
decoder_out = decoder_model(encoder_out)
ae_model = tf.keras.Model(inputs=ae_input, outputs=decoder_out)
return ae_model, encoder_model, decoder_model
def main() -> None:
data, res = gen_data_from_sequence()
dataset_size = len(data)
train_size = (dataset_size // 10) * 7
val_size = (dataset_size - train_size) // 2
train_data, train_res = data[:train_size], res[:train_size]
val_data, val_res = data[train_size:train_size +
val_size], res[train_size:train_size + val_size]
test_data, test_res = data[train_size + val_size:], res[train_size +
val_size:]
model = Sequential()
model.add(
layers.GRU(128,
recurrent_activation='sigmoid',
input_shape=(None, 1),
return_sequences=True))
model.add(
layers.LSTM(64,
activation='relu',
input_shape=(None, 1),
return_sequences=True,
dropout=0.2))
model.add(layers.GRU(32, input_shape=(None, 1), recurrent_dropout=0.2))
model.add(layers.Dense(1))
model.compile(optimizer='nadam', loss='mse')
history = model.fit(train_data,
train_res,
epochs=50,
validation_data=(val_data, val_res))
loss = history.history['loss']
val_loss = history.history['val_loss']
print(np.mean(val_loss))
plt.plot(range(len(loss)), loss)
plt.plot(range(len(val_loss)), val_loss)
plt.show()
predicted_res = model.predict(test_data)
pred_length = range(len(predicted_res))
plt.plot(pred_length, predicted_res, "r")
plt.plot(pred_length, test_res, "b")
plt.legend()
plt.show()
def build_model(input_dim, output_dim, rnn_layers=5, rnn_units=128):
"""Model similar to DeepSpeech2."""
# Model's input
input_spectrogram = layers.Input((None, input_dim), name="input")
# Expand the dimension to use 2D CNN.
x = layers.Reshape((-1, input_dim, 1), name="expand_dim")(input_spectrogram)
# Convolution layer 1
x = layers.Conv2D(
filters=32,
kernel_size=[11, 41],
strides=[2, 2],
padding="same",
use_bias=False,
name="conv_1",
)(x)
x = layers.BatchNormalization(name="conv_1_bn")(x)
x = layers.ReLU(name="conv_1_relu")(x)
# Convolution layer 2
x = layers.Conv2D(
filters=32,
kernel_size=[11, 21],
strides=[1, 2],
padding="same",
use_bias=False,
name="conv_2",
)(x)
x = layers.BatchNormalization(name="conv_2_bn")(x)
x = layers.ReLU(name="conv_2_relu")(x)
# Reshape the resulted volume to feed the RNNs layers
x = layers.Reshape((-1, x.shape[-2] * x.shape[-1]))(x)
# RNN layers
for i in range(1, rnn_layers + 1):
recurrent = layers.GRU(
units=rnn_units,
activation="tanh",
recurrent_activation="sigmoid",
use_bias=True,
return_sequences=True,
reset_after=True,
name=f"gru_{i}",
)
x = layers.Bidirectional(
recurrent, name=f"bidirectional_{i}", merge_mode="concat"
)(x)
if i < rnn_layers:
x = layers.Dropout(rate=0.5)(x)
# Dense layer
x = layers.Dense(units=rnn_units * 2, name="dense_1")(x)
x = layers.ReLU(name="dense_1_relu")(x)
x = layers.Dropout(rate=0.5)(x)
# Classification layer
output = layers.Dense(units=output_dim + 1, activation="softmax")(x)
# Model
model = keras.Model(input_spectrogram, output, name="DeepSpeech_2")
# Optimizer
opt = keras.optimizers.Adam(learning_rate=1e-4)
# Compile the model and return
model.compile(optimizer=opt, loss=CTCLoss)
return model
def get_line_model():
line_input = layers.Input(shape=(LINE_LEN, INPUT_DIM))
masking = layers.Masking(0)(line_input)
bi_seq = layers.Bidirectional(layers.GRU(128),
merge_mode='sum')(masking)
bi_seq = layers.BatchNormalization()(bi_seq)
bi_seq = layers.Activation('relu')(bi_seq)
return line_input, bi_seq
