def build_nlp_model(q_input,rnn_dim,bidirection,dropout,lang_att):
if not lang_att:
q_input = Masking()(q_input)
if bidirection:
bi_rnn=gru_rnn_module_a(q_input,rnn_dim,dropout,lang_att)
else:
bi_rnn = gru_rnn_module_s(q_input, rnn_dim, dropout,lang_att)
# bi_rnn = q_input
if lang_att:
bi_rnn_weights = Dense(K.int_shape(bi_rnn)[-1], activation='tanh')(bi_rnn)
bi_rnn_weights = Dropout(0.1)(bi_rnn_weights)
bi_rnn_weights = Lambda(K.softmax, arguments={'axis': 1})(bi_rnn_weights)
bi_rnn = Multiply()([bi_rnn, bi_rnn_weights])
bi_rnn = Lambda(K.sum, arguments={'axis': 1})(bi_rnn)
return bi_rnn
def build(self):
word2vec_embedding_dim = self.config["params"]["word2vec"][
"embedding_dim"]
embedding_dim = self.config["params"]["sentence_representation"][
"embedding_dim"]
max_sentence_length = self.config["params"]["sentence_representation"][
"max_sentence_length"]
pad_char_index = self.config["params"]["sentence_representation"][
"pad_char_index"]
learning_rate = self.config["hyper_params"]["sentence_representation"][
"lr"]
word_encoder_model, word_decoder_model = self.get_embedding_model()
for layer in word_encoder_model.layers:
layer.trainable = False
for layer in word_decoder_model.layers:
layer.trainable = False
inputs = Input(shape=(max_sentence_length, 1))
mask = Masking(pad_char_index)(inputs)
word_embedding = TimeDistributed(word_encoder_model)(mask)
encoder = LSTM(32, return_sequences=True)(word_embedding)
encoder = LSTM(64, return_sequences=True)(encoder)
encoder = LSTM(128, return_sequences=True)(encoder)
encoded = LSTM(embedding_dim, return_sequences=False)(encoder)
repeated = RepeatVector(max_sentence_length)(encoded)
decoder = LSTM(128, return_sequences=True)(repeated)
decoder = LSTM(256, return_sequences=True)(decoder)
decoder = TimeDistributed(Dense(word2vec_embedding_dim))(decoder)
outputs = TimeDistributed(word_decoder_model)(decoder)
model = Model(inputs=inputs, outputs=outputs)
self.model = model
self.model.summary()
self.model.compile(
loss="sparse_categorical_crossentropy",
optimizer=Adam(learning_rate),
metrics=["accuracy"],
)
def make_rnn(vocab_size, embedding_size, max_len):
input_layer = Input(shape=(None, ), name="input")
mask = Masking(mask_value=0, name="mask")(input_layer)
# embedding
embedding_layer = Embedding(vocab_size, embedding_size,
name="embedding")(mask)
# rnn
rnn_fn = LSTM # GRU/LSTM
rnn_layers = []
lstms = [64]
for i, count in enumerate(lstms):
prev = embedding_layer if i == 0 else rnn_layers[-1]
last = i == len(lstms) - 1
rnn_layers.append(
rnn_fn(
count,
activation="relu",
return_sequences=not last,
name=f"recurrent{i+1}",
)(prev))
# dense
dense_layers = []
denses = [(128, 0.25), (64, 0.25)]
for i, param in enumerate(denses):
prev = rnn_layers[-1] if i == 0 else dense_layers[-1]
dense_layers.append(Dense(param[0], name=f"dense{i+1}")(prev))
if param[1] > 0:
dense_layers.append(
Dropout(param[1], name=f"dropout{i+1}")(dense_layers[-1]))
# out
output_layer = Dense(1, activation="sigmoid",
name="output")(dense_layers[-1])
model = Model(inputs=[input_layer], outputs=[output_layer])
model.compile(
loss="binary_crossentropy",
optimizer="nadam",
metrics=["binary_accuracy"],
)
return model
def make_model(self):
with k.name_scope("SVM_features"):
svm_features = Input(shape = (self.svm_dims,), name = "svm_features")
svm_input = Dense(128, activation = "tanh", name = "svm_dense")(svm_features)
# svm_input = LeakyReLU()(svm_input)
with k.name_scope("LSTM_features"):
lstm_features = Input(shape = (None, self.input_shape), name = "lstm_features")
lstm_mask = Masking(mask_value = Config.MASKING_VALUE, input_shape = (self.time_steps, self.input_shape))(lstm_features)
lstm_output, state_h, state_c = LSTM(Config.LSTM_UNITS, return_sequences = True, return_state = True, name = "lstm_sequence")(lstm_mask)
# lstm_output_last = LSTM(Config.LSTM_UNITS, return_sequences = False, name = "lstm_last_output")(lstm_mask)
with k.name_scope("AttentionLayer_1"):
__, lstm_output_ex_last = Lambda(lambda t: [t, t[:, :-1, :]], name = "lstm_T1_Tn-1")(lstm_output)
lstm_output_last = state_h
attention_weights1 = dot([lstm_output_last, lstm_output_ex_last], name = "attention_weights1", axes = -1) # [B, 1, M]
attention_weights2 = Activation("softmax", name = "attention_weights2")(attention_weights1)
lstm_attention = dot([attention_weights2, lstm_output_ex_last], name = "lstm_attention", axes = 1)
# final_attention = concatenate([lstm_attention, lstm_output_last])
print(lstm_attention)
"""
with k.name_scope("AttentionLayer_2"):
# Attention layer 2 - attention params
input_attention = Input(shape = (Config.ATTENTION_UNITS, ), name = "attention_params")
u = Dense(Config.ATTENTION_UNITS, activation = "softmax", name = "attention_u")(input_attention)
alpha = dot([u, lstm_output], axes = -1)
alpha = Activation("softmax", name = "attention_weights")(alpha)
# weighted pool
lstm_attention = dot([alpha, lstm_output], name = "attention_output", axes = 1)
"""
with k.name_scope("Concatenate"):
x = concatenate([lstm_attention, svm_input])
x_dense = Dense(128, activation = "tanh")(x)
# x_dense = LeakyReLU()(x_dense)
dense_2 = Dense(128, activation = "tanh")(x_dense)
batchnorm2 = BatchNormalization()(dense_2)
dropout = Dropout(rate = 0.3, name = "dropout")(batchnorm2)
pred = Dense(self.num_classes, activation = "softmax", name = "output")(dropout)
self.model = Model(inputs = [svm_features, lstm_features], outputs = [pred])
return self.model
def attention_model(lr=1e-5):
predict_info = Input(shape=(None, 13), name='predict_info')
sequence_data = Input(shape=(None, 14), name='sequence_data')
item_info = Input(shape=(3,), name='item_info')
x = Masking(mask_value=0, input_shape=(None, 19))(sequence_data)
x = GRU(128, name='GRU_layer1', return_sequences=True)(x)
# past sequence understanding
attention_x = attention_mechanism(TRAIN_TIMESEQUENCE, x)
gru_out = Permute((2, 1))(x)
attention_mul = K.batch_dot(gru_out, attention_x)
attention_mul = Permute((2, 1))(attention_mul)
x = GRU(128, name='GRU_layer2', return_sequences=True)(attention_mul)
# item feature added
v = RepeatVector(TRAIN_TIMESEQUENCE)(item_info) # past sequence
x = Concatenate(axis=-1)([v, x])
x = TimeDistributed(Dense(32, activation='relu'))(x)
x = TimeDistributed(Dense(128, activation='relu'))(x)
# get extraction of item feature
x = Flatten()(x)
v = RepeatVector(PREDICT_TIMESEQUENCE)(x) # predict sequence
x = Concatenate(axis=-1)([predict_info, v])
x = GRU(128, name='GRU_predict_layer1', return_sequences=True)(x)
# past sequence understanding
attention_x = attention_mechanism(PREDICT_TIMESEQUENCE, x)
gru_out = Permute((2, 1))(x)
attention_mul = K.batch_dot(gru_out, attention_x)
attention_mul = Permute((2, 1))(attention_mul)
x = GRU(128, name='GRU_predict_layer2', return_sequences=True)(attention_mul)
x = TimeDistributed(Dense(32, activation='relu'))(x)
x = TimeDistributed(Dense(128, activation='relu'))(x)
outputs = TimeDistributed(Dense(1, activation='sigmoid'))(x)
print(outputs.shape)
optimizer = Adam(lr=lr, name='adam')
model = Model([sequence_data, item_info, predict_info], outputs, name='gru_network')
model.compile(optimizer=optimizer, loss='mse')
return model
def version_7(x_shape):
model = Sequential()
model.add(Masking(mask_value=0.,input_shape=(x_shape[1],x_shape[2])))
model.add(Bidirectional(LSTM(128, return_sequences=True)))
model.add(Bidirectional(LSTM(64, return_sequences=True)))
model.add(TimeDistributed(Dense(32, activation=None)))
model.add(BatchNormalization(axis=-1, momentum=0.99))
model.add(Activation('relu'))
model.add(TimeDistributed(Dense(1, activation=None)))
model.add(BatchNormalization(axis=-1, momentum=0.99))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[rec_acc] )
return model
def main(selected_emotion):
x_train, y_train, x_dev, y_dev, x_test, y_test = load_dataset(selected_emotion)
x_train = np.vstack([x_train, x_dev]) # add dev data to train data
y_train = np.append(y_train, y_dev, axis=0) # add dev labels to train labels
new_x_train, new_y_train = subsampling(x_train, y_train)
verbose, epochs, batch_size = 0, 15, 64
n_timesteps, n_features, n_outputs = new_x_train.shape[1], new_x_train.shape[2], 1
model = Sequential()
model.add(Masking(mask_value=-1, input_shape=(n_timesteps, n_features)))
model.add(LSTM(100, dropout=0.5))
model.add(Dense(100, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(n_outputs, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
# fit network
model.fit(new_x_train, new_y_train, epochs=epochs, batch_size=batch_size, verbose=verbose)
# Evaluate
y_pred = model.predict(x_test)
y_pred_class = np.where(y_pred>0.5, 1, 0)
out_dir = os.path.join('results', 'LSTM', 'delaunay', selected_emotion)
os.makedirs(out_dir, exist_ok=True)
data = {'cohen_kappa': cohen_kappa_score(y_test, y_pred_class, weights='linear')}
with open(os.path.join(out_dir, 'cohen_kappa.json'), 'w') as f:
json.dump(data, f)
clf_report = classification_report(
y_test, y_pred_class, labels=[0, 1], output_dict=True)
pd.DataFrame(clf_report).T.to_csv(os.path.join(out_dir, 'classification_report.csv'))
sns.heatmap(pd.DataFrame(clf_report).iloc[:-1, :].T, annot=True)
plt.savefig(os.path.join(out_dir, 'classification_report.png'))
plt.close()
confusion_mtx = confusion_matrix(y_test, y_pred_class, normalize='true')
sns.heatmap(confusion_mtx, annot=True, fmt='.2g', cmap='Blues')
plt.xlabel('Predicted label')
plt.ylabel('True label')
plt.savefig(os.path.join(out_dir, 'confusion_matrix.png'))
def create_discrete_encoder(enc_in_dim,
layer_size=2048,
latent_dim=1024,
**kwargs):
# Input #
inputs = Input(shape=(None, enc_in_dim),
dtype=tf.float32,
name='encoder_in')
# Layers #
x = Masking(mask_value=0.)(inputs)
x = Bidirectional(LSTM(layer_size, return_sequences=True),
merge_mode='concat')(x)
x = Bidirectional(LSTM(layer_size, return_sequences=False),
merge_mode='concat')(x)
logits = Dense(latent_dim, name='to_vocab')(x)
return Model([inputs], logits)
def __init__(self, lstm_units=256, masking_enabled=True):
super(se_comp_BiLSTM, self).__init__()
self.masking_enabled = masking_enabled
if masking_enabled:
self.masking = Masking(mask_value=0.0)
self.lstm_0 = LSTM(
units=lstm_units,
activation='tanh',
recurrent_activation='sigmoid', # check this
return_sequences=True,
stateful=True,
# return_state=True,
name="lstm_0",
# dropout=0.2,
# recurrent_dropout=0.2,
)
self.bilstm_0 = tf.keras.layers.Bidirectional(self.lstm_0,
merge_mode='sum')
def get_audio_lstm(self):
self.embedding_dim = self.train_x.shape[2]
print("Creating Model...")
inputs = Input(shape=(self.sequence_length, self.embedding_dim), dtype='float32')
masked = Masking(mask_value =0)(inputs)
lstm = Bidirectional(LSTM(200, activation='tanh', return_sequences = True, dropout=0.4), name='lstm_a')(masked)
self.audio_lstm_layer = lstm
lstm = Bidirectional(LSTM(200, activation='tanh', return_sequences = True, dropout=0.4), name="utter_a")(lstm)
output = TimeDistributed(Dense(self.classes,activation='softmax',kernel_initializer='uniform'))(lstm)
model = Model(inputs, output)
self.audio_lstm = model
return lstm, inputs
def create_encoder(obs_dim, act_dim, layer_size=2048, latent_dim=256):
# Input #
obs = Input(shape=(None, obs_dim), dtype=tf.float32, name='obs')
acts = Input(shape=(None, act_dim), dtype=tf.float32, name='acts')
# Layers #
x = Concatenate(axis=-1)([obs, acts])
x = Masking(mask_value=0.)(x)
x = Bidirectional(LSTM(layer_size, return_sequences=True),
merge_mode='concat')(x)
x = Bidirectional(LSTM(layer_size, return_sequences=False),
merge_mode='concat')(x)
# Latent Variable #
x = Dense(tfpl.MultivariateNormalTriL.params_size(latent_dim),
activation=None)(x),
z = tfpl.MultivariateNormalTriL(latent_dim, name='latent')(x)
return Model([obs, acts], z)
def create_discrete_encoder(obs_dim,
act_dim,
layer_size=2048,
latent_dim=1024,
**kwargs):
# Input #
obs = Input(shape=(None, obs_dim), dtype=tf.float32, name='obs')
acts = Input(shape=(None, act_dim), dtype=tf.float32, name='acts')
# Layers #
x = Concatenate(axis=-1)([obs, acts])
x = Masking(mask_value=0.)(x)
x = Bidirectional(LSTM(layer_size, return_sequences=True),
merge_mode='concat')(x)
x = Bidirectional(LSTM(layer_size, return_sequences=False),
merge_mode='concat')(x)
logits = Dense(latent_dim, name='to_vocab')(x)
return Model([obs, acts], logits)
def create_model(n_labels: int, batch_size: int) -> object:
nn_size = 300
embed_dim = 300
#print(bpemb_model.vectors.shape)
wm = np.zeros((1000, 300))
for i in range(wm.shape[0]):
# set pad to 0
if i == 999:
wm[0] = bpemb_model.vectors[-1]
else:
wm[i + 1] = bpemb_model.vectors[i]
model = Sequential()
model.add(
Embedding(1000,
embed_dim,
embeddings_initializer=Constant(wm),
batch_input_shape=(batch_size, None)))
model.add(Masking(mask_value=0, input_shape=(batch_size, None)))
#model.add(Input(shape=(bs, None, embed_dim)))
model.add(
Bidirectional(
LSTM(nn_size, return_sequences=True, stateful=False, dropout=0.3)))
model.add(
Bidirectional(
LSTM(nn_size, return_sequences=True, stateful=False, dropout=0.3)))
#model.add(Dense(n_labels-1, activation='softmax'))
model.add(Dense(n_labels - 1, activation='sigmoid'))
#model.compile(loss='binary_crossentropy',#'mean_squared_error',
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print(model.summary())
return model
def get_model(self):
input_text = Input(shape=(self.max_len_question, self.latent_dim))
mask = Masking(mask_value=np.zeros(self.latent_dim))(input_text)
"""
# lstm
lstm = LSTM(self.latent_dim, return_sequences=False, return_state=False)
lstm_output = lstm(input_vec)
layer_norm1 = LayerNormalization(axis=1)(lstm_output)
# dense
dense = Dense(1, activation='sigmoid')
dense_output = dense(layer_norm1)
"""
fe = Conv1D(16, 3, strides=2, padding='same')(mask)
fe = LeakyReLU(alpha=0.2)(fe)
fe = Dropout(0.2)(fe)
# normal
fe = Conv1D(32, 3, strides=2, padding='same')(fe)
fe = BatchNormalization()(fe)
fe = LeakyReLU(alpha=0.2)(fe)
fe = Dropout(0.2)(fe)
# downsample to 7x7
fe = Conv1D(64, 3, strides=2, padding='same')(fe)
fe = BatchNormalization()(fe)
fe = LeakyReLU(alpha=0.2)(fe)
fe = Dropout(0.2)(fe)
# downsample one more
fe = Conv1D(128, 3, strides=2, padding='same')(fe)
fe = BatchNormalization()(fe)
fe = LeakyReLU(alpha=0.2)(fe)
fe = Dropout(0.2)(fe)
# flatten feature maps
fe = Flatten()(fe)
# real/fake output
out1 = Dense(1, activation='sigmoid')(fe)
model = Model(input_text, out1)
return model
def __init__(self,
output_vec_len=300,
masking_enabled=True,
SeATT_enabled=True,
**kwargs):
super(WE_SeAtt_BiLSTM, self).__init__(**kwargs)
self.masking_enabled = masking_enabled
if masking_enabled:
self.masking = Masking(mask_value=0.0)
self.dense_0 = Dense(units=300)
self.lstm_0 = LSTM(
units=output_vec_len,
activation='tanh',
recurrent_activation='sigmoid', # check this
return_sequences=True,
stateful=True,
return_state=True,
name="lstm_0",
# dropout=0.2,
# recurrent_dropout=0.2,
)
self.bilstm_0 = tf.keras.layers.Bidirectional(self.lstm_0,
merge_mode='sum')
self.lstm_1 = LSTM(
units=output_vec_len,
activation='tanh',
recurrent_activation='sigmoid', # check this
return_sequences=True,
stateful=True,
return_state=False,
name="lstm_1",
# dropout=0.2,
# recurrent_dropout=0.2,
)
self.dropout_0 = tf.keras.layers.Dropout(0.2)
self.bilstm_1 = tf.keras.layers.Bidirectional(self.lstm_1,
merge_mode='sum')
if SeATT_enabled:
self.sat = custom_SAT()
else:
self.max_pool = MaxPool2D
def create_model(self, model: Optional[IModel]) -> IModel:
assert model is None
# None here means variable over batches (but not within a batch)
input = Input(shape=(self.params.max_timesteps,
self.params.swipe_feature_count))
masking = Masking(mask_value=myNaN)(input)
d = Dense(50, kernel_initializer='random_uniform',
activation='linear')(masking)
lstm = LSTM(16, kernel_initializer='random_uniform')(d)
middle = Dense(50,
kernel_initializer='random_uniform',
activation='relu')(lstm)
output = Dense(1,
kernel_initializer='random_uniform',
activation='sigmoid')(middle)
model = Model(inputs=[input], outputs=output)
return model
def blstm_architecture(input_shape,
output_shape,
rnn_size=100,
dropout=0,
dense_units=100,
learning_rate=0.01,
**kwargs):
""" The architecture of a bidirectional lstm model. (Dense Layer, forward LSTM Layer, backward LSTM Layer, Dense Output Layer)
Parameters:
input_shape (tuple): contains part of the input shape of the data, namely the maximal length of time points and the number of features
output_shape (int): contains part of the output shape of the data, namely the number of labels
rnn_size (int): how many units the LSTM layers should have
dropout (float): how many percent of the units in the layers should drop out
dense_units (int): how many units the first dense layer should have. Default=100.
learning_rate (float): which learning rate the Adam optimizer should use. Default=0.01.
Return:
model: a compiled model ready for fitting
"""
# architecture
# change this: https://stackoverflow.com/questions/62991082/bidirectional-lstm-merge-mode-explanation
model = Sequential()
model.add(Masking(mask_value=0.0, input_shape=(None, input_shape[1])))
if dense_units > 0:
model.add(Dense(units=dense_units, activation="relu"))
model.add(Dropout(dropout))
forward_layer = LSTM(rnn_size, return_sequences=True)
backward_layer = LSTM(rnn_size, return_sequences=True, go_backwards=True)
model.add(
Bidirectional(forward_layer,
backward_layer=backward_layer,
merge_mode="concat"))
model.add(Dropout(dropout))
model.add(Dense(units=output_shape, activation="relu"))
model.add(Activation("softmax"))
optimizer = keras.optimizers.Adam(learning_rate=learning_rate)
model.compile(optimizer=optimizer,
loss="categorical_crossentropy",
metrics=["acc"])
return model
def create_actor(obs_dim,
act_dim,
goal_dim,
layer_size=1024,
latent_dim=256,
training=True):
# params #
batch_size = None if training else 1
stateful = not training
# Input #
o = Input(shape=(None, obs_dim),
batch_size=batch_size,
dtype=tf.float32,
name='input_obs')
z = Input(shape=(None, latent_dim),
batch_size=batch_size,
dtype=tf.float32,
name='input_latent')
g = Input(shape=(None, goal_dim),
batch_size=batch_size,
dtype=tf.float32,
name='input_goals')
# RNN #
x = Concatenate(axis=-1)([o, z, g])
x = Masking(mask_value=0.)(x)
x = LSTM(layer_size,
return_sequences=True,
stateful=stateful,
name='LSTM_in_1')(x)
x = LSTM(layer_size,
return_sequences=True,
stateful=stateful,
name='LSTM_in_2')(x)
# Deterministic output #
actions = Dense(act_dim, activation='tanh', name='acts')(x)
actions = Lambda(lambda a: a * ACT_LIMITS)(
actions) # scale to action limits
return Model([o, z, g], actions)
def get_LSTM(timesteps,
num_feature,
num_class,
n_hidden=100,
inter_num=0,
inter_nodes=10,
drop_rate=0.5,
mask_value=None,
optimizer='Adam',
learning_rate=1e-2,
loss=None,
final_activation=None):
opt = None
if optimizer == 'SGD': opt = SGD(lr=learning_rate)
if optimizer == 'RMSprop': opt = RMSprop(lr=learning_rate)
if optimizer == 'Adadelta': opt = Adadelta(lr=learning_rate)
if optimizer == 'Adam': opt = Adam(lr=learning_rate)
if loss == None:
loss = ('binary_crossentropy'
if num_class <= 2 else 'categorical_crossentropy')
if final_activation == None:
final_activation = ('sigmoid' if num_class <= 2 else 'softmax')
inputs = Input(shape=(timesteps, num_feature))
x_nn = inputs
if mask_value != None: x_nn = Masking(mask_value=mask_value)(x_nn)
lstm_out = LSTM(n_hidden, return_sequences=True)(x_nn)
inter = lstm_out
for i in range(inter_num):
inter = TimeDistributed(Dense(inter_nodes, activation='relu'))(inter)
inter = TimeDistributed(Dropout(drop_rate))(inter)
outputs = TimeDistributed(Dense(num_class,
activation=final_activation))(inter)
model = Model(inputs=inputs, outputs=outputs)
model.compile(optimizer=optimizer, loss=loss)
return model
def generate_model(self):
# Input
input = Input(shape=[None, self.args.num_mels], name='input_speech')
x = Masking(mask_value=0,
input_shape=(None, self.args.num_mels))(input)
# Bidirectional LSTM
for b in range(self.args.num_lstm_layers):
x = Bidirectional(
GRU(units=self.args.num_units_per_lstm // 2,
return_sequences=True,
recurrent_initializer='glorot_uniform'))(x)
x = LayerNormalization(epsilon=1e-6)(x)
# output layer with softmax
logit = Dense(self.args.num_classes,
name='speech_encoder',
activation='softmax')(x)
# Generate model
self.model = Model(inputs=input, outputs=logit)
def __init__(
self,
num_hidden,
embedding=False,
embedding_len = 50
):
"""
:param num_hidden:
:param embedding:
:param embedding_len:
"""
super(
SequenceAttentionLayer, self
).__init__()
self.num_hidden = num_hidden
self.if_embedding = embedding
self.embedding_len = embedding_len
if embedding:
self.embedding_len = embedding_len
self.embed = Embedding(
input_dim=len(word_vectors),
output_dim=self.embedding_len,
weights=[word_vectors],
mask_zero=True,
trainable=False
)
self.embed_mask = Masking(mask_value=0)
self.lstm = LSTM(
self.num_hidden,
activation="tanh",
return_sequences=True
)
# pass the type of attention
self.attention_layer = AdditiveAttention(
use_scale=True
)
self.attention_layer.__setattr__('supports_masking', True)
def create_model(vocab: Dict[bytes, int], labels: list,
batch_size: int) -> object:
nn_size = 100
embed_dim = 300
model = Sequential()
model.add(
Embedding(10000,
embed_dim,
mask_zero=True,
batch_input_shape=(batch_size, None)))
model.add(Masking(mask_value=0., input_shape=(batch_size, None)))
model.add(
Bidirectional(
LSTM(nn_size, return_sequences=True, stateful=False, dropout=0.3)))
#model.add(Bidirectional(LSTM(nn_size,
# return_sequences=True,
# stateful=False,
# dropout=0.3)))
#model.add(Bidirectional(LSTM(nn_size,
# return_sequences=True,
# stateful=False,
# dropout=0.3)))
#model.add(LSTM(nn_size, return_sequences=True))
model.add(TimeDistributed(Dense(len(labels), activation='softmax')))
model.compile(
loss='sparse_categorical_crossentropy', #'mean_squared_error',
optimizer='adam',
metrics=['sparse_categorical_accuracy'])
print(model.summary())
return model
def my_model(input_shape):
# model
# Δημιουργία μοντέλου με χρήση του keras API
model = Sequential()
# Πρώτο κρυφό επίπεδο
model.add(Dense(794, input_shape=(784, ), activation='relu'))
model.add(Masking(mask_value=0.0, input_shape=input_shape))
# Δεύτερο κρυφό επίπεδο
model.add(Dense(50, activation='relu'))
# Επίπεδο εξόδου
model.add(Dense(10, activation='softmax'))
# compile the model
model.load_weights('my_model_weights.h5')
# model = tensorflow.keras.models.load_model('my_model.h5')
model.compile(loss='categorical_crossentropy',
optimizer=tensorflow.keras.optimizers.SGD(learning_rate=0.1,
momentum=0.6,
decay=0.0,
nesterov=False),
metrics=['accuracy'])
return model
def get_model(self):
input_text = Input(shape=(self.max_len_question, self.embed_size))
mask = Masking(mask_value=np.zeros(self.embed_size))(input_text)
# lstm1
lstm1 = LSTM(256, return_sequences=True, return_state=True)
lstm1_output, state_h1, state_c1 = lstm1(mask)
layer_norm1 = LayerNormalization(axis=1)(lstm1_output)
# lstm2
lstm2 = LSTM(128, return_sequences=True, return_state=True)
lstm2_output, state_h2, state_c2 = lstm2(layer_norm1)
layer_norm2 = LayerNormalization(axis=1)(lstm2_output)
# lstm3
lstm3 = LSTM(64, return_sequences=True, return_state=True)
lstm3_output, state_h3, state_c3 = lstm3(layer_norm2)
layer_norm3 = LayerNormalization(axis=1)(lstm3_output)
# lstm4
lstm4 = LSTM(128, return_sequences=True, return_state=True)
lstm4_output, state_h4, state_c4 = lstm4(layer_norm3)
layer_norm4 = LayerNormalization(axis=1)(lstm4_output)
# lstm5
lstm5 = LSTM(256, return_sequences=True, return_state=True)
lstm5_output, state_h5, state_c5 = lstm5(layer_norm4)
layer_norm5 = LayerNormalization(axis=1)(lstm5_output)
# lstm6
lstm6 = LSTM(self.embed_size,
return_sequences=True,
return_state=True,
activation='tanh')
lstm6_output, state_h6, state_c6 = lstm6(layer_norm5)
model = Model(inputs=input_text, outputs=lstm6_output)
return model
def call(self, inp):
"""
Compute attention mask.
Argument/s:
inp - used to compute sequence mask.
Returns:
Attention mask.
"""
batch_size = tf.shape(inp)[0]
max_seq_len = tf.shape(inp)[1]
flat_seq_mask = Masking(mask_value=0.0).compute_mask(inp)
seq_mask = self.merge_masks(tf.expand_dims(flat_seq_mask, axis=1),
tf.expand_dims(flat_seq_mask, axis=2))
causal_mask = self.lower_triangular_mask([1, max_seq_len, max_seq_len
]) if self.causal else None
logical_mask = self.merge_masks(causal_mask, seq_mask)
att_mask = tf.cast(logical_mask, tf.float32)
att_mask = tf.reshape(att_mask,
[batch_size, 1, max_seq_len, max_seq_len])
return att_mask
def attention_model(input_size, days=21, batch_size=32, epochs=200, lr=1e-3):
country_input = Input(shape=(313, ), name='country_onehot')
inputs = Input(shape=(None, input_size), name='encoder_input')
target_number = Input(shape=(1, ), name='target_input')
flag_input = Input(shape=(1, ), name='flag_input')
x = Masking(mask_value=0, input_shape=(None, input_size))(inputs)
x = GRU(128, name='GRU_layer1', return_sequences=True)(x)
attention_x = attention_mechanism(days, x)
gru_out = Permute((2, 1))(x)
attention_mul = K.batch_dot(gru_out, attention_x)
attention_mul = Permute((2, 1))(attention_mul)
x = GRU(128, name='GRU_layer2', return_sequences=True)(attention_mul)
gru_x = Flatten()(x)
# country onehot concatenate
# x = Concatenate(axis=-1)([country_input, gru_x])
x = Dense(32, activation='relu')(gru_x)
# x = BatchNormalization()(x)
x = Dense(128, activation='relu')(x)
# x = BatchNormalization()(x)
x = Dense(256, activation='relu')(x)
# x = Dense(16, activation='relu')(x)
# x = Concatenate(axis=-1)([x, gru_x])
outputs = Dense(1, activation='sigmoid')(x)
outputs = target_number * (flag_input + outputs)
print(outputs.shape, flag_input.shape, target_number.shape)
optimizer = Adam(lr=lr, name='adam')
model = Model([inputs, country_input, target_number, flag_input],
outputs,
name='gru_network')
model.compile(optimizer=optimizer, loss=rmsle)
# print(self.model.summary())
return model
def Encoder():
inputs = Input(shape=(None, ), name="encoder_inputs")
masking = Masking(mask_value=0, name="Encoder_masking")
embedding = Embedding(input_dim=hps.n_symbols,
output_dim=hps.embedding_dim,
name="Character_embedding")
bidlstm = Bidirectional(LSTM(units=hps.lstm_size,
recurrent_dropout=hps.lstm_drop_rate,
return_sequences=True),
name='Bidir_LSTM')
conv1d_3 = Conv1d_3()
x = masking(inputs)
# making 대신 embedding masking zero 도 있는데 확인은 안해보았다.
x = embedding(x)
x = conv1d_3(x)
outputs = bidlstm(x)
return Model(inputs, x, name="encoder")
def version_8(x_shape):
model = Sequential()
model.add(Masking(mask_value=0., input_shape=(x_shape[1], x_shape[2])))
model.add(
Bidirectional(LSTM(64, return_sequences=True),
input_shape=(x_shape[1], x_shape[2])))
model.add(Bidirectional(LSTM(128)))
model.add(Dense(64, activation=None))
model.add(BatchNormalization(axis=-1, momentum=0.99))
model.add(Activation('relu'))
model.add(Dense(32, activation=None))
model.add(BatchNormalization(axis=-1, momentum=0.99))
model.add(Activation('relu'))
model.add(Dense(1, activation=None))
model.add(BatchNormalization(axis=-1, momentum=0.99))
model.add(Activation('sigmoid'))
model.compile('adam', 'binary_crossentropy', metrics=['accuracy'])
return model
def seqvector_LSTM(obj):
mirrored_strategy = MirroredStrategy()
with mirrored_strategy.scope():
METRICS = [
keras.metrics.CategoricalAccuracy(name='accuracy'),
keras.metrics.Precision(name='precision'),
keras.metrics.Recall(name='recall'),
keras.metrics.AUC(name='auc')
]
model = keras.models.Sequential(name='seqvector_LSTM')
model.add(
Masking(mask_value=0., input_shape=obj.train_set[0].shape[1:]))
model.add(Bidirectional(LSTM(units=128, dropout=.2)))
model.add(Dense(units=128, activation='relu'))
model.add(Dropout(.4))
model.add(Dense(units=64, activation='relu'))
model.add(Dropout(.2))
model.add(Dense(units=21, activation='softmax', dtype='float32'))
model.compile(optimizer=keras.optimizers.Adam(),
loss='categorical_crossentropy',
metrics=METRICS)
model.summary()
return model
def create_goal_space_mapper(input_embedding_dim,
goal_embedding_dim,
layer_size=2048,
**kwargs):
# params #
batch_size = None
# Input #
input_embeddings = Input(
shape=(input_embedding_dim, ),
batch_size=batch_size,
dtype=tf.float32,
name='lang_embeds') # embeddings created by MUSE or equiv
# Layers #
x = Masking(mask_value=0.)(input_embeddings)
x = Dense(layer_size, activation="relu", name='layer_1')(x)
x = Dense(layer_size, activation="relu", name='layer_2')(x)
goal_embeddings = Dense(goal_embedding_dim,
activation=None,
name='goal_space')(x)
return Model(input_embeddings, goal_embeddings)