def rcnn(base_layers,
rois,
num_classes,
image_max_dim,
head_fn,
pool_size=(7, 7),
fc_layers_size=1024):
# RoiAlign
x = RoiAlign(image_max_dim, pool_size=pool_size)([base_layers, rois]) #
# 收缩维度
shared_layer = head_fn(x)
# 分类
class_logits = TimeDistributed(layers.Dense(num_classes,
activation='linear'),
name='rcnn_class_logits')(shared_layer)
# 回归(类别相关)
deltas = TimeDistributed(
layers.Dense(4 * num_classes, activation='linear'),
name='rcnn_deltas')(
shared_layer) # shape (batch_size,roi_num,4*num_classes)
# 变为(batch_size,roi_num,num_classes,4)
roi_num = backend.int_shape(deltas)[
1] # print("roi_num:{}".format(roi_num))
deltas = layers.Reshape((roi_num, num_classes, 4))(deltas)
return deltas, class_logits
def seq_2_seq_att_LSTM(X_embedding, MAX_LEN, num_words,
EMBEDDING_DIM, LSTM_units, LSTM_dropout):
# Encoder
# Encoder input shape is (batch size, max length)
encoder_inputs = Input(shape=(MAX_LEN,))
encoder_embedding = Embedding(input_dim=num_words, output_dim=EMBEDDING_DIM,
input_length = MAX_LEN, embeddings_initializer=Constant(X_embedding),
trainable=False)(encoder_inputs)
# LSTM
encoder_lstm = LSTM(units=LSTM_units, return_state=True, return_sequences=True, recurrent_dropout=LSTM_dropout, dropout=LSTM_dropout)
encoder_outputs, state_h, state_c = encoder_lstm(encoder_embedding)
encoder_states = [state_h, state_c]
# Decoder
decoder_inputs = Input(shape=(None,))
decoder_embedding_layer = Embedding(input_dim=num_words, output_dim=EMBEDDING_DIM, trainable=True)
decoder_embedding = decoder_embedding_layer(decoder_inputs)
decoder_lstm = LSTM(units=LSTM_units, return_state=True, return_sequences=True, recurrent_dropout=LSTM_dropout, dropout=LSTM_dropout)
decoder_outputs, _, _ = decoder_lstm(decoder_embedding, initial_state=encoder_states)
# Attention
attention_weight = dot([decoder_outputs, encoder_outputs], axes=[2, 2], normalize=True) # cosine similarity
attention = Activation('softmax')(attention_weight)
context = dot([attention, encoder_outputs], axes=[2,1])
decoder_combined_context = concatenate([context, decoder_outputs])
att_output = TimeDistributed(Dense(64, activation="tanh"))(decoder_combined_context)
output = TimeDistributed(Dense(num_words, activation="softmax"))(att_output)
model = Model(inputs=[encoder_inputs,decoder_inputs], outputs=output)
return model
def bd_model(input_shape, output_sequence_length, english_vocab_size,
spanish_vocab_size):
"""
Build and train a bidirectional RNN model on x and y
:param input_shape: Tuple of input shape
:param output_sequence_length: Length of output sequence
:param english_vocab_size: Number of unique English words in the dataset
:param french_vocab_size: Number of unique French words in the dataset
:return: Keras model built, but not trained
"""
# TODO: Implement
# Hyperparameters
#learning_rate = 0.003
# TODO: Build the layers
model = Sequential()
model.add(
Bidirectional(GRU(128, return_sequences=True),
input_shape=input_shape[1:]))
model.add(TimeDistributed(Dense(512, activation='relu')))
#model.add(Dropout(0.5))
model.add(TimeDistributed(Dense(spanish_vocab_size, activation='softmax')))
# Compile model
model.compile(loss='sparse_categorical_crossentropy',
optimizer='Adam',
metrics=['accuracy'])
return model
def create_rnn_model(self):
"""
"""
seq_input = Input(shape=(self.dense_input_len, 1))
seq_output = Input(shape=(self.dense_input_len, 1))
# norm_seq_input = BatchNormalization(name = 'Dense_BN_trainable')(seq_input)
rnn_out = Bidirectional(
LSTM(self.rnn_units[0], return_sequences=True,
activation='relu'))(seq_input)
rnn_out = Bidirectional(
LSTM(self.rnn_units[1], return_sequences=True,
activation='relu'))(rnn_out)
seq_pred = TimeDistributed(Dense(self.hidden_dim[0],
activation='relu'))(rnn_out)
seq_pred = TimeDistributed(Dense(1, activation='relu'))(seq_pred)
# seq_pred = Dense(1, activation = 'relu')(rnn_out)
seq_pred = Reshape((self.dense_input_len, ))(seq_pred)
seq_input_reshape = Reshape((self.dense_input_len, ))(seq_input)
model = Model(seq_input, seq_pred)
loss = K.mean(
mean_squared_error(seq_input_reshape[:, 1:], seq_pred[:, :-1]))
model.add_loss(loss)
# def _mean_squared_error(y_true, y_pred):
# return K.mean(K.square(y_pred - y_true))
model.compile(optimizer='adam', loss=None) #_mean_squared_error)
return model
def build_model(self, pos_mode=0, use_mask=False, active_layers=999):
v_input = Input(shape=(self.seq_len, self.d_feature), name='v_input')
d0 = TimeDistributed(Dense(self.d_model))(v_input)
pos_input = Input(shape=(self.seq_len, ),
dtype='int32',
name='pos_input')
if pos_mode == 0: # use fixed pos embedding
pos_embedding = Embedding(self.seq_len, self.d_model, trainable=False,\
weights=[GetPosEncodingMatrix(self.seq_len, self.d_model)])
p0 = pos_embedding(pos_input)
elif pos_mode == 1: # use trainable pos embedding
pos_embedding = Embedding(self.seq_len, self.d_model)
p0 = pos_embedding(pos_input)
else: # no pos embedding
p0 = None
if p0 != None:
combine_input = Add()([d0, p0])
else:
combine_input = d0 # no pos
sub_mask = None
if use_mask:
sub_mask = Lambda(GetSubMask)(pos_input)
enc_output = self.encoder(combine_input,
mask=sub_mask,
active_layers=active_layers)
# score
time_score_dense1 = TimeDistributed(
Dense(self.d_model, activation='tanh'))(enc_output)
time_score_dense2 = TimeDistributed(Dense(1))(time_score_dense1)
flat = Flatten()(time_score_dense2)
score_output = Activation(activation='softmax')(flat)
self.model = Model([pos_input, v_input], score_output)
return self.model
def demo_create_encoder(latent_dim, cat_dim, window_size, input_dim):
input_layer = Input(shape=(window_size, input_dim))
code = TimeDistributed(Dense(64, activation='linear'))(input_layer)
code = Bidirectional(LSTM(128, return_sequences=True))(code)
code = BatchNormalization()(code)
code = ELU()(code)
code = Bidirectional(LSTM(64))(code)
code = BatchNormalization()(code)
code = ELU()(code)
cat = Dense(64)(code)
cat = BatchNormalization()(cat)
cat = PReLU()(cat)
cat = Dense(cat_dim, activation='softmax')(cat)
latent_repr = Dense(64)(code)
latent_repr = BatchNormalization()(latent_repr)
latent_repr = PReLU()(latent_repr)
latent_repr = Dense(latent_dim, activation='linear')(latent_repr)
decode = Concatenate()([latent_repr, cat])
decode = RepeatVector(window_size)(decode)
decode = Bidirectional(LSTM(64, return_sequences=True))(decode)
decode = ELU()(decode)
decode = Bidirectional(LSTM(128, return_sequences=True))(decode)
decode = ELU()(decode)
decode = TimeDistributed(Dense(64))(decode)
decode = ELU()(decode)
decode = TimeDistributed(Dense(input_dim, activation='linear'))(decode)
error = Subtract()([input_layer, decode])
return Model(input_layer, [decode, latent_repr, cat, error])
def get_test_model(seq_len, stacked_features_size, edges_features_matrix_depth,
neighbourhood_size, units):
# INPUTS
stacked_base_features_input = Input(shape=(seq_len, stacked_features_size),
name='stacked_base_features')
adjacency_matrix_input = Input(shape=(seq_len, seq_len),
name='adjacency_matrix')
edges_features_matrix_input = Input(shape=(seq_len, seq_len,
edges_features_matrix_depth),
name='edges_features_matrix')
inputs = (stacked_base_features_input, adjacency_matrix_input,
edges_features_matrix_input)
# ACTUAL MODEL
x = Subgraphing(neighbourhood_size)(inputs)
x = RNN(GraphReduceCell(units), return_sequences=True)(x)
# OUTPUTS
reactivity_pred = TimeDistributed(Dense(1), name='reactivity')(x)
deg_Mg_pH10_pred = TimeDistributed(Dense(1), name='deg_Mg_pH10')(x)
deg_Mg_50C_pred = TimeDistributed(Dense(1), name='deg_Mg_50C')(x)
scored_outputs = [reactivity_pred, deg_Mg_pH10_pred, deg_Mg_50C_pred]
stacked_outputs = Concatenate(axis=2,
name='stacked_outputs')(scored_outputs)
# MODEL DEFINING
model = Model(inputs=inputs,
outputs={'stacked_scored_labels': stacked_outputs},
name='graph_reduce_model')
return model
def model_v2(top_layer_units):
model = Sequential()
model.add(
TimeDistributed(Conv1D(filters=16,
kernel_size=4,
padding='same',
activation=tf.nn.relu,
data_format='channels_last'),
input_shape=(NUM_MFCC, NUM_FRAMES, 1)))
model.add(
TimeDistributed(
Conv1D(filters=8,
kernel_size=2,
padding='same',
activation=tf.nn.relu)))
model.add(TimeDistributed(MaxPooling1D(pool_size=2)))
model.add(TimeDistributed(Flatten()))
model.add(LSTM(50, return_sequences=True))
model.add(Dropout(0.3))
model.add(Flatten())
model.add(Dense(units=512, activation=tf.nn.tanh))
model.add(Dense(units=256, activation=tf.nn.tanh))
model.add(
Dense(units=top_layer_units,
activation=tf.nn.softmax,
name='top_layer'))
return model
def BasicBlock(inputs, filters, strides=1, downsample=None):
residual = inputs
out = TimeDistributed(
Conv2D(filters=filters,
kernel_size=3,
strides=strides,
padding='same',
kernel_initializer='he_uniform'))(inputs)
out = BatchNormalization()(out)
out = Activation('relu')(out)
out = TimeDistributed(
Conv2D(filters=filters,
kernel_size=3,
strides=1,
padding='same',
kernel_initializer='he_uniform'))(out)
out = BatchNormalization()(out)
if downsample is not None:
residual = downsample(inputs)
out = add([out, residual])
out = Activation('relu')(out)
return out
def __init__(self,
n_head,
d_model,
d_k,
d_v,
dropout,
mode=0,
use_norm=True):
self.mode = mode
self.n_head = n_head
self.d_k = d_k
self.d_v = d_v
self.dropout = dropout
if mode == 0:
self.qs_layer = Dense(n_head * d_k, use_bias=False)
self.ks_layer = Dense(n_head * d_k, use_bias=False)
self.vs_layer = Dense(n_head * d_v, use_bias=False)
elif mode == 1:
self.qs_layers = []
self.ks_layers = []
self.vs_layers = []
for _ in range(n_head):
self.qs_layers.append(
TimeDistributed(Dense(d_k, use_bias=False)))
self.ks_layers.append(
TimeDistributed(Dense(d_k, use_bias=False)))
self.vs_layers.append(
TimeDistributed(Dense(d_v, use_bias=False)))
self.attention = ScaledDotProductAttention(d_model)
self.layer_norm = LayerNormalization() if use_norm else None
self.w_o = TimeDistributed(Dense(d_model))
def build(self):
n_steps, n_length = 4, 32
self.train_X = self.train_X.reshape(
(self.train_X.shape[0], n_steps, n_length, self.n_features)) # self.n_features = 6
self.test_X = self.test_X.reshape(
(self.test_X.shape[0], n_steps, n_length, self.n_features))
model = Sequential()
model.add(TimeDistributed(Conv1D(filters=64, kernel_size=3,
activation='relu'), input_shape=(None, n_length, self.n_features)))
model.add(TimeDistributed(
Conv1D(filters=16, kernel_size=3, activation='relu')))
model.add(TimeDistributed(Dropout(0.5)))
model.add(TimeDistributed(MaxPooling1D(pool_size=2)))
model.add(TimeDistributed(Flatten()))
model.add(LSTM(100))
model.add(Dropout(0.5))
model.add(Dense(100, activation='relu'))
model.add(Dense(self.n_outputs, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam', metrics=['accuracy'])
# print(model.summary())
# input()
super().build(model)
def create_decoder(input_timesteps, latent_dim):
decoder_in = K.layers.Input(shape=(
input_timesteps,
latent_dim,
),
name='latent_sample_in')
action_data = Input(shape=(input_timesteps, 1), name='action_data')
observations = TimeDistributed(Dense(units=100, activation='relu'),
name='obs1')(decoder_in)
observations = TimeDistributed(Dense(units=100, activation='relu'),
name='obs2')(observations)
observations = TimeDistributed(Dense(units=100, activation='relu'),
name='obs3')(observations)
observations = TimeDistributed(Dense(units=3 * 2),
name='obs_out')(observations)
rewards = TimeDistributed(Dense(units=100, activation='relu'),
name='rew1')(decoder_in)
rewards = TimeDistributed(Dense(units=100, activation='relu'),
name='rew2')(rewards)
rewards = TimeDistributed(Dense(units=100, activation='relu'),
name='rew3')(rewards)
rewards = TimeDistributed(Dense(units=1 * 2), name='rew_out')(rewards)
state_action = Concatenate()([decoder_in, action_data])
next_state = TimeDistributed(Dense(units=100, activation='relu'),
name='ns1')(state_action)
#next_state = TimeDistributed(Dense(units=100, activation='relu'), name='ns2')(next_state)
#next_state = TimeDistributed(Dense(units=100, activation='relu'), name='ns3')(next_state)
next_state = TimeDistributed(Dense(units=latent_dim * 2),
name='next_state')(next_state)
return Model(inputs=[decoder_in, action_data],
outputs=[observations, rewards, next_state])
def default_final_detection_model(pyramid_feature_size=256,
final_detection_feature_size=256,
roi_size=(14, 14),
name='final_detection_submodel'):
"""Creates a final detection model for 3D RetinaMask models.
Args:
pyramid_feature_size (int): Number of features for the input to the
final detection model.
final_detection_feature_size (int): Number of filters used in the 2D
convolution layers.
roi_size (tuple): Size of the region of interest, serves as the
x and y dimensions of the input to the final detection model.
name (str): Name of the model.
Returns:
tensorflow.keras.Model: a FinalDetection submodel for 3D RetinaMask.
"""
options = {
'kernel_size': 3,
'strides': 1,
'padding': 'same',
'kernel_initializer': normal(mean=0.0, stddev=0.01, seed=None),
'bias_initializer': 'zeros',
'activation': 'relu'
}
if K.image_data_format() == 'channels_first':
input_shape = (None, pyramid_feature_size, roi_size[0], roi_size[1])
else:
input_shape = (None, roi_size[0], roi_size[1], pyramid_feature_size)
inputs = Input(shape=input_shape)
outputs = inputs
for i in range(2):
outputs = TimeDistributed(
Conv2D(filters=final_detection_feature_size, **options),
name='final_detection_submodel_conv1_block{}'.format(i))(outputs)
outputs = TimeDistributed(
Conv2D(filters=final_detection_feature_size, **options),
name='final_detection_submodel_conv2_block{}'.format(i))(outputs)
outputs = TimeDistributed(
MaxPool2D(),
name='final_detection_submodel_pool1_block{}'.format(i))(outputs)
outputs = TimeDistributed(
Conv2D(filters=final_detection_feature_size,
kernel_size=3,
padding='valid',
kernel_initializer=normal(mean=0.0, stddev=0.01, seed=None),
bias_initializer='zeros',
activation='relu'))(outputs)
outputs = TimeDistributed(
Conv2D(filters=1, kernel_size=1, activation='sigmoid'))(outputs)
outputs = Lambda(lambda x: tf.squeeze(x, axis=[2, 3]))(outputs)
return Model(inputs=inputs, outputs=outputs, name=name)
def create_model(self):
hidden_size = 256
enc_timesteps = self.max_encoder_seq_length
#timesteps = self.max_encoder_seq_length #perhaps making timesteps size of max sequence length would work?????""
dec_timesteps = self.max_decoder_seq_length
print(f"embedding size: {self.glove_model.embedding_size}")
# encoder_inputs = Input(shape=(None, self.glove_model.embedding_size), name='encoder_inputs')
# decoder_inputs = Input(shape=(None, self.num_decoder_tokens), name='decoder_inputs')
encoder_inputs = Input(shape=(enc_timesteps, self.glove_model.embedding_size), name='encoder_inputs')
decoder_inputs = Input(shape=(dec_timesteps, self.num_decoder_tokens), name='decoder_inputs')
# Encoder GRU
encoder_gru = Bidirectional(GRU(hidden_size, return_sequences=True, return_state=True, name='encoder_gru'), name='bidirectional_encoder')
encoder_out, encoder_fwd_state, encoder_back_state = encoder_gru(encoder_inputs)
# Set up the decoder GRU, using `encoder_states` as initial state.
decoder_gru = GRU(hidden_size*2, return_sequences=True, return_state=True, name='decoder_gru')
decoder_out, decoder_state = decoder_gru(
decoder_inputs, initial_state=Concatenate(axis=-1)([encoder_fwd_state, encoder_back_state])
)
# Attention layer
attn_layer = AttentionLayer(name='attention_layer')
attn_out, attn_states = attn_layer([encoder_out, decoder_out])
# Concat attention input and decoder GRU output
decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([decoder_out, attn_out])
# Dense layer
dense = Dense(self.num_decoder_tokens, activation='softmax', name='softmax_layer')
dense_time = TimeDistributed(dense, name='time_distributed_layer')
decoder_pred = dense_time(decoder_concat_input)
# Full model
self.model = Model(inputs=[encoder_inputs, decoder_inputs], outputs=decoder_pred)
self.model.compile(optimizer=tf.train.RMSPropOptimizer(learning_rate=0.01), loss='categorical_crossentropy')
self.model.summary()
""" Inference model """
batch_size = 1
""" Encoder (Inference) model """
encoder_inf_inputs = Input(batch_shape=(batch_size, enc_timesteps, self.glove_model.embedding_size), name='encoder_inf_inputs')
encoder_inf_out, encoder_inf_fwd_state, encoder_inf_back_state = encoder_gru(encoder_inf_inputs)
self.encoder_model = Model(inputs=encoder_inf_inputs, outputs=[encoder_inf_out, encoder_inf_fwd_state, encoder_inf_back_state])
""" Decoder (Inference) model """
decoder_inf_inputs = Input(batch_shape=(batch_size, 1, self.num_decoder_tokens), name='decoder_word_inputs')
encoder_inf_states = Input(batch_shape=(batch_size, dec_timesteps, 2*hidden_size), name='encoder_inf_states')
decoder_init_state = Input(batch_shape=(batch_size, 2*hidden_size), name='decoder_init')
decoder_inf_out, decoder_inf_state = decoder_gru(
decoder_inf_inputs, initial_state=decoder_init_state)
attn_inf_out, attn_inf_states = attn_layer([encoder_inf_states, decoder_inf_out])
decoder_inf_concat = Concatenate(axis=-1, name='concat')([decoder_inf_out, attn_inf_out])
decoder_inf_pred = TimeDistributed(dense)(decoder_inf_concat)
self.decoder_model = Model(inputs=[encoder_inf_states, decoder_init_state, decoder_inf_inputs],
outputs=[decoder_inf_pred, attn_inf_states, decoder_inf_state])
def define_nmt(hidden_size, batch_size, en_timesteps, en_vsize, fr_timesteps, fr_vsize):
""" Defining a NMT model """
# Define an input sequence and process it.
if batch_size:
encoder_inputs = Input(batch_shape=(batch_size, en_timesteps, en_vsize), name='encoder_inputs')
decoder_inputs = Input(batch_shape=(batch_size, fr_timesteps - 1, fr_vsize), name='decoder_inputs')
else:
encoder_inputs = Input(shape=(en_timesteps, en_vsize), name='encoder_inputs')
decoder_inputs = Input(shape=(fr_timesteps - 1, fr_vsize), name='decoder_inputs')
# Encoder GRU
encoder_gru = Bidirectional(GRU(hidden_size, return_sequences=True, return_state=True, name='encoder_gru'), name='bidirectional_encoder')
encoder_out, encoder_fwd_state, encoder_back_state = encoder_gru(encoder_inputs)
# Set up the decoder GRU, using `encoder_states` as initial state.
decoder_gru = Bidirectional(GRU(hidden_size, return_sequences=True, return_state=True, name='decoder_gru'), name='bidirectional_decoder')
decoder_out, decoder_fwd_state, decoder_back_state = decoder_gru(decoder_inputs, initial_state=[encoder_fwd_state, encoder_back_state])
# Attention layer
attn_layer = AttentionLayer(name='attention_layer')
attn_out, attn_states = attn_layer([encoder_out, decoder_out])
# Concat attention input and decoder GRU output
decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([decoder_out, attn_out])
# Dense layer
dense = Dense(fr_vsize, activation='softmax', name='softmax_layer')
dense_time = TimeDistributed(dense, name='time_distributed_layer')
decoder_pred = dense_time(decoder_concat_input)
# Full model
full_model = Model(inputs=[encoder_inputs, decoder_inputs], outputs=decoder_pred)
full_model.compile(optimizer='adam', loss='categorical_crossentropy')
full_model.summary()
""" Inference model """
batch_size = 1
""" Encoder (Inference) model """
encoder_inf_inputs = Input(batch_shape=(batch_size, en_timesteps, en_vsize), name='encoder_inf_inputs')
encoder_inf_out, encoder_inf_fwd_state, encoder_inf_back_state = encoder_gru(encoder_inf_inputs)
encoder_model = Model(inputs=encoder_inf_inputs, outputs=[encoder_inf_out, encoder_inf_fwd_state, encoder_inf_back_state])
""" Decoder (Inference) model """
decoder_inf_inputs = Input(batch_shape=(batch_size, 1, fr_vsize), name='decoder_word_inputs')
encoder_inf_states = Input(batch_shape=(batch_size, en_timesteps, 2*hidden_size), name='encoder_inf_states')
decoder_init_fwd_state = Input(batch_shape=(batch_size, hidden_size), name='decoder_fwd_init')
decoder_init_back_state = Input(batch_shape=(batch_size, hidden_size), name='decoder_back_init')
decoder_inf_out, decoder_inf_fwd_state, decoder_inf_back_state = decoder_gru(decoder_inf_inputs, initial_state=[decoder_init_fwd_state, decoder_init_back_state])
attn_inf_out, attn_inf_states = attn_layer([encoder_inf_states, decoder_inf_out])
decoder_inf_concat = Concatenate(axis=-1, name='concat')([decoder_inf_out, attn_inf_out])
decoder_inf_pred = TimeDistributed(dense)(decoder_inf_concat)
decoder_model = Model(inputs=[encoder_inf_states, decoder_init_fwd_state, decoder_init_back_state, decoder_inf_inputs],
outputs=[decoder_inf_pred, attn_inf_states, decoder_inf_fwd_state, decoder_inf_back_state])
return full_model, encoder_model, decoder_model
def getAdaptationModel(modelPath, adaptationVersion, features, seqLen):
fineTuneModel = load_model(modelPath)
# Test optimizer's state:
#print(fineTuneModel.optimizer.get_config())
#print(dir(fineTuneModel.optimizer))
#print(fineTuneModel.optimizer.lr)
fineTuneModel.get_layer('td').trainable = True
if adaptationVersion == 2:
fineTuneModel.get_layer('td').activation = 'relu'
fineTuneModel.get_layer('rnn').trainable = False
if fineTuneModel.get_layer('rnn_2nd_layer') != None:
fineTuneModel.get_layer('rnn_2nd_layer').trainable = False
fineTuneModel.get_layer('nn').trainable = False
fineTuneModel.get_layer('nn_dropout').trainable = False
fineTuneModel.get_layer('output_softmax').trainable = False
if adaptationVersion == 3:
fineTuneModel.get_layer('td').activation = 'relu'
fineTuneModel.name = "existingModel"
newModel = Sequential()
newModel.add(TimeDistributed(Dense(features,
kernel_initializer='identity',
bias_initializer='zeros',
activation='relu'), input_shape=(seqLen, features), name='td0', trainable=True))
newModel.add(fineTuneModel)
fineTuneModel = newModel
if adaptationVersion == 4: # initializer does not work with this initializer cause it is not square
fineTuneModel.get_layer('td').activation = 'relu'
fineTuneModel.name = "existingModel"
newModel = Sequential()
newModel.add(TimeDistributed(Dense(10*features,
kernel_initializer='identity',
bias_initializer='zeros',
activation='relu'), input_shape=(seqLen, features), name='td0', trainable=True))
newModel.add(fineTuneModel)
fineTuneModel = newModel
if onTpu:
multiFineTuneModel.compile(loss="categorical_crossentropy",
optimizer=tf.train.AdamOptimizer(lr=0.001),
metrics=["accuracy"])
multiFineTuneModel = toTpuModel(fineTuneModel)
else:
multiFineTuneModel = toMultiGpuModel(fineTuneModel)
multiFineTuneModel.compile(loss="categorical_crossentropy",
optimizer=optimizers.Adam(lr=0.001),
metrics=["accuracy"])
# Test optimizer's state:
#print(fineTuneModel.optimizer.get_config())
#print(dir(fineTuneModel.optimizer))
#print(fineTuneModel.optimizer.lr)
return fineTuneModel, multiFineTuneModel
def build_model(self):
"""
Function to build the seq2seq model used.
:return: Encoder model, decoder model (used for predicting) and full model (used for training).
"""
# Define model inputs for the encoder/decoder stack
x_enc = Input(shape=(self.seq_len_in, self.input_feature_amount), name="x_enc")
x_dec = Input(shape=(self.seq_len_out, self.output_feature_amount), name="x_dec")
# Add noise
x_dec_t = GaussianNoise(0.2)(x_dec)
# Define the encoder GRU, which only has to return a state
encoder_gru = GRU(self.state_size, return_sequences=True, return_state=True, name="encoder_gru")
encoder_out, encoder_state = encoder_gru(x_enc)
# Decoder GRU
decoder_gru = GRU(self.state_size, return_state=True, return_sequences=True,
name="decoder_gru")
# Use these definitions to calculate the outputs of out encoder/decoder stack
dec_intermediates, decoder_state = decoder_gru(x_dec_t, initial_state=encoder_state)
# Define the attention layer
attn_layer = AttentionLayer(name="attention_layer")
attn_out, attn_states = attn_layer([encoder_out, dec_intermediates])
# Concatenate decoder and attn out
decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([dec_intermediates, attn_out])
# Define the dense layer
dense = Dense(self.output_feature_amount, activation='linear', name='output_layer')
dense_time = TimeDistributed(dense, name='time_distributed_layer')
decoder_pred = dense_time(decoder_concat_input)
# Define the encoder/decoder stack model
encdecmodel = tsModel(inputs=[x_enc, x_dec], outputs=decoder_pred)
# Define the separate encoder model for inferencing
encoder_inf_inputs = Input(shape=(self.seq_len_in, self.input_feature_amount), name="encoder_inf_inputs")
encoder_inf_out, encoder_inf_state = encoder_gru(encoder_inf_inputs)
encoder_model = tsModel(inputs=encoder_inf_inputs, outputs=[encoder_inf_out, encoder_inf_state])
# Define the separate encoder model for inferencing
decoder_inf_inputs = Input(shape=(1, self.output_feature_amount), name="decoder_inputs")
encoder_inf_states = Input(shape=(self.seq_len_in, self.state_size), name="encoder_inf_states")
decoder_init_state = Input(shape=(self.state_size,), name="decoder_init")
decoder_inf_out, decoder_inf_state = decoder_gru(decoder_inf_inputs, initial_state=decoder_init_state)
attn_inf_out, attn_inf_states = attn_layer([encoder_inf_states, decoder_inf_out])
decoder_inf_concat = Concatenate(axis=-1, name='concat')([decoder_inf_out, attn_inf_out])
decoder_inf_pred = TimeDistributed(dense)(decoder_inf_concat)
decoder_model = tsModel(inputs=[encoder_inf_states, decoder_init_state, decoder_inf_inputs],
outputs=[decoder_inf_pred, attn_inf_states, decoder_inf_state])
return encoder_model, decoder_model, encdecmodel
def encoder_identity_block(net,
f,
filters,
stage,
block,
dropout_rate=DROPOUT_RATE):
# Defining name basis
conv_name_base = 'enc_res' + str(stage) + block + '_branch'
bn_name_base = 'enc_bn' + str(stage) + block + '_branch'
# Retrieve Filters
f1, f2, f3 = filters
# Save the input value
net_shortcut = net
# First component of main path
net = TimeDistributed(
ConvSN2D(filters=f1,
kernel_size=(1, 1),
strides=(1, 1),
padding='valid',
name=conv_name_base + '2a',
kernel_initializer=glorot_uniform(seed=0)))(net)
net = BatchNormalization(axis=-1, name=bn_name_base + '2a')(net)
net = SwishLayer()(net)
# Second component of main path
net = TimeDistributed(Dropout(dropout_rate))(net)
net = TimeDistributed(
ConvSN2D(filters=f2,
kernel_size=(f, f),
strides=(1, 1),
padding='same',
name=conv_name_base + '2b',
kernel_initializer=glorot_uniform(seed=0)))(net)
net = BatchNormalization(axis=-1, name=bn_name_base + '2b')(net)
net = SwishLayer()(net)
# Third component of main path
net = TimeDistributed(Dropout(dropout_rate))(net)
net = TimeDistributed(
ConvSN2D(filters=f3,
kernel_size=(1, 1),
strides=(1, 1),
padding='valid',
name=conv_name_base + '2c',
kernel_initializer=glorot_uniform(seed=0)))(net)
net = BatchNormalization(axis=-1, name=bn_name_base + '2c')(net)
# Final step: Add shortcut value to main path, and pass it through a RELU activation
net = Add()([net, net_shortcut])
net = SwishLayer()(net)
return net
def make_decoder(self):
reduced_inputs = layers.Input(shape=(self.time_steps, self.img_size, self.img_size, self.nb_channels_out),
name='reduced_inputs')
x = TimeDistributed(layers.Conv2D(self.interm1, (3, 3), activation=LeakyReLU(0.2), padding='same'))(
reduced_inputs)
x = TimeDistributed(BatchNormalization())(x)
x = TimeDistributed(layers.Conv2D(self.interm2, (3, 3), activation=LeakyReLU(0.2), padding='same'))(x)
x = TimeDistributed(BatchNormalization())(x)
out = TimeDistributed(layers.Conv2D(self.nb_channels_in, (3, 3), activation=LeakyReLU(0.2), padding='same'),
name='predicted_inputs')(x)
return Model(inputs=reduced_inputs, outputs=out, name='decoder')
def __build_anchors(anchor_parameters, features, frames_per_batch=1):
"""Builds anchors for the shape of the features from FPN.
Args:
anchor_parameters (AnchorParameters): Parameters that determine how
anchors are generated.
features (list): The FPN features.
frames_per_batch (int): Size of z axis in generated batches.
If equal to 1, assumes 2D data.
Returns:
tensor: The anchors for the FPN features.
The shape is:
```
(batch_size, num_anchors, 4)
```
"""
if len(features) == 1:
if frames_per_batch > 1:
anchors = TimeDistributed(
Anchors(size=anchor_parameters.sizes[0],
stride=anchor_parameters.strides[0],
ratios=anchor_parameters.ratios,
scales=anchor_parameters.scales,
name='anchors'))(features[0])
else:
anchors = Anchors(size=anchor_parameters.sizes[0],
stride=anchor_parameters.strides[0],
ratios=anchor_parameters.ratios,
scales=anchor_parameters.scales,
name='anchors')(features[0])
return anchors
else:
if frames_per_batch > 1:
anchors = [
TimeDistributed(
Anchors(size=anchor_parameters.sizes[i],
stride=anchor_parameters.strides[i],
ratios=anchor_parameters.ratios,
scales=anchor_parameters.scales,
name='anchors_{}'.format(i)))(f)
for i, f in enumerate(features)
]
else:
anchors = [
Anchors(size=anchor_parameters.sizes[i],
stride=anchor_parameters.strides[i],
ratios=anchor_parameters.ratios,
scales=anchor_parameters.scales,
name='anchors_{}'.format(i))(f)
for i, f in enumerate(features)
]
return Concatenate(axis=-2, name='anchors')(anchors)
def __init__(self, w3, dropout_rate):
# w3 are the window sizes of the convolutions, hyperparameters
super().__init__()
self.logger = logging.getLogger(__name__)
self.conv1 = TimeDistributed(
Conv1D(1,
w3,
activation=None,
padding='causal',
name='atn_weight_conv1'))
self.dropout = Dropout(dropout_rate)
self.softmax = TimeDistributed(Softmax(name='atn_weight_softmax'))
def __init__(self, params=None, is_training=False):
super(Attention, self).__init__()
self.is_training = is_training
self.max_len = params['max_len']
self.emb_dim = params['emb_dim']
self.attn_weights = TimeDistributed(Dense(self.emb_dim,
activation='softmax'),
name='attn_weights')
self.attn_out = TimeDistributed(Dense(self.emb_dim, activation=None),
name='attn_out')
def layers(self):
input_layer = Input(self.real_input_shape, self.batch_size)
# 64x64x3
# Zero-Padding
net = TimeDistributed(ZeroPadding2D((3, 3)))(input_layer)
# 70x70x3
# Stage 0
net = TimeDistributed(Conv2D(64, (7, 7), strides=(2, 2), name='conv0',
kernel_initializer=glorot_uniform(seed=0)))(net)
net = BatchNormalization(axis=-1, name='bn_conv0')(net)
net = Activation('relu')(net)
# 32x32x64
# Stage 1
net = encoder_convolutional_block(net, f=3, filters=[64, 64, 128], stage=1, block='a', s=1)
# net = encoder_identity_block(net, 3, [64, 64, 128], stage=1, block='b')
# 32x32x64
self.mean_32x32x64 = encoder_residual_block(net, depth=64)
self.stddev_32x32x64 = encoder_residual_block(net, depth=64)
# Stage 2
net = encoder_convolutional_block(net, f=3, filters=[64, 64, 128], stage=2, block='a', s=2)
# net = encoder_identity_block(net, 3, [64, 64, 128], stage=2, block='b')
# 16x16x128
self.mean_16x16x128 = encoder_residual_block(net, depth=128)
self.stddev_16x16x128 = encoder_residual_block(net, depth=128)
# Stage 3
net = encoder_convolutional_block(net, f=3, filters=[128, 128, 256], stage=3, block='a', s=2)
# net = encoder_identity_block(net, 3, [128, 128, 256], stage=3, block='b')
# 8x8x256
self.mean_8x8x256 = encoder_residual_block(net, depth=256)
self.stddev_8x8x256 = encoder_residual_block(net, depth=256)
# Stage 4
net = encoder_convolutional_block(net, f=3, filters=[256, 256, 512], stage=4, block='a', s=2)
# net = encoder_identity_block(net, 3, [256, 256, 512], stage=4, block='b')
# 4x4x512
self.mean_4x4x512 = encoder_residual_block(net, depth=512)
self.stddev_4x4x512 = encoder_residual_block(net, depth=512)
return input_layer, [self.mean_4x4x512, self.stddev_4x4x512,
self.mean_8x8x256, self.stddev_8x8x256,
self.mean_16x16x128, self.stddev_16x16x128,
self.mean_32x32x64, self.stddev_32x32x64]
def LocalMultiTaskLayer(x, input_type):
classes_root = 35 if input_type.startswith(
'spelling') else 12 # the twelve notes without enharmonic duplicates
classes_quality = 12 # ['M', 'm', 'd', 'a', 'M7', 'm7', 'D7', 'd7', 'h7', 'Gr+6', 'It+6', 'Fr+6']
classes_inversion = 4 # root position, 1st, 2nd, and 3rd inversion (the last only for seventh chords)
o_qlt = TimeDistributed(Dense(classes_quality, activation='softmax'),
name='quality')(x)
o_inv = TimeDistributed(Dense(classes_inversion, activation='softmax'),
name='inversion')(x)
o_roo = TimeDistributed(Dense(classes_root, activation='softmax'),
name='root')(x)
return [o_qlt, o_inv, o_roo]
def ProgressionMultiTaskLayer(x, input_type):
classes_key = 30 if input_type.startswith(
'spelling') else 24 # Major keys: 0-11, Minor keys: 12-23
classes_degree = 21 # 7 degrees * 3: regular, diminished, augmented
o_key = TimeDistributed(Dense(classes_key, activation='softmax'),
name='key')(x)
z = Concatenate()([x, o_key])
o_dg1 = TimeDistributed(Dense(classes_degree, activation='softmax'),
name='degree_1')(z)
o_dg2 = TimeDistributed(Dense(classes_degree, activation='softmax'),
name='degree_2')(z)
return [o_key, o_dg1, o_dg2]
def variant_default(cls, x_train, y_train, x_val, y_val, params):
model = Sequential()
model.add(
TimeDistributed(
Flatten(input_shape=(x_train.shape[1],
x_train.shape[2] * x_train.shape[3]))))
model.add(
TimeDistributed(
Flatten(input_shape=(x_train.shape[1],
x_train.shape[2] * x_train.shape[3]))))
for i in range(params["lstm_layers"] - 1):
model.add(
LSTM(
params["hidden_size"],
return_sequences=True,
activation=params["activation"],
))
model.add(LSTM(params["hidden_size"], activation=params["activation"]))
if params["dropout"] > 0:
model.add(Dropout(params["dropout"]))
model.add(Dense(10, activation=params["output_activation"]))
model.compile(
optimizer=tf.keras.optimizers.SGD(lr=params["lr"],
momentum=params["momentum"]),
loss="sparse_categorical_crossentropy",
metrics=["accuracy"],
)
history = model.fit(
x_train,
y_train,
validation_data=(x_val, y_val),
epochs=params["epochs"],
batch_size=params["batch_size"],
verbose=1,
callbacks=[
tf.keras.callbacks.TensorBoard(
"./logs/" + "lstm_default/" +
"-".join("=".join((str(k), str(v)))
for k, v in params.items()) +
"-ts={}".format(str(time.time())))
],
)
return history, model
def seq_2_seq_biLSTM_att(X_embedding, MAX_LEN, num_words,
EMBEDDING_DIM, LSTM_units, LSTM_dropout):
# Encoder
# [?, 100]
encoder_inputs = Input(shape=(MAX_LEN,))
# [?, 100, 300]
encoder_embedding = Embedding(input_dim=num_words, output_dim=EMBEDDING_DIM,
input_length = MAX_LEN, embeddings_initializer=Constant(X_embedding),
trainable=False)(encoder_inputs)
# LSTM
encoder_lstm = Bidirectional(LSTM(units=LSTM_units, return_state=True, return_sequences=True, recurrent_dropout=LSTM_dropout, dropout=LSTM_dropout))
# [?, 100, 300]
encoder_outputs, forward_h, forward_c, backward_h, backward_c = encoder_lstm(encoder_embedding)
# [?, 300]
state_h = concatenate([forward_h, backward_h])
state_c = concatenate([forward_c, backward_c])
encoder_states = [state_h, state_c]
# Decoder
# [?, 30]
decoder_inputs = Input(shape=(None,))
decoder_embedding_layer = Embedding(input_dim=num_words, output_dim=EMBEDDING_DIM, trainable=True)
# [?, 30, 300]
decoder_embedding = decoder_embedding_layer(decoder_inputs)
decoder_lstm = LSTM(units=2*LSTM_units, return_state=True, return_sequences=True, recurrent_dropout=LSTM_dropout, dropout=LSTM_dropout)
# [?, 30, 300]
decoder_outputs, _, _ = decoder_lstm(decoder_embedding, initial_state=encoder_states)
# [?, 30, 100]
attention_weight = dot([decoder_outputs, encoder_outputs], axes=[2, 2])
attention = Activation('softmax')(attention_weight)
# [?, 30, 300]
context = dot([attention, encoder_outputs], axes=[2,1]) #[?, 100, 300] = dot([?,?,100] , [?, 100, 300])
# [?, 30, 600]
decoder_combined_context = concatenate([context, decoder_outputs])
# [?, 30, 64]
att_output = TimeDistributed(Dense(128, activation="tanh"))(decoder_combined_context)
# [?, 30, 39093]
output = TimeDistributed(Dense(num_words, activation="softmax"))(att_output)
model = Model(inputs=[encoder_inputs,decoder_inputs], outputs=output)
return model
def build_fpn_mask_graph(rois, feature_maps, image_shape, pool_size,
num_classes):
"""Builds the computation graph of the mask head of Feature Pyramid Network.
rois: [batch, num_rois, (y1, x1, y2, x2)] Proposal boxes in normalized
coordinates.
feature_maps: List of feature maps from diffent layers of the pyramid,
[P2, P3, P4, P5]. Each has a different resolution.
image_shape: [height, width, depth]
pool_size: The width of the square feature map generated from ROI Pooling.
num_classes: number of classes, which determines the depth of the results
Returns: Masks [batch, roi_count, height, width, num_classes]
"""
# ROI Pooling
# Shape: [batch, boxes, pool_height, pool_width, channels]
x = PyramidROIAlign([pool_size, pool_size],
image_shape,
name="roi_align_mask")([rois] + feature_maps)
# Conv layers
x = TimeDistributed(Conv2D(256, (3, 3), padding="same"),
name="mrcnn_mask_conv1")(x)
x = TimeDistributed(BatchNorm(axis=3), name='mrcnn_mask_bn1')(x)
x = Activation('relu')(x)
x = TimeDistributed(Conv2D(256, (3, 3), padding="same"),
name="mrcnn_mask_conv2")(x)
x = TimeDistributed(BatchNorm(axis=3), name='mrcnn_mask_bn2')(x)
x = Activation('relu')(x)
x = TimeDistributed(Conv2D(256, (3, 3), padding="same"),
name="mrcnn_mask_conv3")(x)
x = TimeDistributed(BatchNorm(axis=3), name='mrcnn_mask_bn3')(x)
x = Activation('relu')(x)
x = TimeDistributed(Conv2D(256, (3, 3), padding="same"),
name="mrcnn_mask_conv4")(x)
x = TimeDistributed(BatchNorm(axis=3), name='mrcnn_mask_bn4')(x)
x = Activation('relu')(x)
x = TimeDistributed(Conv2DTranspose(256, (2, 2),
strides=2,
activation="relu"),
name="mrcnn_mask_deconv")(x)
x = TimeDistributed(Conv2D(num_classes, (1, 1),
strides=1,
activation="sigmoid"),
name="mrcnn_mask")(x)
return x
def CRNN(input_shape):
Input_Tr = Input(input_shape, dtype='float', name='Input_Tr')
conv_layer1 = Conv2D(32, kernel_size=3, strides=1,
padding='SAME')(Input_Tr)
batch_layer1 = BatchNormalization(axis=-1)(conv_layer1)
conv_layer1_out = Activation('relu')(batch_layer1)
pooling_layer1 = MaxPooling2D((1, 4))(conv_layer1_out)
dropout_layer1 = Dropout(0.5)(pooling_layer1)
conv_layer2 = Conv2D(64, kernel_size=3, strides=1,
padding='SAME')(dropout_layer1)
batch_layer2 = BatchNormalization(axis=-1)(conv_layer2)
conv_layer2_out = Activation('relu')(batch_layer2)
pooling_layer2 = MaxPooling2D((1, 4))(conv_layer2_out)
dropout_layer2 = Dropout(0.5)(pooling_layer2)
print(dropout_layer2.shape)
reshape_layer3 = Reshape(
(600, 64 * int(round(n_mel / 4 / 4))))(dropout_layer2)
print(reshape_layer3.shape)
bidir_layer3 = Bidirectional(
GRU(64, return_sequences=True, activation='tanh'))(reshape_layer3)
output = TimeDistributed(Dense(1, activation='sigmoid'))(bidir_layer3)
model = Model(inputs=[Input_Tr], outputs=[output])
return model
def create_model(steps_before, steps_after, feature_count):
"""
creates, compiles and returns a RNN model
@param steps_before: the number of previous time steps (input)
@param steps_after: the number of posterior time steps (output or predictions)
@param feature_count: the number of features in the model
@param hidden_neurons: the number of hidden neurons per LSTM layer
"""
DROPOUT = 0.5
LAYERS = 2
hidden_neurons = 300
model = Sequential()
model.add(
LSTM(input_dim=feature_count,
output_dim=hidden_neurons,
return_sequences=False))
model.add(RepeatVector(steps_after))
model.add(LSTM(output_dim=hidden_neurons, return_sequences=True))
model.add(TimeDistributed(Dense(feature_count)))
model.add(Activation('linear'))
model.compile(loss='mean_squared_error',
optimizer='rmsprop',
metrics=['accuracy'])
return model
