def create_network(n_notes, n_durations, embed_size=100, rnn_units=256):
""" create the structure of the neural network """
# There are two inputs to the network: the sequence of previous note names and duration values.
notes_in = layers.Input(shape=(None, ))
durations_in = layers.Input(shape=(None, ))
# The Embedding layers convert the integer values of the note names and durations into vectors.
x1 = layers.Embedding(n_notes, embed_size)(notes_in)
x2 = layers.Embedding(n_durations, embed_size)(durations_in)
# The vectors are concatenated to form one long vector that will be used as input into the recurrent layers.
x = layers.Concatenate()([x1, x2])
# Two stacked LSTM layers are used as the recurrent part of the network. Notice how we set return_sequences to True to make
# each layer pass the full sequence of hidden states to the next layer, rather than just the final hidden state.
x = layers.LSTM(rnn_units, return_sequences=True)(x)
x = layers.LSTM(rnn_units, return_sequences=True)(x)
# The alignment function is just a Dense layer with one output unit and tanh activation. We can use a Reshape layer to
# squash the output to a single vector, of length equal to the length of the input sequence (seq_length).
e = layers.Dense(1, activation='tanh')(x)
e = layers.Reshape([-1])(e)
# The weights are calculated through applying a softmax activation to the alignment values.
alpha = layers.Activation('softmax')(e)
# To get the weighted sum of the hidden states, we need to use a RepeatVector layer to copy the weights rnn_units times
# to form a matrix of shape [rnn_units, seq_length], then transpose this matrix using a Permute layer to get a matrix of
# shape [seq_length, rnn_units]. We can then multiply this matrix pointwise with the hidden states from the final LSTM layer,
# which also has shape [seq_length, rnn_units]. Finally, we use a Lambda layer to perform the summation along the seq_length
# axis, to give the context vector of length rnn_units.
alpha_repeated = layers.Permute([2,
1])(layers.RepeatVector(rnn_units)(alpha))
c = layers.Multiply()([x, alpha_repeated])
c = layers.Lambda(lambda xin: K.sum(xin, axis=1),
output_shape=(rnn_units, ))(c)
# The network has a double-headed output, one for the next note name and one for the next note length.
notes_out = layers.Dense(n_notes, activation='softmax', name='pitch')(c)
durations_out = layers.Dense(n_durations,
activation='softmax',
name='duration')(c)
# The final model accepts the previous note names and note durations as input and outputs a distribution
# for the next note name and next note duration.
model = Model([notes_in, durations_in], [notes_out, durations_out])
# The model is compiled using categorical_crossentropy for both the note name and note duration output heads, as this is a
# multiclass classification problem.
model.compile(
loss=['categorical_crossentropy', 'categorical_crossentropy'],
optimizer=RMSprop(lr=0.001))
return model
def attention_with_count_vector(inputs, count_vectors):
# inputs.shape = (batch_size, time_steps, input_dim)
# count_vectors.shape = (batch_size, sym_count)
time_steps = int(inputs.shape[1])
input_dim = int(inputs.shape[2])
a = layers.Dense(time_steps, activation='softmax')(count_vectors)
a = layers.RepeatVector(input_dim)(a)
a_probs = layers.Permute((2, 1), name='attention_vec')(a)
output_attention_mul = layers.multiply([inputs, a_probs],
name='attention_mul')
return output_attention_mul
def create_q_model(actions):
"""This function creates a convolution neural network"""
# Network defined by the Deepmind paper
inputs = layers.Input(shape=(4, 84, 84))
layer0 = layers.Permute((2, 3, 1))(inputs)
layer1 = layers.Conv2D(32, 8, strides=4, activation="relu")(layer0)
layer2 = layers.Conv2D(64, 4, strides=2, activation="relu")(layer1)
layer3 = layers.Conv2D(64, 3, strides=1, activation="relu")(layer2)
layer4 = layers.Flatten()(layer3)
layer5 = layers.Dense(512, activation="relu")(layer4)
action = layers.Dense(actions, activation="linear")(layer5)
return K.Model(inputs=inputs, outputs=action)
def build_model_cnn(input_shape, actions):
"""
Creates a Convolutional Neural Network.
"""
model = models.Sequential()
model.add(layers.Permute((2, 3, 1), input_shape=input_shape))
model.add(layers.Convolution2D(32, (4, 4), strides=(2, 2), activation="relu"))
model.add(layers.Convolution2D(64, (3, 3), strides=(1, 1), activation="relu"))
model.add(layers.Flatten())
model.add(layers.Dense(32, activation="relu"))
model.add(layers.Dense(actions, activation="linear"))
return model
def duc(x, factor=8, output_shape=(224, 224, 1), name=None):
if K.image_data_format() == 'channels_last':
bn_axis = -1
else:
bn_axis = 1
if name is None:
name = 'duc_%s' % factor
H, W, c, r = output_shape[0], output_shape[1], output_shape[2], factor
h = H // r
w = W // r
x = KL.Conv2D(c * r * r, (3, 3), padding='same', name='conv_%s' % name)(x)
x = KL.BatchNormalization(axis=bn_axis, name='bn_%s' % name)(x)
x = KL.Activation('relu')(x)
x = KL.Permute((3, 1, 2))(x)
x = KL.Reshape((c, r, r, h, w))(x)
x = KL.Permute((1, 4, 2, 5, 3))(x)
x = KL.Reshape((c, H, W))(x)
x = KL.Permute((2, 3, 1))(x)
return x
def attention_block_3(inputs, feature_cnt, dim):
a = Flatten()(inputs)
a = Dense(feature_cnt * dim, activation='softmax')(a)
a = L.Reshape((
feature_cnt,
dim,
))(a)
a = L.Lambda(lambda x: K1.sum(x, axis=2), name='attention')(a)
a = L.RepeatVector(dim)(a)
a_probs = L.Permute((2, 1), name='attention_vec')(a)
attention_out = L.Multiply()([inputs, a_probs])
return attention_out
def build_model(input_shape, actions):
model = models.Sequential()
model.add(layers.Permute((2, 3, 1), input_shape=input_shape))
model.add(
layers.Convolution2D(32, (8, 8), strides=(4, 4), activation="relu"))
model.add(
layers.Convolution2D(64, (4, 4), strides=(2, 2), activation="relu"))
model.add(
layers.Convolution2D(64, (3, 3), strides=(1, 1), activation="relu"))
model.add(layers.Flatten())
model.add(layers.Dense(512, activation="relu"))
model.add(layers.Dense(actions, activation="linear"))
return model
def create_q_model(actions):
inputs = layers.Input(shape=(4, 84, 84))
layer0 = layers.Permute((2, 3, 1))(inputs)
layer1 = layers.Conv2D(32, 8, strides=4, activation="relu",
data_format="channels_last")(layer0)
layer2 = layers.Conv2D(64, 4, strides=2, activation="relu",
data_format="channels_last")(layer1)
layer3 = layers.Conv2D(64, 3, strides=1, activation="relu",
data_format="channels_last")(layer2)
layer4 = layers.Flatten()(layer3)
layer5 = layers.Dense(512, activation="relu")(layer4)
action = layers.Dense(actions, activation="linear")(layer5)
return K.Model(inputs=inputs, outputs=action)
def build_model(sigs_1d, sigs_0d, sigs_predict, rho_length_in, rho_length_out,
lookback, delay, rnn_type, rnn_size, rnn_activation,
num_rnn_layers, dense_0d_size, dense_0d_activation,
dense_final_size, dense_final_activation, num_final_layers):
if (rnn_type == 'LSTM'):
rnn_layer = layers.LSTM
elif (rnn_type == 'GRU'):
rnn_layer = layers.GRU
else:
raise ValueError('rnn_type in conf must be GRU or LSTM')
input_0d = layers.Input(shape=(
delay + 1,
len(sigs_0d),
))
input_1d = layers.Input(shape=(
lookback + 1,
rho_length_in * len(sigs_1d),
))
permuted_0d = layers.Permute((
2,
1,
))(input_0d)
compressed_0d = layers.Dense(dense_0d_size)(permuted_0d)
final_input_0d = layers.Flatten()(compressed_0d)
final_input_1d = input_1d
for i in range(num_rnn_layers - 1):
final_input_1d = rnn_layer(rnn_size,
activation=rnn_activation,
return_sequences=True)(final_input_1d)
final_input_1d = rnn_layer(rnn_size,
activation=rnn_activation)(final_input_1d)
output = layers.Concatenate()([final_input_0d, final_input_1d])
for i in range(num_final_layers):
output = layers.Dense(dense_final_size,
activation=dense_final_activation)(output)
output = layers.Dense(rho_length_out * len(sigs_predict))(output)
model = models.Model(inputs=[input_0d, input_1d], outputs=output)
print(model.summary())
return model
def train_rgbmaps():
sequence_length = dataset_parameters["sequence_length"]
model = models.Sequential()
if sequence_length == 0:
model.add(
layers.Conv2D(64, (3, 3),
activation="relu",
input_shape=(image_size, image_size, 3)))
else:
model.add(
layers.Permute(
(2, 3, 1, 4),
input_shape=(sequence_length, image_size, image_size, 3)))
model.add(layers.Reshape(
(image_size, image_size, sequence_length * 3)))
model.add(layers.Conv2D(64, (3, 3), activation="relu"))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation="relu"))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(128, (3, 3), activation="relu"))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(128, (3, 3), activation="relu"))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Flatten())
model.add(layers.Dropout(0.25))
model.add(layers.Dense(128, activation="relu"))
model.add(layers.Dropout(0.25))
model.add(layers.Dense(1, activation="linear"))
model.summary()
# Compile the model.
optimizer = "rmsprop"
#optimizer = optimizers.Adagrad(lr=0.7, epsilon=None, decay=0.2)
model.compile(optimizer=optimizer, loss="mse", metrics=["mae"])
# Train the model.
history = model.fit_generator(generator_train,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=generator_validate,
validation_steps=validation_steps,
callbacks=[tensorboard_callback])
histories["rgbnet"] = history
modelutils.save_model_and_history(output_path, model, history,
training_details, "rgbnet")
def build_attn_with_layer(seq, controller, layer, cell_size=300):
""" Build attention mechanism in computation graph. """
controller_repeated = layers.RepeatVector(20)(controller)
controller_repeated = layer(controller_repeated)
attention = layers.Lambda(my_dot, output_shape=(20,))([controller_repeated, seq])
attention_s = layers.Flatten()(attention)
attention = layers.Lambda(to_prob, output_shape=(20,))(attention_s)
attention_repeated = layers.RepeatVector(cell_size)(attention)
attention_repeated = layers.Permute((2, 1))(attention_repeated)
weighted = layers.merge([attention_repeated, seq], mode='mul')
summed = layers.Lambda(sum_seq, output_shape=(cell_size,))(weighted)
return summed, attention
def se_block(tensor, ratio=8):
n_channels = K.int_shape(tensor)[-1]
se_feature = layers.GlobalAveragePooling2D()(tensor)
se_feature = layers.Reshape((1, 1, n_channels))(se_feature)
assert K.int_shape(se_feature)[1:] == (1, 1, n_channels)
se_feature = layers.Dense(n_channels // ratio,
activation='relu')(se_feature)
assert K.int_shape(se_feature)[1:] == (1, 1, n_channels // ratio)
se_feature = layers.Dense(n_channels, activation='sigmoid')(se_feature)
assert K.int_shape(se_feature)[1:] == (1, 1, n_channels)
if K.image_data_format() == 'channels_first':
se_feature = layers.Permute((3, 1, 2))(se_feature)
se_feature = layers.multiply([tensor, se_feature])
return se_feature
def call(self, inputs, training=None):
#inflate and permute input tensor (num/dim_capsule axis fipped)
inflate = layers.Reshape((self.h_image, self.w_image, self.input_dim_capsule, self.input_num_capsule))
conv_out = layers.Permute((1, 2, 4, 3))(inflate(inputs))
pose_in = layers.Reshape((self.h_image, self.w_image, self.input_num_capsule, self.h_capsule, self.input_w_capsule))(conv_out[..., :-1])
coeffs_in = K.expand_dims(K.expand_dims(conv_out[..., -1], axis=-1), axis=-1)
pose_votes = tf.einsum('abcmjk,bcmnij->abcmnik', pose_in, self.W)
# pose_votes.shape=[None, self.h_image, self.w_image, input_num_capsule, num_capsule, h_capsule, w_capsule]
assert self.h_capsule == int(pose_votes.shape[-2]), self.w_capsule == int(pose_votes.shape[-1])
votes_in = layers.Reshape((self.h_image, self.w_image, self.input_num_capsule, self.num_capsule, self.dim_capsule -1))(pose_votes)
poses, coeffs = routing_alg(votes_in, coeffs_in, self.bias, self.iters)
#deflate capsules for output
deflate = layers.Reshape((self.h_image, self.w_image, -1))
final = K.concatenate([deflate(poses), deflate(coeffs)], axis=-1)
return final
def build_branch_bilstm_position_am(emb):
"""
build attention model according position infomation
result not good.
"""
m_lstm = klayers.Bidirectional(
klayers.LSTM(args.nencoder, return_sequences=True,
trainable=True))(emb)
attention = klayers.TimeDistributed(klayers.Dense(
1, activation='tanh'))(m_lstm)
attention = klayers.Flatten()(attention)
attention = klayers.Activation('softmax')(attention)
attention = klayers.RepeatVector(args.ndecoder * 2)(attention)
attention = klayers.Permute([2, 1])(attention)
m_lstm_am = klayers.merge([m_lstm, attention], mode='mul')
m_lstm_am = klayers.Lambda(lambda xin: K.sum(xin, axis=1))(m_lstm_am)
return m_lstm_am
def call(self, inputs, training=None):
#inflate and permute input tensor (num/dim_capsule axis fipped)
inflate = layers.Reshape((self.h_image, self.w_image, self.input_dim_capsule, self.input_num_capsule))
conv_out = layers.Permute((1, 2, 4, 3))(inflate(inputs))
pose_in = layers.Reshape((self.h_image, self.w_image, self.input_num_capsule, self.h_capsule, self.input_w_capsule))(conv_out[..., :-1])
coeffs_in = K.expand_dims(K.expand_dims(conv_out[..., -1], axis=-1), axis=-1)
pose_votes = tf.einsum('abcmjk,bcmnij->abcmnik', pose_in, self.W)
# pose_votes.shape=[None, self.h_image, self.w_image, input_num_capsule, num_capsule, h_capsule, w_capsule]
assert self.h_capsule == int(pose_votes.shape[-2]), self.w_capsule == int(pose_votes.shape[-1])
votes_in = layers.Reshape((self.h_image, self.w_image, self.input_num_capsule, self.num_capsule, self.dim_capsule -1))(pose_votes)
aug_votes = self.coord_add(votes_in, self.h_image, self.w_image)
# collapsed_coeffs.shape=[None, 1, 1, self.h_image*self.w_image*input_num_capsule]
# collapsed_votes.shape=[None, 1, 1, self.h_image*self.w_image*input_num_capsule, num_capsule, h_capsule*w_capsule]
collapsed_coeffs = layers.Reshape((1, 1, self.h_image*self.w_image*self.input_num_capsule, 1, 1))(coeffs_in)
collapsed_votes = layers.Reshape((1, 1, self.h_image*self.w_image*self.input_num_capsule, self.num_capsule, \
self.h_capsule*self.w_capsule))(aug_votes)
poses, coeffs = routing_alg(collapsed_votes, collapsed_coeffs, self.bias, self.iters)
return tf.squeeze(coeffs, [1, 2, -3, -1])
def tdcnn(main_input, time):
pool_size1 = (1, 4, 2)
pool_size2 = (1, 2, 4)
activation_name = 'elu'
# Input
main_batch_norm = BatchNormalization()(main_input)
conv3d = Conv3D(kernel_size=(5, 4, 2), strides=(1, 1, 1), filters=128, padding='same')(main_input)
batch_norm = BatchNormalization()(conv3d)
conv = Activation(activation_name)(batch_norm)
up_sample1 = AveragePooling3D(pool_size=pool_size1, strides=(1, 1, 1), padding='same')(conv)
up_sample2 = AveragePooling3D(pool_size=pool_size2, strides=(1, 1, 1), padding='same')(conv)
conv_concat = concatenate([main_batch_norm, conv, up_sample1, up_sample2])
conv_1d = Conv3D(kernel_size=(1, 1, 1), strides=(1, 1, 1), filters=16)(conv_concat)
batch_norm = BatchNormalization()(conv_1d)
activation = Activation(activation_name)(batch_norm)
bidir_rnn = Reshape((time, -1))(activation)
bidir_rnn1 = Bidirectional(LSTM(100, dropout=0.5, recurrent_dropout=0.5, return_sequences=True))(bidir_rnn)
bidir_rnn2 = Bidirectional(LSTM(100, dropout=0.5, recurrent_dropout=0.5, return_sequences=True))(bidir_rnn1)
# bidir_rnn1 = Bidirectional(GRU(100, dropout=0.5, recurrent_dropout=0.5, return_sequences=True))(bidir_rnn)
# bidir_rnn2 = Bidirectional(GRU(100, dropout=0.5, recurrent_dropout=0.5, return_sequences=True))(bidir_rnn1)
permute = Permute((2, 1))(bidir_rnn2)
dense_1 = Dense(X_train.shape[1:], activation='softmax')(permute)
prob = layers.Permute((2, 1), name='attention_vec')(dense_1)
attention_mul = layers.multiply([main_input, prob])
attention_mul = layers.Flatten()(attention_mul)
flat = Flatten()(attention_mul)
drop = Dropout(0.5)(flat)
dense_2 = Dense(200)(drop)
main_output = Dense(2, activation='softmax')(dense_2)
return main_output
def squeeze_excite_block(input, ratio=16):
init = input
channel_axis = 1 if K.image_data_format() == "channels_first" else -1
filters = init._keras_shape[channel_axis]
se_shape = (1, 1, filters)
se = layers.GlobalAveragePooling2D()(init)
se = layers.Reshape(se_shape)(se)
se = layers.Dense(filters // ratio,
activation='relu',
kernel_initializer='he_normal',
use_bias=False)(se)
se = layers.Dense(filters,
activation='sigmoid',
kernel_initializer='he_normal',
use_bias=False)(se)
if K.image_data_format() == 'channels_first':
se = layers.Permute((3, 1, 2))(se)
x = layers.multiply([init, se])
return x
def build_models(input_shape, actions):
cnn_model = models.Sequential()
cnn_model.add(layers.Permute((2, 3, 1), input_shape=input_shape))
cnn_model.add(
layers.Convolution2D(32, (8, 8), strides=(4, 4), activation="relu"))
cnn_model.add(
layers.Convolution2D(64, (4, 4), strides=(2, 2), activation="relu"))
cnn_model.add(
layers.Convolution2D(64, (3, 3), strides=(1, 1), activation="relu"))
cnn_model.add(layers.Flatten())
v_model_input = layers.Input(shape=input_shape)
v_model_output = cnn_model(v_model_input)
v_model_output = layers.Dense(512, activation="relu")(v_model_output)
v_model_output = layers.Dense(1, activation="linear")(v_model_output)
v_model = models.Model(v_model_input, v_model_output)
mu_model_input = layers.Input(shape=input_shape)
mu_model_output = cnn_model(mu_model_input)
mu_model_output = layers.Dense(512, activation="relu")(mu_model_output)
mu_model_output = layers.Dense(actions,
activation="linear")(mu_model_output)
mu_model = models.Model(mu_model_input, mu_model_output)
l_model_action_input = layers.Input(shape=(actions, ))
l_model_action_output = l_model_action_input
l_model_observation_input = layers.Input(shape=input_shape)
l_model_observation_output = cnn_model(l_model_observation_input)
l_model_output = layers.Concatenate()(
[l_model_action_output, l_model_observation_output])
l_model_output = layers.Dense(512, activation="relu")(l_model_output)
l_model_output = layers.Dense(((actions * actions + actions) // 2),
activation="linear")(l_model_output)
l_model = models.Model([l_model_action_input, l_model_observation_input],
l_model_output)
return v_model, mu_model, l_model
def model_CNN(embedding_matrix, max_sent_len, n_out):
print(config.params_dict)
# Take sentence encoded as indices split in three parts and convert it to embeddings
sentence_input = layers.Input(shape=(max_sent_len,), dtype='int32', name='sentence_input')
word_embeddings = layers.Embedding(output_dim=embedding_matrix.shape[1],
input_dim=embedding_matrix.shape[0],
input_length=max_sent_len, weights=[embedding_matrix],
mask_zero=True, trainable=False)(sentence_input)
word_embeddings = layers.Dropout(config.Params.dropout1)(word_embeddings)
# Take token markers that identify entity positions, convert to position embeddings
entity_markers = layers.Input(shape=(2, max_sent_len,), dtype='int8', name='entity_markers')
pos_embeddings = layers.wrappers.TimeDistributed(
layers.Embedding(output_dim=config.Params.position_emb, input_dim=(max_sent_len * 2) + 1, input_length=max_sent_len,
mask_zero=False, embeddings_regularizer=regularizers.l2(), trainable=True),
name='pos_embedding')(entity_markers)
pos_embeddings = layers.Permute((2, 1, 3))(pos_embeddings)
pos_embeddings = layers.Reshape((max_sent_len, config.Params.position_emb * 2))(pos_embeddings)
# Merge word and position embeddings and apply the specified amount of CNN layers
x = layers.concatenate([word_embeddings, pos_embeddings])
x = MaskedConvolution1D(nb_filter=config.Params.units1, filter_length=config.Params.window_size, border_mode='same')(x)
sentence_vector = MaskedGlobalMaxPooling1D()(x)
sentence_vector = layers.Lambda(lambda l: K.tanh(l))(sentence_vector)
# Apply softmax
sentence_vector = layers.Dropout(config.Params.dropout1)(sentence_vector)
main_output = layers.Dense(n_out, activation="softmax", name='main_output')(sentence_vector)
model = models.Model(input=[sentence_input, entity_markers], output=[main_output])
model.compile(optimizer=config.Params.optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
return model
def buildAttention(self, seq, controller):
controller_repeated = layers.RepeatVector(
self._config['context_window_size'])(controller)
controller_repeated = layers.TimeDistributedDense(
self._w2v.conceptEmbeddingsSz)(controller_repeated)
attention = layers.Lambda(
my_dot, output_shape=(self._config['context_window_size'], ))(
[controller_repeated, seq])
attention = layers.Flatten()(attention)
attention = layers.Lambda(
to_prob,
output_shape=(self._config['context_window_size'], ))(attention)
attention_repeated = layers.RepeatVector(
self._w2v.conceptEmbeddingsSz)(attention)
attention_repeated = layers.Permute((2, 1))(attention_repeated)
weighted = layers.merge([attention_repeated, seq], mode='mul')
summed = layers.Lambda(
sum_seq, output_shape=(self._w2v.conceptEmbeddingsSz, ))(weighted)
return summed, attention
def tensor_product_vector(y_coords, x_coords, side_size):
'''
Hardcoded helper function to calculate the Cartesian Product
or Tensor Product, as there does not exist a TensorDot layer
in Keras as of writing.
We achieve the TensorDot by duplicating the values into a new dimension
and muliply the tensors.
'''
# Duplicates vertically, so coords needs transposing.
y_coords_dup = Duplicate(side_size, slice_axis=-2,
name="duplicator_y")(y_coords)
# Transpose the Y tensor
y_coords_dup = Kl.Permute((1, 3, 2))(y_coords_dup)
x_coords_dup = Duplicate(side_size, slice_axis=-2,
name="duplicator_x")(x_coords)
# Multiplying to obtain single coordinate.
tensor_product = Kl.multiply([y_coords_dup, x_coords_dup])
return tensor_product
def create_model(input_shape, n_class, n_instance, n_part, routings):
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param n_instance: number of instance of each class
:param n_part: number of parts in each instance
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
`eval_model` can also be used for training.
"""
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=32, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1, dim_capsule_attr=1, num_capsule=32, kernel_size=9, strides=2, padding='valid')
# Layer 3: Capsule layer. Attention algorithm works here.
digitcaps = CAN(num_capsule=n_class, dim_capsule_attr=1, routings=routings, num_instance=n_instance, num_part=n_part,
name='digitcaps')(primarycaps)
# Layer 4: Convert capsule probabilities to a classification
out_caps = Lambda(lambda x: x[:, :, :, 0],name='select_probability')(digitcaps)
out_caps = layers.Permute([2, 1], name='capsnet')(out_caps) # for clasification we swap order to be instance,class
# Capture the pose
out_pose = Lambda(lambda x: x[:, :, :, 1:1+canlayer.dim_geom],name='select_pose')(digitcaps)
# Models for training and evaluation (prediction)
model = models.Model([x], [out_caps,out_pose])
return model #
def create_q_model(window, n_actions):
'''
Creates a DQN model.
Args.
window: DQN inputs.
n_actrions: DQN outputs.
Returns.
The DQN model.
'''
inputs = layers.Input(shape=(window, 84, 84))
# rearranges layers
dqn_layers = layers.Permute((2, 3, 1))(inputs)
dqn_layers = layers.Conv2D(
filters=16,
activation='relu',
kernel_size=8,
strides=(4, 4),
)(dqn_layers)
dqn_layers = layers.Conv2D(
filters=32,
activation='relu',
kernel_size=4,
strides=(2, 2),
)(dqn_layers)
dqn_layers = layers.Conv2D(
filters=64,
activation='relu',
kernel_size=2,
)(dqn_layers)
dqn_layers = layers.Flatten()(dqn_layers)
dqn_layers = layers.Dense(512, activation="relu")(dqn_layers)
dqn_layers = layers.Dense(256, activation="relu")(dqn_layers)
outputs = layers.Dense(n_actions, activation="linear")(dqn_layers)
return Model(inputs, outputs)
def _create_Kao_Rnet(self, weight_path='./models/mtcnn/24net.h5'):
'''
'''
input = KL.Input(shape=[24, 24, 3]) # change this shape to [None,None,3] to enable arbitraty shape input
x = KL.Conv2D(28, (3, 3), strides=1, padding='valid', name='conv1', data_format="channels_last")(input)
x = KL.PReLU(shared_axes=[1, 2], name='prelu1')(x)
x = KL.MaxPool2D(pool_size=3, strides=2, padding='same', data_format="channels_last")(x)
x = KL.Conv2D(48, (3, 3), strides=1, padding='valid', name='conv2', data_format="channels_last")(x)
x = KL.PReLU(shared_axes=[1, 2], name='prelu2')(x)
x = KL.MaxPool2D(pool_size=3, strides=2, data_format="channels_last")(x)
x = KL.Conv2D(64, (2, 2), strides=1, padding='valid', name='conv3', data_format="channels_last")(x)
x = KL.PReLU(shared_axes=[1, 2], name='prelu3')(x)
x = KL.Permute((3, 2, 1))(x)
x = KL.Flatten()(x)
x = KL.Dense(128, name='conv4')(x)
x = KL.PReLU(name='prelu4')(x)
classifier = KL.Dense(2, activation='softmax', name='conv5-1')(x)
bbox_regress = KL.Dense(4, name='conv5-2')(x)
model = Model([input], [classifier, bbox_regress])
model.load_weights(weight_path, by_name=True)
return model
def my_model(num_actions, window):
"""
model to use in the deep Q_learning process
We use the same model that was described by
Mnih et al. (2015).
"""
# change sequencial model to input style
input = layers.Input(shape=(window, 84, 84))
process_input = layers.Permute((2, 3, 1))(input)
layer1 = layers.Conv2D(32,
8,
strides=4,
activation="relu",
data_format="channels_last")(process_input)
layer2 = layers.Conv2D(64,
4,
strides=2,
activation="relu",
data_format="channels_last")(layer1)
layer3 = layers.Conv2D(64,
3,
strides=1,
activation="relu",
data_format="channels_last")(layer2)
layer4 = layers.Flatten()(layer3)
layer5 = layers.Dense(512, activation="relu")(layer4)
nb_actions = layers.Dense(num_actions, activation="linear")(layer5)
model = K.Model(inputs=input, outputs=nb_actions)
return model
def __call__(self, inputs):
"""CNN block and flattening to time distributed layer from
the original image of shape ( height, width, 3) will be mapped to (width/4, 128)"""
#transpose because the "time axis" is width
inputs = layers.Permute(dims=(2, 1, 3))(inputs)
# flatten the height and channels to one dimension, after this the dimension
# is batch, width, len(height)*len(channels)
encoder_features = layers.TimeDistributed(layers.Flatten())(inputs)
x = layers.Bidirectional(
layers.LSTM(self.rnn_hidden_size,
return_sequences=True,
use_bias=True,
recurrent_activation='sigmoid'))(encoder_features)
x = layers.TimeDistributed(layers.Dense(self.rnn_hidden_size))(x)
x = layers.Bidirectional(
layers.LSTM(self.rnn_hidden_size,
return_sequences=True,
use_bias=True,
recurrent_activation='sigmoid'))(x)
# the +1 stands for a blank character needed in CTC
fc = layers.TimeDistributed(layers.Dense(self.char_num + 1),
name="DenseSoftmax")(x)
return fc
def model(self, train):
Input = layers.Input(shape=(self.time_step, 1))
lstm = layers.Bidirectional(
layers.LSTM(self.units,
return_sequences=True,
recurrent_initializer='truncated_normal'))(Input)
attention = layers.TimeDistributed(layers.Dense(
1, activation='tanh'))(lstm)
attention = layers.Flatten()(attention)
attention = layers.Activation('softmax')(attention)
attention = layers.RepeatVector(self.units * 2)(attention)
attention = layers.Permute([2, 1])(attention)
#lstm = layers.Bidirectional(layers.LSTM(self.units,return_sequences = True,recurrent_initializer='truncated_normal'))(attention)
sent_representation = layers.Multiply()([lstm, attention])
sent_representation = layers.Lambda(
lambda xin: K.sum(xin, axis=-2),
output_shape=(self.units * 2, ))(sent_representation)
Wh = layers.Dense(self.units)(sent_representation)
# self.Ws = tf.keras.layers.Dense(self.units)
# self.Wx = tf.keras.layers.Dense(1)
# self.V = tf.keras.layers.Dense(1)
O = layers.Dense(self.output_size)(Wh)
model = keras.models.Model(inputs=Input, outputs=O)
if train == False:
model = keras.models.load_model('model/model.h5')
optimizer = keras.optimizers.Adam(0.0001)
model.compile(loss='mse', optimizer=optimizer)
return model
def test_permute(self):
x = Normal(loc=tf.zeros([100, 10, 5]), scale=tf.ones([100, 10, 5]))
y = layers.Permute((2, 1))(x.value())
with self.test_session():
self.assertEqual(y.eval().shape, (100, 5, 10))
class RetroProcessor(Processor):
def process_observation(self, obs):
img = Image.fromarray(obs)
img = img.resize(INPUT_SHAPE).convert('L')
return np.array(img, dtype=np.uint8)
def process_state_batch(self, batch):
return batch.astype(np.float32) / 255.
def process_action(self, action):
return actual_actions[action]
x = input_layer = layers.Input((1, ) + INPUT_SHAPE)
x = layers.Permute((2, 3, 1))(x)
# weights = K.constant( [ [ [ [0.21 , 0.72 , 0.07] ] ] ] )
# grayscale = K.sum( x * weights, axis=-1, keepdims=True )
# x = grayscale
x = layers.Conv2D(filters=32, kernel_size=(8, 8), strides=4,
activation='relu')(x)
x = layers.Conv2D(filters=64, kernel_size=(4, 4), strides=2,
activation='relu')(x)
x = layers.Conv2D(filters=64, kernel_size=(3, 3), strides=1,
activation='relu')(x)
x = layers.Flatten()(x)
def build_model(max_encoder_seq_length, max_decoder_seq_length,
num_encoder_tokens, num_decoder_tokens, latent_dim):
# 인코더 생성
encoder_inputs = layers.Input(shape=(max_encoder_seq_length,
num_encoder_tokens))
encoder = layers.GRU(latent_dim, return_sequences=True, return_state=True)
encoder_outputs, state_h = encoder(encoder_inputs)
# 디코더 생성.
decoder_inputs = layers.Input(shape=(max_decoder_seq_length,
num_decoder_tokens))
decoder = layers.GRU(latent_dim, return_sequences=True, return_state=True)
decoder_outputs, _ = decoder(decoder_inputs, initial_state=state_h)
# 어텐션 매커니즘.
repeat_d_layer = RepeatVectorLayer(max_encoder_seq_length, 2)
repeat_d = repeat_d_layer(decoder_outputs)
repeat_e_layer = RepeatVectorLayer(max_decoder_seq_length, 1)
repeat_e = repeat_e_layer(encoder_outputs)
concat_for_score_layer = layers.Concatenate(axis=-1)
concat_for_score = concat_for_score_layer([repeat_d, repeat_e])
dense1_t_score_layer = layers.Dense(latent_dim // 2, activation='tanh')
dense1_score_layer = layers.TimeDistributed(dense1_t_score_layer)
dense1_score = dense1_score_layer(concat_for_score)
dense2_t_score_layer = layers.Dense(1)
dense2_score_layer = layers.TimeDistributed(dense2_t_score_layer)
dense2_score = dense2_score_layer(dense1_score)
dense2_score = layers.Reshape(
(max_decoder_seq_length, max_encoder_seq_length))(dense2_score)
softmax_score_layer = layers.Softmax(axis=-1)
softmax_score = softmax_score_layer(dense2_score)
repeat_score_layer = RepeatVectorLayer(latent_dim, 2)
repeat_score = repeat_score_layer(softmax_score)
permute_e = layers.Permute((2, 1))(encoder_outputs)
repeat_e_layer = RepeatVectorLayer(max_decoder_seq_length, 1)
repeat_e = repeat_e_layer(permute_e)
attended_mat_layer = layers.Multiply()
attended_mat = attended_mat_layer([repeat_score, repeat_e])
context_layer = layers.Lambda(lambda x: K.sum(x, axis=-1),
lambda x: tuple(x[:-1]))
context = context_layer(attended_mat)
concat_context_layer = layers.Concatenate(axis=-1)
concat_context = concat_context_layer([context, decoder_outputs])
attention_dense_output_layer = layers.Dense(latent_dim, activation='tanh')
attention_output_layer = layers.TimeDistributed(
attention_dense_output_layer)
attention_output = attention_output_layer(concat_context)
decoder_dense = layers.Dense(num_decoder_tokens, activation='softmax')
decoder_outputs = decoder_dense(attention_output)
# 모델 생성
model = models.Model([encoder_inputs, decoder_inputs], decoder_outputs)
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['acc'])
model.summary()
return model
