def FTA_Module(x, shape, kt, kf):
x = BatchNormalization()(x)
## Residual
x_r = Conv2D(shape[2], (1, 1), padding='same', activation='relu')(x)
## Time Attention
# Attn Map (1, T, C), FC
a_t = Lambda(K.mean, arguments={'axis': -3})(x)
a_t = Conv1D(shape[2], kt, padding='same', activation='selu')(a_t)
a_t = Conv1D(shape[2], kt, padding='same', activation='selu')(a_t) #2
a_t = Softmax(axis=-2)(a_t)
a_t = Reshape((1, shape[1], shape[2]))(a_t)
# Reweight
x_t = Conv2D(shape[2], (3, 3), padding='same', activation='selu')(x)
x_t = Conv2D(shape[2], (5, 5), padding='same', activation='selu')(x_t)
x_t = Multiply()([x_t, a_t])
###TODO:参考Strip pooling，将两个map相乘后再使用
# Frequency Attention
# Attn Map (F, 1, C), Conv1D
a_f = Lambda(K.mean, arguments={'axis': -2})(x)
a_f = Conv1D(shape[2], kf, padding='same', activation='selu')(a_f)
a_f = Conv1D(shape[2], kf, padding='same', activation='selu')(a_f)
a_f = Softmax(axis=-2)(a_f)
a_f = Reshape((shape[0], 1, shape[2]))(a_f)
# Reweight
x_f = Conv2D(shape[2], (3, 3), padding='same', activation='selu')(x)
x_f = Conv2D(shape[2], (5, 5), padding='same', activation='selu')(x_f)
x_f = Multiply()([x_f, a_f])
x = Add()([x_r, x_t, x_f])
return x
def make_ner_model(embedding_tensor, words_vocab_size, tags_vocab_size,
num_hidden_units=128, attention_units=64):
EMBEDDING_DIM = embedding_tensor.shape[1]
words_input = Input(dtype='int32', shape=[MAX_SEQUENCE_LENGTH])
x = Embedding(words_vocab_size + 1,
EMBEDDING_DIM,
weights=[embedding_tensor],
input_length=MAX_SEQUENCE_LENGTH,
trainable=False)(words_input)
outputs = GRU(num_hidden_units,
return_sequences=True,
dropout=0.5,
name='RNN_Layer')(x)
# Simple attention
hidden_layer = Dense(attention_units, activation='tanh')(outputs)
hidden_layer = Dropout(0.25)(hidden_layer)
hidden_layer = Dense(1, activation=None)(hidden_layer)
hidden_layer = Flatten()(hidden_layer)
attention_vector = Softmax(name='attention_vector')(hidden_layer)
attention = RepeatVector(num_hidden_units)(attention_vector)
attention = Permute([2, 1])(attention)
encoding = Multiply()([outputs, attention])
encoding = Dropout(0.25)(encoding)
ft1 = Dense(num_hidden_units)(encoding)
ft1 = Dropout(0.25)(ft1)
ft2 = Dense(tags_vocab_size)(ft1)
out = Softmax(name='Final_Sofmax')(ft2)
model = Model(inputs=words_input, outputs=out)
return model
def _build_model(self):
# Neural Net for Deep-Q learning
input_ = Input(shape=(self.board_size,self.board_size,5))
print("input:",input_.shape)
red = Maximum()([input_[:,:,:,0],input_[:,:,:,1]])
print("red:",red.shape)
#blue = Maximum()([input_[:,:,2],input_[:,:,3]])
conv_1 = Conv2D(32, kernel_size=3, activation='relu')(input_)
conv_2 = Conv2D(32, kernel_size=3, activation='relu')(conv_1)
deconv_3 = Conv2DTranspose(32, kernel_size=3, activation='relu')(conv_2)
deconv_4 = Conv2DTranspose(32, kernel_size=3, activation='relu')(deconv_3)
flatten_1 = Flatten()(deconv_4)
print("flatten:", flatten_1.shape)
dense_6a = Dense(self.board_size**2, activation='relu')(flatten_1[:,0:self.board_size**2])
dense_6b = Dense(self.board_size**2, activation='relu')(flatten_1[:,self.board_size**2:])
print("dense:", dense_6b.shape)
reshape_7a = Reshape((self.board_size, self.board_size))(dense_6a)
print("reshape a:", reshape_7a.shape)
reshape_7b = Reshape((self.board_size, self.board_size,1))(dense_6b)
min_a = Minimum()([reshape_7a, red])
flatten_2a = Flatten()(min_a)
softmax_a = Softmax(axis=1)(flatten_2a)
flatten_2b = Flatten()(reshape_7b)
softmax_b = Softmax(axis=1)(flatten_2b)
reshape_a = Reshape((self.board_size, self.board_size,1))(softmax_a)
reshape_b = Reshape((self.board_size, self.board_size,1))(softmax_b)
concat = Concatenate(axis=-1)([reshape_a, reshape_b])
model = Model(inputs=input_, outputs=concat)
model.compile(loss='mse', optimizer=Adam(lr=self.learning_rate))
return model
def MINI_MTL(inputs, filters, numClasses, i):
x_edge = RA(inputs, inputs, filters)
x_mask = RA(inputs, inputs, filters)
x_edge = Conv3D(filters, (3, 3, 3), padding='same')(x_edge)
x_edge = BatchNormalization(axis=-1)(x_edge)
x_edge = Activation('relu')(x_edge)
x_mask = Conv3D(filters, (3, 3, 3), padding='same')(x_mask)
x_mask = BatchNormalization(axis=-1)(x_mask)
x_mask = Activation('relu')(x_mask)
out_edge = Conv3D(numClasses - 1, (1, 1, 1), padding='same')(x_edge)
out_edge = UpSampling3D(pow(2, i))(out_edge)
out_edge = Softmax(axis=-1, dtype='float32',
name='out_edge_{}'.format(i))(out_edge)
out_mask = Conv3D(numClasses, (1, 1, 1), padding='same')(x_mask)
out_mask = UpSampling3D(pow(2, i))(out_mask)
out_mask = Softmax(axis=-1, dtype='float32',
name='out_mask_{}'.format(i))(out_mask)
# out_mtl = Add()([x_mask, x_edge])
out_mtl = Concatenate()([x_mask, x_edge])
out_mtl = Conv3D(filters, (1, 1, 1), padding='same')(out_mtl)
return out_mtl, out_edge, out_mask
def Linear(num_outputs=2, input_shape=(60, 160, 3), drop=0.2, l4_stride=1):
"""
:param img_in: input layer of network
:param drop: dropout rate
:param l4_stride: 4-th layer stride, default 1
"""
inputs = Input(shape=input_shape, name='img_in')
x = conv2d_relu(inputs, 16, 3, 1, 0)
x = Dropout(drop)(x)
x = conv2d_relu(x, 32, 3, 2, 1)
x = Dropout(drop)(x)
x = conv2d_relu(x, 32, 3, 2, 2)
x = Dropout(drop)(x)
x = conv2d_relu(x, 32, 3, 2, 3)
x = Dropout(drop)(x)
x = conv2d_relu(x, 64, 3, l4_stride, 4)
x = Dropout(drop)(x)
x = conv2d_relu(x, 16, 1, 1, 5)
x = Dropout(drop)(x)
x = Flatten(name='flattened')(x)
x = Dense(16, name='dense_1')(x)
x = ReLU()(x)
x = Dropout(drop)(x)
outputs = []
_x = Dense(16, name='throttle')(x)
_x = Softmax()(_x)
outputs.append(_x)
_x = Dense(20, name='steer')(x)
_x = Softmax()(_x)
outputs.append(_x)
model = Model(inputs=inputs, outputs=outputs)
return model
def get_layers(model_type,
nclasses=10,
hidden_layer_1_neurons=400,
hidden_layer_2_neurons=100,
dropout_rate=0.25):
"""
Construct layers for keras model based on a dict of model types.
"""
model_layers = {
'linear': [Flatten(), Dense(nclasses),
Softmax()],
'dnn': [
Flatten(),
Dense(hidden_layer_1_neurons, activation='relu'),
Dense(hidden_layer_2_neurons, activation='relu'),
Dense(nclasses),
Softmax()
],
'dnn_dropout': [
Flatten(),
Dense(hidden_layer_1_neurons, activation='relu'),
Dense(hidden_layer_2_neurons, activation='relu'),
Dropout(dropout_rate),
Dense(nclasses),
Softmax()
]
}
return model_layers[model_type]
def build_MINI_MTL(input_shape, filters, numClasses, i):
input_layer = Input(shape=(input_shape, input_shape, input_shape, filters))
x_edge = RA(input_layer, input_layer, filters)
x_mask = RA(input_layer, input_layer, filters)
x_edge = Conv3D(filters, (3, 3, 3), padding='same')(x_edge)
x_edge = BatchNormalization(axis=-1)(x_edge)
x_edge = Activation('relu')(x_edge)
x_mask = Conv3D(filters, (3, 3, 3), padding='same')(x_mask)
x_mask = BatchNormalization(axis=-1)(x_mask)
x_mask = Activation('relu')(x_mask)
out_edge = Conv3D(numClasses, (1, 1, 1), padding='same')(x_edge)
out_edge = Softmax(axis=-1)(out_edge)
out_edge = UpSampling3D(pow(2, i), name='out_edge_{}'.format(i))(out_edge)
out_mask = Conv3D(numClasses, (1, 1, 1), padding='same')(x_mask)
out_mask = Softmax(axis=-1)(out_mask)
out_mask = UpSampling3D(pow(2, i), name='out_mask_{}'.format(i))(out_mask)
out_mtl = Concatenate()([x_mask, x_edge])
out_mtl = Conv3D(filters, (1, 1, 1), padding='same')(out_mtl)
mtl_model = Model(inputs=[input_layer], outputs=[out_edge, out_mask])
return mtl_model, out_mtl
def keras_model(action_dim, z_dim):
from tensorflow.keras.layers import Conv2D, Flatten, Dense, Input, Dot, Reshape, Softmax
from beta_regularizer import BetaRegularization
s = Input(shape=(110, 84, 1), name='input_s')
z = Input(shape=(z_dim,), name='input_z')
conv1 = Conv2D(16, (8, 8), strides=(4, 4), activation='relu')(s)
conv2 = Conv2D(32, (4, 4), strides=(2, 2), activation='relu')(conv1)
conv2_flat = Flatten()(conv2)
h1 = Dense(256, activation='relu')(conv2_flat)
# Predict the next latent
h2_z = Dense(z_dim * z_dim, activation=None)(h1) # (B x Z * Z)
h2_z_reshaped = Reshape((z_dim, z_dim))(h2_z) # (B x Z x Z)
z_tp1_matrix = Softmax(axis=-1, name='latent_matrix')(h2_z_reshaped)
z_tp1 = Dot(axes=1, name='latent')([z_tp1_matrix, z])
# Predict the next action
h2_a = Dense(action_dim * z_dim, activation=None)(h1) # (B x A * Z)
h2_a_reshaped = Reshape((z_dim, action_dim))(h2_a) # (B x Z x A)
a_matrix = Softmax(axis=-1, name='action_matrix')(h2_a_reshaped)
a = Dot(axes=1, name='action')([a_matrix, z])
# Predict termination
h2_b = Dense(z_dim * 2, activation=None)(h1) # (B x 2 * Z)
h2_b_reshaped = Reshape((z_dim, 2))(h2_b) # (B x Z x 2)
b_matrix = Softmax(axis=-1, name='termination_matrix')(h2_b_reshaped)
b_matrix = BetaRegularization(1.0, 99.)(b_matrix)
b = Dot(axes=1, name='termination')([b_matrix, z])
return tf.keras.Model(inputs=[s, z], outputs=[a, z_tp1, b])
def multitask_attention_model(output_size,
pos_vocab_size,
lex_vocab_size,
config_params,
visualize=False,
plot=False):
hidden_size = int(config_params['hidden_size'])
batch_size = int(config_params['batch_size'])
embedding_size = 768
max_seq_len = 512
in_id = Input(shape=(max_seq_len, ), name="input_ids")
in_mask = Input(shape=(max_seq_len, ), name="input_masks")
in_segment = Input(shape=(max_seq_len, ), name="segment_ids")
bert_inputs = [in_id, in_mask, in_segment]
bert_output_ = BertEmbeddingLayer(n_fine_tune_layers=3,
pooling="mean")(bert_inputs)
bert_output = Reshape((max_seq_len, embedding_size))(bert_output_)
in_mask = Input(shape=(None, output_size),
batch_size=batch_size,
name='Candidate_Synsets_Mask')
bert_inputs.append(in_mask)
bilstm = Bidirectional(LSTM(hidden_size,
dropout=0.2,
recurrent_dropout=0.2,
return_sequences=True,
input_shape=(None, None, embedding_size)),
merge_mode='sum')(bert_output)
attention = SeqSelfAttention(units=128,
attention_activation='sigmoid',
name='Attention')(bilstm)
logits = TimeDistributed(Dense(output_size))(attention)
logits_mask = Add()([logits, in_mask])
pos_logits = TimeDistributed(Dense(pos_vocab_size),
name='POS_logits')(attention)
lex_logits = TimeDistributed(Dense(lex_vocab_size),
name='LEX_logits')(attention)
wsd_output = Softmax(name="WSD_output")(logits_mask)
pos_output = Softmax(name="POS_output")(pos_logits)
lex_output = Softmax(name="LEX_output")(lex_logits)
model = Model(inputs=bert_inputs,
outputs=[wsd_output, pos_output, lex_output],
name='Bert_BiLSTM_ATT_MultiTask')
model.compile(loss="sparse_categorical_crossentropy",
optimizer=Adadelta(),
metrics=['acc'])
visualize_plot_mdl(visualize, plot, model)
return model
def get_genotype(self, random_search_flag=False):
"""get genotype"""
def _parse(weights, edge_weights,random_search_flag=False):
n = 2
start = 0
gene = []
for i in range(self.steps):
end = start + n
w = weights[start:end].copy()
ew = edge_weights[start:end].copy()
# fused weights
for j in range(n):
w[j, :] = w[j, :] * ew[j]
if random_search_flag==False:
# pick the top 2 edges (k = 2).
edges = sorted(
range(i + 2),
key=lambda x: -max(w[x][k] for k in range(len(w[x]))
if k != PRIMITIVES.index('none'))
)[:2]
else:
# Randomly set the edges so that it could get the genotype
rng = np.random.default_rng()
# TODO : release the constraint of only two precedents
edges = rng.choice(range(i+1),2)
# print(f"edges = {edges}")
# pick the top best op, and append into genotype.
# This is used to avoid the none edges
for j in edges:
k_best = None
for k in range(len(w[j])):
if k != PRIMITIVES.index('none'):
if k_best is None or w[j][k] > w[j][k_best]:
k_best = k
gene.append((PRIMITIVES[k_best], j))
start = end
n += 1
# print(f'gene is {gene}')
return gene
gene_reduce = _parse(
Softmax()(self.alphas_reduce).numpy(),
SplitSoftmax(range(2, 2 + self.steps))(self.betas_reduce).numpy(),
random_search_flag=random_search_flag)
gene_normal = _parse(
Softmax()(self.alphas_normal).numpy(),
SplitSoftmax(range(2, 2 + self.steps))(self.betas_normal).numpy(),
random_search_flag=random_search_flag)
concat = range(2 + self.steps - self.multiplier, self.steps + 2)
genotype = Genotype(normal=gene_normal, normal_concat=concat,
reduce=gene_reduce, reduce_concat=concat)
return genotype
def attention_model(vocabulary_size, config_params,
output_size, pos_vocab_size,
lex_vocab_size, visualize=False,
plot=False, tokenizer=None):
hidden_size = int(config_params['hidden_size'])
batch_size = int(config_params['batch_size'])
input_type = 'string' if tokenizer is not None else None
in_sentences = Input(shape=(None,), dtype=input_type,
batch_size=batch_size)
if tokenizer is not None:
embedding = ElmoEmbeddingLayer()(in_sentences)
embedding_size = 1024
else:
embedding_size = int(config_params['embedding_size'])
embedding = Embedding(input_dim=vocabulary_size,
output_dim=embedding_size,
mask_zero=True,
name="Embeddings")(in_sentences)
bilstm = Bidirectional(LSTM(hidden_size, dropout=0.2,
recurrent_dropout=0.2,
return_sequences=True,
input_shape=(None, None, embedding_size)
),
merge_mode='sum')(embedding)
attention = SeqSelfAttention(attention_activation='sigmoid',
name='Attention')(bilstm)
logits = TimeDistributed(Dense(output_size))(attention)
in_mask = Input(shape=(None, output_size), batch_size=batch_size,
name='Candidate_Synsets_Mask')
logits_mask = Add()([logits, in_mask])
pos_logits = TimeDistributed(Dense(pos_vocab_size),
name='POS_logits')(attention)
lex_logits = TimeDistributed(Dense(lex_vocab_size),
name='LEX_logits')(attention)
wsd_output = Softmax(name="WSD_output")(logits_mask)
pos_output = Softmax(name="POS_output")(pos_logits)
lex_output = Softmax(name="LEX_output")(lex_logits)
model = Model(inputs=[in_sentences, in_mask],
outputs=[wsd_output, pos_output, lex_output],
name='BiLSTM_ATT_MultiTask')
model.compile(loss="sparse_categorical_crossentropy",
optimizer=Adadelta(), metrics=['acc'])
visualize_plot_mdl(visualize, plot, model)
return model
def __init__(self,
picture_shape,
action_size,
hidden_size=128,
learning_rate=3e-4):
super(SAC__Policy, self).__init__(picture_shape, hidden_size)
self.d1 = Dense(units=hidden_size, activation='relu', dtype=tf.float32)
self.d3 = Dense(units=action_size, dtype=tf.float32)
self.soft_max = Softmax(axis=1)
self.soft_max_gumbel = Softmax(axis=1)
self.optimizer = Adam(learning_rate=learning_rate)
def get_layers(model_type,
nclasses=10,
hidden_layer_1_neurons=400,
hidden_layer_2_neurons=100,
dropout_rate=0.25,
num_filters_1=64,
kernel_size_1=3,
pooling_size_1=2,
num_filters_2=32,
kernel_size_2=3,
pooling_size_2=2):
"""
Construct layers for keras model based on a dict of model types.
"""
model_layers = {
'linear': [Flatten(), Dense(nclasses),
Softmax()],
'dnn': [
Flatten(),
Dense(hidden_layer_1_neurons, activation='relu'),
Dense(hidden_layer_2_neurons, activation='relu'),
Dense(nclasses),
Softmax()
],
'dnn_dropout': [
Flatten(),
Dense(hidden_layer_1_neurons, activation='relu'),
Dense(hidden_layer_2_neurons, activation='relu'),
Dropout(dropout_rate),
Dense(nclasses),
Softmax()
],
'cnn': [
Conv2D(num_filters_1,
kernel_size=kernel_size_1,
activation='relu',
input_shape=(WIDTH, HEIGHT, 1)),
MaxPooling2D(pooling_size_1),
Conv2D(num_filters_2, kernel_size=kernel_size_2,
activation='relu'),
MaxPooling2D(pooling_size_2),
Flatten(),
Dense(hidden_layer_1_neurons, activation='relu'),
Dense(hidden_layer_2_neurons, activation='relu'),
Dropout(dropout_rate),
Dense(nclasses),
Softmax()
]
}
return model_layers[model_type]
def __init__(self,
config:BertConfig,
with_nsp=False,
with_mlm=False,
is_pretrain=False,
name="bert_model",
**kwargs):
super(BertModel, self).__init__(**kwargs)
self.config = config
self.with_nsp = with_nsp
self.with_mlm = with_mlm
self.is_pretrain = is_pretrain
self.embedding_layer = EmbeddingLayer(
vocab_size=config.vocab_size,
max_position_length=config.max_position_embeddings,
embedding_size=config.hidden_size,
type_vocab_size=config.type_vocab_size,
dropout_prob=config.hidden_dropout_prob,
stddev=config.initializer_range)
self.encoder_layers = [
Encoder(hidden_size=config.hidden_size,
num_attention_heads=config.num_attention_heads,
attention_probs_dropout_prob=config.attention_probs_dropout_prob,
hidden_dropout_prob=config.hidden_dropout_prob,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
initializer_range=config.initializer_range)
for _ in range(config.num_hidden_layers)
]
self.pooled_dense_layer = Dense(config.hidden_size,
activation="tanh",
kernel_initializer=create_initializer(config.initializer_range),
name="pooled_dense_layer")
if self.with_nsp:
self.nsp_dense_layer = Dense(2,
kernel_initializer=create_initializer(config.initializer_range),
name="nsp_dense_layer")
self.nsp_softmax_layer = Softmax(name="nsp_softmax_layer")
if self.with_mlm:
self.mlm_dense_layer = Dense(config.hidden_size,
activation=config.hidden_act,
kernel_initializer=create_initializer(config.initializer_range),
name="mlm_dense_layer")
self.mlm_layerNorm_layer = LayerNormalization(name="mlm_layerNorm_layer")
self.mlm_softmax_layer = Softmax(name="mlm_softmax_layer")
def get_genotype(self):
"""get genotype"""
def _parse(weights, edge_weights):
n = 2
start = 0
gene = []
for i in range(self.steps):
end = start + n
w = weights[start:end].copy()
ew = edge_weights[start:end].copy()
# fused weights
for j in range(n):
w[j, :] = w[j, :] * ew[j]
# pick the top 2 edges (k = 2).
edges = sorted(
range(i + 2),
key=lambda x: -max(w[x][k] for k in range(len(w[x]))
if k != PRIMITIVES.index('none')))[:2]
# pick the top best op, and append into genotype.
for j in edges:
k_best = None
for k in range(len(w[j])):
if k != PRIMITIVES.index('none'):
if k_best is None or w[j][k] > w[j][k_best]:
k_best = k
gene.append((PRIMITIVES[k_best], j))
start = end
n += 1
return gene
gene_reduce = _parse(
Softmax()(self.alphas_reduce).numpy(),
SplitSoftmax(range(2, 2 + self.steps))(self.betas_reduce).numpy())
gene_normal = _parse(
Softmax()(self.alphas_normal).numpy(),
SplitSoftmax(range(2, 2 + self.steps))(self.betas_normal).numpy())
concat = range(2 + self.steps - self.multiplier, self.steps + 2)
genotype = Genotype(normal=gene_normal,
normal_concat=concat,
reduce=gene_reduce,
reduce_concat=concat)
return genotype
def getPursuitAgent(action_space, beta, temp):
# Get number of action
n_actions = len(action_space)
# Define constants
# Define all possible discrete returns
I = tf.constant(np.eye(n_actions))
# Rate of increment/decrement
beta = tf.constant(beta)
# Make input state tensor
in_action_vals = Input(batch_shape=(None, n_actions),
name='action_vals_GPolicy')
# Get action probabilities directly proportional to Q(·)
p = Softmax()(in_action_vals / temp)
# Get greedy action
greedy_actions = tf.argmax(in_action_vals, axis=-1, output_type="int32")
# Increment value of greedy action
increment = tf.cast(tf.gather(I, greedy_actions),
"float32") * (1 - p) * beta
decrement = (1 - tf.cast(tf.gather(I, greedy_actions), "float32")) * (
-p) * beta
return Model(in_action_vals,
p + increment + decrement,
name='PursuitPolicyModel')
def call(self, inputs):
merged_context, modeled_passage = inputs
span_begin_input = K.concatenate([merged_context, modeled_passage])
span_begin_weights = TimeDistributed(self.dense_1)(span_begin_input)
span_begin_probabilities = Softmax()(K.squeeze(span_begin_weights,
axis=-1))
return span_begin_probabilities
def get_tcn(seq_len,
feature_dim,
output_dim=2,
nb_filters=8,
nb_stacks=3,
use_skip_connections=True,
return_sequences=True,
use_batch_norm=True,
dilation_stages=5):
dilations = [2**x for x in range(dilation_stages)]
i = Input(shape=(seq_len, feature_dim))
x = Reshape(target_shape=(1, seq_len, feature_dim))(i)
x = MyConv1D(filters=nb_filters,
kernel_size=3,
name='initial_conv',
kernel_initializer='glorot_normal')(x)
x = TCN(nb_filters=nb_filters,
kernel_size=3,
dilations=dilations,
nb_stacks=nb_stacks,
use_skip_connections=use_skip_connections,
return_sequences=return_sequences,
use_batch_norm=use_batch_norm,
kernel_initializer='glorot_normal')(x)
if return_sequences:
x = Reshape(target_shape=(seq_len, nb_filters))(x)
else:
x = Reshape(target_shape=(nb_filters, ))(x)
x = Dense(output_dim)(x)
x = Softmax()(x)
model = Model(inputs=[i], outputs=[x])
return model
def CNN(inputs, num_classes):
x = Conv2D(32, 3, 1, padding="same", kernel_regularizer=l2(0.00))(inputs)
x = BatchNormalization()(x)
x = Activation(tf.nn.relu)(x)
x = Conv2D(64, 3, 1, padding="same", kernel_regularizer=l2(0.00))(x)
x = BatchNormalization()(x)
x = Activation(tf.nn.relu)(x)
x = Conv2D(128, 3, 1, padding="same", kernel_regularizer=l2(0.00))(x)
x = BatchNormalization()(x)
x = Activation(tf.nn.relu)(x)
x = MaxPooling2D(2, padding="same")(x)
x = Dropout(0.5)(x)
x = Conv2D(192, 3, 1, padding="same", kernel_regularizer=l2(0.00))(x)
x = BatchNormalization()(x)
x = Activation(tf.nn.relu)(x)
x = Dropout(0.5)(x)
x = SKNet(384, 3, activation=tf.nn.relu)(x)
x = GlobalAveragePooling2D()(x)
x = Dropout(0.5)(x)
x = Dense(num_classes, kernel_regularizer=l2(0))(x)
x = BatchNormalization()(x)
x = Softmax()(x)
return x
def PNet(input_shape=None):
if input_shape is None:
input_shape = (None, None, 3)
input_ = Input(input_shape)
# Conv2D ---- 1
x = Conv2D(10, kernel_size=(3, 3), strides=(1, 1), padding="valid")(input_)
x = PReLU(shared_axes=[1, 2])(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding="same")(x)
# Conv2D --- 2
x = Conv2D(16, kernel_size=(3, 3), strides=(1, 1), padding="valid")(x)
x = PReLU(shared_axes=[1, 2])(x)
# Conv2D --- 3
x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding="valid")(x)
x = PReLU(shared_axes=[1, 2])(x)
output_1 = Conv2D(2, kernel_size=(1, 1), strides=(1, 1))(x)
output_1 = Softmax(axis=3)(output_1)
output_2 = Conv2D(4, kernel_size=(1, 1), strides=(1, 1))(x)
pnet = Model(input_, [output_2, output_1])
return pnet
def __init__(self):
super(SNP_model, self).__init__()
self.conv1_1 = Conv2D(16, kernel_size=[1,5], strides=[1,1], activation='selu', name='C1_1', use_bias=True, padding='same')
self.conv1_2 = Conv2D(16, kernel_size=[5,1], strides=[1,1], activation='selu', name='C1_2', use_bias=True, padding='same')
self.conv1_3 = Conv2D(16, kernel_size=[5,5], strides=[1,1], activation='selu', name='C1_3', use_bias=True, padding='same')
self.conv2 = Conv2D(32, kernel_size=[2,3], strides=[1,2], activation='selu', name='C2', use_bias=True, padding='valid')
self.conv3 = Conv2D(64, kernel_size=[2,3], strides=[1,2], activation='selu', name='C3', use_bias=True, padding='valid')
self.flatten = Flatten()
self.dropout = Dropout(0.5)
self.fc1 = Dense(48, activation='selu',name='C4',use_bias=True)
self.fa = Dense(16, activation='selu',name='C5',use_bias=True)
self.A = Dense(2, activation=None ,name='C9',use_bias=True)
self.G = Dense(2, activation=None ,name='C10',use_bias=True)
self.T = Dense(2, activation=None ,name='C11',use_bias=True)
self.C = Dense(2, activation=None ,name='C12',use_bias=True)
self.fc2 = Dense(16, activation='selu',name='C6',use_bias=True)
self.fc3 = Dense(8, activation='selu',name='C7',use_bias=True)
self.GT = Dense(2, activation=None,name='C8',use_bias=True)
self.softmax=Softmax()
def CNN(opts, input_tensor=None):
""" Initializes the CNN backbone of the DCGMM model.
Arguments:
- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
- input_shape: shape of input image. format HxWxC
- num_clusters: number of clusters to use
"""
if input_tensor is None:
input_shape = (opts.img_size, opts.img_size, 1)
img_input = Input(shape=input_shape)
else:
img_input = input_tensor
x1 = Conv2D(filters=32, kernel_size=(3, 3), padding='same',
data_format='channels_last', strides=(1, 1), activation='relu')(img_input)
x2 = Conv2D(filters=64, kernel_size=(3, 3), padding='same',
data_format='channels_last', strides=(1, 1), activation='relu')(x1)
x2 = MaxPool2D(pool_size=(2, 2), padding='same')(x2)
x3 = Conv2D(filters=64, kernel_size=(3, 3), padding='same',
data_format='channels_last', strides=(1, 1), activation='relu')(x2)
x4 = Conv2D(filters=128, kernel_size=(3, 3), padding='same',
data_format='channels_last', strides=(1, 1), activation='relu')(x3)
x4 = MaxPool2D(pool_size=(2, 2), padding='same')(x4)
x5 = Conv2D(filters=128, kernel_size=(3, 3), padding='same',
data_format='channels_last', strides=(1, 1), activation='relu')(x4)
# Upscale the H and W dimension by a factor 2
output_shape = x5.shape.as_list()
output_shape[1] *= 2
output_shape[2] *= 2
x5 = tf.image.resize(x5, output_shape[1:3], method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
x6 = Conv2D(filters=128, kernel_size=(3, 3), padding='same',
data_format='channels_last', strides=(1, 1), activation='relu')(x5)
# Upscale the H and W dimension by a factor 2
output_shape = x6.shape.as_list()
output_shape[1] *= 2
output_shape[2] *= 2
x6 = tf.image.resize(x6, output_shape[1:3], method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
x7 = Conv2D(filters=64, kernel_size=(3, 3), padding='same',
data_format='channels_last', strides=(1, 1), activation='relu')(x6)
x8 = Conv2D(filters=32, kernel_size=(3, 3), padding='same',
data_format='channels_last', strides=(1, 1), activation='relu')(x7)
x9 = Conv2D(filters=opts.num_clusters, kernel_size=(3, 3), padding='same',
data_format='channels_last', strides=(1, 1), activation='relu')(x8)
gamma = Softmax(axis=-1)(x9)
model = Model(img_input, gamma, name='CNN_backbone')
return model
def call(self,decoder_hidden_state,encoder_output, enc_mask):
''' Attention mechanism takes two inputs current step -- decoder_hidden_state and all the encoder_outputs. '''
decoder_hidden_state = tf.expand_dims(decoder_hidden_state, axis=1)
# mask from encoder
enc_mask = tf.expand_dims(enc_mask, axis=-1)
# score shape: (batch_size, input_length, 1)
if self.scoring_function == 'dot':
# Implementing Dot score function
score = self.dot([encoder_output, decoder_hidden_state])
elif self.scoring_function == 'general':
# Implementing General score function here
score = tf.keras.layers.Dot(axes=[2, 2])([self.wa(encoder_output), decoder_hidden_state])
elif self.scoring_function == 'concat':
# Implementing General score function here
decoder_output = tf.tile(decoder_hidden_state, [1, encoder_output.shape[1], 1])
score = self.va(self.wa(tf.concat((decoder_output, encoder_output), axis=-1)))
score = score + (tf.cast(tf.math.equal(enc_mask, False), score.dtype)*-1e9)
# shape: (batch_size, input_length, 1)
attention_weights = Softmax(axis=1)(score)
enc_mask = tf.cast(enc_mask, attention_weights.dtype)
# masking attention weights
attention_weights = attention_weights * enc_mask
context_vector = attention_weights * encoder_output
# shape = (batch_size, dec lstm units)
context_vector = tf.reduce_sum(context_vector, axis=1)
return context_vector, attention_weights
def init_model(params):
""" Params dictionary based model builder. """
input_layer = Input(shape=params['input_shape'])
x = conv_block(input_layer, params['init_filters'], (3, 3), (1, 1),
params['use_batchnorm'])
x = conv_block(x, params['init_filters'] * 2, (3, 3), (1, 1),
params['use_batchnorm'])
x = MaxPooling2D()(x)
if params['depth'] > 1:
filter_mult = 2
for depth in range(2, params['depth']):
x = conv_block(x, params['init_filters'] * filter_mult, (3, 3),
(1, 1), params['use_batchnorm'])
filter_mult += 1
x = conv_block(x, params['init_filters'] * filter_mult, (3, 3),
(1, 1), params['use_batchnorm'])
x = MaxPooling2D()(x)
x = Flatten()(x)
x = Dense(params['dense_neurons'])(x)
x = Activation('relu')(x)
x = Dropout(params['dropout'])(x)
x = Dense(params['output_shape'])(x)
x = Activation('relu')(x)
output_layer = Softmax()(x)
model = Model(input_layer, output_layer)
return model
def cal_landmarks(self, x):
_, h, w, _ = x.shape.as_list()
# Assume normalized coordinate [0, 1] for numeric stability
ref_xs, ref_ys = np.meshgrid(
np.linspace(0, 1.0, num=w, endpoint=True),
np.linspace(0, 1.0, num=h, endpoint=True),
indexing='xy'
)
ref_xs = np.reshape(ref_xs, [-1, h*w])
ref_ys = np.reshape(ref_ys, [-1, h*w])
# Assuming NHWC
beta = 1e2
# Transpose x from NHWC to NCHW
x = tf.transpose(x, (0, 3, 1, 2))
x = tf.reshape(x, [-1, self._hg_num_landmarks, h*w])
x = Softmax(axis=-1)(beta*x)
lmrk_xs = tf.math.reduce_sum(ref_xs * x, axis=[2])
lmrk_ys = tf.math.reduce_sum(ref_ys * x, axis=[2])
# Return to actual coordinates ranges
return tf.stack([
lmrk_xs * (w - 1.0) + 0.5,
lmrk_ys * (h - 1.0) + 0.5
], axis=2) # N x 18 x 2
def load_KerasGraph(path):
print("> ====== loading Keras model for classification")
thread_graph = Graph()
with thread_graph.as_default():
thread_session = Session()
with thread_session.as_default():
input_shape = (28, 28, 1)
num_classes = 6
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3), input_shape=input_shape))
model.add(ReLU())
model.add(Conv2D(32, kernel_size=(3, 3)))
model.add(ReLU())
model.add(Conv2D(32, kernel_size=(3, 3)))
model.add(ReLU())
model.add(Conv2D(32, kernel_size=(3, 3)))
model.add(ReLU())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128))
model.add(ReLU())
model.add(Dropout(0.5))
model.add(Dense(num_classes))
model.add(Softmax())
model.load_weights(path)
graph = tf.get_default_graph()
print("> ====== Keras model loaded")
return model, graph, thread_session
def call(self, inputs, **kwargs):
states_encoder = inputs[0]
states_decoder = inputs[1]
# (0) adjust the size of encoder state to the size of decoder state
if isinstance(self.n_state, int):
# Key Vector와 Value Vector을 다르게 둚
key_vector = self.key_dense(states_encoder)
val_vector = self.val_dense(states_encoder)
else:
key_vector = states_encoder
val_vector = states_encoder
# (1) Calculate Score
expanded_states_encoder = key_vector[:, None, ...]
# >>> (batch size, 1, length of encoder sequence, num hidden)
expanded_states_decoder = states_decoder[..., None, :]
# >>> (batch size, length of decoder sequence, 1, num hidden)
score = K.sum(expanded_states_encoder * expanded_states_decoder,
axis=-1)
# >>> (batch size, length of decoder input, length of encoder input)
# (2) Normalize score
attention = Softmax(axis=-1, name='attention')(score)
# (3) Calculate Context Vector
expanded_val_vector = val_vector[:, None, ...]
context = K.sum(expanded_val_vector * attention[..., None], axis=2)
# >>> (batch size, length of decoder input, num hidden)
return context, attention
def RNet(input_shape=None):
if input_shape is None:
input_shape = (24, 24, 3)
input_ = Input(input_shape)
# Conv2D --- 1
x = Conv2D(28, kernel_size=(3, 3), strides=(1, 1), padding="valid")(input_)
x = PReLU(shared_axes=[1, 2])(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding="same")(x)
# Conv2D --- 2
x = Conv2D(48, kernel_size=(3, 3), strides=(1, 1), padding="valid")(x)
x = PReLU(shared_axes=[1, 2])(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding="valid")(x)
# Conv2D --- 3
x = Conv2D(64, kernel_size=(2, 2), strides=(1, 1), padding="valid")(x)
x = PReLU(shared_axes=[1, 2])(x)
x = Flatten()(x)
x = Dense(128)(x)
x = PReLU()(x)
output_1 = Dense(2)(x)
output_1 = Softmax(axis=1)(output_1)
output_2 = Dense(4)(x)
rnet = Model(input_, [output_2, output_1])
return rnet
def SF_Module(x_list, n_channel, reduction, limitation):
## Split
fused = None
for x_s in x_list:
if fused==None:
fused = x_s
else:
fused = Add()([fused, x_s])
## Fuse
fused = GlobalAveragePooling2D()(fused)
fused = BatchNormalization()(fused)
fused = Dense(max(n_channel // reduction, limitation), activation='selu')(fused)
## Select
masks = []
for i in range(len(x_list)):
masks.append(Dense(n_channel)(fused))
mask_stack = Lambda(K.stack, arguments={'axis': -1})(masks)
mask_stack = Softmax(axis=-2)(mask_stack) # (n_channel, n_kernel)
selected = None
for i, x_s in enumerate(x_list):
mask = Lambda(lambda z: z[:, :, i])(mask_stack)
mask = Reshape((1, 1, n_channel))(mask)
x_s = Multiply()([x_s, mask])
if selected==None:
selected = x_s
else:
selected = Add()([selected, x_s])
return selected
def learn_model(log, attributes, epochs, early_stop):
num_activities = len(log.values[log.activity]) + 1
# Input + Embedding layer for every attribute
input_layers = []
embedding_layers = []
for attr in attributes:
if attr not in log.ignoreHistoryAttributes and attr != log.time and attr != log.trace:
for k in range(log.k):
i = Input(shape=(1, ),
name=attr.replace(" ", "_").replace("(", "").replace(
")", "").replace(":", "_") + "_Prev%i" % k)
input_layers.append(i)
# e = Embedding(len(log.values[attr]) + 1, 32, embeddings_initializer="zeros")(i)
e = Embedding(len(log.values[attr]) + 1,
len(log.values[attr]) + 1,
embeddings_initializer="zeros")(i)
embedding_layers.append(e)
concat = Concatenate()(embedding_layers)
drop = Dropout(0.2)(concat)
dense2 = Dense(num_activities)(drop)
flat = Flatten()(dense2)
output = Softmax(name="output")(flat)
model = Model(inputs=input_layers, outputs=[output])
opt = Nadam(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
schedule_decay=0.004,
clipvalue=3)
model.compile(loss={'output': 'categorical_crossentropy'}, optimizer=opt)
model.summary()
early_stopping = EarlyStopping(monitor='val_loss', patience=early_stop)
outfile = 'tmp/model_{epoch:03d}-{val_loss:.2f}.h5'
model_checkpoint = ModelCheckpoint(outfile,
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='auto')
x, y, vals = transform_data(
log, [a for a in attributes if a != log.time and a != log.trace])
if len(y) < 10:
split = 0
else:
split = 0.2
model.fit(x=x,
y=y,
validation_split=split,
verbose=2,
callbacks=[early_stopping],
batch_size=32,
epochs=epochs)
return model
