def myLSTMCell(units, input):
h_ = Input(shape=(units, ))
c_ = Input(shape=(units, ))
zf = Dense(units=units,
kernel_initializer='glorot_uniform',
bias_initializer='ones')(input)
z_ = Dense(units=units * 3,
kernel_initializer='glorot_uniform',
bias_initializer='zeros')(input)
z0 = Concatenate(axis=-1)([zf, z_])
z1 = Dense(units=units * 4,
kernel_initializer='orthogonal',
use_bias=False)(h_)
z = Add()([z0, z1])
z_f = Slicer(units=units, iPart=0)(z)
z_i = Slicer(units=units, iPart=1)(z)
z_o = Slicer(units=units, iPart=2)(z)
z_c = Slicer(units=units, iPart=3)(z)
f = Activation('hard_sigmoid')(z_f)
i = Activation('hard_sigmoid')(z_i)
o = Activation('hard_sigmoid')(z_o)
c = Activation('tanh')(z_c)
c0 = Multiply()([f, c_])
c1 = Multiply()([i, c])
c = Add()([c0, c1])
h = Activation('tanh')(c)
h = Multiply()([o, h])
return h_, c_, h, c
def stageT_block(x, x1, x2, num_p, stage, branch, weight_decay):
# Block 1
x = conv(x, 128, 7, "Mconv1_stage%d_L%d" % (stage, branch),
(weight_decay, 0))
x = relu(x)
x = conv(x, 128, 7, "Mconv2_stage%d_L%d" % (stage, branch),
(weight_decay, 0))
x = relu(x)
x = conv(x, 128, 7, "Mconv3_stage%d_L%d" % (stage, branch),
(weight_decay, 0))
x = relu(x)
x = conv(x, 128, 7, "Mconv4_stage%d_L%d" % (stage, branch),
(weight_decay, 0))
x = relu(x)
x = conv(x, 128, 7, "Mconv5_stage%d_L%d" % (stage, branch),
(weight_decay, 0))
x = relu(x)
x = conv(x, 128, 1, "Mconv6_stage%d_L%d" % (stage, branch),
(weight_decay, 0))
x = relu(x)
x = conv(x, num_p, 1, "Mconv7_stage%d_L%d" % (stage, branch),
(weight_decay, 0))
w_name = "weight_stage%d_L%d" % (stage, branch)
if num_p == 38:
w = Multiply(name=w_name)([x, x1]) # vec_weight
else:
w = Multiply(name=w_name)([x, x2]) # vec_heat
return x, w
def obsModule(self, kpm=True, neighEnc=True, visEnc=False, vStt=False):
if (neighEnc): self.neighStateEncoder(vStt=vStt)
if (visEnc): self.neighVisEncoder()
if (kpm):
self.kpmGate()
mod_inp = []
mod_out = []
if (neighEnc):
mod_inp += [self.pose, self.gaze, self.modeN]
mod_out = Multiply()([
self.kpm_mod([self.pose, self.gaze]),
self.nstate_mod([self.modeN])
])
if (visEnc):
mult_out = Multiply()([
self.kpm_mod([self.pose, self.gaze]),
self.nvis_mod([self.visP, self.visG])
])
if (mod_out != []): mod_out = Cat()([mod_out, mult_out])
else: mod_out = mult_out
mod_inp += [self.visP, self.visG]
self.obs_mod = Container(mod_inp, mod_out)
else:
if (neighEnc and not visEnc): self.obs_mod = self.nstate_mod
elif (not neighEnc and visEnc): self.obs_mod = self.nvis_mod
elif (neighEnc and visEnc):
mod_out = Cat()([
self.nstate_mod([self.modeN]),
self.nvis_mod([self.visP, self.visG])
])
self.obs_mod = Container([self.modeN, self.visP, self.visG],
mod_out)
self.obs_mod.name = 'ObsMod'
def apply_mask(x, mask1, mask2, num_p, stage, branch):
w_name = "weight_stage%d_L%d" % (stage, branch)
if num_p == 38:
w = Multiply(name=w_name)([x, mask1]) # vec_weight
else:
w = Multiply(name=w_name)([x, mask2]) # vec_heat
return w
def apply_mask(x, mask1, mask2, num_p, stage, branch):
w_name = "weight_stage%d_L%d" % (stage, branch)
if num_p == np_branch1:
w = Multiply(name=w_name)([x, mask1]) # vec_weight
elif num_p == np_branch2:
w = Multiply(name=w_name)([x, mask2]) # vec_heat
else:
assert False, "wrong number of layers num_p=%d " % num_p
return w
def my_loss(y_true, y_pred):
pixels1 = K.sum(K.cast(K.greater(y_pred, 0.5), 'float32'))
pixels2 = K.sum(K.cast(K.less_equal(y_pred, 0.5), 'float32'))
mask1 = Multiply()([K.cast(K.greater(y_pred, 0.5), 'float32'),
y_pred]) # values greater than 0.5
mask0 = Multiply()([K.cast(K.less_equal(y_pred, 0.5), 'float32'),
y_pred]) # values greater than 0.5
return -K.log((K.sum(mask1) / pixels1 - K.sum(mask0) / pixels2) / 255)
def mask_block(x, mask1, mask2, num_p, stage, branch):
w_name = "weight_stage%d_L%d" % (stage, branch)
if num_p == 38:
w = Multiply(name=w_name)([x, mask1])
elif num_p == 19:
w = Multiply(name=w_name)([x, mask2])
else:
assert False, "wrong number of layers num_p=%d " % num_p
return w
def apply_mask(x, mask1, mask2, num_p, stage, branch, np_branch1, np_branch2):
w_name = "weight_stage%d_L%d" % (stage, branch)
assert np_branch1 != np_branch2
if num_p == np_branch1:
w = Multiply(name=w_name)([x, mask1])
elif num_p == np_branch2:
w = Multiply(name=w_name)([x, mask2])
else:
assert False, "wrong number of layers num_p=%d " % num_p
# w = slice_layer(name=s_name)(w)
return w
def apply_mask(x, mask1, mask2, num_p, stage, branch, np_branch1, np_branch2):
w_name = "weight_stage%d_L%d" % (stage, branch)
# TODO: we have branch number here why we made so strange check
assert np_branch1 != np_branch2 # we selecting branches by number of pafs, if they accidentally became the same it will be disaster
if num_p == np_branch1:
w = Multiply(name=w_name)([x, mask1]) # vec_weight
elif num_p == np_branch2:
w = Multiply(name=w_name)([x, mask2]) # vec_heat
else:
assert False, "wrong number of layers num_p=%d " % num_p
return w
def generate_model(self, X):
""" Generate the model.
return: model
"""
# LSTM encoders
inp = [
Input(shape=(X[0].shape[1], X[0].shape[2])),
Input(shape=(X[1].shape[1], X[1].shape[2])),
Input(shape=(X[1].shape[1], X[1].shape[2]))
]
if (self.net_type == "LSTM"):
enc = [
LSTM(self.nUnit)(inp[0]),
LSTM(self.nUnit)(inp[1]),
LSTM(self.nUnit)(inp[2])
]
else:
enc = [
GRU(self.nUnit)(inp[0]),
GRU(self.nUnit)(inp[1]),
GRU(self.nUnit)(inp[2])
]
# Attention layer for observed ship
at_probs_o = Dense(self.nUnit, activation='softmax')(enc[0])
at_mul_o = Multiply()([enc[0], at_probs_o])
# Attention layer for neighbours
con_n = Concatenate()([enc[1], enc[2]])
at_probs_n = Dense((len(X) - 1) * self.nUnit,
activation='softmax')(con_n)
at_mul_n = Multiply()([con_n, at_probs_n])
# Attention layer for all
con_a = Concatenate()([at_mul_o, at_mul_n])
at_probs_a = Dense(len(X) * self.nUnit, activation='softmax')(con_a)
at_mul_a = Multiply()([con_a, at_probs_a])
# LSTM decoder
rep = RepeatVector(self.inputWindowSize)(at_mul_a)
if (self.net_type == "LSTM"):
dec = LSTM(self.nUnit, return_sequences=True)(rep)
else:
dec = GRU(self.nUnit, return_sequences=True)(rep)
tdis = TimeDistributed(Dense(self.nFeatures))(dec)
model = Model(inputs=inp, outputs=tdis)
return model
def scse_block(blockInput, bottle=4, ssigmoid=False, alpha=0.0001):
channel_cnt = int(blockInput.shape[-1])
img_size = int(blockInput.shape[1])
img_size2 = img_size * img_size
x = GlobalMaxPooling2D(data_format="channels_last")(blockInput)
x = Dense(int(channel_cnt / bottle))(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Dense(channel_cnt, kernel_regularizer=l2(alpha))(x)
x = Activation("sigmoid")(x)
x = Reshape((1, 1, channel_cnt))(x)
x = Multiply()([blockInput, x])
y = Conv2D(1, (1, 1), padding="same")(blockInput)
if ssigmoid:
y = Activation("elu")(y)
if img_size >= 50:
y_pooling = (4, 4) if img_size % 2 == 1 else (2, 2)
kernel_size = (5, 5) if img_size % 2 == 1 else (3, 3)
padding = "valid" if img_size % 2 == 1 else "same"
y = MaxPooling2D(y_pooling)(y)
pool_size = int(y.shape[1])
pool_size2 = pool_size * pool_size
y = Reshape((pool_size2, ))(y)
y = Dense(int(pool_size2 / bottle))(y)
y = BatchNormalization()(y)
y = Activation("relu")(y)
y = Dense(pool_size2, kernel_regularizer=l2(alpha))(y)
y = Activation("sigmoid")(y)
y = Reshape((pool_size, pool_size, 1))(y)
y = Conv2DTranspose(1, kernel_size, strides=y_pooling,
padding=padding)(y)
else:
y = Reshape((img_size2, ))(y)
y = Dense(int(img_size2 / bottle))(y)
y = BatchNormalization()(y)
y = Activation("relu")(y)
y = Dense(img_size2, kernel_regularizer=l2(alpha))(y)
y = Activation("sigmoid")(y)
y = Reshape((img_size, img_size, 1))(y)
y = Multiply()([blockInput, y])
z = Add()([x, y])
return z
def result(input):
emb_c = emb_layer(input)
conv_results_list = []
for cl in conv2d_layers:
conv_results_list.append(cl(emb_c))
emb_c = Lambda(lambda x: K.concatenate(x, axis=3))(conv_results_list)
emb_c = Lambda(lambda x: K.max(x, axis=2))(emb_c)
if highway_on_top:
sigmoid_gate = dense1(emb_c)
sigmoid_gate = Activation('sigmoid')(sigmoid_gate)
deeper_units = dense2(emb_c)
emb_c = Add()([Multiply()([sigmoid_gate, deeper_units]),
Multiply()([Lambda(lambda x: K.constant(1., shape=K.shape(x)) - x)(sigmoid_gate), emb_c])])
emb_c = Activation('relu')(emb_c)
return emb_c
def get_training_model(weight_decay, np_branch2, stages=6, gpus=None):
img_input_shape = (None, None, 3)
heat_input_shape = (None, None, np_branch2)
inputs = []
outputs = []
img_input = Input(shape=img_input_shape)
heat_weight_input = Input(shape=heat_input_shape)
inputs.append(img_input)
#inputs.append(heat_weight_input)
#img_normalized = img_input # will be done on augmentation stage
# VGG
stage0_out = vgg_block(img_input, weight_decay)
# stage 1
new_x = []
stage1_branch2_out = stage1_block(stage0_out, np_branch2, 2, weight_decay)
w = Multiply()([stage1_branch2_out, heat_weight_input])
#w2 = apply_mask(stage1_branch2_out, vec_weight_input, heat_weight_input, np_branch2, 1, 2, np_branch1, np_branch2)
outputs.append(stage1_branch2_out)
new_x.append(stage1_branch2_out)
new_x.append(stage0_out)
x = Concatenate()(new_x)
# stage sn >= 2, repeating stage
for sn in range(2, stages + 1):
new_x = []
# stage T
stageT_branch2_out = stageT_block(x, np_branch2, sn, 2, weight_decay)
w = Multiply()([stageT_branch2_out, heat_weight_input])
#w2 = apply_mask(stageT_branch2_out, vec_weight_input, heat_weight_input, np_branch2, sn, 2, np_branch1, np_branch2)
outputs.append(stageT_branch2_out)
new_x.append(stageT_branch2_out)
new_x.append(stage0_out)
if sn < stages:
x = Concatenate()(new_x)
model = Model(inputs=inputs, outputs=outputs)
return model
def build(self, generator=False, **kwargs):
if generator:
input = Input((1,), dtype='int32')
embedded = Embedding(self.input_dim,
self.output_dim,
input_length=1)(input)
embed_model = Model(input, embedded)
return embed_model
else:
input = Input((self.seq_len,), dtype='int32')
embedded = Embedding(self.input_dim,
self.output_dim,
input_length=self.seq_len)(input)
embed_model = Model(input, embedded)
input1 = Input((self.seq_len,), dtype='int32')
input2 = Input((self.seq_len,), dtype='int32')
emb1 = embed_model(input1) # (bs, seq_len, output_dim)
emb2 = embed_model(input2) # (bs, seq_len, output_dim)
sim = Multiply()([emb1, emb2])
sim = Lambda(lambda x: K.sum(x, 2),
output_shape=lambda x: x[:-1])(sim)
sim = Activation('sigmoid')(sim)
self.siamese = Model([input1, input2], sim)
self.embed_model = embed_model
self.siamese.summary()
def InitializeModel():
query_input = Input(shape=(300, 1), dtype='float32', name='query_input')
query_conv = Conv1D(filters=50,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(query_input)
query_pool = MaxPooling1D(pool_size=2)(query_conv)
#query_conv1 = Conv1D(filters=50, kernel_size=3, strides=1, padding='same', activation='relu')(query_pool)
#query_pool1 = MaxPooling1D(pool_size=2)(query_conv1)
query_flat = Flatten()(query_pool)
query_dense = Dense(units=25, activation='relu')(query_flat)
doc_input = Input(shape=(300, 1), dtype='float32', name='doc_input')
doc_conv = Conv1D(filters=50,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(doc_input)
doc_pool = MaxPooling1D(pool_size=2)(doc_conv)
doc_flat = Flatten()(doc_pool)
doc_dense = Dense(units=25, activation='relu')(doc_flat)
query_doc_vector = Multiply()([query_dense, doc_dense])
query_doc_dense = Dense(units=5, activation='softmax',
name='relevance')(query_doc_vector)
model = Model(inputs=[query_input, doc_input], outputs=query_doc_dense)
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=[metrics.mae, metrics.categorical_accuracy])
print model.summary()
return model
def step(self, inputs2, states):
h_tm1 = states[0]
c_tm1 = states[1]
d_tm1 = states[2]
inputs = states[3]
batch_size, samples, input_dim = K.int_shape(inputs)
x1 = TimeDistributed(self.W_1)(inputs)
x2 = self.W_2(d_tm1)
#broadcast (W_1 * H_T) + (W_2 * d->)
x2 = K.expand_dims(x2, axis=-2)
x2 = K.repeat_elements(x2, samples, axis=-2)
x = Add()([x1, x2])
x = Activation('tanh')(x)
#broadcast v_t * tanh(x: (samples, input_dim))
v = K.expand_dims(self.v, axis=-1)
x = K.dot(x, v)
x = K.sum(x, axis=-1)
x = Activation('softmax')(x)
#element wise multiplication of attention vector
a_t = K.expand_dims(x, axis=-1)
a_t = K.repeat_elements(a_t, input_dim, -1)
x = Multiply()([inputs, a_t])
d_t = K.sum(x, axis=-2)
lstm_states = [h_tm1, c_tm1] + list(states[4:])
h, (h, c) = super(AttentionLSTM,
self).step(d_t,
lstm_states) #pass in only lstm states
return h, [h, c, d_t]
def get_model(inputdim, outputdim, regularization_strength=0.01, lr=0.001, cosine=False, **kwargs):
inp1 = Input(name='embeddings1', shape=(inputdim,))
inp2 = Input(name='embeddings2', shape=(inputdim,))
transformation = Dense(inputdim, kernel_initializer='identity',
kernel_constraint=Orthogonal())
transform1 = transformation(inp1)
transform2 = transformation(inp2)
projected1 = Lambda(lambda x: x[:, :outputdim], name='projected1')(transform1)
projected2 = Lambda(lambda x: -x[:, :outputdim], name='negprojected2')(transform2)
if cosine:
normalized1 = Lambda(lambda x: x / K.reshape(K.sqrt(K.sum(x * x, axis=1)), (x.shape[0], 1)),
name='normalized1')(projected1)
normalized2 = Lambda(lambda x: x / K.reshape(K.sqrt(K.sum(x * x, axis=1)), (x.shape[0], 1)),
name='normalized2')(projected2)
merged_norm = Multiply()([normalized1, normalized2])
distances = Lambda(lambda x: K.reshape(K.sum(x, axis=1), (x.shape[0], 1)), name='distances')(merged_norm)
else:
merged_norm = Add()([projected1, projected2])
distances = Lambda(lambda x: K.sqrt(K.sum(x * x, axis=1)),
name='distances')(merged_norm)
model = Model(inputs=[inp1, inp2], outputs=[distances])
model.compile(loss=mean_mul_loss, optimizer=SGD(lr=lr))
return model
def multiplicative_self_attention(units,
n_hidden=None,
n_output_features=None,
activation=None):
"""
Compute multiplicative self attention for time series of vectors (with batch dimension)
the formula: score(h_i, h_j) = <W_1 h_i, W_2 h_j>, W_1 and W_2 are learnable matrices
with dimensionality [n_hidden, n_input_features]
Args:
units: tf tensor with dimensionality [batch_size, time_steps, n_input_features]
n_hidden: number of units in hidden representation of similarity measure
n_output_features: number of features in output dense layer
activation: activation at the output
Returns:
output: self attended tensor with dimensionality [batch_size, time_steps, n_output_features]
"""
n_input_features = K.int_shape(units)[2]
if n_hidden is None:
n_hidden = n_input_features
if n_output_features is None:
n_output_features = n_input_features
exp1 = Lambda(lambda x: expand_tile(x, axis=1))(units)
exp2 = Lambda(lambda x: expand_tile(x, axis=2))(units)
queries = Dense(n_hidden)(exp1)
keys = Dense(n_hidden)(exp2)
scores = Lambda(lambda x: K.sum(queries * x, axis=3, keepdims=True))(keys)
attention = Lambda(lambda x: softvaxaxis2(x))(scores)
mult = Multiply()([attention, exp1])
attended_units = Lambda(lambda x: K.sum(x, axis=2))(mult)
output = Dense(n_output_features, activation=activation)(attended_units)
return output
def __init__(self, h_size):
self.inputs = Input(shape=(84,84,3))
self.actions = Input(shape=(1,), dtype='int32')
self.actions_onehot = Lambda(K.one_hot, arguments={'num_classes':env.actions}, output_shape=(None, env.actions))(self.actions)
x = Conv2D(filters=32, kernel_size=[8,8], strides=[4,4], input_shape=(84, 84, 3))(self.inputs)
x = Conv2D(filters=64, kernel_size=[4,4],strides=[2,2])(x)
x = Conv2D(filters=64, kernel_size=[3,3],strides=[1,1])(x)
x = Conv2D(filters=h_size, kernel_size=[7,7],strides=[1,1])(x)
#Splice outputs of last conv layer using lambda layer
x_value = Lambda(lambda x: x[:,:,:,:h_size//2])(x)
x_advantage = Lambda(lambda x: x[:,:,:,h_size//2:])(x)
#Process spliced data stream into value and advantage function
value = Dense(env.actions, activation="linear")(x_value)
advantage = Dense(env.actions, activation="linear")(x_advantage)
#Recombine value and advantage layers into Q layer
q = QLayer()([value, advantage])
self.q_out = Multiply()([q, self.actions_onehot])
self.q_out = Lambda(lambda x: K.cumsum(x, axis=3), output_shape=(1,))(self.q_out)
#need to figure out how to represent actions within training
self.model = Model(inputs=[self.inputs, self.actions], outputs=[q, self.q_out])
self.model.compile(optimizer="Adam", loss="mean_squared_error")
self.model.summary()
def shift_conv_attention(self, att, merged_modules, predicate_masks):
"""Takes an intial attention map and shifts it using the predicate convs.
Args:
att: An initial attention by the object or the subject.
merged_modules: A list containing `self.num_predicate` elements
where each element is a list of `self.nb_conv_att_map` convs.
predicate_masks: A helpful tensor indicating which predicate
is involved for relationship element in the batch.
Returns:
The shifted attention.
"""
conv_outputs = []
for group in merged_modules:
att_map = att
for conv_module in group:
att_map = conv_module(att_map)
conv_outputs.append(att_map)
merged_output = Concatenate(axis=3)(conv_outputs)
predicate_att = Multiply()([predicate_masks, merged_output
]) # mask是one-hot吗，如果是这里其他组的通道就是0了啊
predicate_att = Lambda(lambda x: K.sum(x, axis=3, keepdims=True))(
predicate_att)
predicate_att = Activation("tanh")(predicate_att)
return predicate_att
def apply_mask(x, mask3, num_p, stage, branch):
w_name = "weight_stage%d_L%d" % (stage, branch)
if num_p == np_branch3:
w = Multiply(name=w_name)([x, mask3]) # seg
else:
assert False, "wrong number of layers num_p=%d " % num_p
return w
def call(self, inputs):
from keras.layers import Input
from keras.layers.core import Dense, Activation, Lambda
from keras.layers.wrappers import TimeDistributed
from keras.layers.merge import Add, Multiply
sequence = inputs[:, :self.num_focus]
contexts = inputs[:, self.num_focus:]
sequence_shape = sequence.shape.as_list()
num_timesteps = sequence_shape[1]
num_features = sequence_shape[-1]
# Bulid. Layers are necessary for RecurrentWrapper to manage '_keras_history'.
SourceFrames = Input(shape=(num_timesteps, num_features))
state = Input(shape=(num_features, ))
# 1) Calculate score for each timestep.
ScoreFrames = TimeDistributed(Dense(units=1,
use_bias=False))(SourceFrames)
ScoreState = Dense(units=1)(state)
AttenScores = Add()([ScoreFrames,
ScoreState]) # (samples, timesteps, 1)
AttenScores = Lambda(
lambda x: K.squeeze(x, axis=-1),
output_shape=(num_timesteps, ) # (samples, timesteps)
)(AttenScores)
#
# ScoreFrames = TimeDistributed(Dense(units=num_features, use_bias=False))(SourceFrames)
# AttenScores = Multiply()([ScoreFrames, state]) # (samples, timesteps, features)
# AttenScores = Lambda(lambda x: K.sum(x, axis=-1),
# output_shape=(num_timesteps, )
# )(AttenScores) # (samples, timesteps)
# 2) Apply Softmax to obtain weights of vectors in sequence.
AttenWeights = Activation('softmax')(
AttenScores) # Along the last (i.e., timestep) axis.
AttenWeights = Lambda(
lambda x: K.expand_dims(x, axis=-1),
output_shape=(num_timesteps, 1) # (samples, timesteps, 1)
)(AttenWeights)
# 3) Get weighted sum.
AttenContext = Multiply()([AttenWeights, SourceFrames
]) # (samples, timesteps, features)
AttenContext = Lambda(
lambda x: K.sum(x, axis=1),
output_shape=(num_features, ) # (samples, features)
)(AttenContext)
self.model = RecurrentWrapper(input=[state],
output=[AttenContext],
bind={},
sequence_input=[SourceFrames],
input_shape=(None, num_features),
return_sequences=True)
Atten_in = [contexts] + [sequence]
Atten_out = self.model(Atten_in)
# Build after call().
self.build(tuple(inputs.shape.as_list()))
return K.concatenate([sequence, Atten_out], axis=1)
def _build_full(inp,gender,network='resnet'):
max_filt = 2 **2
conv1 = _conv_bn_relu(filters=32,kernel_size=(3,3),strides=(1,1))(inp)
conv2 = _conv_bn_relu(filters=64,kernel_size=(3,3),strides=(1,1))(conv1)
conv3 = _conv_bn_relu(filters=max_filt,kernel_size=(3,3),strides=(1,1))(conv2)
sub = SubpixelConv2D(input_shape=conv3.get_shape().as_list()[1:],scale=2)(conv3)
#Give the network one channel with the resized input image as well as a concatenated super resolution layer
inp_resize = Lambda(lambda x: K.tf.image.resize_images(x,sub.get_shape().as_list()[1:3]))(inp)
#We need three channels for the networks (as they are originally trained for color)
#so we concatenate the input resize and the subpixel layers output
inp_conc = concatenate([inp_resize,sub,sub])
if network == 'resnet':
m = keras.applications.ResNet50(include_top = False,weights='imagenet',input_shape=inp_conc.get_shape().as_list()[1:],pooling=None)
else:
m = keras.applications.VGG16(include_top=False,weights='imagenet',input_shape=inp_conc.get_shape().as_list()[1:])
m.trainable = False
m_out = m(inp_conc)
attn_layer1 = Conv2D(2048,kernel_size=[1,1],activation='sigmoid')(m_out)
attn_layer2 = Conv2D(m_out.get_shape().as_list()[-1],kernel_size=[1,1],activation='linear',use_bias=False)(attn_layer1)
attn_layer3 = Multiply()([m_out,attn_layer2])
attn_layer3 = Lambda(lambda x: x,name='attention')(attn_layer3)
flat = Flatten()(attn_layer3)
gender_out = Dense(32)(gender)
flat = concatenate([flat,gender_out])
fcl = Dense(512,activation='linear')(flat)
fcl = Dense(512,activation='linear')(fcl)
out = Dense(1,activation='linear')(fcl)
return Model(inputs=[inp,gender],outputs=out)
def apply_transformation(
x_source,
x_transformation,
output_shape,
conditioning_input_shape,
transform_name,
flow_indexing='xy',
color_transform_type='WB',
):
n_dims = len(conditioning_input_shape) - 1
transformation_shape = x_transformation.get_shape().as_list()[1:]
x_transformation = Reshape(
transformation_shape,
name='{}_dec_out'.format(transform_name))(x_transformation)
# apply color transform
print('Applying color transform {}'.format(color_transform_type))
if color_transform_type == 'delta':
x_color_out = Add()([x_source, x_transformation])
elif color_transform_type == 'mult':
x_color_out = Multiply()([x_source, x_transformation])
else:
raise NotImplementedError(
'Only color transform types delta and mult are supported!')
im_out = Reshape(output_shape, name='color_transformer')(x_color_out)
return im_out, x_transformation
def AttetionWrapper(num_timesteps, num_features, state):
SourceFrames = Input(shape=(num_timesteps, num_features))
# 1) Calculate score for each timestep.
ScoreFrames = TimeDistributed(Dense(units=1,
use_bias=False))(SourceFrames)
ScoreState = Dense(units=1)(state)
AttenScores = Add()([ScoreFrames,
ScoreState]) # (samples, timesteps, 1)
AttenScores = Lambda(
lambda x: K.squeeze(x, axis=-1),
output_shape=(num_timesteps, ) # (samples, timesteps)
)(AttenScores)
# 2) Apply Softmax to obtain weights of vectors in sequence.
# AttenWeights = Activation('tanh')(AttenScores)
AttenWeights = Activation('softmax')(
AttenScores) # Along the last (i.e., timestep) axis.
AttenWeights = Lambda(
lambda x: K.expand_dims(x, axis=-1),
output_shape=(num_timesteps, 1) # (samples, timesteps, 1)
)(AttenWeights)
# 3) Get weighted sum.
AttenContext = Multiply()([AttenWeights, SourceFrames
]) # (samples, timesteps, features)
AttenContext = Lambda(
lambda x: K.sum(x, axis=1),
output_shape=(num_features, ) # (samples, features)
)(AttenContext)
return SourceFrames, AttenContext
def get_conv_rnn_model(config):
num_classes = config.n_classes
(_, n_time, n_freq) = X_train.shape ## (N, 240, 64)
input_logmel = Input(shape=(n_time, n_freq),
name='in_layer') # (N, 240, 64)
a1 = Reshape((n_time, n_freq, 1))(input_logmel) # (N, 240, 64, 1)
a1 = block(a1)
a1 = block(a1)
a1 = MaxPooling2D(pool_size=(1, 2))(a1) # (N, 240, 32, 128)
a1 = Dropout(rate=0.5)(a1)
a1 = block(a1)
a1 = block(a1)
a1 = MaxPooling2D(pool_size=(1, 2))(a1) # (N, 240, 16, 128)
a1 = Dropout(rate=0.5)(a1)
a1 = block(a1)
a1 = block(a1)
a1 = MaxPooling2D(pool_size=(1, 2))(a1) # (N, 240, 8, 128)
a1 = Dropout(rate=0.5)(a1)
a1 = block(a1)
a1 = block(a1)
a1 = MaxPooling2D(pool_size=(1, 2))(a1) # (N, 240, 4, 128)
a1 = Dropout(rate=0.5)(a1)
a1 = Conv2D(256, (3, 3), padding="same", activation="relu",
use_bias=True)(a1)
a1 = MaxPooling2D(pool_size=(1, 4))(a1) # (N, 240, 1, 256)
a1 = Dropout(rate=0.5)(a1)
a1 = Reshape((n_time, 256))(a1) # (N, 240, 256)
# Gated BGRU
rnnout = Bidirectional(GRU(128, activation='linear',
return_sequences=True))(a1)
rnnout_gate = Bidirectional(
GRU(128, activation='sigmoid', return_sequences=True))(a1)
a2 = Multiply()([rnnout, rnnout_gate])
# Attention
cla = TimeDistributed(Dense(num_classes, activation='sigmoid'),
name='localization_layer')(a2)
att = TimeDistributed(Dense(num_classes, activation='softmax'))(a2)
out = Lambda(outfunc, output_shape=(num_classes, ))([cla, att])
# out = Lambda(att, output_shape=(num_classes,))
model = Model(inputs=input_logmel, outputs=out)
opt = optimizers.Adam(lr=config.learning_rate,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08)
# model.summary()
# model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['acc'])
model.compile(optimizer=opt,
loss=losses.categorical_crossentropy,
metrics=['acc'])
return model
def build_ca_model(self):
DeepCpf1_Input_SEQ = Input(shape=(34, 4))
DeepCpf1_C1 = Convolution1D(80, 5,
activation='relu')(DeepCpf1_Input_SEQ)
DeepCpf1_P1 = AveragePooling1D(2)(DeepCpf1_C1)
DeepCpf1_F = Flatten()(DeepCpf1_P1)
DeepCpf1_DO1 = Dropout(0.3)(DeepCpf1_F)
DeepCpf1_D1 = Dense(80, activation='relu')(DeepCpf1_DO1)
DeepCpf1_DO2 = Dropout(0.3)(DeepCpf1_D1)
DeepCpf1_D2 = Dense(40, activation='relu')(DeepCpf1_DO2)
DeepCpf1_DO3 = Dropout(0.3)(DeepCpf1_D2)
DeepCpf1_D3_SEQ = Dense(40, activation='relu')(DeepCpf1_DO3)
DeepCpf1_Input_CA = Input(shape=(1, ))
DeepCpf1_D3_CA = Dense(40, activation='relu')(DeepCpf1_Input_CA)
DeepCpf1_M = Multiply()([DeepCpf1_D3_SEQ, DeepCpf1_D3_CA])
DeepCpf1_DO4 = Dropout(0.3)(DeepCpf1_M)
DeepCpf1_Output = Dense(1, activation='linear')(DeepCpf1_DO4)
DeepCpf1 = Model(inputs=[DeepCpf1_Input_SEQ, DeepCpf1_Input_CA],
outputs=[DeepCpf1_Output])
# Load scores from the data file into the model
try:
self.logger.info("Loading scores for the CA model")
DeepCpf1.load_weights(
os.path.join(os.path.dirname(__file__), 'kimsong_ca_scores.h5')
) # Renamed 'DeepCpf1_weights.h5' file
except FileNotFoundError:
raise Exception(
"Could not find DeepCpf1 data files containing scores")
return DeepCpf1
def generator_model_mlp_cnn():
input_noise = Input(shape=(100,))
model = Dense(128)(input_noise)
model_1 = Reshape((128,1))(model)
model_1 = UpSampling1D(2) (model_1)
model_1 = Convolution1D(64,35,border_mode='same')(model_1)
model_1 = BatchNormalization()(model_1)
model_1 = LeakyReLU()(model_1)
model_1 = UpSampling1D(2) (model_1)
model_1 = Convolution1D(64,25,border_mode='same')(model_1)
model_1 = BatchNormalization()(model_1)
model_1 = LeakyReLU()(model_1)
model_1 = UpSampling1D(2) (model_1)
model_1 = Convolution1D(64,15,border_mode='same')(model_1)
model_1 = BatchNormalization()(model_1)
model_1 = LeakyReLU()(model_1)
model_1 = UpSampling1D(2) (model_1)
model_1 = Convolution1D(64,15,border_mode='same')(model_1)
model_1 = BatchNormalization()(model_1)
model_1 = LeakyReLU()(model_1)
model_1 = UpSampling1D(2) (model_1)
model_1 = Convolution1D(64,15,border_mode='same')(model_1)
model_1 = BatchNormalization()(model_1)
model_1 = LeakyReLU()(model_1)
model_1 = UpSampling1D(2) (model_1)
model_1 = Convolution1D(64,15,border_mode='same')(model_1)
model_1_1 = Convolution1D(64,7,border_mode='same')(model_1)
model_1_1_1 = Convolution1D(64,4,border_mode='same')(model_1)
model_1 = Add()([model_1,model_1_1,model_1_1_1])
model_1 = BatchNormalization()(model_1)
model_1 = LeakyReLU()(model_1)
model_1 = Convolution1D(1,1,border_mode='same')(model_1)
model_1 = Activation('tanh')(model_1)
model_2 = Dense(8192)(model)
model_2 = Activation('tanh')(model_2)
model_2 = Reshape((8192,1))(model_2)
model = Multiply()([model_1,model_2])
from keras.layers import Lambda
def mean_computation(x):
return K.mean(x,axis=1)
def mean_computation_output_shape(input_shape):
new_shape = tuple([input_shape[0],input_shape[-1]])
return new_shape
def std_computation(x):
return K.std(x,axis=1)
def std_computation_output_shape(input_shape):
new_shape = tuple([input_shape[0],input_shape[-1]])
return new_shape
mean_layer = Lambda(mean_computation,output_shape=mean_computation_output_shape)
std_layer = Lambda(std_computation,output_shape=std_computation_output_shape)
mean = mean_layer(model)
std = std_layer(model)
model = Model(input_noise,model)
model_statistics = Model(input_noise,[mean,std])
return model,model_statistics
def layer(input_tensor):
channel_cnt = int(input_tensor.shape[-1])
x = GlobalMaxPooling2D(data_format="channels_last")(input_tensor)
x = Dense(int(channel_cnt // re))(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Dense(channel_cnt)(x)
x = Activation("sigmoid")(x)
x = Reshape((1, 1, channel_cnt))(x)
x = Multiply()([blockInput, x])
y = Conv2D(1, (1, 1), padding="same")(input_tensor)
y = Activation("sigmoid")(y)
y = Multiply()([blockInput, y])
z = Add()([x, y])
return z
def cab(input1,input2,fn):
x = concatenate([input1,input2],axis=3)
x = AveragePooling2D((x.get_shape().as_list()[1],x.get_shape().as_list()[1]))(x)
x = Conv2D(fn,(1,1),activation="relu", padding="valid")(x)
x = Conv2D(input2.get_shape().as_list()[-1],(1,1),padding="valid",activation="sigmoid")(x)
x = Multiply()([input2,x])
return x
