def test_Residual():
from vnv.keras import LayerFactory as LF, Residual, Stacked, Gated
f_conv = LF("base_conv", kl.Conv2D, 128, (3, 3), padding="same")
f_pool = LF("res_pool", kl.MaxPooling2D, (2, 2))
t_res = Residual(Gated(f_conv) * 2, activation="relu") + f_pool
t_stk = t_res * 3
i = km.Input(Xt.shape[1:])
y = Stacked(
t_stk,
kl.Flatten(),
kl.Dense(128, activation="relu"),
kl.Dropout(0.5),
kl.Dense(128, activation="relu"),
kl.Dense(yt.shape[-1], activation="softmax"),
)(i)
with TFLOCK:
m = compile_model(i, y, loss="cxe", metrics="acc")
m.summary()
m.fit(Xt, yt, epochs=5)
yp = np.argmax(m.predict(Xv), axis=-1)
acc = accuracy_score(np.argmax(yv, axis=-1), yp)
assert acc > 0.8
def build_discriminator():
dropout = 0.4
model = Sequential()
model.add(
layers.Conv2D(64,
kernel_size=5,
strides=2,
input_shape=img_shape,
padding="same"))
model.add(layers.LeakyReLU(alpha=lrelu_alpha))
model.add(layers.Dropout(dropout))
model.add(layers.Conv2D(128, kernel_size=5, strides=2, padding="same"))
model.add(layers.BatchNormalization(momentum=norm_momentum))
model.add(layers.LeakyReLU(alpha=lrelu_alpha))
model.add(layers.Dropout(dropout))
model.add(layers.Conv2D(256, kernel_size=5, strides=2, padding="same"))
model.add(layers.BatchNormalization(momentum=norm_momentum))
model.add(layers.LeakyReLU(alpha=lrelu_alpha))
model.add(layers.Dropout(dropout))
model.add(layers.Conv2D(512, kernel_size=5, strides=1, padding="same"))
model.add(layers.BatchNormalization(momentum=norm_momentum))
model.add(layers.LeakyReLU(alpha=lrelu_alpha))
model.add(layers.Dropout(dropout))
model.add(layers.Flatten())
model.add(layers.Dense(1, activation='sigmoid'))
model.summary()
img = models.Input(shape=img_shape)
validity = model(img)
return models.Model(img, validity)
def build_model_lstm_crf(vocabulary_words,
embedding_matrix,
embedding_dim,
nb_classes,
unit_multiplier=1,
input_dropout=0.0,
dropout=0.0,
recurrent_dropout=0.0,
ouptput_dropout=0.0,
optimizer='rmsprop'):
input = models.Input(shape=(None, ))
model = layers.Embedding(len(vocabulary_words) + 2,
embedding_dim,
weights=[embedding_matrix],
trainable=True,
mask_zero=True)(input)
model = layers.Dropout(input_dropout)(model)
model = layers.Bidirectional(
layers.LSTM(unit_multiplier * (nb_classes + 1),
return_sequences=True,
dropout=dropout,
recurrent_dropout=recurrent_dropout))(model)
model = layers.Dropout(ouptput_dropout)(model)
crf = CRF((nb_classes + 1)) # CRF layer
out = crf(model) # output
model = models.Model(input, out)
model.compile(optimizer=optimizer,
loss=crf.loss_function,
metrics=[crf.accuracy])
return model
def build_model(x, y):
i = models.Input(batch_shape=(None, None, x.shape[2]))
o = tcn.TCN(nb_filters=32, dropout_rate=0.30, return_sequences=True)(i)
o = Dense(y.max() + 1)(o)
o = Activation('softmax')(o)
m = models.Model(i, o)
return m
def load(folder):
history = np.load(os.path.join(folder, 'history.npy')).tolist()
generator = models.load_model(os.path.join(folder, 'gan_g.h5'))
discriminator = models.load_model(os.path.join(folder, 'gan_d.h5'))
discriminator.trainable = False
input_noise = models.Input(shape=(latent_dim, ))
combined = models.Model(input_noise, discriminator(generator(input_noise)))
combined.compile(loss='binary_crossentropy', optimizer=optimizer)
return history, generator, discriminator, combined
def test_compile_model():
from vnv.keras import compile_model
i0 = km.Input(shape=(16, ))
i1 = km.Input(shape=(16, ))
h0 = kl.Dense(16)(i0)
h1 = kl.Dense(16)(i1)
y0 = kl.Dense(2)(h0)
y01 = kl.Dense(2)(h0)
ym = kl.Dense(2)(kl.Concatenate()([h0, h1]))
y1 = kl.Dense(2)(h1)
compile_model(i0, y0)
compile_model([i0, i1], [y0, y1], loss="mse")
compile_model([i0, i1], [y0, y1], loss=["mse", "cxe"])
compile_model(i0, [y0, y01], loss="cxe")
compile_model([i0, i1], ym, loss=["cxe"])
def regressor(input_dim):
model_path = '/home/gautam-admin/EEG/deepspeech_from_brain/models/model_1_100_True.hdf5'
i = models.Input(batch_shape=(None, None, input_dim))
o = TCN(return_sequences=True)(i) # The TCN layers are here.
o = layers.TimeDistributed(layers.Dense(13, activation='linear'))(o)
regressor = models.Model(inputs=[i], outputs=[o])
regressor.load_weights(model_path)
return regressor
def ctc_model(input_dim, nb_labels, padding_value, regression=True):
reg = regressor(input_dim)
i = models.Input(batch_shape=(None, None, input_dim))
o = layers.Masking(mask_value=padding_value)(i)
if regression:
o = reg(o)
o = layers.Bidirectional(
layers.GRU(128, return_sequences=True, dropout=0.1))(o)
o = layers.Bidirectional(layers.GRU(64, return_sequences=True,
dropout=0.1))(o)
o = layers.Bidirectional(layers.GRU(32, return_sequences=True,
dropout=0.1))(o)
o = layers.TimeDistributed(layers.Dense(nb_labels,
activation='softmax'))(o)
model = CTCModel([i], [o])
model.compile(optimizer=optimizers.Adam(lr=1e-2))
return model
def build_generator():
model = Sequential()
model.add(
layers.Dense(512 * 6 * 6, activation='relu',
input_dim=latent_dim)) # 输入维度为100
model.add(layers.Reshape((6, 6, 512)))
model.add(layers.Conv2DTranspose(256, 5, strides=2, padding='same'))
model.add(layers.BatchNormalization(momentum=norm_momentum))
model.add(layers.Activation("relu"))
model.add(layers.Conv2DTranspose(128, 5, strides=2, padding='same'))
model.add(layers.BatchNormalization(momentum=norm_momentum))
model.add(layers.Activation("relu"))
model.add(layers.Conv2DTranspose(64, 5, strides=2, padding='same'))
model.add(layers.BatchNormalization(momentum=norm_momentum))
model.add(layers.Activation("relu"))
model.add(
layers.Conv2DTranspose(img_shape[-1], 5, strides=2, padding='same'))
model.add(layers.Activation("tanh"))
model.summary() # 打印网络参数
noise = models.Input(shape=(latent_dim, ))
img = model(noise)
return models.Model(noise, img) # 定义一个 一个输入noise一个输出img的模型
def test_LayerFactory():
from vnv.keras import LayerFactory, Stacked
cfac = LayerFactory("base_conv", kl.Conv2D, 64, (3, 3), activation="relu")
fpac = LayerFactory("base_pool", kl.MaxPooling2D, (2, 2))
i = km.Input(Xt.shape[1:])
y = Stacked(
(cfac + fpac) * 2,
kl.Flatten(),
kl.Dense(128, activation="relu"),
kl.Dense(yv.shape[-1], activation="softmax"),
)(i)
with TFLOCK:
m = compile_model(i, y, loss="cxe", metrics="acc")
m.summary()
m.fit(Xt, yt, epochs=1)
yp = np.argmax(m.predict(Xv), axis=-1)
acc = accuracy_score(np.argmax(yv, axis=-1), yp)
assert acc > 0.8
def test_Gated():
from vnv.keras import Gated, LayerFactory as LF, Stacked
f_s1 = LF("base_conv", kl.Conv2D, 128, (3, 3), strides=1)
f_s2 = LF("base_conv", kl.Conv2D, 128, (3, 3), strides=2)
# f_pool = LF('base_pool', kl.MaxPooling2D, (2, 2))
t_gate = Gated(f_s1) + Gated(f_s2)
# t_gwp = Stacked(t_gate, f_pool)
# stk = Stacked(t_gwp, t_gwp, t_gwp)
i = km.Input(Xt.shape[1:])
y = Stacked(
f_s1,
kl.Activation("relu"),
t_gate * 2,
kl.Flatten(),
kl.Dense(128, activation="relu"),
kl.Dropout(0.5),
kl.Dense(128, activation="relu"),
kl.Dropout(0.5),
kl.Dense(128, activation="relu"),
kl.Dropout(0.5),
kl.Dense(yt.shape[-1], activation="softmax"),
)(i)
with TFLOCK:
m = compile_model(i, y, loss="cxe", metrics="acc")
m.summary()
m.fit(Xt, yt, epochs=5, validation_data=(Xv, yv))
yp = np.argmax(m.predict(Xv), axis=-1)
acc = accuracy_score(np.argmax(yv, axis=-1), yp)
assert acc > 0.7
def Inception_mainNet():
inpt = KM.Input(shape=(224,224,3))
#padding = 'same'，填充为(步长-1）/2,还可以用ZeroPadding2D((3,3))
x = Conv2d_BN(inpt,64,(7,7),strides=(2,2),padding='same')
x = KL.MaxPooling2D(pool_size=(3,3),strides=(2,2),padding='same')(x)
x = Conv2d_BN(x,192,(3,3),strides=(1,1),padding='same')
x = KL.MaxPooling2D(pool_size=(3,3),strides=(2,2),padding='same')(x)
x = Inception(x,64)#256
x = Inception(x,120)#480
x = KL.MaxPooling2D(pool_size=(3,3),strides=(2,2),padding='same')(x)
x = Inception(x,128)#512
x = Inception(x,128)
x = Inception(x,128)
x = Inception(x,132)#528
x = Inception(x,208)#832
x = KL.MaxPooling2D(pool_size=(3,3),strides=(2,2),padding='same')(x)
x = Inception(x,208)
x = Inception(x,256)#1024
x = KL.AveragePooling2D(pool_size=(7,7),strides=(7,7),padding='same')(x)
x = KL.Dropout(0.4)(x)
x = KL.Dense(1000,activation='relu')(x)
x = KL.Dense(1000,activation='softmax')(x)
model = KM.Model(inpt,x,name='inception')
return model
ae.add(kl.Dense(25, activation='elu', input_shape=(input_dim, )))
ae.add(kl.Dense(16, activation='elu'))
ae.add(
kl.Dense(4, activation='linear', name='bottleneck')
) #This layer contains the compressed representation of the input data ('bottleneck layer')
#Deocder layers
ae.add(kl.Dense(16, activation='elu'))
ae.add(kl.Dense(25, activation='elu'))
ae.add(kl.Dense(34, activation='sigmoid'))
# output dimension of compressed layer from encoder
encoding_output_dim = 4
#specify the size of the input data to the decoder or the size of the output from the encoder
decoder_input = km.Input(shape=(encoding_output_dim, ))
#first layer of the decoder, pass in the size of the output of the encoder to the decoder
decoder = ae.layers[-3](decoder_input)
#second layer of the decoder, pass in the output from the first layer of the decoder
decoder = ae.layers[-2](decoder)
#third and last layer of the decoder, pass in the output from the second layer of the decoder
decoder = ae.layers[-1](decoder)
# build the encoder model with the autoencoder as the input and the output is the compressed data
encoder = km.Model(ae.input,
ae.get_layer('bottleneck').output) # encoder output
#Build the decoder model given the input as the encoder output, and the output of the decoder
def build_birnn_feature_coattention_cnn_model(voca_dim,
time_steps,
num_features,
feature_dim,
output_dim,
model_dim,
mlp_dim,
num_filters,
filter_sizes,
item_embedding=None,
rnn_depth=1,
mlp_depth=1,
drop_out=0.5,
rnn_drop_out=0.,
rnn_state_drop_out=0.,
cnn_drop_out=0.5,
pooling='max',
trainable_embedding=False,
gpu=False,
return_customized_layers=False):
"""
Create A Bidirectional Attention Model.
:param voca_dim: vocabulary dimension size.
:param time_steps: the length of input
:param output_dim: the output dimension size
:param model_dim: rrn dimension size
:param mlp_dim: the dimension size of fully connected layer
:param item_embedding: integer, numpy 2D array, or None (default=None)
If item_embedding is a integer, connect a randomly initialized embedding matrix to the input tensor.
If item_embedding is a matrix, this matrix will be used as the embedding matrix.
If item_embedding is None, then connect input tensor to RNN layer directly.
:param rnn_depth: rnn depth
:param mlp_depth: the depth of fully connected layers
:param num_att_channel: the number of attention channels, this can be used to mimic multi-head attention mechanism
:param drop_out: dropout rate of fully connected layers
:param rnn_drop_out: dropout rate of rnn layers
:param rnn_state_drop_out: dropout rate of rnn state tensor
:param trainable_embedding: boolean
:param gpu: boolean, default=False
If True, CuDNNLSTM is used instead of LSTM for RNN layer.
:param return_customized_layers: boolean, default=False
If True, return model and customized object dictionary, otherwise return model only
:return: keras model
"""
if model_dim % 2 == 1:
model_dim += 1
if item_embedding is not None:
inputs = models.Input(shape=(time_steps, ),
dtype='int32',
name='input0')
x1 = inputs
# item embedding
if isinstance(item_embedding, np.ndarray):
assert voca_dim == item_embedding.shape[0]
x1 = layers.Embedding(voca_dim,
item_embedding.shape[1],
input_length=time_steps,
weights=[
item_embedding,
],
trainable=trainable_embedding,
mask_zero=False,
name='embedding_layer0')(x1)
elif utils.is_integer(item_embedding):
x1 = layers.Embedding(voca_dim,
item_embedding,
input_length=time_steps,
trainable=trainable_embedding,
mask_zero=False,
name='embedding_layer0')(x1)
else:
raise ValueError(
"item_embedding must be either integer or numpy matrix")
else:
inputs = models.Input(shape=(time_steps, voca_dim),
dtype='float32',
name='input0')
x1 = inputs
inputs1 = models.Input(shape=(num_features, feature_dim),
dtype='float32',
name='input1')
x2 = layers.Dense(feature_dim, name="feature_map_layer",
activation="relu")(inputs1)
if gpu:
# rnn encoding
for i in range(rnn_depth):
x1 = layers.Bidirectional(layers.CuDNNLSTM(int(model_dim / 2),
return_sequences=True),
name='bi_lstm_layer' + str(i))(x1)
x1 = layers.BatchNormalization(name='rnn_batch_norm_layer' +
str(i))(x1)
x1 = layers.Dropout(rnn_drop_out,
name="rnn_dropout_layer" + str(i))(x1)
else:
# rnn encoding
for i in range(rnn_depth):
x1 = layers.Bidirectional(layers.LSTM(
int(model_dim / 2),
return_sequences=True,
dropout=rnn_drop_out,
recurrent_dropout=rnn_state_drop_out),
name='bi_lstm_layer' + str(i))(x1)
x1 = layers.BatchNormalization(name='rnn_batch_norm_layer' +
str(i))(x1)
# attention
attens = clayers.CoAttentionWeight(name="coattention_weights_layer")(
[x1, x2])
attens1 = clayers.FeatureNormalization(
name="normalized_coattention_weights_layer1", axis=1)(attens)
attens2 = clayers.FeatureNormalization(
name="normalized_coattention_weights_layer2", axis=2)(attens)
# compare
focus1 = layers.Dot((1, 1), name="focus_layer1")([attens1, x1])
focus2 = layers.Dot((2, 1), name="focus_layer2")([attens2, x2])
pair1 = layers.Concatenate(axis=-1, name="pair_layer1")([x1, focus2])
pair2 = layers.Concatenate(axis=-1, name="pair_layer2")([x2, focus1])
x1 = layers.TimeDistributed(layers.Dense(model_dim, activation="relu"),
name="compare_layer1")(pair1)
x2 = layers.TimeDistributed(layers.Dense(model_dim, activation="relu"),
name="compare_layer2")(pair2)
# Multi-Channel CNN for x1
pooled_outputs = []
for i in range(len(filter_sizes)):
conv = layers.Conv1D(num_filters,
kernel_size=filter_sizes[i],
padding='valid',
activation='relu')(x1)
if pooling == 'max':
conv = layers.MaxPooling1D(pool_size=time_steps - filter_sizes[i] +
1,
strides=1,
padding='valid')(conv)
else:
conv = layers.AveragePooling1D(pool_size=time_steps -
filter_sizes[i] + 1,
strides=1,
padding='valid')(conv)
pooled_outputs.append(conv)
x1 = layers.Concatenate(name='concated_layer')(pooled_outputs)
x1 = layers.Flatten()(x1)
x1 = layers.Dropout(cnn_drop_out, name='conv_dropout_layer')(x1)
x1 = layers.BatchNormalization(name="batch_norm_layer")(x1)
# Average Pool for x2
x2 = layers.GlobalAveragePooling1D(name="average_pool_layer")(x2)
x = layers.Concatenate(axis=1, name="concat_deep_feature_layer")([x1, x2])
# MLP Layers
for i in range(mlp_depth - 1):
x = layers.Dense(mlp_dim,
activation='selu',
kernel_initializer='lecun_normal',
name='selu_layer' + str(i))(x)
x = layers.AlphaDropout(drop_out, name='alpha_layer' + str(i))(x)
outputs = layers.Dense(output_dim,
activation="softmax",
name="softmax_layer0")(x)
model = models.Model(inputs, outputs)
if return_customized_layers:
return model, {
'CoAttentionWeight': clayers.CoAttentionWeight,
"FeatureNormalization": clayers.FeatureNormalization
}
return model
def build_inter_coattention_cnn_model(num_feature_channels1,
num_feature_channels2,
num_features1,
num_features2,
feature_dim1,
output_dim,
num_filters,
filter_sizes,
atten_dim,
model_dim,
mlp_dim,
mlp_depth=1,
drop_out=0.5,
pooling='max',
padding='valid',
return_customized_layers=False):
"""
Create A Multi-Layer Perceptron Model with Coattention Mechanism.
inputs:
embeddings: [batch, num_embed_feature, embed_dims] * 3 ## pronoun, A, B
positional_features: [batch, num_pos_feature] * 2 ## pronoun-A, pronoun-B
outputs:
[batch, num_classes] # in our case there should be 3 output classes: A, B, None
:param output_dim: the output dimension size
:param model_dim: rrn dimension size
:param mlp_dim: the dimension size of fully connected layer
:param mlp_depth: the depth of fully connected layers
:param drop_out: dropout rate of fully connected layers
:param return_customized_layers: boolean, default=False
If True, return model and customized object dictionary, otherwise return model only
:return: keras model
"""
def _mlp_channel1(feature_dropout_layer, x):
#x = feature_dropout_layer(x)
return x
def _mlp_channel2(feature_map_layer, x):
x = feature_map_layer(x)
return x
# inputs
inputs1 = list()
for fi in range(num_feature_channels1):
inputs1.append(
models.Input(shape=(num_features1, feature_dim1),
dtype='float32',
name='input1_' + str(fi)))
inputs2 = list()
for fi in range(num_feature_channels2):
inputs2.append(
models.Input(shape=(num_features2, ),
dtype='float32',
name='input2_' + str(fi)))
# define feature map layers
# MLP Layers
feature_dropout_layer1 = layers.TimeDistributed(
layers.Dropout(rate=drop_out, name="input_dropout_layer"))
feature_map_layer2 = layers.Dense(feature_dim1,
name="feature_map_layer2",
activation="relu")
x1 = [_mlp_channel1(feature_dropout_layer1, input_) for input_ in inputs1]
x2 = [_mlp_channel2(feature_map_layer2, input_) for input_ in inputs2]
# From mention-pair embeddings
reshape_layer = layers.Reshape((1, feature_dim1), name="reshape_layer")
x2 = [reshape_layer(x2_) for x2_ in x2]
pair1 = layers.Concatenate(
axis=1, name="concate_pair1_layer")([x1[0], x1[1], x2[0]])
pair2 = layers.Concatenate(
axis=1, name="concate_pair2_layer")([x1[0], x1[2], x2[1]])
coatten_layer = RemappedCoAttentionWeight(atten_dim,
name="coattention_weights_layer")
featnorm_layer1 = FeatureNormalization(
name="normalized_coattention_weights_layer1", axis=1)
featnorm_layer2 = FeatureNormalization(
name="normalized_coattention_weights_layer2", axis=2)
focus_layer1 = layers.Dot((1, 1), name="focus_layer1")
focus_layer2 = layers.Dot((2, 1), name="focus_layer2")
pair_layer1 = layers.Concatenate(axis=-1, name="pair_layer1")
pair_layer2 = layers.Concatenate(axis=-1, name="pair_layer2")
# attention
attens = coatten_layer([pair1, pair2])
attens1 = featnorm_layer1(attens)
attens2 = featnorm_layer2(attens)
# compare
focus1 = focus_layer1([attens1, pair1])
focus2 = focus_layer2([attens2, pair2])
pair1 = pair_layer1([pair1, focus2])
pair2 = pair_layer2([pair2, focus1])
x = layers.Concatenate(axis=1, name="concate_layer")([pair1, pair2])
x = layers.TimeDistributed(
layers.Dropout(rate=drop_out, name="pair_dropout_layer"))(x)
x = layers.TimeDistributed(
layers.Dense(mlp_dim, name="pair_feature_map_layer",
activation="relu"))(x)
x = layers.Flatten(name="pair_feature_flatten_layer1")(x)
# pooled_outputs = []
# for i in range(len(filter_sizes)):
# conv = layers.Conv1D(num_filters[i], kernel_size=filter_sizes[i], padding=padding, activation='relu')(x)
# if pooling == 'max':
# conv = layers.GlobalMaxPooling1D(name='global_pooling_layer' + str(i))(conv)
# else:
# conv = layers.GlobalAveragePooling1D(name='global_pooling_layer' + str(i))(conv)
# pooled_outputs.append(conv)
# if len(pooled_outputs) > 1:
# x = layers.Concatenate(name='concated_layer')(pooled_outputs)
# else:
# x = conv
# MLP Layers
x = layers.BatchNormalization(name='batch_norm_layer')(x)
x = layers.Dropout(rate=drop_out, name="dropout_layer")(x)
for i in range(mlp_depth - 1):
x = layers.Dense(mlp_dim,
activation='selu',
kernel_initializer='lecun_normal',
name='selu_layer' + str(i))(x)
x = layers.AlphaDropout(drop_out, name='alpha_layer' + str(i))(x)
outputs = layers.Dense(output_dim,
activation="softmax",
name="softmax_layer0")(x)
model = models.Model(inputs1 + inputs2, outputs)
if return_customized_layers:
return model, {
'RemappedCoAttentionWeight': RemappedCoAttentionWeight,
"FeatureNormalization": FeatureNormalization
}
return model
def build_multi_channel_cnn_model(num_feature_channels1,
num_feature_channels2,
num_features1,
num_features2,
feature_dim1,
output_dim,
num_filters,
filter_sizes,
model_dim,
mlp_dim,
mlp_depth=1,
drop_out=0.5,
pooling='max',
padding='valid',
return_customized_layers=False):
"""
Create A Multi-Layer Perceptron Model.
inputs:
embeddings: [batch, num_embed_feature, embed_dims] * 3 ## pronoun, A, B
positional_features: [batch, num_pos_feature] * 2 ## pronoun-A, pronoun-B
outputs:
[batch, num_classes] # in our case there should be 3 output classes: A, B, None
:param output_dim: the output dimension size
:param num_filters: list of integers
The number of filters.
:param filter_sizes: list of integers
The kernel size.
:param pooling: str, either 'max' or 'average'
Pooling method.
:param padding: One of "valid", "causal" or "same" (case-insensitive).
Padding method.
:param model_dim: rrn dimension size
:param mlp_dim: the dimension size of fully connected layer
:param mlp_depth: the depth of fully connected layers
:param drop_out: dropout rate of fully connected layers
:param return_customized_layers: boolean, default=False
If True, return model and customized object dictionary, otherwise return model only
:return: keras model
"""
def _mlp_channel1(feature_dropout_layer, cnns, pools, concate_layer1, x):
x = feature_dropout_layer(x)
pooled_outputs = []
for i in range(len(cnns)):
conv = cnns[i](x)
if pooling == 'max':
conv = pools[i](conv)
else:
conv = pools[i](conv)
pooled_outputs.append(conv)
if len(cnns) == 1:
x = conv
else:
x = concate_layer1(pooled_outputs)
return x
def _mlp_channel2(feature_map_layer, x):
x = feature_map_layer(x)
return x
# inputs
inputs1 = list()
for fi in range(num_feature_channels1):
inputs1.append(
models.Input(shape=(num_features1, feature_dim1),
dtype='float32',
name='input1_' + str(fi)))
inputs2 = list()
for fi in range(num_feature_channels2):
inputs2.append(
models.Input(shape=(num_features2, ),
dtype='float32',
name='input2_' + str(fi)))
# define feature map layers
# CNN Layers
cnns = []
pools = []
feature_dropout_layer1 = layers.TimeDistributed(
layers.Dropout(rate=drop_out, name="input_dropout_layer"))
for i in range(len(filter_sizes)):
cnns.append(
layers.Conv1D(num_filters[i],
kernel_size=filter_sizes[i],
padding=padding,
activation='relu',
name="cc_layer1" + str(i)))
if pooling == 'max':
pools.append(
layers.GlobalMaxPooling1D(name='global_pooling_layer1' +
str(i)))
else:
pools.append(
layers.GlobalAveragePooling1D(name='global_pooling_layer1' +
str(i)))
concate_layer1 = layers.Concatenate(name='concated_layer')
feature_map_layer2 = layers.Dense(model_dim,
name="feature_map_layer2",
activation="relu")
x1 = [
_mlp_channel1(feature_dropout_layer1, cnns, pools, concate_layer1,
input_) for input_ in inputs1
]
x2 = [_mlp_channel2(feature_map_layer2, input_) for input_ in inputs2]
x = layers.Concatenate(axis=1, name="concate_layer")(x1 + x2)
# MLP Layers
x = layers.BatchNormalization(name='batch_norm_layer')(x)
x = layers.Dropout(rate=drop_out, name="dropout_layer")(x)
for i in range(mlp_depth - 1):
x = layers.Dense(mlp_dim,
activation='selu',
kernel_initializer='lecun_normal',
name='selu_layer' + str(i))(x)
x = layers.AlphaDropout(drop_out, name='alpha_layer' + str(i))(x)
outputs = layers.Dense(output_dim,
activation="softmax",
name="softmax_layer0")(x)
model = models.Model(inputs1 + inputs2, outputs)
if return_customized_layers:
return model, {}
return model
def build_mlp_model(num_feature_channels1,
num_feature_channels2,
num_features1,
num_features2,
feature_dim1,
output_dim,
model_dim,
mlp_dim,
mlp_depth=1,
drop_out=0.5,
return_customized_layers=False):
"""
Create A Multi-Layer Perceptron Model.
inputs:
embeddings: [batch, num_embed_feature, embed_dims] * 3 ## pronoun, A, B
positional_features: [batch, num_pos_feature] * 2 ## pronoun-A, pronoun-B
outputs:
[batch, num_classes] # in our case there should be 3 output classes: A, B, None
:param output_dim: the output dimension size
:param model_dim: rrn dimension size
:param mlp_dim: the dimension size of fully connected layer
:param mlp_depth: the depth of fully connected layers
:param drop_out: dropout rate of fully connected layers
:param return_customized_layers: boolean, default=False
If True, return model and customized object dictionary, otherwise return model only
:return: keras model
"""
def _mlp_channel1(feature_dropout_layer, feature_map_layer, flatten_layer,
x):
x = feature_dropout_layer(x)
x = feature_map_layer(x)
x = flatten_layer(x)
return x
def _mlp_channel2(feature_map_layer, x):
x = feature_map_layer(x)
return x
# inputs
inputs1 = list()
for fi in range(num_feature_channels1):
inputs1.append(
models.Input(shape=(num_features1, feature_dim1),
dtype='float32',
name='input1_' + str(fi)))
inputs2 = list()
for fi in range(num_feature_channels2):
inputs2.append(
models.Input(shape=(num_features2, ),
dtype='float32',
name='input2_' + str(fi)))
# define feature map layers
# MLP Layers
feature_dropout_layer1 = layers.TimeDistributed(
layers.Dropout(rate=drop_out, name="input_dropout_layer"))
feature_map_layer1 = layers.TimeDistributed(
layers.Dense(model_dim, name="feature_map_layer1", activation="relu"))
flatten_layer1 = layers.Flatten(name="feature_flatten_layer1")
feature_map_layer2 = layers.Dense(model_dim,
name="feature_map_layer2",
activation="relu")
x1 = [
_mlp_channel1(feature_dropout_layer1, feature_map_layer1,
flatten_layer1, input_) for input_ in inputs1
]
x2 = [_mlp_channel2(feature_map_layer2, input_) for input_ in inputs2]
x = layers.Concatenate(axis=1, name="concate_layer")(x1 + x2)
# MLP Layers
x = layers.BatchNormalization(name='batch_norm_layer')(x)
x = layers.Dropout(rate=drop_out, name="dropout_layer")(x)
for i in range(mlp_depth - 1):
x = layers.Dense(mlp_dim,
activation='selu',
kernel_initializer='lecun_normal',
name='selu_layer' + str(i))(x)
x = layers.AlphaDropout(drop_out, name='alpha_layer' + str(i))(x)
outputs = layers.Dense(output_dim,
activation="softmax",
name="softmax_layer0")(x)
model = models.Model(inputs1 + inputs2, outputs)
if return_customized_layers:
return model, {}
return model
def build_birnn_multifeature_coattention_model(voca_dim,
time_steps,
num_feature_channels,
num_features,
feature_dim,
output_dim,
model_dim,
atten_dim,
mlp_dim,
item_embedding=None,
rnn_depth=1,
mlp_depth=1,
drop_out=0.5,
rnn_drop_out=0.,
rnn_state_drop_out=0.,
trainable_embedding=False,
gpu=False,
return_customized_layers=False):
"""
Create A Bidirectional Attention Model.
:param voca_dim: vocabulary dimension size.
:param time_steps: the length of input
:param output_dim: the output dimension size
:param model_dim: rrn dimension size
:param mlp_dim: the dimension size of fully connected layer
:param item_embedding: integer, numpy 2D array, or None (default=None)
If item_embedding is a integer, connect a randomly initialized embedding matrix to the input tensor.
If item_embedding is a matrix, this matrix will be used as the embedding matrix.
If item_embedding is None, then connect input tensor to RNN layer directly.
:param rnn_depth: rnn depth
:param mlp_depth: the depth of fully connected layers
:param num_feature_channels: the number of attention channels, this can be used to mimic multi-head attention mechanism
:param drop_out: dropout rate of fully connected layers
:param rnn_drop_out: dropout rate of rnn layers
:param rnn_state_drop_out: dropout rate of rnn state tensor
:param trainable_embedding: boolean
:param gpu: boolean, default=False
If True, CuDNNLSTM is used instead of LSTM for RNN layer.
:param return_customized_layers: boolean, default=False
If True, return model and customized object dictionary, otherwise return model only
:return: keras model
"""
if model_dim % 2 == 1:
model_dim += 1
if item_embedding is not None:
inputs = models.Input(shape=(time_steps, ),
dtype='int32',
name='input0')
x1 = inputs
# item embedding
if isinstance(item_embedding, np.ndarray):
assert voca_dim == item_embedding.shape[0]
x1 = layers.Embedding(voca_dim,
item_embedding.shape[1],
input_length=time_steps,
weights=[
item_embedding,
],
trainable=trainable_embedding,
mask_zero=False,
name='embedding_layer0')(x1)
elif utils.is_integer(item_embedding):
x1 = layers.Embedding(voca_dim,
item_embedding,
input_length=time_steps,
trainable=trainable_embedding,
mask_zero=False,
name='embedding_layer0')(x1)
else:
raise ValueError(
"item_embedding must be either integer or numpy matrix")
else:
inputs = models.Input(shape=(time_steps, voca_dim),
dtype='float32',
name='input0')
x1 = inputs
inputs1 = list()
for fi in range(num_feature_channels):
inputs1.append(
models.Input(shape=(num_features, feature_dim),
dtype='float32',
name='input1' + str(fi)))
feature_map_layer = layers.TimeDistributed(layers.Dense(
model_dim, name="feature_map_layer", activation="sigmoid"),
name="td_feature_map_layer")
x2s = list(map(lambda input_: feature_map_layer(input_), inputs1))
if gpu:
# rnn encoding
for i in range(rnn_depth):
x1 = layers.Bidirectional(layers.CuDNNLSTM(int(model_dim / 2),
return_sequences=True),
name='bi_lstm_layer' + str(i))(x1)
x1 = layers.BatchNormalization(name='rnn_batch_norm_layer' +
str(i))(x1)
x1 = layers.Dropout(rnn_drop_out,
name="rnn_dropout_layer" + str(i))(x1)
else:
# rnn encoding
for i in range(rnn_depth):
x1 = layers.Bidirectional(layers.LSTM(
int(model_dim / 2),
return_sequences=True,
dropout=rnn_drop_out,
recurrent_dropout=rnn_state_drop_out),
name='bi_lstm_layer' + str(i))(x1)
x1 = layers.BatchNormalization(name='rnn_batch_norm_layer' +
str(i))(x1)
coatten_layer = clayers.CoAttentionWeight(name="coattention_weights_layer")
featnorm_layer1 = clayers.FeatureNormalization(
name="normalized_coattention_weights_layer1", axis=1)
featnorm_layer2 = clayers.FeatureNormalization(
name="normalized_coattention_weights_layer2", axis=2)
focus_layer1 = layers.Dot((1, 1), name="focus_layer1")
focus_layer2 = layers.Dot((2, 1), name="focus_layer2")
pair_layer1 = layers.Concatenate(axis=-1, name="pair_layer1")
pair_layer2 = layers.Concatenate(axis=-1, name="pair_layer2")
compare_layer1 = layers.TimeDistributed(layers.Dense(model_dim,
activation="relu"),
name="compare_layer1")
compare_layer2 = layers.TimeDistributed(layers.Dense(model_dim,
activation="relu"),
name="compare_layer2")
flatten_layer = layers.Flatten(name="flatten_layer")
xs = list()
for x2_ in x2s:
xs += _coatten_compare_aggregate(coatten_layer, featnorm_layer1,
featnorm_layer2, focus_layer1,
focus_layer2, pair_layer1,
pair_layer2, compare_layer1,
compare_layer2, flatten_layer, x1,
x2_)
x = layers.Concatenate(axis=1, name="concat_feature_layer")(xs)
# MLP Layers
for i in range(mlp_depth - 1):
x = layers.Dense(mlp_dim,
activation='selu',
kernel_initializer='lecun_normal',
name='selu_layer' + str(i))(x)
x = layers.AlphaDropout(drop_out, name='alpha_layer' + str(i))(x)
outputs = layers.Dense(output_dim,
activation="softmax",
name="softmax_layer0")(x)
model = models.Model([inputs] + inputs1, outputs)
if return_customized_layers:
return model, {
'CoAttentionWeight': clayers.CoAttentionWeight,
"FeatureNormalization": clayers.FeatureNormalization
}
return model
def network(x_train, y_train, x_val, y_val, x_test, y_test):
"""
This code execute the network and give as output the trained model
x_train : input train samples
y_train : output train samples
x_val : input validation samples
y_val : output validation samples
x_test : input test samples
y_test : output test samples
"""
from keras import layers, models, optimizers, callbacks
# Network parameters
epochs = 200
batch_size = 32
lr = 4e-3
optimizer = optimizers.Adam(lr=lr)
loss = 'mse'
metrics = 'mean_absolute_error'
# Network
inputs = []
conv_1 = []
pool_1 = []
for n in range(len(x_train)):
inputs.append(models.Input(shape=x_train[n].shape[1:]))
conv_1.append(layers.Conv2D(128, (5, 5), padding='same', activation='relu')(inputs[n]))
pool_1.append(layers.MaxPooling2D((2, 2))(conv_1[n]))
if len(x_train) != 1:
cont = layers.add([pool_1[n] for n in range(len(x_train))])
conv_2 = layers.Conv2D(64, (5, 5), padding='same', activation='relu')(cont)
else:
conv_2 = layers.Conv2D(64, (5, 5), padding='same', activation='relu')(pool_1[0])
pool_2 = layers.MaxPooling2D((2, 2))(conv_2)
conv_3 = layers.Conv2D(32, (5, 5), padding='same', activation='relu')(pool)
pool_3 = layers.MaxPooling2D((2, 2))(conv_3)
flat = layers.Flatten()(pool_3)
dense1 = layers.Dense(80)(flat)
norm1 = layers.BatchNormalization()(dense1)
act1 = layers.Activation('elu')(norm1)
dense2 = layers.Dense(32)(act1)
norm2 = layers.BatchNormalization()(dense2)
act2 = layers.Activation('elu')(norm2)
dense3 = layers.Dense(16)(act2)
norm3 = layers.BatchNormalization()(dense3)
act3 = layers.Activation('elu')(norm3)
output = layers.Dense(len(y_train[0]))(act3)
model = models.Model(inputs=[inputs[n] for n in range(len(x_train))], outputs=output)
model.compile(optimizer=optimizer, loss=loss, metrics=[metrics])
model.summary()
# Function that reduce a learning rate during a run
reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=.25, patience=25, min_lr=5e-8)
history = model.fit([x_train[n] for n in range(len(x_train))], y_train,
batch_size=batch_size,
epochs=epochs,
verbose=2,
callbacks=[reduce_lr],
validation_data=([x_val[n] for n in range(len(x_train))], y_val)
)
score = model.evaluate([x_test[n] for n in range(len(x_train))], y_test, verbose=0)
print("Error: %f" % (score[1]))
return model
X2i = genX2
genX2 = pd.DataFrame(columns=features.columns)
yield [
X1i[0],
tf.convert_to_tensor(X2i.values, dtype=tf.float32)
], X1i[1]
X1i = validation_generator.next()
X2i = genX2
i = 0
yield [X1i[0],
tf.convert_to_tensor(X2i.values, dtype=tf.float32)], X1i[1]
# Model
# Inputs
images = km.Input(shape=(64, 64, 3))
features = km.Input(shape=(200, ))
# Transfer Learning Bases
vgg19 = ka.VGG19(weights='imagenet', include_top=False)
vgg19.trainable = False
# Image Classification Branch
x = vgg19(images)
x = kl.GlobalAveragePooling2D()(x)
x = kl.Dense(32, activation='relu')(x)
x = kl.Dropout(rate=0.25)(x)
x = km.Model(inputs=images, outputs=x)
# Text Classification Branch
y = kl.Embedding(vocab_size, EMBEDDING_LENGTH, input_length=200)(features)
return history, generator, discriminator, combined
# ************************** Load Data
# 数据来源:https://drive.google.com/drive/folders/1mCsY5LEsgCnc0Txv0rpAUhKVPWVkbw5I?usp=sharing
# x = load_dir_img(r'C:\dataset\faces3m96')
print('正在加载数据')
x = np.load(r'C:\Users\78753\Desktop\DL\picpro\faces5m96.npy')
# ************************** 建模
# 对判别器进行构建和编译
discriminator = build_discriminator()
discriminator.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
# 对生成器进行构造
generator = build_generator()
# 构造对抗模型
# 总体模型只对生成器进行训练
discriminator.trainable = False
input_noise = models.Input(shape=(latent_dim, ))
combined = models.Model(input_noise, discriminator(generator(input_noise)))
combined.compile(loss='binary_crossentropy', optimizer=optimizer)
# ************************** 运行
history = run()
# 从断点开始训练
# history, generator, discriminator, combined=load(r'C:\Users\78753\Desktop\DL\picpro\moegirlgandone52g+')
# history = run(history=history)
def build_birnn_cnn_model(voca_dim,
time_steps,
output_dim,
rnn_dim,
mlp_dim,
num_filters,
filter_sizes,
item_embedding=None,
rnn_depth=1,
mlp_depth=1,
drop_out=0.5,
rnn_drop_out=0.5,
rnn_state_drop_out=0.5,
cnn_drop_out=0.5,
pooling='max',
trainable_embedding=False,
gpu=False,
return_customized_layers=False):
"""
Create A Bidirectional CNN Model.
:param voca_dim: vocabulary dimension size.
:param time_steps: the length of input
:param output_dim: the output dimension size
:param rnn_dim: rrn dimension size
:param num_filters: the number of filters
:param filter_sizes: list of integers
The kernel size.
:param mlp_dim: the dimension size of fully connected layer
:param item_embedding: integer, numpy 2D array, or None (default=None)
If item_embedding is a integer, connect a randomly initialized embedding matrix to the input tensor.
If item_embedding is a matrix, this matrix will be used as the embedding matrix.
If item_embedding is None, then connect input tensor to RNN layer directly.
:param rnn_depth: rnn depth
:param mlp_depth: the depth of fully connected layers
:param num_att_channel: the number of attention channels, this can be used to mimic multi-head attention mechanism
:param drop_out: dropout rate of fully connected layers
:param rnn_drop_out: dropout rate of rnn layers
:param rnn_state_drop_out: dropout rate of rnn state tensor
:param cnn_drop_out: dropout rate of between cnn layer and fully connected layers
:param pooling: str, either 'max' or 'average'
Pooling method.
:param trainable_embedding: boolean
:param gpu: boolean, default=False
If True, CuDNNLSTM is used instead of LSTM for RNN layer.
:param return_customized_layers: boolean, default=False
If True, return model and customized object dictionary, otherwise return model only
:return: keras model
"""
if item_embedding is not None:
inputs = models.Input(shape=(time_steps, ),
dtype='int32',
name='input0')
x = inputs
# item embedding
if isinstance(item_embedding, np.ndarray):
assert voca_dim == item_embedding.shape[0]
x = layers.Embedding(voca_dim,
item_embedding.shape[1],
input_length=time_steps,
weights=[
item_embedding,
],
trainable=trainable_embedding,
mask_zero=False,
name='embedding_layer0')(x)
elif utils.is_integer(item_embedding):
x = layers.Embedding(voca_dim,
item_embedding,
input_length=time_steps,
trainable=trainable_embedding,
mask_zero=False,
name='embedding_layer0')(x)
else:
raise ValueError(
"item_embedding must be either integer or numpy matrix")
else:
inputs = models.Input(shape=(time_steps, voca_dim),
dtype='float32',
name='input0')
x = inputs
if gpu:
# rnn encoding
for i in range(rnn_depth):
x = layers.Bidirectional(layers.CuDNNLSTM(rnn_dim,
return_sequences=True),
name='bi_lstm_layer' + str(i))(x)
x = layers.BatchNormalization(name='rnn_batch_norm_layer' +
str(i))(x)
x = layers.Dropout(rnn_drop_out,
name="rnn_dropout_layer" + str(i))(x)
else:
# rnn encoding
for i in range(rnn_depth):
x = layers.Bidirectional(layers.LSTM(
rnn_dim,
return_sequences=True,
dropout=rnn_drop_out,
recurrent_dropout=rnn_state_drop_out),
name='bi_lstm_layer' + str(i))(x)
x = layers.BatchNormalization(name='rnn_batch_norm_layer' +
str(i))(x)
pooled_outputs = []
for i in range(len(filter_sizes)):
conv = layers.Conv1D(num_filters,
kernel_size=filter_sizes[i],
padding='valid',
activation='relu')(x)
if pooling == 'max':
conv = layers.MaxPooling1D(pool_size=time_steps - filter_sizes[i] +
1,
strides=1,
padding='valid')(conv)
else:
conv = layers.AveragePooling1D(pool_size=time_steps -
filter_sizes[i] + 1,
strides=1,
padding='valid')(conv)
pooled_outputs.append(conv)
x = layers.Concatenate(name='concated_layer')(pooled_outputs)
x = layers.Flatten()(x)
x = layers.Dropout(cnn_drop_out, name='conv_dropout_layer')(x)
x = layers.BatchNormalization(name="batch_norm_layer")(x)
# MLP Layers
for i in range(mlp_depth - 1):
x = layers.Dense(mlp_dim,
activation='selu',
kernel_initializer='lecun_normal',
name='selu_layer' + str(i))(x)
x = layers.AlphaDropout(drop_out, name='alpha_layer' + str(i))(x)
outputs = layers.Dense(output_dim,
activation="softmax",
name="softmax_layer0")(x)
model = models.Model(inputs, outputs)
if return_customized_layers:
return model, dict()
return model
import keras from keras.datasets import imdb from keras.preprocessing.sequence import pad_sequences from keras import models from keras import layers max_features = 10000 max_len = 500 (X_train, y_train), (X_test, y_test) = imdb.load_data(num_words=max_features) X_train_pad = pad_sequences(X_train, max_len, padding='post') X_test_pad = pad_sequences(X_test, max_len, padding='post') inputs = models.Input(shape=(500, )) #channel #1 embedding1 = layers.Embedding(input_dim=max_features, output_dim=100)(inputs) conv1 = layers.Conv1D(32, 4, activation='relu')(embedding1) drop1 = layers.Dropout(0.5)(conv1) mp1 = layers.MaxPool1D()(drop1) flat1 = layers.Flatten()(mp1) #channel #2 embedding2 = layers.Embedding(input_dim=max_features, output_dim=100)(inputs) conv2 = layers.Conv1D(32, 6, activation='relu')(embedding2) drop2 = layers.Dropout(0.5)(conv2) mp2 = layers.MaxPool1D()(drop2) flat2 = layers.Flatten()(mp2)
from keras import models, layers
from keras import backend as K
from keras.callbacks import ModelCheckpoint
from src.data_generator import DataGenerator
from src.image_utils import get_ids
def dice_coef(y_true, y_pred, smooth=1):
y_true_f = K.flatten(y_true)
y_pred_f = K.flatten(y_pred)
intersection = K.sum(y_true_f * y_pred_f)
return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) +
smooth)
inputs = models.Input((256, 1600, 1))
c1 = layers.Conv2D(16, (3, 3), activation='elu', padding='same')(inputs)
c1 = layers.Conv2D(16, (3, 3), activation='elu', padding='same')(c1)
m1 = layers.MaxPooling2D((2, 2))(c1)
c2 = layers.Conv2D(32, (3, 3), activation='elu', padding='same')(m1)
c2 = layers.Conv2D(32, (3, 3), activation='elu', padding='same')(c2)
m2 = layers.MaxPooling2D((2, 2))(c2)
c_ = layers.Conv2D(64, (3, 3), activation='elu', padding='same')(m2)
c_ = layers.Conv2D(64, (3, 3), activation='elu', padding='same')(c_)
u2 = layers.Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c_)
u2 = layers.concatenate([u2, c2])
c3 = layers.Conv2D(32, (3, 3), activation='elu', padding='same')(u2)
c3 = layers.Conv2D(32, (3, 3), activation='elu', padding='same')(c3)
def build_model_resUnet(self):
def Tanimoto_loss(label,pred):
square=tf.square(pred)
sum_square=tf.reduce_sum(square,axis=-1)
product=tf.multiply(pred,label)
sum_product=tf.reduce_sum(product,axis=-1)
denomintor=tf.subtract(tf.add(sum_square,1),sum_product)
loss=tf.divide(sum_product,denomintor)
loss=tf.reduce_mean(loss)
return 1.0-loss
def Tanimoto_dual_loss(label,pred):
loss1=Tanimoto_loss(pred,label)
pred=tf.subtract(1.0,pred)
label=tf.subtract(1.0,label)
loss2=Tanimoto_loss(label,pred)
loss=(loss1+loss2)/2
def ResBlock(input,filter,kernel_size,dilation_rates,stride):
def branch(dilation_rate):
x=KL.BatchNormalization()(input)
x=KL.Activation('relu')(x)
x=KL.Conv2D(filter,kernel_size,strides=stride,dilation_rate=dilation_rate,padding='same')(x)
x=KL.BatchNormalization()(x)
x=KL.Activation('relu')(x)
x=KL.Conv2D(filter,kernel_size,strides=stride,dilation_rate=dilation_rate,padding='same')(x)
return x
out=[]
for d in dilation_rates:
out.append(branch(d))
if len(dilation_rates)>1:
out=KL.Add()(out)
else:
out=out[0]
return out
def PSPPooling(input,filter):
x1=KL.MaxPooling2D(pool_size=(2,2))(input)
x2=KL.MaxPooling2D(pool_size=(4,4))(input)
x3=KL.MaxPooling2D(pool_size=(8,8))(input)
x4=KL.MaxPooling2D(pool_size=(16,16))(input)
x1=KL.Conv2D(int(filter/4),(1,1))(x1)
x2=KL.Conv2D(int(filter/4),(1,1))(x2)
x3=KL.Conv2D(int(filter/4),(1,1))(x3)
x4=KL.Conv2D(int(filter/4),(1,1))(x4)
x1=KL.UpSampling2D(size=(2,2))(x1)
x2=KL.UpSampling2D(size=(4,4))(x2)
x3=KL.UpSampling2D(size=(8,8))(x3)
x4=KL.UpSampling2D(size=(16,16))(x4)
x=KL.Concatenate()([x1,x2,x3,x4,input])
x=KL.Conv2D(filter,(1,1))(x)
return x
def combine(input1,input2,filter):
x=KL.Activation('relu')(input1)
x=KL.Concatenate()([x,input2])
x=KL.Conv2D(filter,(1,1))(x)
return x
inputs=KM.Input(shape=(self.config.IMAGE_H, self.config.IMAGE_W, self.config.IMAGE_C))
c1=x=KL.Conv2D(32,(1,1),strides=(1,1),dilation_rate=1)(inputs)
c2=x=ResBlock(x,32,(3,3),[1,3,15,31],(1,1))
x=KL.Conv2D(64,(1,1),strides=(2,2))(x)
c3=x=ResBlock(x,64,(3,3),[1,3,15,31],(1,1))
x=KL.Conv2D(128,(1,1),strides=(2,2))(x)
c4=x=ResBlock(x,128,(3,3),[1,3,15],(1,1))
x=KL.Conv2D(256,(1,1),strides=(2,2))(x)
c5=x=ResBlock(x,256,(3,3),[1,3,15],(1,1))
x=KL.Conv2D(512,(1,1),strides=(2,2))(x)
c6=x=ResBlock(x,512,(3,3),[1],(1,1))
x=KL.Conv2D(1024,(1,1),strides=(2,2))(x)
x=ResBlock(x,1024,(3,3),[1],(1,1))
x=PSPPooling(x,1024)
x=KL.Conv2D(512,(1,1))(x)
x=KL.UpSampling2D()(x)
x=combine(x,c6,512)
x=ResBlock(x,512,(3,3),[1],1)
x=KL.Conv2D(256,(1,1))(x)
x=KL.UpSampling2D()(x)
x=combine(x,c5,256)
x=ResBlock(x,256,(3,3),[1,3,15],1)
x=KL.Conv2D(128,(1,1))(x)
x=KL.UpSampling2D()(x)
x=combine(x,c4,128)
x=ResBlock(x,128,(3,3),[1,3,15],1)
x=KL.Conv2D(64,(1,1))(x)
x=KL.UpSampling2D()(x)
x=combine(x,c3,64)
x=ResBlock(x,64,(3,3),[1,3,15,31],1)
x=KL.Conv2D(32,(1,1))(x)
x=KL.UpSampling2D()(x)
x=combine(x,c2,32)
x=ResBlock(x,32,(3,3),[1,3,15,31],1)
x=combine(x,c1,32)
x=PSPPooling(x,32)
x=KL.Conv2D(self.config.CLASSES_NUM,(1,1))(x)
x=KL.Activation('softmax')(x)
model=KM.Model(inputs=inputs,outputs=x)
model.compile(optimizer=keras.optimizers.SGD(lr=0.001,momentum=0.8),loss=Tanimoto_loss,metrics=['accuracy'])
model.summary()
return model
batch = K.shape(z_mean)[0]
dim = K.int_shape(z_mean)[1]
# By default, random_normal has mean=0 and std=1.0
epsilon = K.random_normal(shape=(batch, dim))
# Return the reparameterized result.
return z_mean + K.exp(0.5 * z_log_var) * epsilon
####################################################################
# The Encoder
# The image inputted into the encoder.
input_image = km.Input(shape=(28, 28, 1))
# Add a 2D convolutional layer, with 16 feature maps.
# input size = 28 x 28 x 1
# output size = 28 x 28 x 16
x = kl.Conv2D(16, kernel_size=(3, 3), activation='relu',
padding='same')(input_image)
# Add a max pooling layer.
# input size 28 x 28 x 16
# output size 14 x 14 x 16
x = kl.MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)
# Add a 2D convolutional layer, with 32 feature maps.
# input size = 14 x 14 x 16
# output size = 14 x 14 x 32
def GAN(self):
net = NetworkComponents(CONFIG_FILE=self.config_file,
PATH_OUTPUT=self.path_output)
# built generator networks
if (self.conf.type_of_gen == 'auto'):
self.generator = net.Autoencoder()
elif (self.conf.type_of_gen == 'auto31'):
self.generator = net.Auto3Encoder1Decoder()
elif (self.conf.type_of_gen == 'unet'):
self.generator = net.Unet()
# built adversary network
if (self.nr_discr == 1):
self.discriminator = net.Discriminator()
else:
self.globaldiscriminator = net.GlobalDiscriminator()
self.localdiscriminator = net.LocalDiscriminator()
# resume models weights from last checkpoint
if (self.conf.resume_path != None and self.conf.resume_epoch != 0):
self.generator.load_weights(
'%sweights/model_weights-ep%d.h5' %
(self.conf.resume_path, self.conf.resume_epoch))
self.discriminator.load_weights(
'%sweights/model_weights-ep%d.h5' %
(self.conf.resume_path, self.conf.resume_epoch))
print(
'Adversary and Generator Model weights resumed from:\tmodel_weights-ep%d.h5'
% (self.conf.resume_epoch))
# copy logs checkpoints
os.system('cp %s*ep-%d.txt %scheckpoints/' %
(self.conf.resume_path, self.conf.resume_epoch,
self.path_output))
else:
print('GAN Model Created')
# freeze discriminator and its layers, un-freeze generator and its layers
UnFreezeNetwork(net=self.generator, state=True)
if (self.nr_discr == 1):
UnFreezeNetwork(net=self.discriminator, state=False)
else:
UnFreezeNetwork(net=self.globaldiscriminator, state=False)
UnFreezeNetwork(net=self.localdiscriminator, state=False)
# Create GAN network by stucking the generator and adversary network
input_img = models.Input(shape=self.conf.img_shape, name='input_img')
input_mask = models.Input(shape=self.conf.img_shape, name='input_mask')
reconstr_img = self.generator([input_img,
input_mask]) # generator layers
if (self.nr_discr == 1):
label_output = self.discriminator(
reconstr_img) # discriminator layers
self.gan = models.Model([input_img, input_mask],
label_output,
name='GAN')
self.gan.compile(loss=self.lossGAN, optimizer=self.optimizer)
else:
global_label_output = self.globaldiscriminator(reconstr_img)
local_label_output = self.localdiscriminator(
[reconstr_img, input_mask])
self.gan = models.Model([input_img, input_mask],
[global_label_output, local_label_output],
name='GAN')
self.gan.compile(loss=self.lossGAN,
loss_weights=self.wlossGAN,
optimizer=self.optimizer)
"""
if(len(self.lossGAN) == 1 and len(self.wlossGAN) == 1):
self.gan = models.Model([input_img, input_mask], label_output)
self.gan.compile(loss=self.lossGAN, optimizer=self.optimizer)
else:
self.gan = models.Model([input_img, input_mask], [reconstr_img, label_output])
self.gan.compile(loss=self.lossGAN, loss_weights=self.wlossGAN, optimizer=self.optimizer)
"""
# un-freeze discriminator and its layers, freeze generator and its layers
UnFreezeNetwork(net=self.generator, state=False)
if (self.nr_discr == 1):
UnFreezeNetwork(net=self.discriminator, state=True)
# compile discriminator
self.discriminator.compile(loss=self.lossD,
optimizer=self.optimizer)
else:
UnFreezeNetwork(net=self.globaldiscriminator, state=True)
UnFreezeNetwork(net=self.localdiscriminator, state=True)
# compile discriminators
self.globaldiscriminator.compile(loss=self.lossD,
optimizer=self.optimizer)
self.localdiscriminator.compile(loss=self.lossD,
optimizer=self.optimizer)
# Save model visualization
plot_model(self.generator,
to_file=self.path_output +
'model/generator_visualization.png',
show_shapes=True,
show_layer_names=True)
if (self.nr_discr == 1):
plot_model(self.discriminator,
to_file=self.path_output +
'model/adversary_visualization.png',
show_shapes=True,
show_layer_names=True)
else:
plot_model(self.globaldiscriminator,
to_file=self.path_output +
'model/global_adversary_visualization.png',
show_shapes=True,
show_layer_names=True)
plot_model(self.localdiscriminator,
to_file=self.path_output +
'model/local_adversary_visualization.png',
show_shapes=True,
show_layer_names=True)
plot_model(self.gan,
to_file=self.path_output + 'model/gan_visualization.png',
show_shapes=True,
show_layer_names=True)
return self.gan
def build_birnn_attention_model(voca_dim,
time_steps,
output_dim,
rnn_dim,
mlp_dim,
item_embedding=None,
rnn_depth=1,
mlp_depth=1,
num_att_channel=1,
drop_out=0.5,
rnn_drop_out=0.,
rnn_state_drop_out=0.,
trainable_embedding=False,
gpu=False,
return_customized_layers=False):
"""
Create A Bidirectional Attention Model.
:param voca_dim: vocabulary dimension size.
:param time_steps: the length of input
:param output_dim: the output dimension size
:param rnn_dim: rrn dimension size
:param mlp_dim: the dimension size of fully connected layer
:param item_embedding: integer, numpy 2D array, or None (default=None)
If item_embedding is a integer, connect a randomly initialized embedding matrix to the input tensor.
If item_embedding is a matrix, this matrix will be used as the embedding matrix.
If item_embedding is None, then connect input tensor to RNN layer directly.
:param rnn_depth: rnn depth
:param mlp_depth: the depth of fully connected layers
:param num_att_channel: the number of attention channels, this can be used to mimic multi-head attention mechanism
:param drop_out: dropout rate of fully connected layers
:param rnn_drop_out: dropout rate of rnn layers
:param rnn_state_drop_out: dropout rate of rnn state tensor
:param trainable_embedding: boolean
:param gpu: boolean, default=False
If True, CuDNNLSTM is used instead of LSTM for RNN layer.
:param return_customized_layers: boolean, default=False
If True, return model and customized object dictionary, otherwise return model only
:return: keras model
"""
if item_embedding is not None:
inputs = models.Input(shape=(time_steps, ),
dtype='int32',
name='input0')
x = inputs
# item embedding
if isinstance(item_embedding, np.ndarray):
assert voca_dim == item_embedding.shape[0]
x = layers.Embedding(voca_dim,
item_embedding.shape[1],
input_length=time_steps,
weights=[
item_embedding,
],
trainable=trainable_embedding,
mask_zero=False,
name='embedding_layer0')(x)
elif utils.is_integer(item_embedding):
x = layers.Embedding(voca_dim,
item_embedding,
input_length=time_steps,
trainable=trainable_embedding,
mask_zero=False,
name='embedding_layer0')(x)
else:
raise ValueError(
"item_embedding must be either integer or numpy matrix")
else:
inputs = models.Input(shape=(time_steps, voca_dim),
dtype='float32',
name='input0')
x = inputs
if gpu:
# rnn encoding
for i in range(rnn_depth):
x = layers.Bidirectional(layers.CuDNNLSTM(rnn_dim,
return_sequences=True),
name='bi_lstm_layer' + str(i))(x)
x = layers.BatchNormalization(name='rnn_batch_norm_layer' +
str(i))(x)
x = layers.Dropout(rnn_drop_out,
name="rnn_dropout_layer" + str(i))(x)
else:
# rnn encoding
for i in range(rnn_depth):
x = layers.Bidirectional(layers.LSTM(
rnn_dim,
return_sequences=True,
dropout=rnn_drop_out,
recurrent_dropout=rnn_state_drop_out),
name='bi_lstm_layer' + str(i))(x)
x = layers.BatchNormalization(name='rnn_batch_norm_layer' +
str(i))(x)
# attention
attention_heads = []
x_per = layers.Permute((2, 1), name='permuted_attention_x')(x)
for h in range(max(1, num_att_channel)):
attention = clayers.AttentionWeight(name="attention_weights_layer" +
str(h))(x)
xx = layers.Dot([2, 1], name='focus_head' + str(h) +
'_layer0')([x_per, attention])
attention_heads.append(xx)
if num_att_channel > 1:
x = layers.Concatenate(name='focus_layer0')(attention_heads)
else:
x = attention_heads[0]
x = layers.BatchNormalization(name='focused_batch_norm_layer')(x)
# MLP Layers
for i in range(mlp_depth - 1):
x = layers.Dense(mlp_dim,
activation='selu',
kernel_initializer='lecun_normal',
name='selu_layer' + str(i))(x)
x = layers.AlphaDropout(drop_out, name='alpha_layer' + str(i))(x)
outputs = layers.Dense(output_dim,
activation="softmax",
name="softmax_layer0")(x)
model = models.Model(inputs, outputs)
if return_customized_layers:
return model, {'AttentionWeight': clayers.AttentionWeight}
return model
