def unet_resnet101(img_rows, img_cols, color_type, num_classes=1):
encode_model, layers = resnet101_model(img_rows, img_cols, color_type)
input = encode_model.input
# layer1 = encode_model.get_layer('conv1_relu')
# layer2 = encode_model.get_layer('res2c_relu')
# layer3 = encode_model.get_layer('res3b2_relu')
# layer4 = encode_model.get_layer('res4b22_relu')
layer1, layer2, layer3, layer4 = layers
x = encode_model.output
x = upsample(x, layer4, 1024 // 2)
x = upsample(x, layer3, 512 // 2)
x = upsample(x, layer2, 256 // 2)
x = upsample(x, layer1, 128 // 2)
x = upsample(x, input, 64 // 2)
output1 = Conv2D(num_classes, (1, 1), activation='sigmoid')(x)
if not USE_REFINE_NET:
model = Model(inputs=input, outputs=[output1])
# model.load_weights('../weights/head-segmentation-model.h5')
# model.compile(optimizer=SGD(lr=0.01, momentum=0.9), loss='binary_crossentropy', metrics=[dice_loss])
return model
else:
inputs2 = concatenate([input, output1])
outputs2 = block2(inputs2, 64)
outputs2 = average([output1, outputs2])
model2 = Model(inputs=input, outputs=[output1, outputs2])
return model2
def getModel():
inputShape = (32, 128, 1)
rnnUnits = 256
maxStringLen = 32
inputs = Input(name='inputX', shape=inputShape, dtype='float32')
inner = res_block(inputs, 64)
inner = res_block(inner, 64)
inner = MaxPooling2D(pool_size=(2, 2), name='MaxPoolName1')(inner)
inner = res_block(inner, 128)
inner = res_block(inner, 128)
inner = MaxPooling2D(pool_size=(2, 2), name='MaxPoolName2')(inner)
inner = res_block(inner, 256)
inner = res_block(inner, 256)
inner = MaxPooling2D(pool_size=(1, 2), strides=(2, 2),
name='MaxPoolName4')(inner)
inner = res_block(inner, 512)
inner = res_block(inner, 512)
inner = MaxPooling2D(pool_size=(1, 2), strides=(2, 2),
name='MaxPoolName6')(inner)
inner = res_block(inner, 512)
inner = Reshape(target_shape=(maxStringLen, rnnUnits),
name='reshape')(inner)
LSF = LSTM(rnnUnits,
return_sequences=True,
kernel_initializer='he_normal',
name='LSTM1F')(inner)
LSB = LSTM(rnnUnits,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='LSTM1B')(inner)
LSB = Lambda(lambda inputTensor: K.reverse(inputTensor, axes=1))(LSB)
LS1 = average([LSF, LSB])
LS1 = BatchNormalization()(LS1)
LSF = LSTM(rnnUnits,
return_sequences=True,
kernel_initializer='he_normal',
name='LSTM2F')(LS1)
LSB = LSTM(rnnUnits,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='LSTM2B')(LS1)
LSB = Lambda(lambda inputTensor: K.reverse(inputTensor, axes=1))(LSB)
LS2 = concatenate([LSF, LSB])
LS2 = BatchNormalization()(LS2)
yPred = Dense(len(unicodes) + 1,
kernel_initializer='he_normal',
name='dense2')(LS2)
yPred = Activation('softmax')(yPred)
return Model(inputs=[inputs], outputs=yPred)
def get_csv_plus_image_model(network,
num_classes,
features,
img_width,
img_height,
merge_type='concat'):
"""Returns a model that uses images and csv as Input
Arguments:
network {string} -- Name of the network to use pretrained on imagenet
num_classes {int} -- Number of classes in the dataset
features {int} -- Number of features used from csv data
img_width {int} -- Image width
img_height {int} -- Image height
Keyword Arguments:
merge_type {str} -- [Type of merge process for inner neural networks] (default: {'concat'})
Returns:
[tuple] -- Keras Model with dual input and last layer number from cnn part.
"""
input_shape = (img_width, img_height, 3)
image_input = Input(shape=input_shape)
base_model, last_layer_number = get_cnn_model(network, input_shape,
image_input)
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(512, activation='relu')(x)
# Create MLP using features from csv files
aux_input = Input(shape=(features, ))
aux = Dense(512, activation='relu')(aux_input)
aux = Dropout(0.3)(aux)
aux = Dense(512, activation='relu')(aux)
aux = Dropout(0.3)(aux)
aux = Dense(512, activation='relu')(aux)
# Merge both inputs
if merge_type == 'concat':
merge = concatenate([x, aux])
elif merge_type == 'add':
merge = add([x, aux])
elif merge_type == 'mul':
merge = multiply([x, aux])
elif merge_type == 'avg':
merge = average([x, aux])
predictions = Dense(num_classes, activation='softmax')(merge)
return Model(inputs=[base_model.input, aux_input],
output=predictions), last_layer_number
def call(self, xs, mask=None):
assert len(xs) == 2
# separate out input matrices
# x1, x2: (BATCH_SIZE, MAX_TIMESTEPS, EMBED_SIZE)
x1, x2 = xs
# build alignment matrix
# alpha: (BATCH_SIZE, MAX_TIMESTEPS, MAX_TIMESTEPS)
alpha = K.softmax(
K.batch_dot(K.dot(x2, self.W), K.permute_dimensions(x1,
(0, 2, 1))))
# build context vectors
# c1, c2: (BATCH_SIZE, MAX_TIMESTEPS, EMBED_SIZE)
c1 = K.repeat_elements(K.sum(K.batch_dot(alpha, x2),
axis=1,
keepdims=True),
self.max_timesteps,
axis=1)
c2 = K.repeat_elements(K.sum(K.batch_dot(
K.permute_dimensions(alpha, (0, 2, 1)), x1),
axis=1,
keepdims=True),
self.max_timesteps,
axis=1)
# build attention vector
# o1t, o2t: (BATCH_SIZE, MAX_TIMESTEPS, EMBED_SIZE)
o1t = K.tanh(K.dot(K.concatenate([c1, x1], axis=2), self.U1))
o2t = K.tanh(K.dot(K.concatenate([c2, x2], axis=2), self.U2))
# masking
if mask is not None and mask[0] is not None:
o1t *= K.cast(
K.repeat_elements(K.expand_dims(mask[0], axis=2), o1t.shape[2],
2), K.floatx())
if mask is not None and mask[1] is not None:
o2t *= K.cast(
K.repeat_elements(K.expand_dims(mask[0], axis=2), o2t.shape[2],
2), K.floatx())
# sum over timesteps
# o1, o2: (BATCH_SIZE, EMBED_SIZE)
o1 = K.sum(o1t, axis=1)
o2 = K.sum(o2t, axis=1)
# merge the attention vectors according to merge_mode
if self.merge_mode == "concat":
return concatenate([o1, o2], axis=1)
elif self.merge_mode == "diff":
return add([o1, -o2])
elif self.merge_mode == "prod":
return multiply([o1, o2])
elif self.merge_mode == "avg":
return average([o1, o2])
else: # max
return maximum([o1, o2])
def net(channel_axis):
# Determine proper input shape
in_shape = [(24, 42, 42, 1), (42, 24, 42, 1), (42, 42, 24, 1)]
strides = [(1, 2, 2), (2, 1, 2), (2, 2, 1)]
kernels = [(5, 5, 5), (5, 5, 5), (5, 5, 5)]
dropout_conv = .2
dropout_dence = .2
inputs = [keras.layers.Input(shape=inx) for inx in in_shape]
coders = [
_encoder(in_tensor=tnsr,
stride=stride,
bn_axis=channel_axis,
kernel=kernel,
dropout=dropout_conv)
for tnsr, stride, kernel in zip(inputs, strides, kernels)
]
x = average(coders)
# shape: 128, 9, 10, 10
x = _conv_block(x,
3, [128, 128],
dropout=dropout_conv,
bn_axis=channel_axis)
x = _identity_block(x,
3, [128, 128],
dropout=dropout_conv,
bn_axis=channel_axis)
x = _conv_block(x,
3, [256, 256],
stride=(2, 2, 2),
dropout=dropout_conv,
bn_axis=channel_axis)
x = _identity_block(x,
3, [256, 256],
dropout=dropout_conv,
bn_axis=channel_axis)
x = keras.layers.BatchNormalization(axis=channel_axis)(x)
x = keras.layers.Activation('relu')(x)
x = keras.layers.AveragePooling3D((2, 2, 2))(x)
x = keras.layers.Flatten()(x)
x = keras.layers.Dropout(dropout_dence)(x)
x = keras.layers.Dense(256)(x)
x = keras.layers.LeakyReLU(.3)(x)
x = keras.layers.Dropout(dropout_dence)(x)
x = keras.layers.Dense(2, activation='softmax', name='is_nodule')(x)
model = keras.models.Model(inputs, x)
model.compile('adam', 'categorical_crossentropy')
return model
def show_basic_eval_for_snapshot_combo(snapshot_weight_list,
scratch_model_func, weight_dirname,
test_data):
# Seperate test_data
x_test, y_test = test_data
nb_classes = y_test.shape[1]
# Create Snapshot Ensemble model
chkpt_models = []
for i, filename in enumerate(snapshot_weight_list):
model = scratch_model_func()
model.load_weights(weight_dirname + filename)
for j, cur_layer in enumerate(model.layers):
cur_layer.name = ("tmpy_%d_%d" % (i, j))
chkpt_models.append(model)
chkpt_inputs = [model.input for model in chkpt_models]
chkpt_outputs = [model.output for model in chkpt_models]
if len(chkpt_models) > 1:
final_output = average(chkpt_outputs)
else:
final_output = chkpt_outputs[0]
snapshot_model = Model(inputs=chkpt_inputs, outputs=final_output)
snapshot_model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy', metrics.top_k_categorical_accuracy])
# Display eval metrics
snapshot_size = len(snapshot_weight_list)
print("\nEvaluating the %d model shotshot" % snapshot_size)
print(snapshot_weight_list)
eval_metrics = snapshot_model.evaluate([x_test] * snapshot_size,
y_test,
batch_size=int(
math.ceil(48 / snapshot_size)),
verbose=1)
print(snapshot_model.metrics_names)
print(eval_metrics)
# Return nothing
return None
def Model_sent2tag_MLP_1(sentvocabsize, tagvocabsize, sent_W, tag_W, s2v_k,
tag2v_k):
input_sent = Input(shape=(1, ), dtype='int32')
sent_embedding = Embedding(input_dim=sentvocabsize,
output_dim=s2v_k,
input_length=1,
mask_zero=False,
trainable=False,
weights=[sent_W])(input_sent)
input_tag = Input(shape=(1, ), dtype='int32')
tag_embedding = Embedding(input_dim=tagvocabsize,
output_dim=tag2v_k,
input_length=1,
mask_zero=False,
trainable=False,
weights=[tag_W])(input_tag)
x1_1 = Flatten()(sent_embedding)
x2_0 = Flatten()(tag_embedding)
# x1_1 = Dense(100, activation='tanh')(x1_0)
sub = subtract([x2_0, x1_1])
mul = multiply([x2_0, x1_1])
max = maximum([x2_0, x1_1])
avg = average([x2_0, x1_1])
class_input = concatenate([x2_0, x1_1, sub, mul, max, avg], axis=-1)
# class_input = Flatten()(class_input)
class_mlp1 = Dense(200, activation='tanh')(class_input)
class_mlp1 = Dropout(0.5)(class_mlp1)
class_mlp2 = Dense(2)(class_mlp1)
class_output = Activation('softmax', name='CLASS')(class_mlp2)
# distance = Lambda(euclidean_distance, output_shape=eucl_dist_output_shape)([mlp_x1_2, x2_0])
# distance = dot([x1_0, x2_0], axes=-1, normalize=True)
mymodel = Model([input_sent, input_tag], class_output)
mymodel.compile(loss='categorical_crossentropy',
optimizer=optimizers.Adam(lr=0.001),
metrics=['acc'])
return mymodel
def construct_model(self):
input1 = Input(shape=(225, 225, 1))
input2 = Input(shape=(225, 225, 1))
input3 = Input(shape=(225, 225, 1))
input4 = Input(shape=(225, 225, 1))
input5 = Input(shape=(225, 225, 1))
y1 = build_model(input1)
y2 = build_model(input2)
y3 = build_model(input3)
y4 = build_model(input4)
y5 = build_model(input5)
out = merge.average([y1, y2, y3, y4, y5])
# out = keras.layers.concatenate([y1, y2], axis=0)
self.model = Model(inputs=[input1, input2, input3, input4, input5],
outputs=out)
print(self.model.summary())
def get_res_layer2(input, output):
'''
26-30个残差块
'''
dense_res11 = Dense(20,
activation='selu',
kernel_initializer='lecun_normal')(output)
dense_res12 = Dense(24,
activation='linear',
kernel_initializer='lecun_normal')(dense_res11)
dense_res21 = Dense(20,
activation='selu',
kernel_initializer='lecun_normal')(dense_res12)
dense_res22 = Dense(24,
activation='linear',
kernel_initializer='lecun_normal')(dense_res21)
dense_res31 = Dense(20,
activation='selu',
kernel_initializer='lecun_normal')(dense_res22)
dense_res32 = Dense(24,
activation='linear',
kernel_initializer='lecun_normal')(dense_res31)
dense_res41 = Dense(20,
activation='selu',
kernel_initializer='lecun_normal')(dense_res32)
dense_res42 = Dense(24,
activation='linear',
kernel_initializer='lecun_normal')(dense_res41)
dense_res51 = Dense(20,
activation='selu',
kernel_initializer='lecun_normal')(dense_res42)
dense_res52 = Dense(24,
activation='linear',
kernel_initializer='lecun_normal')(dense_res51)
output_new = average([dense_res52, input, output])
return output_new
def _create_model(self, shape):
n_objects, n_features = shape[1].value, shape[2].value
if hasattr(self, "n_features"):
if self.n_features != n_features:
logger.error("Number of features is not consistent.")
input_layer = Input(shape=(n_objects, n_features))
inputs = [create_input_lambda(i)(input_layer) for i in range(n_objects)]
# Connect input tensors with set mapping layer:
set_mappings = []
for i in range(n_objects):
curr = inputs[i]
for j in range(len(self.set_mapping_layers)):
curr = self.set_mapping_layers[j](curr)
set_mappings.append((i, curr))
# TODO: is feature_repr used outside?
feature_repr = average([x for (j, x) in set_mappings])
self.cached_models[n_objects] = Model(inputs=input_layer, outputs=feature_repr)
def call(self, xs, mask=None):
assert len(xs) == 2
# separate out input matrices
# x1.shape == (BATCH_SIZE, MAX_TIMESTEPS, EMBED_SIZE)
# x2.shape == (BATCH_SIZE, MAX_TIMESTEPS, EMBED_SIZE)
x1, x2 = xs
# build alignment matrix
alpha = K.softmax(K.batch_dot(x1, x2, axes=(2, 2)))
# align inputs
# a1t, a2t: (BATCH_SIZE, MAX_TIMESTEPS, EMBED_SIZE)
a1t = K.batch_dot(alpha, x2, axes=(1, 1))
a2t = K.batch_dot(alpha, x1, axes=(2, 1))
# produce aligned outputs
# o1t, o2t: (BATCH_SIZE, MAX_TIMESTEPS*2, EMBED_SIZE)
o1t = K.tanh(K.dot(x1, self.U1) + K.dot(a1t, self.V1))
o2t = K.tanh(K.dot(x2, self.U2) + K.dot(a2t, self.V2))
# masking
if mask is not None and mask[0] is not None:
o1t *= K.cast(
K.repeat_elements(K.expand_dims(mask[0], axis=2), o1t.shape[2],
2), K.floatx())
if mask is not None and mask[1] is not None:
o2t *= K.cast(
K.repeat_elements(K.expand_dims(mask[1], axis=2), o2t.shape[2],
2), K.floatx())
# o1, o2: (BATCH_SIZE, EMBED_SIZE)
o1 = K.mean(o1t, axis=1)
o2 = K.mean(o2t, axis=1)
# merge the attention vectors according to merge_mode
if self.merge_mode == "concat":
return concatenate([o1, o2], axis=1)
elif self.merge_mode == "diff":
return add([o1, -o2])
elif self.merge_mode == "prod":
return multiply([o1, o2])
elif self.merge_mode == "avg":
return average([o1, o2])
else: # max
return maximum([o1, o2])
def __init__(self,
unit=64,
dropout=0.2,
max_len=39,
update_num=3,
regularization=0.1,
embedding_matrix=None,
use_cudnn=False,
use_share=False,
use_one_cell=False):
self.unit = unit
self.dropout = dropout
self.use_share = use_share
self.use_one_cell = use_one_cell
self.regularization = l2(regularization)
Q1_input = Input(shape=(max_len, ), dtype='int32', name='Q1') # (?, L)
Q2_input = Input(shape=(max_len, ), dtype='int32', name='Q2') # (?, L)
# Q1_m = Input(shape=(max_len,), dtype='int32', name='mask1')
# Q2_m = Input(shape=(max_len,), dtype='int32', name='mask2')
# magic = Input(shape=(4,), dtype='float32', name='magic')
embedding = Embedding(input_dim=embedding_matrix.shape[0],
output_dim=embedding_matrix.shape[1],
mask_zero=True,
weights=[embedding_matrix],
trainable=False)
# bn = BatchNormalization()
Q1 = embedding(Q1_input)
Q2 = embedding(Q2_input)
GRULayer = CuDNNGRU if use_cudnn else GRU
for i in range(update_num):
Q1, Q2 = self.update_module(Q1, Q2, GRULayer)
Q1, Q2 = self.attention(Q1, Q2, GRULayer, implementation=3)
# bn1 = BatchNormalization()
# bnm = BatchNormalization()
# bns = BatchNormalization()
# regression = Bilinear(implementation=0, activation='tanh')([Q1, Q2])
att = SelfAttention(1, activation='tanh')
Q1 = att(Q1)
Q2 = att(Q2)
vector = concatenate([ # Q1, Q2,
merge.multiply([Q1, Q2]),
# merge.subtract([Q1, Q2], use_abs=True),
merge.subtract([Q1, Q2]),
merge.average([Q1, Q2])
])
# vector = merge.subtract([Q1, Q2])
# vector = merge.add([Q1, Q2])
# vector = Dropout(self.dropout)(vector)
# vector = Dense(units=512, activation='tanh')(vector)
# magic_new = Dense(units=64, activation='tanh')(magic)
# vector = concatenate([vector, magic_new])
vector = Dropout(self.dropout)(vector)
vector = Dense(units=256, activation='tanh')(vector)
vector = Dropout(self.dropout)(vector)
regression = Dense(units=1, activation='sigmoid')(vector)
super(IAM, self).__init__(inputs=[Q1_input, Q2_input],
outputs=regression)
def Model_BiLSTM_CRF_multi3_1(sourcevocabsize,
targetvocabsize,
source_W,
input_seq_lenth,
output_seq_lenth,
hidden_dim,
emd_dim,
sourcecharsize,
character_W,
input_word_length,
char_emd_dim,
targetpossize,
batch_size=32,
loss='categorical_crossentropy',
optimizer='rmsprop'):
word_input = Input(shape=(input_seq_lenth, ), dtype='int32')
char_input = Input(shape=(
input_seq_lenth,
input_word_length,
),
dtype='int32')
char_embedding = Embedding(input_dim=sourcecharsize,
output_dim=char_emd_dim,
batch_input_shape=(batch_size, input_seq_lenth,
input_word_length),
mask_zero=False,
trainable=True,
weights=[character_W])
char_embedding2 = TimeDistributed(char_embedding)(char_input)
char_cnn = TimeDistributed(
Conv1D(50, 3, activation='relu', padding='valid'))(char_embedding2)
char_macpool = TimeDistributed(GlobalMaxPooling1D())(char_cnn)
char_macpool = Dropout(0.5)(char_macpool)
word_embedding = Embedding(input_dim=sourcevocabsize + 1,
output_dim=emd_dim,
input_length=input_seq_lenth,
mask_zero=False,
trainable=True,
weights=[source_W])(word_input)
word_embedding_dropout = Dropout(0.5)(word_embedding)
embedding = concatenate([word_embedding_dropout, char_macpool], axis=-1)
BiLSTM = Bidirectional(LSTM(
hidden_dim,
return_sequences=True,
),
merge_mode='concat')(embedding)
# BiLSTM = Bidirectional(LSTM(hidden_dim, return_sequences=True))(word_embedding_dropout)
BiLSTM = BatchNormalization(axis=1)(BiLSTM)
BiLSTM_dropout = Dropout(0.5)(BiLSTM)
mlp3_hidden1 = Bidirectional(LSTM(hidden_dim, return_sequences=True),
merge_mode='concat')(BiLSTM_dropout)
mlp3_hidden1 = Dropout(0.5)(mlp3_hidden1)
mlp3_hidden2 = TimeDistributed(Dense(100, activation='tanh'))(mlp3_hidden1)
output3 = TimeDistributed(Dense(targetpossize + 1, activation='softmax'),
name='pos')(mlp3_hidden2)
embedding2 = concatenate([BiLSTM_dropout, mlp3_hidden1], axis=-1)
mlp1_hidden1 = Bidirectional(LSTM(hidden_dim, return_sequences=True),
merge_mode='concat')(embedding2)
mlp1_hidden1 = Dropout(0.5)(mlp1_hidden1)
mlp1_hidden2 = TimeDistributed(Dense(100, activation='tanh'))(mlp1_hidden1)
# output1 = TimeDistributed(Dense(5+1, activation='softmax'), name='BIOES')(decodelayer1)
mlp1_hidden3 = TimeDistributed(Dense(5 + 1, activation=None))(mlp1_hidden2)
crflayer1 = CRF(5 + 1,
sparse_target=False,
learn_mode='marginal',
name='BIOES')
output1 = crflayer1(mlp1_hidden3)
mlp2_hidden1 = Bidirectional(LSTM(hidden_dim, return_sequences=True),
merge_mode='concat')(embedding2)
mlp2_hidden1 = Dropout(0.5)(mlp2_hidden1)
mlp2_hidden2 = TimeDistributed(Dense(100, activation='tanh'))(mlp2_hidden1)
# output2 = TimeDistributed(Dense(5+1, activation='softmax'), name='Type')(decodelayer2)
mlp2_hidden3 = TimeDistributed(Dense(5 + 1, activation=None))(mlp2_hidden2)
mlp2_hidden4 = average([mlp1_hidden3, mlp2_hidden3])
crflayer2 = CRF(5 + 1,
sparse_target=False,
name='Type',
learn_mode='marginal')
output2 = crflayer2(mlp2_hidden4)
Models = Model([word_input, char_input], [output1, output2, output3])
# Models.compile(optimizer=optimizers.RMSprop(lr=0.001),
# loss={'finall': crflayer.loss_function, 'BIOES': 'categorical_crossentropy', 'Type': 'categorical_crossentropy'},
# loss_weights={'finall': 1., 'BIOES': 1., 'Type': 1.},
# metrics={'finall': [crflayer.accuracy], 'BIOES': ['acc'], 'Type': ['acc']})
Models.compile(optimizer=optimizers.RMSprop(lr=0.001),
loss={
'BIOES': crflayer1.loss_function,
'Type': crflayer2.loss_function,
'pos': 'categorical_crossentropy'
},
loss_weights={
'BIOES': 1.,
'Type': 1.,
'pos': 0.5
},
metrics={
'BIOES': [crflayer2.accuracy],
'Type': [crflayer2.accuracy],
'pos': ['acc']
})
return Models
def model_mlp_mk_rff(input_dimensions, embedding_dim,
hidden_classification_layers, n_classes):
"""Build a match kernel-like model that computes the average of RFF feature for a set and then classifies it using
specified fully connected layers.
Parameters
----------
input_dimensions : dict
Dictionary of dimensions of the input features: {dim0: n_features0, dim1: n_features1, ...}
embedding_dim : int
Dimension of the RFF output space (approximation of the RKHS associated to the best kernel)
hidden_classification_layers : list
Number of units per hidden layer in the fully connected MLP classification part of the model
n_classes : int
Number of classes for the classification problem
Returns
-------
Model
The full model (including the Logistic Regression step), but not compiled
Examples
--------
>>> m = model_mlp_mk_rff(input_dimensions={2: 5}, embedding_dim=10, hidden_classification_layers=[5, 5], n_classes=2)
>>> m.count_params()
127
"""
inputs = []
for d in sorted(input_dimensions.keys()):
n_features = input_dimensions[d]
inputs.extend([Input(shape=(d, )) for _ in range(n_features)])
if len(inputs) == 1:
rff_layer = RFFLayer(units=embedding_dim)
concatenated_avg_rffs = rff_layer(inputs[0])
elif len(input_dimensions) == 1:
rff_layer = RFFLayer(units=embedding_dim)
rffs = [rff_layer(input_feature) for input_feature in inputs]
concatenated_avg_rffs = average(rffs)
else:
avg_rffs = []
idx0 = 0
rff_layers = {}
for d in sorted(input_dimensions.keys()):
n_features = input_dimensions[d]
rff_layers[d] = RFFLayer(units=embedding_dim)
rffs = [
rff_layers[d](input_feature)
for input_feature in inputs[idx0:idx0 + n_features]
]
avg_rffs.append(average(rffs))
idx0 += n_features
concatenated_avg_rffs = concatenate(avg_rffs)
hidden_layers = [concatenated_avg_rffs]
for n_units in hidden_classification_layers:
hidden_layers.append(
Dense(units=n_units, activation="tanh")(hidden_layers[-1]))
predictions = Dense(units=n_classes,
activation="softmax")(hidden_layers[-1])
return Model(inputs=inputs, outputs=predictions)
def CompoundNet_VGG19(include_top=True,
weights=None,
input_tensor=None,
input_shape=None,
fusion_strategy='concatenate',
mode='fine_tuning',
pooling_mode='avg',
classes=9,
data_augm_enabled=False):
"""Instantiates the CompoundNet VGG19 architecture fine-tuned (2 steps) on Human Rights Archive dataset.
Optionally loads weights pre-trained on Human Rights Archive Database.
# Arguments
include_top: whether to include the 3 fully-connected
layers at the top of the network.
weights: one of `None` (random initialization),
'HRA' (pre-training on Human Rights Archive),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(224, 224, 3)` (with `channels_last` data format)
or `(3, 224, 224)` (with `channels_first` data format).
It should have exactly 3 input channels,
and width and height should be no smaller than 48.
E.g. `(200, 200, 3)` would be one valid value.
fusion_strategy: one of `concatenate` (feature vectors of different sources are concatenated into one super-vector),
`average` (the feature set is averaged)
or `maximum` (selects the highest value from the corresponding features).
mode: one of `feature_extraction` (freeze all but the penultimate layer and re-train the last Dense layer)
or `fine_tuning` (unfreeze the lower convolutional layers and retrain more layers).
pooling_mode: Optional pooling_mode mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling_mode
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling_mode will
be applied.
classes: optional number of classes to classify images into,
only to be specified if `weights` argument is `None`.
data_augm_enabled: whether to use the augmented samples during training.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`.
"""
if not (weights in {'HRA', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `HRA` '
'(pre-training on Human Rights Archive), '
'or the path to the weights file to be loaded.')
if not (fusion_strategy in {'concatenate', 'average', 'maximum'}):
raise ValueError(
'The `fusion_strategy` argument should be either '
'`concatenate` (feature vectors of different sources are concatenated into one super-vector), '
'`average` (the feature set is averaged) '
'or `maximum` (selects the highest value from the corresponding features).'
)
if not (pooling_mode in {'avg', 'max', 'flatten'}):
raise ValueError('The `pooling_mode` argument should be either '
'`avg` (GlobalAveragePooling2D), `max` '
'(GlobalMaxPooling2D), '
'or `flatten` (Flatten).')
if weights == 'HRA' and classes != 9:
raise ValueError(
'If using `weights` as Human Rights Archive, `classes` should be 9.'
)
cache_subdir = 'HRA_models'
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_size=224,
min_size=48,
data_format=K.image_data_format(),
require_flatten=include_top,
weights=weights)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
input_tensor = Input(shape=(224, 224, 3))
object_centric_model = VGG19(input_tensor=input_tensor,
weights='imagenet',
include_top=False)
scene_centric_model = VGG16_Places365(input_tensor=input_tensor,
weights='places',
include_top=False)
# retrieve the ouputs
object_model_output = object_centric_model.output
scene_model_output = scene_centric_model.output
# We will feed the extracted features to a merging layer
if fusion_strategy == 'concatenate':
merged = concatenate([object_model_output, scene_model_output])
elif fusion_strategy == 'average':
merged = average([object_model_output, scene_model_output])
else:
merged = maximum([object_model_output, scene_model_output])
if include_top:
if pooling_mode == 'avg':
x = GlobalAveragePooling2D(name='GAP')(merged)
elif pooling_mode == 'max':
x = GlobalMaxPooling2D(name='GMP')(merged)
elif pooling_mode == 'flatten':
x = Flatten(name='FLATTEN')(merged)
x = Dense(256, activation='relu',
name='FC1')(x) # let's add a fully-connected layer
# When random init is enabled, we want to include Dropout,
# otherwise when loading a pre-trained HRA model we want to omit
# Dropout layer so the visualisations are done properly (there is an issue if it is included)
if weights is None:
x = Dropout(0.5, name='DROPOUT')(x)
# and a logistic layer with the number of classes defined by the `classes` argument
x = Dense(classes, activation='softmax',
name='PREDICTIONS')(x) # new softmax layer
# Ensure that the model takes into account any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# this is the transfer learning model we will train
model = Model(inputs=inputs, outputs=x, name='CompoundNet-VGG19')
# load weights
if weights == 'HRA':
if include_top:
if mode == 'feature_extraction':
for layer in object_centric_model.layers:
layer.trainable = False
for layer in scene_centric_model.layers:
layer.trainable = False
model.compile(optimizer=SGD(lr=0.0001, momentum=0.9),
loss='categorical_crossentropy')
if data_augm_enabled:
if fusion_strategy == 'concatenate':
if pooling_mode == 'avg':
weights_path = get_file(
AUGM_FEATURE_EXTRACTION_CONCATENATE_FUSION_AVG_POOL_fname,
AUGM_FEATURE_EXTRACTION_CONCATENATE_FUSION_AVG_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'flatten':
weights_path = get_file(
AUGM_FEATURE_EXTRACTION_CONCATENATE_FUSION_FLATTEN_fname,
AUGM_FEATURE_EXTRACTION_CONCATENATE_FUSION_FLATTEN_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'max':
weights_path = get_file(
AUGM_FEATURE_EXTRACTION_CONCATENATE_FUSION_MAX_POOL_fname,
AUGM_FEATURE_EXTRACTION_CONCATENATE_FUSION_MAX_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif fusion_strategy == 'average':
if pooling_mode == 'avg':
weights_path = get_file(
AUGM_FEATURE_EXTRACTION_AVERAGE_FUSION_AVG_POOL_fname,
AUGM_FEATURE_EXTRACTION_AVERAGE_FUSION_AVG_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'flatten':
weights_path = get_file(
AUGM_FEATURE_EXTRACTION_AVERAGE_FUSION_FLATTEN_fname,
AUGM_FEATURE_EXTRACTION_AVERAGE_FUSION_FLATTEN_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'max':
weights_path = get_file(
AUGM_FEATURE_EXTRACTION_AVERAGE_FUSION_MAX_POOL_fname,
AUGM_FEATURE_EXTRACTION_AVERAGE_FUSION_MAX_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif fusion_strategy == 'maximum':
if pooling_mode == 'avg':
weights_path = get_file(
AUGM_FEATURE_EXTRACTION_MAXIMUM_FUSION_AVG_POOL_fname,
AUGM_FEATURE_EXTRACTION_MAXIMUM_FUSION_AVG_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'flatten':
weights_path = get_file(
AUGM_FEATURE_EXTRACTION_MAXIMUM_FUSION_FLATTEN_fname,
AUGM_FEATURE_EXTRACTION_MAXIMUM_FUSION_FLATTEN_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'max':
weights_path = get_file(
AUGM_FEATURE_EXTRACTION_MAXIMUM_FUSION_MAX_POOL_fname,
AUGM_FEATURE_EXTRACTION_MAXIMUM_FUSION_MAX_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
else:
if fusion_strategy == 'concatenate':
if pooling_mode == 'avg':
weights_path = get_file(
FEATURE_EXTRACTION_CONCATENATE_FUSION_AVG_POOL_fname,
FEATURE_EXTRACTION_CONCATENATE_FUSION_AVG_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'flatten':
weights_path = get_file(
FEATURE_EXTRACTION_CONCATENATE_FUSION_FLATTEN_fname,
FEATURE_EXTRACTION_CONCATENATE_FUSION_FLATTEN_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'max':
weights_path = get_file(
FEATURE_EXTRACTION_CONCATENATE_FUSION_MAX_POOL_fname,
FEATURE_EXTRACTION_CONCATENATE_FUSION_MAX_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif fusion_strategy == 'average':
if pooling_mode == 'avg':
weights_path = get_file(
FEATURE_EXTRACTION_AVERAGE_FUSION_AVG_POOL_fname,
FEATURE_EXTRACTION_AVERAGE_FUSION_AVG_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'flatten':
weights_path = get_file(
FEATURE_EXTRACTION_AVERAGE_FUSION_FLATTEN_fname,
FEATURE_EXTRACTION_AVERAGE_FUSION_FLATTEN_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'max':
weights_path = get_file(
FEATURE_EXTRACTION_AVERAGE_FUSION_MAX_POOL_fname,
FEATURE_EXTRACTION_AVERAGE_FUSION_MAX_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif fusion_strategy == 'maximum':
if pooling_mode == 'avg':
weights_path = get_file(
FEATURE_EXTRACTION_MAXIMUM_FUSION_AVG_POOL_fname,
FEATURE_EXTRACTION_MAXIMUM_FUSION_AVG_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'flatten':
weights_path = get_file(
FEATURE_EXTRACTION_MAXIMUM_FUSION_FLATTEN_fname,
FEATURE_EXTRACTION_MAXIMUM_FUSION_FLATTEN_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'max':
weights_path = get_file(
FEATURE_EXTRACTION_MAXIMUM_FUSION_MAX_POOL_fname,
FEATURE_EXTRACTION_MAXIMUM_FUSION_MAX_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif mode == 'fine_tuning':
for layer in model.layers[:36]:
layer.trainable = False
for layer in model.layers[36:]:
layer.trainable = True
model.compile(optimizer=SGD(lr=0.0001, momentum=0.9),
loss='categorical_crossentropy')
if data_augm_enabled:
if fusion_strategy == 'concatenate':
if pooling_mode == 'avg':
weights_path = get_file(
AUGM_FINE_TUNING_CONCATENATE_FUSION_AVG_POOL_fname,
AUGM_FINE_TUNING_CONCATENATE_FUSION_AVG_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'flatten':
weights_path = get_file(
AUGM_FINE_TUNING_CONCATENATE_FUSION_FLATTEN_fname,
AUGM_FINE_TUNING_CONCATENATE_FUSION_FLATTEN_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'max':
weights_path = get_file(
AUGM_FINE_TUNING_CONCATENATE_FUSION_MAX_POOL_fname,
AUGM_FINE_TUNING_CONCATENATE_FUSION_MAX_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif fusion_strategy == 'average':
if pooling_mode == 'avg':
weights_path = get_file(
AUGM_FINE_TUNING_AVERAGE_FUSION_AVG_POOL_fname,
AUGM_FINE_TUNING_AVERAGE_FUSION_AVG_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'flatten':
weights_path = get_file(
AUGM_FINE_TUNING_AVERAGE_FUSION_FLATTEN_fname,
AUGM_FINE_TUNING_AVERAGE_FUSION_FLATTEN_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'max':
weights_path = get_file(
AUGM_FINE_TUNING_AVERAGE_FUSION_MAX_POOL_fname,
AUGM_FINE_TUNING_AVERAGE_FUSION_MAX_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif fusion_strategy == 'maximum':
if pooling_mode == 'avg':
weights_path = get_file(
AUGM_FINE_TUNING_MAXIMUM_FUSION_AVG_POOL_fname,
AUGM_FINE_TUNING_MAXIMUM_FUSION_AVG_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'flatten':
weights_path = get_file(
AUGM_FINE_TUNING_MAXIMUM_FUSION_FLATTEN_fname,
AUGM_FINE_TUNING_MAXIMUM_FUSION_FLATTEN_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'max':
weights_path = get_file(
AUGM_FINE_TUNING_MAXIMUM_FUSION_MAX_POOL_fname,
AUGM_FINE_TUNING_MAXIMUM_FUSION_MAX_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
else:
if fusion_strategy == 'concatenate':
if pooling_mode == 'avg':
weights_path = get_file(
FINE_TUNING_CONCATENATE_FUSION_AVG_POOL_fname,
FINE_TUNING_CONCATENATE_FUSION_AVG_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'flatten':
weights_path = get_file(
FINE_TUNING_CONCATENATE_FUSION_FLATTEN_fname,
FINE_TUNING_CONCATENATE_FUSION_FLATTEN_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'max':
weights_path = get_file(
FINE_TUNING_CONCATENATE_FUSION_MAX_POOL_fname,
FINE_TUNING_CONCATENATE_FUSION_MAX_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif fusion_strategy == 'average':
if pooling_mode == 'avg':
weights_path = get_file(
FINE_TUNING_AVERAGE_FUSION_AVG_POOL_fname,
FINE_TUNING_AVERAGE_FUSION_AVG_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'flatten':
weights_path = get_file(
FINE_TUNING_AVERAGE_FUSION_FLATTEN_fname,
FINE_TUNING_AVERAGE_FUSION_FLATTEN_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'max':
weights_path = get_file(
FINE_TUNING_AVERAGE_FUSION_MAX_POOL_fname,
FINE_TUNING_AVERAGE_FUSION_MAX_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif fusion_strategy == 'maximum':
if pooling_mode == 'avg':
weights_path = get_file(
FINE_TUNING_MAXIMUM_FUSION_AVG_POOL_fname,
FINE_TUNING_MAXIMUM_FUSION_AVG_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'flatten':
weights_path = get_file(
FINE_TUNING_MAXIMUM_FUSION_FLATTEN_fname,
FINE_TUNING_MAXIMUM_FUSION_FLATTEN_WEIGHTS_PATH,
cache_subdir=cache_subdir)
elif pooling_mode == 'max':
weights_path = get_file(
FINE_TUNING_MAXIMUM_FUSION_MAX_POOL_fname,
FINE_TUNING_MAXIMUM_FUSION_MAX_POOL_WEIGHTS_PATH,
cache_subdir=cache_subdir)
else:
if fusion_strategy == 'average':
weights_path = get_file(
FINE_TUNING_AVERAGE_FUSION_NO_TOP_fname,
FINE_TUNING_AVERAGE_FUSION_WEIGHTS_PATH_NO_TOP,
cache_subdir=cache_subdir)
elif fusion_strategy == 'concatenate':
weights_path = get_file(
FINE_TUNING_CONCATENATE_FUSION_NO_TOP_fname,
FINE_TUNING_CONCATENATE_FUSION_WEIGHTS_PATH_NO_TOP,
cache_subdir=cache_subdir)
elif fusion_strategy == 'maximum':
weights_path = get_file(
FINE_TUNING_MAXIMUM_FUSION_NO_TOP_fname,
FINE_TUNING_MAXIMUM_FUSION_WEIGHTS_PATH_NO_TOP,
cache_subdir=cache_subdir)
model.load_weights(weights_path)
return model
def compoundNet_feature_extraction(object_centric_model='VGG16',
scene_centric_model='VGG16_Places365',
fusion_strategy='concatenate',
pooling_mode='avg',
classes=9,
data_augm_enabled=False):
"""ConvNet as fixed feature extractor, consist of taking the convolutional base of a previously-trained network,
running the new data through it, and training a new classifier on top of the output.
(i.e. train only the randomly initialized top layers while freezing all convolutional layers of the original model).
# Arguments
object_centric_model: one of `VGG16`, `VGG19` or `ResNet50`
scene_centric_model: `VGG16_Places365`
fusion_strategy: one of `concatenate` (feature vectors of different sources are concatenated into one super-vector),
`average` (the feature set is averaged) or `maximum` (selects the highest value from the corresponding features).
pooling_mode: Optional pooling_mode mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling_mode
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling_mode will
be applied.
classes: optional number of classes to classify images into,
only to be specified if `weights` argument is `None`.
data_augm_enabled: whether to use the augmented samples during training.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `object_centric_model`, `pooling_mode`,
`fusion_strategy` , `scene_centric_model` or invalid input shape.
"""
if not (object_centric_model in {'VGG16', 'VGG19', 'ResNet50'}):
raise ValueError(
'The `scene_centric_model` argument should be either '
'`VGG16`, `VGG19` or `ResNet50`. Other models will be supported in future releases. '
)
if not (pooling_mode in {'avg', 'max', 'flatten'}):
raise ValueError('The `pooling_mode` argument should be either '
'`avg` (GlobalAveragePooling2D), `max` '
'(GlobalMaxPooling2D), '
'or `flatten` (Flatten).')
if not (fusion_strategy in {'concatenate', 'average', 'maximum'}):
raise ValueError(
'The `fusion_strategy` argument should be either '
'`concatenate` (feature vectors of different sources are concatenated into one super-vector),'
' `average` (the feature set is averaged) '
'or `maximum` (selects the highest value from the corresponding features).'
)
if not (scene_centric_model in {'VGG16_Places365'}):
raise ValueError(
'The `scene_centric_model` argument should be '
'`VGG16_Places365`. Other models will be supported in future releases.'
)
# Define the name of the model and its weights
weights_name = 'compoundNet_feature_extraction_' \
+ object_centric_model + '_' \
+ fusion_strategy + '_fusion_' \
+ pooling_mode + '_pool_weights_tf_dim_ordering_tf_kernels.h5'
augm_samples_weights_name = 'augm_compoundNet_feature_extraction_' \
+ object_centric_model + '_' \
+ fusion_strategy + '_fusion_' \
+ pooling_mode + '_pool_weights_tf_dim_ordering_tf_kernels.h5'
model_log = logs_dir + 'compoundNet_feature_extraction_' \
+ object_centric_model + '_' \
+ fusion_strategy + '_fusion_' \
+ pooling_mode + '_pool_log.csv'
csv_logger = CSVLogger(model_log, append=True, separator=',')
input_tensor = Input(shape=(224, 224, 3))
# create the base object_centric_model pre-trained model for warm-up
if object_centric_model == 'VGG16':
object_base_model = VGG16(input_tensor=input_tensor,
weights='imagenet',
include_top=False)
elif object_centric_model == 'VGG19':
object_base_model = VGG19(input_tensor=input_tensor,
weights='imagenet',
include_top=False)
elif object_centric_model == 'ResNet50':
tmp_model = ResNet50(input_tensor=input_tensor,
weights='imagenet',
include_top=False)
object_base_model = Model(
inputs=tmp_model.input,
outputs=tmp_model.get_layer('activation_48').output)
print('\n \n')
print('The plain, object-centric `' + object_centric_model +
'` pre-trained convnet was successfully initialised.\n')
scene_base_model = VGG16_Places365(input_tensor=input_tensor,
weights='places',
include_top=False)
print('The plain, scene-centric `' + scene_centric_model +
'` pre-trained convnet was successfully initialised.\n')
# retrieve the ouputs
object_base_model_output = object_base_model.output
scene_base_model_output = scene_base_model.output
# We will feed the extracted features to a merging layer
if fusion_strategy == 'concatenate':
merged = concatenate(
[object_base_model_output, scene_base_model_output])
elif fusion_strategy == 'average':
merged = average([object_base_model_output, scene_base_model_output])
else:
merged = maximum([object_base_model_output, scene_base_model_output])
if pooling_mode == 'avg':
x = GlobalAveragePooling2D(name='GAP')(merged)
elif pooling_mode == 'max':
x = GlobalMaxPooling2D(name='GMP')(merged)
elif pooling_mode == 'flatten':
x = Flatten(name='FLATTEN')(merged)
x = Dense(256, activation='relu',
name='FC1')(x) # let's add a fully-connected layer
# When random init is enabled, we want to include Dropout,
# otherwise when loading a pre-trained HRA model we want to omit
# Dropout layer so the visualisations are done properly (there is an issue if it is included)
x = Dropout(0.5, name='DROPOUT')(x)
# and a logistic layer with the number of classes defined by the `classes` argument
predictions = Dense(classes, activation='softmax',
name='PREDICTIONS')(x) # new softmax layer
# this is the transfer learning model we will train
model = Model(inputs=object_base_model.input, outputs=predictions)
print(
'Randomly initialised classifier was successfully added on top of the merged outputs. \n'
)
print(
'Number of trainable weights before freezing the conv. bases of the respective original models: '
'' + str(len(model.trainable_weights)))
# first: train only the top layers (which were randomly initialized)
# i.e. freeze all convolutional layers of the preliminary base model
for layer in object_base_model.layers:
layer.trainable = False
for layer in scene_base_model.layers:
layer.trainable = False
print(
'Number of trainable weights after freezing the conv. bases of the respective original models: '
'' + str(len(model.trainable_weights)))
print('\n')
# compile the warm_up_model (should be done *after* setting layers to non-trainable)
model.compile(optimizer=SGD(lr=0.0001, momentum=0.9),
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
# # The attribute model.metrics_names will give you the display labels for the scalar outputs.
# print warm_up_model.metrics_names
if data_augm_enabled:
print(
'Using augmented samples for training. This may take a while ! \n')
t = now()
history = model.fit_generator(augmented_train_generator,
steps_per_epoch=nb_train_samples //
batch_size,
epochs=feature_extraction_epochs,
callbacks=[csv_logger],
class_weight=class_weight)
print(
'Training time for re-training the last Dense layer using augmented samples: %s'
% (now() - t))
model.save_weights(feature_extraction_dir + augm_samples_weights_name)
print('Model weights using augmented samples were saved as `' +
augm_samples_weights_name + '`')
print('\n')
else:
t = now()
history = model.fit_generator(train_generator,
steps_per_epoch=nb_train_samples //
batch_size,
epochs=feature_extraction_epochs,
callbacks=[csv_logger],
class_weight=class_weight)
print('Training time for re-training the last Dense layer: %s' %
(now() - t))
model.save_weights(feature_extraction_dir + weights_name)
print('Model weights were saved as `' + weights_name + '`')
print('\n')
return model
def MI_Net_with_DS(dataset):
"""Train and evaluate on MI-Net with deep supervision.
Parameters
-----------------
dataset : dict
A dictionary contains all dataset information. We split train/test by keys.
Returns
-----------------
test_acc : float
Testing accuracy of MI-Net with deep supervision.
"""
# load data and convert type
train_bags = dataset['train']
test_bags = dataset['test']
# convert bag to batch
train_set = convertToBatch(train_bags)
test_set = convertToBatch(test_bags)
dimension = train_set[0][0].shape[1]
weight = [1.0, 1.0, 1.0, 0.0]
# data: instance feature, n*d, n = number of training instance
data_input = Input(shape=(dimension, ), dtype='float32', name='input')
# fully-connected
fc1 = Dense(256,
activation='relu',
kernel_regularizer=l2(args.weight_decay))(data_input)
fc2 = Dense(128,
activation='relu',
kernel_regularizer=l2(args.weight_decay))(fc1)
fc3 = Dense(64,
activation='relu',
kernel_regularizer=l2(args.weight_decay))(fc2)
# dropout
dropout1 = Dropout(rate=0.5)(fc1)
dropout2 = Dropout(rate=0.5)(fc2)
dropout3 = Dropout(rate=0.5)(fc3)
# features pooling
fp1 = Feature_pooling(output_dim=1,
kernel_regularizer=l2(args.weight_decay),
pooling_mode=args.pooling_mode,
name='fp1')(dropout1)
fp2 = Feature_pooling(output_dim=1,
kernel_regularizer=l2(args.weight_decay),
pooling_mode=args.pooling_mode,
name='fp2')(dropout2)
fp3 = Feature_pooling(output_dim=1,
kernel_regularizer=l2(args.weight_decay),
pooling_mode=args.pooling_mode,
name='fp3')(dropout3)
# score average
mg_ave = average([fp1, fp2, fp3], name='ave')
model = Model(inputs=[data_input], outputs=[fp1, fp2, fp3, mg_ave])
sgd = SGD(lr=args.init_lr,
decay=1e-4,
momentum=args.momentum,
nesterov=True)
model.compile(loss={
'fp1': bag_loss,
'fp2': bag_loss,
'fp3': bag_loss,
'ave': bag_loss
},
loss_weights={
'fp1': weight[0],
'fp2': weight[1],
'fp3': weight[2],
'ave': weight[3]
},
optimizer=sgd,
metrics=[bag_accuracy])
# train model
t1 = time.time()
num_batch = len(train_set)
for epoch in range(args.max_epoch):
train_loss, train_acc = train_eval(model, train_set)
test_loss, test_acc = test_eval(model, test_set)
print 'epoch=', epoch, ' train_loss= {:.3f}'.format(
train_loss), ' train_acc= {:.3f}'.format(
train_acc), ' test_loss={:.3f}'.format(
test_loss), ' test_acc= {:.3f}'.format(test_acc)
t2 = time.time()
print 'run time:', (t2 - t1) / 60, 'min'
print 'test_acc={:.3f}'.format(test_acc)
return test_acc
def getModel(training):
inputShape = (128, 32, 1)
kernelVals = [5, 5, 3, 3, 3]
convFilters = [32, 64, 128, 128, 256]
strideVals = [(2, 2), (2, 2), (1, 2), (1, 2), (1, 2)]
rnnUnits = 256
maxStringLen = 32
inputs = Input(name='inputX', shape=inputShape, dtype='float32')
inner = inputs
for i in range(len(kernelVals)):
inner = Conv2D(convFilters[i],(kernelVals[i],kernelVals[i]),padding = 'same',\
name = 'conv'+str(i), kernel_initializer = 'he_normal')(inner)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = MaxPooling2D(pool_size=strideVals[i],
name='max' + str(i + 1))(inner)
inner = Reshape(target_shape=(maxStringLen, rnnUnits),
name='reshape')(inner)
LSF = LSTM(rnnUnits,
return_sequences=True,
kernel_initializer='he_normal',
name='LSTM1F')(inner)
LSB = LSTM(rnnUnits,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='LSTM1B')(inner)
LSB = Lambda(lambda inputTensor: K.reverse(inputTensor, axes=1))(LSB)
LS1 = average([LSF, LSB])
LS1 = BatchNormalization()(LS1)
LSF = LSTM(rnnUnits,
return_sequences=True,
kernel_initializer='he_normal',
name='LSTM2F')(LS1)
LSB = LSTM(rnnUnits,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='LSTM2B')(LS1)
LSB = Lambda(lambda inputTensor: K.reverse(inputTensor, axes=1))(LSB)
LS2 = concatenate([LSF, LSB])
LS2 = BatchNormalization()(LS2)
yPred = Dense(len(unicodes) + 1,
kernel_initializer='he_normal',
name='dense2')(LS2)
yPred = Activation('softmax', name='softmax')(yPred)
#Model(inputs = inputs,outputs = yPred).summary()
labels = Input(name='label', shape=[32], dtype='float32')
inputLength = Input(name='inputLen', shape=[1], dtype='int64') # (None, 1)
labelLength = Input(name='labelLen', shape=[1], dtype='int64')
lossOut = Lambda(ctcLambdaFunc, output_shape=(1, ),
name='ctc')([yPred, labels, inputLength, labelLength])
if training:
return Model(inputs=[inputs, labels, inputLength, labelLength],
outputs=[lossOut, yPred])
return Model(inputs=[inputs], outputs=yPred)
def get_unet_512(input_shape=(512, 512, 3), num_classes=1):
inputs = Input(shape=input_shape)
# 512
down0 = block(inputs, 16)
down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0)
# 256
down1 = block(down0_pool, 32)
down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)
# 128
down2 = block(down1_pool, 64)
down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)
# 64
down3 = block(down2_pool, 128)
down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3)
# 32
down4 = block(down3_pool, 256)
down4_pool = MaxPooling2D((2, 2), strides=(2, 2))(down4)
# 16
down5 = block(down4_pool, 512)
down5_pool = MaxPooling2D((2, 2), strides=(2, 2))(down5)
# 8
center = block(down5_pool, 1024)
# stacked dilated convolution
# dilate1 = Conv2D(1024, (3, 3), activation='relu', padding='same', dilation_rate=1)(down5_pool)
# dilate2 = Conv2D(1024, (3, 3), activation='relu', padding='same', dilation_rate=2)(dilate1)
# dilate3 = Conv2D(1024, (3, 3), activation='relu', padding='same', dilation_rate=4)(dilate2)
# dilate4 = Conv2D(1024, (3, 3), activation='relu', padding='same', dilation_rate=8)(dilate3)
# dilate5 = Conv2D(1024, (3, 3), activation='relu', padding='same', dilation_rate=16)(dilate4)
# dilate6 = Conv2D(1024, (3, 3), activation='relu', padding='same', dilation_rate=32)(dilate5)
# center = add([dilate1, dilate2, dilate3, dilate4, dilate5, dilate6])
# center
up5 = UpSampling2D((2, 2))(center)
up5 = concatenate([down5, up5], axis=3)
up5 = block(up5, 512)
# 16
up4 = UpSampling2D((2, 2))(up5)
up4 = concatenate([down4, up4], axis=3)
up4 = block(up4, 256)
# 32
up3 = UpSampling2D((2, 2))(up4)
up3 = concatenate([down3, up3], axis=3)
up3 = block(up3, 128)
# 64
up2 = UpSampling2D((2, 2))(up3)
up2 = concatenate([down2, up2], axis=3)
up2 = block(up2, 64)
# 128
up1 = UpSampling2D((2, 2))(up2)
up1 = concatenate([down1, up1], axis=3)
up1 = block(up1, 32)
# 256
up0 = UpSampling2D((2, 2))(up1)
up0 = concatenate([down0, up0], axis=3)
up0 = block(up0, 16)
# 512
output1 = Conv2D(num_classes, (1, 1), activation='sigmoid')(up0)
if not USE_REFINE_NET:
model = Model(inputs=inputs, outputs=[output1])
# model.load_weights('../weights/head-segmentation-model.h5')
# model.compile(optimizer=SGD(lr=0.01, momentum=0.9), loss='binary_crossentropy', metrics=[dice_loss])
return model
else:
inputs2 = concatenate([inputs, output1])
outputs2 = block2(inputs2, 64)
outputs2 = average([output1, outputs2])
model2 = Model(inputs=inputs, outputs=[output1, outputs2])
return model2
def save_basic_ensemble_evals(sequences,
test_data,
scratch_model_func,
weight_dirname,
eval_dirname,
force=False):
# Get test data
x_test, y_test = test_data
nb_classes = y_test.shape[1]
# Iterate through all provided Snapshot models
for cur_seq, init_lrs, iter_epochs, restart_indices in sequences:
base_string = snap_train.get_checkpoint_base_string(
cur_seq, init_lrs, iter_epochs, restart_indices)
max_size = len(cur_seq)
if max_size < 2:
continue
# Get biggest sized ensemble and ensemble omitting first model
for cur_size in range(max_size - 1, max_size + 1):
eval_filename = eval_dirname + base_string + (".%d.eval" %
cur_size)
if not os.path.isfile(eval_filename) or force:
# Get checkpoint weights
tmp_chkpt_names = []
first_checkpt = base_string + ".0.hdf5"
for filename in os.listdir(weight_dirname):
if filename.startswith(base_string):
tmp_chkpt_names.append(weight_dirname + filename)
# Check to ensure that correct number of files were found
chkpt_names = trim_snap_weight_list(tmp_chkpt_names,
base_string, cur_size)
if not chkpt_names:
continue
# Get individual models and their averages
print("\n\nGetting averages of individual Snapshot models... ")
running_vals = [0.0, 0.0, 0.0]
chkpt_models = []
for i, name in enumerate(chkpt_names):
model = scratch_model_func()
model.load_weights(name)
for j, cur_layer in enumerate(model.layers):
cur_layer.name = ("%s_%d_%d" % (base_string, i, j))
# Keep track of model averages
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=[
'accuracy',
metrics.top_k_categorical_accuracy
])
for j, val in enumerate(
model.evaluate(x_test,
y_test,
batch_size=64,
verbose=0)):
running_vals[j] += (float(val) / cur_size)
# Add checkpoint model to list of models
chkpt_models.append(model)
# Create Snapshot Ensemble model
chkpt_inputs = [model.input for model in chkpt_models]
chkpt_outputs = [model.output for model in chkpt_models]
# Usage of "average" vs. "keras.layers.merge.Average":
# https://github.com/fchollet/keras/issues/3921
final_output = average(chkpt_outputs)
snapshot_model = Model(inputs=chkpt_inputs,
outputs=final_output)
snapshot_model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy', metrics.top_k_categorical_accuracy])
# Save eval metrics (in eval_dirname)
print("\n\nEvaluating the following Snapshot Ensemble: %s" %
eval_filename)
eval_metrics = snapshot_model.evaluate([x_test] * cur_size,
y_test,
batch_size=12,
verbose=1)
print(snapshot_model.metrics_names)
print(eval_metrics)
running_vals.extend(eval_metrics)
pickle.dump(running_vals, open(eval_filename, "wb"))
# Return nothing
return
def miccai2018_net(vol_size, enc_nf, dec_nf, int_steps=7, use_miccai_int=False, indexing='ij', bidir=False, vel_resize=1/2):
"""
architecture for probabilistic diffeomoprhic VoxelMorph presented in the MICCAI 2018 paper.
You may need to modify this code (e.g., number of layers) to suit your project needs.
The stationary velocity field operates in a space (0.5)^3 of vol_size for computational reasons.
:param vol_size: volume size. e.g. (256, 256, 256)
:param enc_nf: list of encoder filters. right now it needs to be 1x4.
e.g. [16,32,32,32]
:param dec_nf: list of decoder filters. right now it must be 1x6, see unet function.
:param use_miccai_int: whether to use the manual miccai implementation of scaling and squaring integration
note that the 'velocity' field outputted in that case was
since then we've updated the code to be part of a flexible layer. see neuron.layers.VecInt
**This param will be phased out (set to False behavior)**
:param int_steps: the number of integration steps
:param indexing: xy or ij indexing. we recommend ij indexing if training from scratch.
miccai 2018 runs were done with xy indexing.
**This param will be phased out (set to 'ij' behavior)**
:return: the keras model
"""
ndims = len(vol_size)
assert ndims in [1, 2, 3], "ndims should be one of 1, 2, or 3. found: %d" % ndims
# get unet
unet_model = unet_model_3d(vol_size, batch_normalization=True)
[src, tgt] = unet_model.inputs
x_out, up0, up1, up2, flow_params0, flow_params1, flow_params2, y0, y1, y2 = unet_model.outputs
# print("unet input", src.shape)
# print("unet output", x_out.shape)
# velocity mean and logsigma layers
Conv = getattr(KL, 'Conv%dD' % ndims)
flow_mean = Conv(ndims, kernel_size=3, padding='same',
kernel_initializer=RandomNormal(mean=0.0, stddev=1e-5), name='flow', data_format="channels_last")(x_out)
# we're going to initialize the velocity variance very low, to start stable.
flow_log_sigma = Conv(ndims, kernel_size=3, padding='same',
kernel_initializer=RandomNormal(mean=0.0, stddev=1e-10),
bias_initializer=keras.initializers.Constant(value=-10),
name='log_sigma',
data_format="channels_last")(x_out)
flow_params = concatenate([flow_mean, flow_log_sigma])
# velocity sample
flow = Sample(name="z_sample")([flow_mean, flow_log_sigma])
from keras.layers.merge import average
flow = average([flow, up0, up1, up2])
# integrate if diffeomorphic (i.e. treating 'flow' above as stationary velocity field)
if use_miccai_int:
v = flow
for _ in range(int_steps):
v1 = nrn_layers.SpatialTransformer(interp_method='linear', indexing=indexing)([v, v])
v = keras.layers.add([v, v1])
flow = v
else:
# new implementation in neuron is cleaner.
z_sample = flow
flow = nrn_layers.VecInt(method='ss', name='flow-int', int_steps=int_steps)(z_sample)
if bidir:
rev_z_sample = Negate()(z_sample)
neg_flow = nrn_layers.VecInt(method='ss', name='neg_flow-int', int_steps=int_steps)(rev_z_sample)
# get up to final resolution
flow = trf_resize(flow, vel_resize, name='diffflow')
if bidir:
neg_flow = trf_resize(neg_flow, vel_resize, name='neg_diffflow')
# transform
y = nrn_layers.SpatialTransformer(interp_method='linear', indexing=indexing)([src, flow])
if bidir:
y_tgt = nrn_layers.SpatialTransformer(interp_method='linear', indexing=indexing)([tgt, neg_flow])
# prepare outputs and losses
outputs = [y, flow_params, flow_params0, flow_params1, flow_params2, y0, y1, y2]
if bidir:
outputs = [y, y_tgt, flow_params]
# build the model
return Model(inputs=[src, tgt], outputs=outputs)
outs = [] #the list of ensemble outputs
for i in range(ens_models):
# Conv[32] -> Conv[32] -> Pool (with dropout on the pooling layer), applying BN in between
conv1 = Convolution2D(conv_depth, kernel_size, kernel_size, border_mode = 'same', init = 'he_uniform', W_regularizer = l2(l2_lambda), activation = 'relu')(inp_norm)
conv1 = BatchNormalization(axis = 1)(conv1)
conv2 = Convolution2D(conv_depth, kernel_size, kernel_size, border_mode = 'same', init = 'he_uniform', W_regularizer = l2(l2_lambda), activation = 'relu')(conv1)
conv2 = BatchNormalization(axis = 1)(conv2)
pool_1 = MaxPooling2D(pool_size = (pool_size, pool_size), dim_ordering="th")(conv2)
drop_1 = Dropout(drop_prob_1)(pool_1)
flat = Flatten()(drop_1)
hidden = Dense(hidden_size, init = 'he_uniform', W_regularizer = l2(l2_lambda), activation = 'relu')(flat) # Hidden ReLU layer
hidden = BatchNormalization(axis = 1)(hidden)
drop = Dropout(drop_prob_2)(hidden)
outs.append(Dense(num_classes, init = 'glorot_uniform', W_regularizer = l2(l2_lambda), activation = 'softmax')(drop)) # Output softmax layer
out = average(outs) # Avarage the predictioins to obtain the final output
model = Model(input = inp, output = out)
model.compile(loss = 'categorical_crossentropy', optimizer = 'adam', metrics = ['accuracy'])
datagen = ImageDataGenerator(width_shift_range = 0.1, height_shift_range = 0.1) # randomly shift images horizontally and vertically
datagen.fit(X_train)
model.fit_generator(datagen.flow(X_train, Y_train, batch_size = batch_size), samples_per_epoch = X_train.shape[0], nb_epoch = num_epochs, validation_data = (X_val, Y_val), verbose = 1,
callbacks = [EarlyStopping(monitor = 'val_loss', patience = 5)])
res = model.evaluate(X_test, Y_test, verbose = 1)
print("Res = ", res)
def get_unet_128(input_shape=(128, 128, 3), num_classes=1):
inputs = Input(shape=input_shape)
# 128
down1 = Conv2D(16, (3, 3), padding='same')(inputs)
down1 = BatchNormalization()(down1)
down1 = Activation('relu')(down1)
down1 = Conv2D(16, (3, 3), padding='same')(down1)
down1 = BatchNormalization()(down1)
down1 = Activation('relu')(down1)
down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)
# 64
down2 = Conv2D(32, (3, 3), padding='same')(down1_pool)
down2 = BatchNormalization()(down2)
down2 = Activation('relu')(down2)
down2 = Conv2D(32, (3, 3), padding='same')(down2)
down2 = BatchNormalization()(down2)
down2 = Activation('relu')(down2)
down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)
# 32
down3 = Conv2D(64, (3, 3), padding='same')(down2_pool)
down3 = BatchNormalization()(down3)
down3 = Activation('relu')(down3)
down3 = Conv2D(64, (3, 3), padding='same')(down3)
down3 = BatchNormalization()(down3)
down3 = Activation('relu')(down3)
down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3)
# 16
down4 = Conv2D(128, (3, 3), padding='same')(down3_pool)
down4 = BatchNormalization()(down4)
down4 = Activation('relu')(down4)
down4 = Conv2D(128, (3, 3), padding='same')(down4)
down4 = BatchNormalization()(down4)
down4 = Activation('relu')(down4)
down4_pool = MaxPooling2D((2, 2), strides=(2, 2))(down4)
# 8
center = Conv2D(256, (3, 3), padding='same')(down4_pool)
center = BatchNormalization()(center)
center = Activation('relu')(center)
center = Conv2D(256, (3, 3), padding='same')(center)
center = BatchNormalization()(center)
center = Activation('relu')(center)
# center
up4 = UpSampling2D((2, 2))(center)
up4 = concatenate([down4, up4], axis=3)
up4 = Conv2D(128, (3, 3), padding='same')(up4)
up4 = BatchNormalization()(up4)
up4 = Activation('relu')(up4)
up4 = Conv2D(128, (3, 3), padding='same')(up4)
up4 = BatchNormalization()(up4)
up4 = Activation('relu')(up4)
up4 = Conv2D(128, (3, 3), padding='same')(up4)
up4 = BatchNormalization()(up4)
up4 = Activation('relu')(up4)
# 16
up3 = UpSampling2D((2, 2))(up4)
up3 = concatenate([down3, up3], axis=3)
up3 = Conv2D(64, (3, 3), padding='same')(up3)
up3 = BatchNormalization()(up3)
up3 = Activation('relu')(up3)
up3 = Conv2D(64, (3, 3), padding='same')(up3)
up3 = BatchNormalization()(up3)
up3 = Activation('relu')(up3)
up3 = Conv2D(64, (3, 3), padding='same')(up3)
up3 = BatchNormalization()(up3)
up3 = Activation('relu')(up3)
# 32
up2 = UpSampling2D((2, 2))(up3)
up2 = concatenate([down2, up2], axis=3)
up2 = Conv2D(32, (3, 3), padding='same')(up2)
up2 = BatchNormalization()(up2)
up2 = Activation('relu')(up2)
up2 = Conv2D(32, (3, 3), padding='same')(up2)
up2 = BatchNormalization()(up2)
up2 = Activation('relu')(up2)
up2 = Conv2D(32, (3, 3), padding='same')(up2)
up2 = BatchNormalization()(up2)
up2 = Activation('relu')(up2)
# 64
up1 = UpSampling2D((2, 2))(up2)
up1 = concatenate([down1, up1], axis=3)
up1 = Conv2D(16, (3, 3), padding='same')(up1)
up1 = BatchNormalization()(up1)
up1 = Activation('relu')(up1)
up1 = Conv2D(16, (3, 3), padding='same')(up1)
up1 = BatchNormalization()(up1)
up1 = Activation('relu')(up1)
up1 = Conv2D(16, (3, 3), padding='same')(up1)
up1 = BatchNormalization()(up1)
up1 = Activation('relu')(up1)
# 128
output1 = Conv2D(num_classes, (1, 1), activation='sigmoid')(up1)
if not USE_REFINE_NET:
model = Model(inputs=inputs, outputs=[output1])
# model.load_weights('../weights/head-segmentation-model.h5')
# model.compile(optimizer=SGD(lr=0.01, momentum=0.9), loss='binary_crossentropy', metrics=[dice_loss])
return model
else:
inputs2 = concatenate([inputs, output1])
outputs2 = block2(inputs2, 64)
outputs2 = average([output1, outputs2])
model2 = Model(inputs=inputs, outputs=[output1, outputs2])
return model2
return model
def Model3_LSTM_BiLSTM_LSTM(wordvocabsize,
targetvocabsize,
charvobsize,
word_W,
char_W,
input_fragment_lenth,
input_leftcontext_lenth,
input_rightcontext_lenth,
input_maxword_length,
w2v_k,
c2v_k,
hidden_dim=200,
batch_size=32,
optimizer='rmsprop'):
hidden_dim = 100
word_input_fragment = Input(shape=(input_fragment_lenth, ), dtype='int32')
word_embedding_fragment = Embedding(input_dim=wordvocabsize + 1,
output_dim=w2v_k,
input_length=input_fragment_lenth,
mask_zero=False,
trainable=True,
weights=[word_W])(word_input_fragment)
word_embedding_fragment = Dropout(0.5)(word_embedding_fragment)
char_input_fragment = Input(shape=(
input_fragment_lenth,
input_maxword_length,
),
dtype='int32')
char_embedding_fragment = TimeDistributed(
Embedding(input_dim=charvobsize,
output_dim=c2v_k,
batch_input_shape=(batch_size, input_fragment_lenth,
input_maxword_length),
mask_zero=False,
trainable=True,
weights=[char_W]))(char_input_fragment)
char_cnn_fragment = TimeDistributed(
Conv1D(50, 3, activation='relu', padding='valid'))
char_embedding_fragment = char_cnn_fragment(char_embedding_fragment)
char_embedding_fragment = TimeDistributed(
GlobalMaxPooling1D())(char_embedding_fragment)
char_embedding_fragment = Dropout(0.25)(char_embedding_fragment)
word_input_leftcontext = Input(shape=(input_leftcontext_lenth, ),
dtype='int32')
word_embedding_leftcontext = Embedding(
input_dim=wordvocabsize + 1,
output_dim=w2v_k,
input_length=input_leftcontext_lenth,
mask_zero=True,
trainable=True,
weights=[word_W])(word_input_leftcontext)
word_embedding_leftcontext = Dropout(0.5)(word_embedding_leftcontext)
char_input_leftcontext = Input(shape=(
input_leftcontext_lenth,
input_maxword_length,
),
dtype='int32')
char_input_rightcontext = Input(shape=(
input_rightcontext_lenth,
input_maxword_length,
),
dtype='int32')
word_input_rightcontext = Input(shape=(input_rightcontext_lenth, ),
dtype='int32')
word_embedding_rightcontext = Embedding(
input_dim=wordvocabsize + 1,
output_dim=w2v_k,
input_length=input_rightcontext_lenth,
mask_zero=True,
trainable=True,
weights=[word_W])(word_input_rightcontext)
word_embedding_rightcontext = Dropout(0.5)(word_embedding_rightcontext)
embedding_fragment = concatenate(
[word_embedding_fragment, char_embedding_fragment], axis=-1)
embedding_leftcontext = word_embedding_leftcontext
embedding_rightcontext = word_embedding_rightcontext
LSTM_leftcontext = LSTM(hidden_dim, go_backwards=False,
activation='tanh')(embedding_leftcontext)
Rep_LSTM_leftcontext = RepeatVector(input_fragment_lenth)(LSTM_leftcontext)
LSTM_rightcontext = LSTM(hidden_dim, go_backwards=True,
activation='tanh')(embedding_rightcontext)
Rep_LSTM_rightcontext = RepeatVector(input_fragment_lenth)(
LSTM_rightcontext)
BiLSTM_fragment = Bidirectional(LSTM(hidden_dim // 2,
activation='tanh',
return_sequences=True),
merge_mode='concat')(embedding_fragment)
context_ADD = add([LSTM_leftcontext, BiLSTM_fragment, LSTM_rightcontext])
context_subtract_l = subtract([BiLSTM_fragment, LSTM_leftcontext])
context_subtract_r = subtract([BiLSTM_fragment, LSTM_rightcontext])
context_average = average(
[LSTM_leftcontext, BiLSTM_fragment, LSTM_rightcontext])
context_maximum = maximum(
[LSTM_leftcontext, BiLSTM_fragment, LSTM_rightcontext])
embedding_mix = concatenate([
embedding_fragment, BiLSTM_fragment, context_ADD, context_subtract_l,
context_subtract_r, context_average, context_maximum
],
axis=-1)
# BiLSTM_fragment = Bidirectional(LSTM(hidden_dim // 2, activation='tanh'), merge_mode='concat')(embedding_fragment)
decoderlayer1 = Conv1D(50, 1, activation='relu', strides=1,
padding='same')(embedding_mix)
decoderlayer2 = Conv1D(50, 2, activation='relu', strides=1,
padding='same')(embedding_mix)
decoderlayer3 = Conv1D(50, 3, activation='relu', strides=1,
padding='same')(embedding_mix)
decoderlayer4 = Conv1D(50, 4, activation='relu', strides=1,
padding='same')(embedding_mix)
CNNs_fragment = concatenate(
[decoderlayer1, decoderlayer2, decoderlayer3, decoderlayer4], axis=-1)
CNNs_fragment = Dropout(0.5)(CNNs_fragment)
CNNs_fragment = GlobalMaxPooling1D()(CNNs_fragment)
concat = Dropout(0.3)(CNNs_fragment)
output = Dense(targetvocabsize, activation='softmax')(concat)
Models = Model([
word_input_fragment, word_input_leftcontext, word_input_rightcontext,
char_input_fragment, char_input_leftcontext, char_input_rightcontext
], output)
Models.compile(loss='categorical_crossentropy',
optimizer=optimizers.RMSprop(lr=0.001),
metrics=['acc'])
return Models
