def mean_squared_error(inputs):
if len(inputs) == 2:
[mu1, mu2] = inputs
if isinstance(mu2, list):
return average([mse(mu1, pred) for pred in mu2])
else:
return mse(mu1, mu2)
else:
true = inputs[0]
return average(
[mse(true, inputs[pred]) for pred in range(1, len(inputs))])
def categorical_crossentropy(inputs):
print("Categorical cross entropy inputs : ", inputs)
if len(inputs) == 2:
[mu1, mu2] = inputs
if isinstance(mu2, list):
return average(
[K.categorical_crossentropy(mu1, pred) for pred in mu2])
else:
return K.categorical_crossentropy(mu1, mu2)
else:
true = inputs[0]
return average([
K.categorical_crossentropy(true, inputs[pred])
for pred in range(1, len(inputs))
])
def binary_crossentropy(inputs):
if len(inputs) == 2:
[mu1, mu2] = inputs
if isinstance(mu2, list):
return average([K.binary_crossentropy(mu1, pred) for pred in mu2])
else:
return K.binary_crossentropy(mu1, mu2)
else:
raise Exception(
"BINARY CROSS ENTROPY HAS MORE THAN 2 ARGUMENTS. ENSURE THIS IS DESIRED"
)
true = inputs[0]
return average([
K.binary_crossentropy(true, inputs[pred])
for pred in range(1, len(inputs))
])
def _embedder(inputs):
assert len(inputs) == len(input_names), \
f'Expected {len(input_names)} input(s) {input_names}, ' \
f'but got {len(inputs)}'
x = []
for n, inp in zip(input_names, inputs):
for layer in embedder_layers[n]:
inp = layer(inp)
x += [inp]
if len(x) > 1:
if merge_mode == 'concat':
x = layers.concatenate(x)
elif merge_mode == 'sum':
x = layers.add(x)
elif merge_mode == 'ave':
x = layers.average(x)
else:
raise ValueError(
'Unrecognized merge mode {}'.format(merge_mode))
else:
x = x[0]
if project_dim is not None:
x = layers.Dense(project_dim, input_shape=(emb_dim, ))(x)
return layers.Dropout(rate=dropout)(x)
def DeepModel(size_set=640):
img_input = Input(shape=(size_set, size_set, 3))
conv1 = Conv2D(32, (3, 3), activation='relu', padding='same', name='block1_conv1')(img_input)
conv1 = Conv2D(32, (3, 3), activation='relu', padding='same', name='block1_conv2')(conv1)
pool1 = MaxPool2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(64, (3, 3), activation='relu', padding='same', name='block2_conv1')(pool1)
conv2 = Conv2D(64, (3, 3), activation='relu', padding='same', name='block2_conv2')(conv2)
pool2 = MaxPool2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(128, (3, 3), activation='relu', padding='same', name='block3_conv1')(pool2)
conv3 = Conv2D(128, (3, 3), activation='relu', padding='same', name='block3_conv2')(conv3)
pool3 = MaxPool2D(pool_size=(2, 2))(conv3)
conv4 = Conv2D(256, (3, 3), activation='relu', padding='same', name='block4_conv1')(pool3)
conv4 = Conv2D(256, (3, 3), activation='relu', padding='same', name='block4_conv2')(conv4)
pool4 = MaxPool2D(pool_size=(2, 2))(conv4)
conv5 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv1')(pool4)
conv5 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv2')(conv5)
up6 = concatenate(
[Conv2DTranspose(256, (2, 2), strides=(2, 2), padding='same', name='block6_dconv')(conv5), conv4],
axis=3)
conv6 = Conv2D(256, (3, 3), activation='relu', padding='same', name='block6_conv1')(up6)
conv6 = Conv2D(256, (3, 3), activation='relu', padding='same', name='block6_conv2')(conv6)
up7 = concatenate(
[Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same', name='block7_dconv')(conv6), conv3],
axis=3)
conv7 = Conv2D(128, (3, 3), activation='relu', padding='same', name='block7_conv1')(up7)
conv7 = Conv2D(128, (3, 3), activation='relu', padding='same', name='block7_conv2')(conv7)
up8 = concatenate([Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same', name='block8_dconv')(conv7), conv2],
axis=3)
conv8 = Conv2D(64, (3, 3), activation='relu', padding='same', name='block8_conv1')(up8)
conv8 = Conv2D(64, (3, 3), activation='relu', padding='same', name='block8_conv2')(conv8)
up9 = concatenate([Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same', name='block9_dconv')(conv8), conv1],
axis=3)
conv9 = Conv2D(32, (3, 3), activation='relu', padding='same', name='block9_conv1')(up9)
conv9 = Conv2D(32, (3, 3), activation='relu', padding='same', name='block9_conv2')(conv9)
side6 = UpSampling2D(size=(8, 8))(conv6)
side7 = UpSampling2D(size=(4, 4))(conv7)
side8 = UpSampling2D(size=(2, 2))(conv8)
out6 = Conv2D(1, (1, 1), activation='sigmoid', name='side_6')(side6)
out7 = Conv2D(1, (1, 1), activation='sigmoid', name='side_7')(side7)
out8 = Conv2D(1, (1, 1), activation='sigmoid', name='side_8')(side8)
out9 = Conv2D(1, (1, 1), activation='sigmoid', name='side_9')(conv9)
out10 = average([out6, out7, out8, out9])
return Model(inputs=img_input, outputs=out10)
def main():
model1 = get_model()
model2 = get_model()
model3 = get_model()
inputs = keras.Input(shape=(128,))
y1 = model1(inputs)
y2 = model2(inputs)
y3 = model3(inputs)
outputs = layers.average([y1, y2, y3])
ensemble_model = keras.Model(inputs=inputs, outputs=outputs, name="function_define_model")
ensemble_model.summary()
def two_stream_fuse(self):
# spatial stream (frozen)
cnn_spatial_multi = self.cnn_spatial_multi()
# temporal stream (frozen)
cnn_temporal_multi = self.cnn_temporal_multi()
# fused by taking average
outputs = average([cnn_spatial_multi.output, cnn_temporal_multi.output])
model = Model([cnn_spatial_multi.input, cnn_temporal_multi.input], outputs)
return model
def __new__(cls, mode: str) \
-> KM.Model:
assert mode in ['base', 'skip', 'bn']
inputs = KL.Input(shape=[None, None, 3], name="input_image")
x = inputs
x = KL.Conv2D(64, (3, 3),
padding="SAME",
kernel_initializer='random_uniform',
bias_initializer='zeros',
name="layer1")(x)
if mode == 'bn':
x = KL.BatchNormalization()(x)
x = KL.ReLU()(x)
x = KL.Conv2D(64, (3, 3),
padding="SAME",
kernel_initializer='random_uniform',
bias_initializer='zeros',
name="layer2")(x)
if mode == 'bn':
x = KL.BatchNormalization()(x)
x = KL.ReLU()(x)
x = KL.Conv2D(64, (3, 3),
padding="SAME",
kernel_initializer='random_uniform',
bias_initializer='zeros',
name="layer3")(x)
if mode == 'bn':
x = KL.BatchNormalization()(x)
x = KL.ReLU()(x)
x = KL.Conv2D(64, (3, 3),
padding="SAME",
kernel_initializer='random_uniform',
bias_initializer='zeros',
name="layer4")(x)
if mode == 'bn':
x = KL.BatchNormalization()(x)
x = KL.ReLU()(x)
x = KL.Conv2D(3, (3, 3),
padding="SAME",
kernel_initializer='random_uniform',
bias_initializer='zeros',
name="layer5")(x)
if mode == 'bn':
x = KL.BatchNormalization()(x)
x = KL.ReLU()(x)
if mode == 'skip' or mode == 'bn':
x = KL.average([x, inputs])
return KM.Model(inputs=inputs, outputs=x, name='denoising')
def create_ensemble(models):
if len(models) == 1:
return models[0]
else:
inputs = Input(shape=[N_FEATURES])
predictions = [model(inputs) for model in models]
outputs = average(predictions)
model = Model(inputs=inputs, outputs=outputs)
model.compile(
loss='mse',
metrics=[RootMeanSquaredError(),
MeanAbsoluteError(),
RSquare()])
return model
def load_weights(self, atom_type):
saved_models_dir = os.path.join(self.path, atom_type)
model_0_path = os.path.join(saved_models_dir, 'model_10_n1_8.h5')
model_1_path = os.path.join(saved_models_dir, 'model_1_n1_8.h5')
model_2_path = os.path.join(saved_models_dir, 'model_2_n1_8.h5')
model_3_path = os.path.join(saved_models_dir, 'model_3_n1_8.h5')
model_4_path = os.path.join(saved_models_dir, 'model_4_n1_8.h5')
model_5_path = os.path.join(saved_models_dir, 'model_5_n1_8.h5')
model_6_path = os.path.join(saved_models_dir, 'model_6_n1_8.h5')
model_7_path = os.path.join(saved_models_dir, 'model_7_n1_8.h5')
model_8_path = os.path.join(saved_models_dir, 'model_8_n1_8.h5')
model_9_path = os.path.join(saved_models_dir, 'model_9_n1_8.h5')
# Load all the models
model_0 = keras.models.load_model(model_0_path, compile=False)
model_1 = keras.models.load_model(model_1_path, compile=False)
model_2 = keras.models.load_model(model_2_path, compile=False)
model_3 = keras.models.load_model(model_3_path, compile=False)
model_4 = keras.models.load_model(model_4_path, compile=False)
model_5 = keras.models.load_model(model_5_path, compile=False)
model_6 = keras.models.load_model(model_6_path, compile=False)
model_7 = keras.models.load_model(model_7_path, compile=False)
model_8 = keras.models.load_model(model_8_path, compile=False)
model_9 = keras.models.load_model(model_9_path, compile=False)
inputs = keras.Input(shape=(384, ))
# Define each sub model
y0 = model_0(inputs)
y1 = model_1(inputs)
y2 = model_2(inputs)
y3 = model_3(inputs)
y4 = model_4(inputs)
y5 = model_5(inputs)
y6 = model_6(inputs)
y7 = model_7(inputs)
y8 = model_8(inputs)
y9 = model_9(inputs)
# Finally generate ensemble model
ml_corrs = [y0, y1, y2, y3, y4, y5, y6, y7, y8, y9]
outputs = layers.average([y0, y1, y2, y3, y4, y5, y6, y7, y8, y9])
print(f"Loaded ensemble {atom_type}")
return keras.Model(inputs=inputs, outputs=([outputs, ml_corrs]))
def ensembleModels(models, ):
"""
This function builds an ensemble model from a list of models
Arguments:
models: list of Keras models
Return:
modelEns: ensemble Keras model
"""
model_input = Input(shape=models[0].input_shape[1:])
# collect outputs of models in a list
yModels=[model(model_input) for model in models]
# averaging outputs
yAvg=layers.average(yModels)
# build model from same input and avg output
modelEns = Model(inputs=model_input, outputs=yAvg, name='ensemble')
return modelEns
def avg_ensemble(model_dir: str):
sub_models = []
model_files = Path(model_dir).glob('*.h5')
for model in model_files:
# skip ensemble model
if model.stem == 'ensemble':
continue
sub_models.append(load_model(model, compile=False))
text_input = layers.Input(shape=(sub_models[0].input['text'].shape[1:]),
name='text')
kw_input = layers.Input(shape=(sub_models[0].input['keyword'].shape[1:]),
name='keyword')
loc_input = layers.Input(shape=(sub_models[0].input['location'].shape[1:]),
name='location')
hashtags_input = layers.Input(
shape=(sub_models[0].input['hashtags'].shape[1:]), name='hashtags')
inputs = {
'text': text_input,
'keyword': kw_input,
'location': loc_input,
'hashtags': hashtags_input
}
model_outputs = []
for model in sub_models:
# remove last layer which should be the activation layer
model.layers.pop(-1)
output = model.layers[-1].output
model = Model(inputs=model.input, outputs=output)
model_outputs.append(model(inputs))
average = layers.average([model for model in model_outputs])
softmax = layers.Activation(tf.nn.softmax)(average)
ensemble = Model(inputs=inputs, outputs=softmax)
save_model(ensemble, os.path.join(model_dir, 'ensemble.h5'))
plot_model(ensemble,
to_file=os.path.join(model_dir, 'ensemble.png'),
show_shapes=True)
return ensemble
for layer in object_mg_model.layers:
layer._name = 'object_' + layer.name
object_output = object_mg_model.get_layer(name='object_dense_1').output
##------------------------------------------------------------------------------------
## get layer from places_mg_model (change directory to load best places_mg_model from folder 'training_models')
places_mg_model = load_model('./pretrained_models/places_mg_model.hdf5')
for layer in places_mg_model.layers:
layer._name = 'places_' + layer.name
places_output = places_mg_model.get_layer(name='places_dense_2').output
## Average both models output
average_layer = average([object_output, places_output])
## Create late_fusion2_model
late_fusion2_model = Model(
inputs=[Object_MG_Model.input, Places_MG_Model.input],
outputs=average_layer)
ground_truth = test_datagen1.classes
## Data generator
dgenerator = doubleGenerator(test_datagen1, test_datagen2)
## Predictions
predictions = late_fusion2_model.predict_generator(
dgenerator, steps=math.ceil(test_datagen1.samples / BATCH_SIZE), verbose=1)
predicted_classes = np.argmax(predictions, axis=1)
def get_model():
inputs = keras.Input(shape=(128, ))
outputs = keras.layers.Dense(1, activation='sigmoid')(inputs)
return keras.Model(inputs, outputs)
model1 = get_model()
model2 = get_model()
model3 = get_model()
inputs = keras.Input(shape=(128, ))
y1 = model1(inputs)
y2 = model2(inputs)
y3 = model3(inputs)
outputs = layers.average([y1, y2, y3])
ensemble_model = keras.Model(inputs, outputs)
#3.复杂网络结构构建
#3.1 多输入与多输出网络
#构建一个根据定制标题、内容和标签， 预测票证优先级和执行部门的网络
#超参数
num_words = 2000
num_tags = 12
num_department = 4
#输入
body_input = keras.Input(shape=(None, ), name='body')
title_input = keras.Input(shape=(None, ), name='title')
tag_input = keras.Input(shape=(num_tags, ), name='tag')
def _create_multicol_model():
inputs = Input(shape=(None, None, 3))
cols = [_create_column(d, inputs) for d in [3, 5, 9]]
model = KModel(inputs=inputs, outputs=average(cols))
return model
def create_object_mg_model(best_object_basic_model):
model = load_model(best_object_basic_model)
MobileNet_model_out = model.get_layer('conv_pw_1_relu').output
conv_1 = Conv2D(filters=64,
kernel_size=(1, 1),
strides=(1, 1),
padding='valid',
data_format=None,
dilation_rate=(1, 1),
activation=None,
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None)(MobileNet_model_out)
branch_1_out = GlobalAveragePooling2D(name='gap1')(conv_1)
MobileNet_model_out = model.get_layer('conv_pw_3_relu').output
conv_2 = Conv2D(filters=64,
kernel_size=(1, 1),
strides=(1, 1),
padding='valid',
data_format=None,
dilation_rate=(1, 1),
activation=None,
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None)(MobileNet_model_out)
branch_2_out = GlobalAveragePooling2D(name='gap2')(conv_2)
MobileNet_model_out = model.get_layer('conv_pw_5_relu').output
conv_3 = Conv2D(filters=64,
kernel_size=(1, 1),
strides=(1, 1),
padding='valid',
data_format=None,
dilation_rate=(1, 1),
activation=None,
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None)(MobileNet_model_out)
branch_3_out = GlobalAveragePooling2D(name='gap3')(conv_3)
MobileNet_model_out = model.get_layer('conv_pw_11_relu').output
conv_4 = Conv2D(filters=64,
kernel_size=(1, 1),
strides=(1, 1),
padding='valid',
data_format=None,
dilation_rate=(1, 1),
activation=None,
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None)(MobileNet_model_out)
branch_4_out = GlobalAveragePooling2D(name='gap4')(conv_4)
MobileNet_model_out = model.get_layer('conv_pw_13_relu').output
conv_5 = Conv2D(filters=64,
kernel_size=(1, 1),
strides=(1, 1),
padding='valid',
data_format=None,
dilation_rate=(1, 1),
activation=None,
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None)(MobileNet_model_out)
branch_5_out = GlobalAveragePooling2D(name='gap5')(conv_5)
merge = average(
[branch_1_out, branch_2_out, branch_3_out, branch_4_out, branch_5_out])
output = Dense(8, activation='softmax')(merge)
model = Model(inputs=model.input, outputs=[output])
return model
# PermutationalEncoder(
# PairwiseModel((8,), repeat_layers(Dense, [16, 4], activation="relu")), 3
# ),
# name="permutational_layer2",
# )
# outputs = perm_layer2(outputs)
# perm_layer3 = PermutationalLayer(
# PermutationalEncoder(
# PairwiseModel((4,), repeat_layers(Dense, [16, 4], activation="tanh")), 3
# ),
# name="permutational_layer3",
# )
# outputs = perm_layer3(outputs)
# # I can even reuse this layer because the input shape and output shape of it is identical. but don't try this at home unless you know what you are doing.
# outputs = perm_layer3(outputs)
output = average(outputs) # let's average the output for single tensor
model = Model(inputs, output)
print("# Multi-layer model summary")
model.summary()
# let's predict with the big mult layers model on some similar data
print("# Multi-layer Prediction")
print(
"Because the layer is preserving permutation invariance. "
"The output will be the same no matter how many times you permute the input order."
)
print("## Input/Output 1")
x = [[[0, 0, 0, 0]], [[1, 2, 3, 4]], [[4, 3, 2, 1]]]
print(x)
print(model.predict(x))
print("## Input/Output 2")
def CNorm(vst_onlyTokens,
dl_terms,
dl_associations,
vso,
nbEpochs=30,
batchSize=64,
l_numberOfFilters=[4000],
l_filterSizes=[1],
phraseMaxSize=15):
# Preparing data for SLFNN and S-CNN components:
dataSCNN, labels, l_unkownTokens, l_uncompleteExpressions = prepare2D_data(
vst_onlyTokens, dl_terms, dl_associations, vso, phraseMaxSize)
dataSLFNN = numpy.zeros((dataSCNN.shape[0], dataSCNN.shape[2]))
for i in range(dataSCNN.shape[0]):
numberOfToken = 0
for embedding in dataSCNN[i]:
if not numpy.any(embedding):
pass
else:
numberOfToken += 1
dataSLFNN[i] += embedding
if numberOfToken > 0:
dataSLFNN[i] = dataSLFNN[i] / numberOfToken
# Input layers:
inputLP = Input(shape=dataSLFNN.shape[1])
inputCNN = Input(shape=[dataSCNN.shape[1], dataSCNN.shape[2]])
# SLFNN component:
ontoSpaceSize = labels.shape[2]
denseLP = layers.Dense(
units=ontoSpaceSize,
use_bias=True,
kernel_initializer=initializers.GlorotUniform())(inputLP)
modelLP = Model(inputs=inputLP, outputs=denseLP)
# Shallow-CNN component:
l_subLayers = list()
for i, filterSize in enumerate(l_filterSizes):
convLayer = (layers.Conv1D(
l_numberOfFilters[i],
filterSize,
strides=1,
kernel_initializer=initializers.GlorotUniform()))(inputCNN)
outputSize = phraseMaxSize - filterSize + 1
pool = (layers.MaxPool1D(pool_size=outputSize))(convLayer)
activationLayer = (layers.LeakyReLU(alpha=0.3))(pool)
l_subLayers.append(activationLayer)
if len(l_filterSizes) > 1:
concatenateLayer = (layers.Concatenate(axis=-1))(
l_subLayers) # axis=-1 // concatenating on the last dimension
else:
concatenateLayer = l_subLayers[0]
denseLayer = layers.Dense(
ontoSpaceSize,
kernel_initializer=initializers.GlorotUniform())(concatenateLayer)
modelCNN = Model(inputs=inputCNN, outputs=denseLayer)
convModel = Model(inputs=inputCNN, outputs=concatenateLayer)
fullmodel = models.Sequential()
fullmodel.add(convModel)
# Combination of the two components:
combinedLayer = layers.average([modelLP.output, modelCNN.output])
fullModel = Model(inputs=[inputLP, inputCNN], outputs=combinedLayer)
fullModel.summary()
# Compile and train:
fullModel.compile(
optimizer=optimizers.Nadam(),
loss=losses.LogCosh(),
metrics=[metrics.CosineSimilarity(),
metrics.MeanSquaredError()])
fullModel.fit([dataSLFNN, dataSCNN],
labels,
epochs=nbEpochs,
batch_size=batchSize)
return fullModel, vso, l_unkownTokens
def _create_msb(filters, dimensions, inputs):
"""
Multi-scale Blob as described in https://arxiv.org/pdf/1702.02359.pdf
"""
cols = [Conv2D(filters, kernel_size=(d, d), activation='relu', padding='same', kernel_initializer=_msb_initializer)(inputs) for d in dimensions]
return average(cols)
def MT_model(perm):
#def funx1(i,maxL):
#if i<maxL:
#return Input(shape=(fpSize,),name=f"MRg_{i}")
#elif i<2*maxL-3 and i!=maxL+2:
# elif maxL<=i< 2*maxL:
#return Input(shape=(FpLen,),name=f"Ml_{i}")
#else:
# return Input(shape=(FpLen1,),name=f"Sq_{i}")
def funx1(i, maxL):
if i < maxL:
return Input(shape=(256, ), name=f"MoR_{i}")
x2 = [funx1(i, maxL) for i in range(1 * maxL)]
# x2_1=Concatenate()([x2[0],x2[1],x2[2]])
xs = []
xs = [x2[perm[0]], x2[perm[1]], x2[perm[2]]]
#xs=[x2[1],x2[2],x2[0]]
#xs=[x2[1],x2[0],x2[2]]
#xs=[x2[2],x2[0],x2[1]]
#xs=[x2[2],x2[1],x2[0]]
#xs=[x2[2],x2[0],x2[1]]
#xs1=[]
xs2 = []
#xs.append(x2)
#for i in range(maxL):
# xs.append(x2[i])
# xs.append(x2[maxL+i])
# xs.append(x2[2*maxL+i])
list1 = [100, 100, 100, 100]
#list1=[100,58,58,58]
# list2=[112,112,112,112]
pairwise_model = pl.PairwiseModel((256, ),
pl.repeat_layers(Dense,
list1,
name="hidden",
activation='relu'),
name="pairwise_model")
perm_encoder = pl.PermutationalEncoder(pairwise_model,
maxL,
name="permutational_encoder")
perm_layer = pl.PermutationalLayer1(perm_encoder,
name="permutational_layer")
outputs = perm_layer.model(xs)
outputs = average(outputs)
#outputs = maximum(outputs)
# print("x2",x2)
output_51 = Dense(100, activation='relu',
kernel_initializer=my_init)(outputs)
# output_51 = Dense(200, activation='relu',kernel_initializer=my_init)(output_51)
# output_51 = Dense(200, activation='relu',kernel_initializer=my_init)(output_51)
# output_51 = Dense(200, activation='relu',kernel_initializer=my_init)(output_51)
# output_51 = Dense(200, activation='relu',kernel_initializer=my_init)(output_51)
# output_51 = Dense(200, activation='relu',kernel_initializer=my_init)(output_51)
# #output_51 = Dropout(0.5)(output_51)
# output_51 = Dense(200, activation='relu',kernel_initializer=my_init)(output_51)
#
output_Loss = Dense(17,
name='Loss_output',
activation='softmax',
kernel_initializer=my_init)(output_51)
# output_Loss=Dense(17,name='Loss_output',activation='linear',kernel_initializer=my_init)(output_51)
#output_Loss=Dense(17,name='Loss_output',activation='linear',kernel_initializer=my_init)(output_51)
model = Model(inputs=x2, outputs=output_Loss)
# model.compile(loss={'Loss_output':'categorical_crossentropy'},
# optimizer=Adam(lr=0.00005, beta_1=0.9, beta_2=0.999, epsilon=1e-8), loss_weights = {'Loss_output':1.}
# ,metrics=['accuracy']
# )
model.compile(
loss={'Loss_output': 'categorical_crossentropy'},
optimizer=Adam(lr=0.00276, beta_1=0.9, beta_2=0.999, epsilon=1e-8),
loss_weights={'Loss_output': 1.}
#optimizer=Adam(lr=0.00005, beta_1=0.9, beta_2=0.999, epsilon=1e-8), loss_weights = {'Loss_output':1.}
,
metrics=['accuracy'])
model.summary()
plot_model(model, 'Config3Mod.png', show_shapes=True)
return model
def MNet(input_shape):
"""Code copied from `Model_MNet <https://github.com/HzFu/MNet_DeepCDR/blob/master/mnet_deep_cdr/Model_MNet.py>`_.
Made some minor changes to adapt the network to our specific case.
Based on `Joint Optic Disc and Cup Segmentation Based on Multi-label Deep
Network and Polar Transformation <https://arxiv.org/abs/1801.00926>`_.
Parameters
----------
input_shape : 3 int tuple
Shape of input images.
Returns
-------
model : Keras model
MNet network model.
Here is a picture of the network extracted from the original paper:
.. image:: img/MNet.jpg
:width: 100%
:align: center
"""
img_input = Input(input_shape)
scale_img_2 = AveragePooling2D(pool_size=(2, 2))(img_input)
scale_img_3 = AveragePooling2D(pool_size=(2, 2))(scale_img_2)
scale_img_4 = AveragePooling2D(pool_size=(2, 2))(scale_img_3)
conv1 = Conv2D(32, (3, 3),
padding='same',
activation='relu',
name='block1_conv1')(img_input)
conv1 = Conv2D(32, (3, 3),
padding='same',
activation='relu',
name='block1_conv2')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
input2 = Conv2D(64, (3, 3),
padding='same',
activation='relu',
name='block2_input1')(scale_img_2)
input2 = concatenate([input2, pool1], axis=3)
conv2 = Conv2D(64, (3, 3),
padding='same',
activation='relu',
name='block2_conv1')(input2)
conv2 = Conv2D(64, (3, 3),
padding='same',
activation='relu',
name='block2_conv2')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
input3 = Conv2D(128, (3, 3),
padding='same',
activation='relu',
name='block3_input1')(scale_img_3)
input3 = concatenate([input3, pool2], axis=3)
conv3 = Conv2D(128, (3, 3),
padding='same',
activation='relu',
name='block3_conv1')(input3)
conv3 = Conv2D(128, (3, 3),
padding='same',
activation='relu',
name='block3_conv2')(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
input4 = Conv2D(256, (3, 3),
padding='same',
activation='relu',
name='block4_input1')(scale_img_4)
input4 = concatenate([input4, pool3], axis=3)
conv4 = Conv2D(256, (3, 3),
padding='same',
activation='relu',
name='block4_conv1')(input4)
conv4 = Conv2D(256, (3, 3),
padding='same',
activation='relu',
name='block4_conv2')(conv4)
pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)
conv5 = Conv2D(512, (3, 3),
padding='same',
activation='relu',
name='block5_conv1')(pool4)
conv5 = Conv2D(512, (3, 3),
padding='same',
activation='relu',
name='block5_conv2')(conv5)
up6 = concatenate([
Conv2DTranspose(
256, (2, 2), strides=(2, 2), padding='same',
name='block6_dconv')(conv5), conv4
],
axis=3)
conv6 = Conv2D(256, (3, 3),
padding='same',
activation='relu',
name='block6_conv1')(up6)
conv6 = Conv2D(256, (3, 3),
padding='same',
activation='relu',
name='block6_conv2')(conv6)
up7 = concatenate([
Conv2DTranspose(
128, (2, 2), strides=(2, 2), padding='same',
name='block7_dconv')(conv6), conv3
],
axis=3)
conv7 = Conv2D(128, (3, 3),
padding='same',
activation='relu',
name='block7_conv1')(up7)
conv7 = Conv2D(128, (3, 3),
padding='same',
activation='relu',
name='block7_conv2')(conv7)
up8 = concatenate([
Conv2DTranspose(
64, (2, 2), strides=(2, 2), padding='same',
name='block8_dconv')(conv7), conv2
],
axis=3)
conv8 = Conv2D(64, (3, 3),
padding='same',
activation='relu',
name='block8_conv1')(up8)
conv8 = Conv2D(64, (3, 3),
padding='same',
activation='relu',
name='block8_conv2')(conv8)
up9 = concatenate([
Conv2DTranspose(
32, (2, 2), strides=(2, 2), padding='same',
name='block9_dconv')(conv8), conv1
],
axis=3)
conv9 = Conv2D(32, (3, 3),
padding='same',
activation='relu',
name='block9_conv1')(up9)
conv9 = Conv2D(32, (3, 3),
padding='same',
activation='relu',
name='block9_conv2')(conv9)
side6 = UpSampling2D(size=(8, 8))(conv6)
side7 = UpSampling2D(size=(4, 4))(conv7)
side8 = UpSampling2D(size=(2, 2))(conv8)
# Changes here to have just one class and use BCE as loss function
out6 = Conv2D(1, (1, 1), activation='sigmoid', name='side_63')(side6)
out7 = Conv2D(1, (1, 1), activation='sigmoid', name='side_73')(side7)
out8 = Conv2D(1, (1, 1), activation='sigmoid', name='side_83')(side8)
out9 = Conv2D(1, (1, 1), activation='sigmoid', name='side_93')(conv9)
out10 = average([out6, out7, out8, out9])
return Model(inputs=[img_input], outputs=[out6, out7, out8, out9, out10])
def call(self, x):
# three scale imgs
scale_img_1 = self.scale1(x)
scale_img_2 = self.scale2(scale_img_1)
scale_img_3 = self.scale3(scale_img_2)
conv1 = self.block1_conv1(x)
conv1 = self.block1_conv2(conv1)
pool1 = self.block1_pool1(conv1)
input2 = self.block2_input1(scale_img_1)
input2 = tf.concat([input2, pool1], axis=3)
conv2 = self.block2_conv1(input2)
conv2 = self.block2_conv2(conv2)
pool2 = self.block2_pool1(conv2)
input3 = self.block3_input1(scale_img_2)
input3 = tf.concat([input3, pool2], axis=3)
conv3 = self.block3_conv1(input3)
conv3 = self.block3_conv2(conv3)
pool3 = self.block3_pool1(conv3)
input4 = self.block4_input1(scale_img_3)
input4 = tf.concat([input4, pool3], axis=3)
conv4 = self.block4_conv1(input4)
conv4 = self.block4_conv2(conv4)
pool4 = self.block4_pool1(conv4)
conv5 = self.block5_conv1(pool4)
conv5 = self.block5_conv2(conv5)
temp6 = self.block6_dconv(conv5)
up6 = tf.concat([temp6, conv4], axis=3)
conv6 = self.block6_conv1(up6)
conv6 = self.block6_conv2(conv6)
temp7 = self.block7_dconv(conv6)
up7 = tf.concat([temp7, conv3], axis=3)
conv7 = self.block7_conv1(up7)
conv7 = self.block7_conv2(conv7)
temp8 = self.block8_dconv(conv7)
up8 = tf.concat([temp8, conv2], axis=3)
conv8 = self.block8_conv1(up8)
conv8 = self.block8_conv2(conv8)
temp9 = self.block9_dconv(conv8)
up9 = tf.concat([temp9, conv1], axis=3)
conv9 = self.block9_conv1(up9)
conv9 = self.block9_conv2(conv9)
side6 = UpSampling2D(size=(8, 8))(conv6)
side7 = UpSampling2D(size=(4, 4))(conv7)
side8 = UpSampling2D(size=(2, 2))(conv8)
out6 = self.side63(side6)
out7 = self.side73(side7)
out8 = self.side83(side8)
out9 = self.side93(conv9)
out10 = average([out6, out7, out8, out9])
return [out6, out7, out8, out9, out10]
def prepare_encoder_output_state(self, inputs):
if 'encoder_output_state' in inputs:
encoder_output_state = inputs['encoder_output_state']
else:
hidden = inputs['hidden']
if len(hidden.shape) == 3: # encoder_output is a sequence
# reduce_sequence returns a [b, h]
encoder_output_state = self.reduce_sequence(hidden)
elif len(hidden.shape) == 2:
# this returns a [b, h]
encoder_output_state = hidden
else:
raise ValueError("Only works for 1d or 2d encoder_output")
# now we have to deal with the fact that the state needs to be a list
# in case of lstm or a tensor otherwise
if (self.cell_type == 'lstm' and
isinstance(encoder_output_state, list)):
if len(encoder_output_state) == 2:
# this maybe a unidirectionsl lstm or a bidirectional gru / rnn
# there is no way to tell
# If it is a unidirectional lstm, pass will work fine
# if it is bidirectional gru / rnn, the output of one of
# the directions will be treated as the inital c of the lstm
# which is weird and may lead to poor performance
# todo future: try to find a way to distinguish among these two cases
pass
elif len(encoder_output_state) == 4:
# the encoder was a bidirectional lstm
# a good strategy is to average the 2 h and the 2 c vectors
encoder_output_state = [
average(
[encoder_output_state[0], encoder_output_state[2]]
),
average(
[encoder_output_state[1], encoder_output_state[3]]
)
]
else:
# no idea how lists of length different than 2 or 4
# might have been originated, we can either rise an ValueError
# or deal with it averaging everything
# raise ValueError(
# "encoder_output_state has length different than 2 or 4. "
# "Please doublecheck your encoder"
# )
average_state = average(encoder_output_state)
encoder_output_state = [average_state, average_state]
elif (self.cell_type == 'lstm' and
not isinstance(encoder_output_state, list)):
encoder_output_state = [encoder_output_state, encoder_output_state]
elif (self.cell_type != 'lstm' and
isinstance(encoder_output_state, list)):
# here we have a couple options,
# either reuse part of the input encoder state,
# or just use its output
if len(encoder_output_state) == 2:
# using h and ignoring c
encoder_output_state = encoder_output_state[0]
elif len(encoder_output_state) == 4:
# using average of hs and ignoring cs
encoder_output_state + average(
[encoder_output_state[0], encoder_output_state[2]]
)
else:
# no idea how lists of length different than 2 or 4
# might have been originated, we can either rise an ValueError
# or deal with it averaging everything
# raise ValueError(
# "encoder_output_state has length different than 2 or 4. "
# "Please doublecheck your encoder"
# )
encoder_output_state = average(encoder_output_state)
# this returns a [b, h]
# decoder_input_state = reduce_sequence(eo, self.reduce_input)
elif (self.cell_type != 'lstm' and
not isinstance(encoder_output_state, list)):
# do nothing, we are good
pass
# at this point decoder_input_state is either a [b,h]
# or a list([b,h], [b,h]) if the decoder cell is an lstm
# but h may not be the same as the decoder state size,
# so we may need to project
if isinstance(encoder_output_state, list):
for i in range(len(encoder_output_state)):
if (encoder_output_state[i].shape[1] !=
self.state_size):
encoder_output_state[i] = self.project(
encoder_output_state[i]
)
else:
if encoder_output_state.shape[1] != self.state_size:
encoder_output_state = self.project(
encoder_output_state
)
return encoder_output_state
def main():
arg_list = None
args = parseArgs(arg_list)
# grab training data
filepath = 'data/train_sample_videos'
datapath = os.path.join(filepath, 'metadata.json')
data = pd.read_json(datapath).T
if args.sample:
files = [os.path.join(filepath, f) for f in data.index][:20]
labels = data.label.values[:20]
else:
files = [os.path.join(filepath, f) for f in data.index]
labels = data.label.values
x_train, x_test, y_train, y_test = train_test_split(
files, labels, test_size=float(args.test_split))
class_weights = compute_class_weight(
'balanced', np.unique(y_train), y_train)
for k, v in zip(np.unique(y_train), class_weights):
print(k, v)
y_train = list(map(lambda x: 0 if x == 'REAL' else 1, y_train))
y_test = list(map(lambda x: 0 if x == 'REAL' else 1, y_test))
y_train = to_categorical(y_train, num_classes=2)
y_test = to_categorical(y_test, num_classes=2)
print(len(x_train), len(y_train), len(x_test), len(y_test))
# validation data
val_path = 'data/test_videos'
if args.sample:
val_files = [os.path.join(val_path, f)
for f in os.listdir(val_path)][:8]
else:
val_files = [os.path.join(val_path, f) for f in os.listdir(val_path)]
print('number of validation files', len(val_files))
# generate datasets
batch_size = args.batch_size
segment_size = args.segment_size
rsz = (128, 128)
train_data = input_fn(
x_train,
y_train,
segment_size=segment_size,
batch_size=batch_size,
rsz=rsz)
test_data = input_fn(
x_test,
y_test,
segment_size=segment_size,
batch_size=batch_size,
rsz=rsz)
val_data = input_fn(
files=val_files,
segment_size=segment_size,
batch_size=batch_size,
rsz=rsz)
rgb_input = tf.keras.Input(
shape=(segment_size, rsz[0], rsz[1], 3),
name='rgb_input')
flow_input = tf.keras.Input(
shape=(segment_size - 1, rsz[0], rsz[1], 2),
name='flow_input')
# TODO: make OO
# RGB MODEL
# block 1
x = layers.Conv3D(
filters=8,
kernel_size=3,
strides=(1, 1, 1),
padding='same',
data_format='channels_last',
activation='relu',
)(rgb_input)
x = layers.Conv3D(
filters=8,
kernel_size=4,
strides=(1, 1, 1),
padding='same',
data_format='channels_last',
activation='relu',
)(x)
block1_output = layers.MaxPool3D(
pool_size=(2, 2, 2),
strides=(2, 2, 2),
padding='same'
)(x)
# block 2
x = layers.Conv3D(
filters=8,
kernel_size=3,
strides=(1, 1, 1),
padding='same',
data_format='channels_last',
activation='relu',
)(block1_output)
x = layers.Conv3D(
filters=8,
kernel_size=4,
strides=(1, 1, 1),
padding='same',
data_format='channels_last',
activation='relu',
)(x)
block2_output = layers.add([x, block1_output])
# block 3
x = layers.Conv3D(
filters=8,
kernel_size=3,
strides=(1, 1, 1),
padding='same',
data_format='channels_last',
activation='relu',
)(block2_output)
x = layers.Conv3D(
filters=8,
kernel_size=4,
strides=(1, 1, 1),
padding='same',
data_format='channels_last',
activation='relu',
)(x)
block3_output = layers.add([x, block2_output])
x = layers.Conv3D(
filters=8,
kernel_size=3,
strides=(1, 1, 1),
padding='same',
data_format='channels_last',
activation='relu',
)(block3_output)
x = layers.GlobalAveragePooling3D()(x)
x = layers.Dense(64, activation='relu')(x)
x = layers.Dropout(0.5)(x)
rgb_outputs = layers.Dense(2, activation='softmax')(x)
rgb_model = Model(inputs=rgb_input, outputs=rgb_outputs)
rgb_model.summary()
# FLOW MODEL
x = layers.ConvLSTM2D(
filters=8,
kernel_size=3,
strides=1,
padding='same',
data_format='channels_last',
return_sequences=True,
dropout=0.5
)(flow_input)
x = layers.BatchNormalization()(x)
x = layers.ConvLSTM2D(
filters=8,
kernel_size=3,
strides=1,
padding='same',
data_format='channels_last',
return_sequences=True,
dropout=0.5
)(x)
x = layers.BatchNormalization()(x)
x = layers.ConvLSTM2D(
filters=8,
kernel_size=3,
strides=1,
padding='same',
data_format='channels_last',
return_sequences=False,
dropout=0.5
)(x)
x = layers.BatchNormalization()(x)
x = layers.Flatten()(x)
x = layers.Dense(128, activation='relu')(x)
x = layers.Dense(128, activation='relu')(x)
x = layers.Dense(128, activation='relu')(x)
x = layers.Dropout(0.5)(x)
flow_output = layers.Dense(2)(x)
flow_model = Model(inputs=flow_input, outputs=flow_output)
flow_model.summary()
# FINAL MODEL
final_average = layers.average([rgb_outputs, flow_output])
x = layers.Flatten()(final_average)
final_output = layers.Dense(
2, activation='softmax', name='final_output')(x)
model = Model(
inputs={"rgb_input": rgb_input, "flow_input": flow_input},
outputs=final_output,
name='my_model'
)
model.summary()
# tf.keras.utils.plot_model(
# model,
# to_file='model.png',
# show_shapes=True,
# show_layer_names=True
# )
# TRAIN
dt = datetime.now().strftime('%Y%m%d_%H%M%S')
opt = tf.keras.optimizers.Adam()
if args.save_checkpoints:
save_path = f'data/model_checkpoints/{dt}/ckpt'
ckpt = tf.keras.callbacks.ModelCheckpoint(
filepath=save_path,
save_best_only=False,
save_weights_only=True
)
ckpt = [ckpt]
else:
ckpt = []
model.compile(
optimizer=opt,
loss='categorical_crossentropy',
metrics=['acc'])
model.fit(
x=train_data.repeat(),
validation_data=test_data.repeat(),
epochs=args.epochs,
verbose=1,
class_weight=class_weights,
steps_per_epoch=len(x_train) // batch_size,
validation_steps=len(x_test) // batch_size,
callbacks=ckpt
)
# EVAL
print('\n\n---------------------------------------------------------')
print('predicting on validation data')
start = time.time()
preds = model.predict(
val_data,
verbose=1,
steps=len(val_files) // batch_size
)
print('prediction time: ', time.time() - start)
preds = np.argmax(preds, axis=1)
df = pd.DataFrame(columns=['filename', 'label'])
df.filename = [v.split('/')[-1] for v in val_files]
df.label = preds
df.to_csv(f'data/submission_{dt}.csv', index=False)
