def build_model(self, predict, custom_batch_size=None):
conf = self.conf
model_conf = conf['model']
rnn_size = model_conf['rnn_size']
rnn_type = model_conf['rnn_type']
regularization = model_conf['regularization']
dense_regularization = model_conf['dense_regularization']
use_batch_norm = False
if 'use_batch_norm' in model_conf:
use_batch_norm = model_conf['use_batch_norm']
dropout_prob = model_conf['dropout_prob']
length = model_conf['length']
pred_length = model_conf['pred_length']
# skip = model_conf['skip']
stateful = model_conf['stateful']
return_sequences = model_conf['return_sequences']
# model_conf['output_activation']
output_activation = conf['data']['target'].activation
use_signals = conf['paths']['use_signals']
num_signals = sum([sig.num_channels for sig in use_signals])
num_conv_filters = model_conf['num_conv_filters']
# num_conv_layers = model_conf['num_conv_layers']
size_conv_filters = model_conf['size_conv_filters']
pool_size = model_conf['pool_size']
dense_size = model_conf['dense_size']
batch_size = self.conf['training']['batch_size']
if predict:
batch_size = self.conf['model']['pred_batch_size']
# so we can predict with one time point at a time!
if return_sequences:
length = pred_length
else:
length = 1
if custom_batch_size is not None:
batch_size = custom_batch_size
if rnn_type == 'LSTM':
rnn_model = LSTM
elif rnn_type == 'CuDNNLSTM':
rnn_model = CuDNNLSTM
elif rnn_type == 'SimpleRNN':
rnn_model = SimpleRNN
else:
print('Unkown Model Type, exiting.')
exit(1)
batch_input_shape = (batch_size, length, num_signals)
# batch_shape_non_temporal = (batch_size, num_signals)
indices_0d, indices_1d, num_0D, num_1D = self.get_0D_1D_indices()
def slicer(x, indices):
return x[:, indices]
def slicer_output_shape(input_shape, indices):
shape_curr = list(input_shape)
assert len(shape_curr) == 2 # only valid for 3D tensors
shape_curr[-1] = len(indices)
return tuple(shape_curr)
pre_rnn_input = Input(shape=(num_signals,))
if num_1D > 0:
pre_rnn_1D = Lambda(lambda x: x[:, len(indices_0d):],
output_shape=(len(indices_1d),))(pre_rnn_input)
pre_rnn_0D = Lambda(lambda x: x[:, :len(indices_0d)],
output_shape=(len(indices_0d),))(pre_rnn_input)
# slicer(x,indices_0d),lambda s:
# slicer_output_shape(s,indices_0d))(pre_rnn_input)
pre_rnn_1D = Reshape((num_1D, len(indices_1d)//num_1D))(pre_rnn_1D)
pre_rnn_1D = Permute((2, 1))(pre_rnn_1D)
if ('simple_conv' in model_conf.keys()
and model_conf['simple_conv'] is True):
for i in range(model_conf['num_conv_layers']):
pre_rnn_1D = Convolution1D(
num_conv_filters, size_conv_filters,
padding='valid', activation='relu')(pre_rnn_1D)
pre_rnn_1D = MaxPooling1D(pool_size)(pre_rnn_1D)
else:
for i in range(model_conf['num_conv_layers']):
div_fac = 2**i
'''The first conv layer learns `num_conv_filters//div_fac`
filters (aka kernels), each of size
`(size_conv_filters, num1D)`. Its output will have shape
(None, len(indices_1d)//num_1D - size_conv_filters + 1,
num_conv_filters//div_fac), i.e., for
each position in the input spatial series (direction along
radius), the activation of each filter at that position.
'''
'''For i=1 first conv layer would get:
(None, (len(indices_1d)//num_1D - size_conv_filters
+ 1)/pool_size-size_conv_filters + 1,
num_conv_filters//div_fac)
'''
pre_rnn_1D = Convolution1D(
num_conv_filters//div_fac, size_conv_filters,
padding='valid')(pre_rnn_1D)
if use_batch_norm:
pre_rnn_1D = BatchNormalization()(pre_rnn_1D)
pre_rnn_1D = Activation('relu')(pre_rnn_1D)
'''The output of the second conv layer will have shape
(None, len(indices_1d)//num_1D - size_conv_filters + 1,
num_conv_filters//div_fac),
i.e., for each position in the input spatial series
(direction along radius), the activation of each filter
at that position.
For i=1, the second layer would output
(None, (len(indices_1d)//num_1D - size_conv_filters + 1)/
pool_size-size_conv_filters + 1,num_conv_filters//div_fac)
'''
pre_rnn_1D = Convolution1D(
num_conv_filters//div_fac, 1, padding='valid')(
pre_rnn_1D)
if use_batch_norm:
pre_rnn_1D = BatchNormalization()(pre_rnn_1D)
pre_rnn_1D = Activation('relu')(pre_rnn_1D)
'''Outputs (None, (len(indices_1d)//num_1D - size_conv_filters
+ 1)/pool_size, num_conv_filters//div_fac)
For i=1, the pooling layer would output:
(None,((len(indices_1d)//num_1D- size_conv_filters
+ 1)/pool_size-size_conv_filters+1)/pool_size,
num_conv_filters//div_fac)
'''
pre_rnn_1D = MaxPooling1D(pool_size)(pre_rnn_1D)
pre_rnn_1D = Flatten()(pre_rnn_1D)
pre_rnn_1D = Dense(
dense_size,
kernel_regularizer=l2(dense_regularization),
bias_regularizer=l2(dense_regularization),
activity_regularizer=l2(dense_regularization))(pre_rnn_1D)
if use_batch_norm:
pre_rnn_1D = BatchNormalization()(pre_rnn_1D)
pre_rnn_1D = Activation('relu')(pre_rnn_1D)
pre_rnn_1D = Dense(
dense_size//4,
kernel_regularizer=l2(dense_regularization),
bias_regularizer=l2(dense_regularization),
activity_regularizer=l2(dense_regularization))(pre_rnn_1D)
if use_batch_norm:
pre_rnn_1D = BatchNormalization()(pre_rnn_1D)
pre_rnn_1D = Activation('relu')(pre_rnn_1D)
pre_rnn = Concatenate()([pre_rnn_0D, pre_rnn_1D])
else:
pre_rnn = pre_rnn_input
if model_conf['rnn_layers'] == 0 or (
'extra_dense_input' in model_conf.keys()
and model_conf['extra_dense_input']):
pre_rnn = Dense(
dense_size,
activation='relu',
kernel_regularizer=l2(dense_regularization),
bias_regularizer=l2(dense_regularization),
activity_regularizer=l2(dense_regularization))(pre_rnn)
pre_rnn = Dense(
dense_size//2,
activation='relu',
kernel_regularizer=l2(dense_regularization),
bias_regularizer=l2(dense_regularization),
activity_regularizer=l2(dense_regularization))(pre_rnn)
pre_rnn = Dense(
dense_size//4,
activation='relu',
kernel_regularizer=l2(dense_regularization),
bias_regularizer=l2(dense_regularization),
activity_regularizer=l2(dense_regularization))(pre_rnn)
pre_rnn_model = Model(inputs=pre_rnn_input, outputs=pre_rnn)
# TODO(KGF): uncomment following lines to get summary of pre-RNN model
# from mpi4py import MPI
# comm = MPI.COMM_WORLD
# task_index = comm.Get_rank()
# if not predict and task_index == 0:
# print('Printing out pre_rnn model...')
# fr = open('model_architecture.log', 'w')
# ori = sys.stdout
# sys.stdout = fr
# pre_rnn_model.summary()
# sys.stdout = ori
# fr.close()
# pre_rnn_model.summary()
x_input = Input(batch_shape=batch_input_shape)
# TODO(KGF): Ge moved this inside a new conditional in Dec 2019. check
# x_in = TimeDistributed(pre_rnn_model)(x_input)
if (num_1D > 0 or (
'extra_dense_input' in model_conf.keys()
and model_conf['extra_dense_input'])):
x_in = TimeDistributed(pre_rnn_model)(x_input)
else:
x_in = x_input
# ==========
# TCN MODEL
# ==========
if ('keras_tcn' in model_conf.keys()
and model_conf['keras_tcn'] is True):
print('Building TCN model....')
tcn_layers = model_conf['tcn_layers']
tcn_dropout = model_conf['tcn_dropout']
nb_filters = model_conf['tcn_hidden']
kernel_size = model_conf['kernel_size_temporal']
nb_stacks = model_conf['tcn_nbstacks']
use_skip_connections = model_conf['tcn_skip_connect']
activation = model_conf['tcn_activation']
use_batch_norm = model_conf['tcn_batch_norm']
for _ in range(model_conf['tcn_pack_layers']):
x_in = TCN(
use_batch_norm=use_batch_norm, activation=activation,
use_skip_connections=use_skip_connections,
nb_stacks=nb_stacks, kernel_size=kernel_size,
nb_filters=nb_filters, num_layers=tcn_layers,
dropout_rate=tcn_dropout)(x_in)
x_in = Dropout(dropout_prob)(x_in)
else: # end TCN model
# ==========
# RNN MODEL
# ==========
# LSTM in ONNX: "The maximum opset needed by this model is only 9."
model_kwargs = dict(return_sequences=return_sequences,
# batch_input_shape=batch_input_shape,
stateful=stateful,
kernel_regularizer=l2(regularization),
recurrent_regularizer=l2(regularization),
bias_regularizer=l2(regularization),
)
if rnn_type != 'CuDNNLSTM':
# Dropout is unsupported in CuDNN library
model_kwargs['dropout'] = dropout_prob
model_kwargs['recurrent_dropout'] = dropout_prob
for _ in range(model_conf['rnn_layers']):
x_in = rnn_model(rnn_size, **model_kwargs)(x_in)
x_in = Dropout(dropout_prob)(x_in)
if return_sequences:
# x_out = TimeDistributed(Dense(100,activation='tanh')) (x_in)
x_out = TimeDistributed(
Dense(1, activation=output_activation))(x_in)
model = Model(inputs=x_input, outputs=x_out)
# bug with tensorflow/Keras
# TODO(KGF): what is this bug? this is the only direct "tensorflow"
# import outside of mpi_runner.py and runner.py
if (conf['model']['backend'] == 'tf'
or conf['model']['backend'] == 'tensorflow'):
first_time = "tensorflow" not in sys.modules
import tensorflow as tf
if first_time:
K.get_session().run(tf.global_variables_initializer())
model.reset_states()
return model
def Build_Model():
_num_classes = Number_of_Classes() # Classes used in this NN
# Convolution methods for SynapticNeuronUnit layers
# _ENC = ('Normal', 'MaxPooling', 'same')
_ENC = ('Atrous', 'None', 'valid')
_NNv = ('Normal', 'None', 'valid')
_SNs = ('Separable', 'None', 'same')
_SNv = ('Separable', 'None', 'valid')
_TUs = ('Transpose', 'UpSampling', 'same')
# Rate for dropout and noise layers
# NOTE: The larger values (around 0.5) may be better when segmentation will have large area such as the heart
# On the other hand, for small segmentation area such as calcium deposits on a valve,
# the rate for SpatialDropout2D should be ZERO and the stddev for GaussianNoise should be small
_dropout_rate1 = (
0.20, 0.0) # (rate for GaussianDropout, rate for SpatialDropout2D)
_dropout_rate2 = (
0.40, 0.0) # (rate for GaussianDropout, rate for SpatialDropout2D)
_stddev_g_noise = 0.10 # std.deviation for GaussianNoise
# Do not change "name='input'", because the name is used to identify the input layer in A.I.Segmentation.
inputs = Input(shape=(200, 200, 1), name='input')
# Cerate stimulation from inputs
stimulation = Conv2D(filters=8,
kernel_size=9,
kernel_initializer='glorot_uniform',
use_bias=False)(inputs)
stimulation = BatchNormalization(momentum=0.95)(stimulation)
stimulation = Activation('sigmoid')(stimulation) # Skip connection
# Main neural network
# def SynapticNeuronUnit(dendrites, filter_size, kernel_size, CRP, d_rate, use_STR)
enc_potential_96 = SynapticNeuronUnit(stimulation, 16, 3, _ENC,
_dropout_rate1, False) # 96
enc_potential_96A = SynapticNeuronUnit(enc_potential_96, 16, 1, _NNv,
_dropout_rate2, False)
enc_potential_96B = SynapticNeuronUnit(enc_potential_96, 16, 5, _SNs,
_dropout_rate2, False)
enc_potential_96 = Concatenate()([enc_potential_96A,
enc_potential_96B]) # Skip connection
enc_potential_48 = SynapticNeuronUnit(enc_potential_96, 32, 3, _ENC,
_dropout_rate1, False) # 48
enc_potential_48A = SynapticNeuronUnit(enc_potential_48, 32, 1, _NNv,
_dropout_rate2, False)
enc_potential_48B = SynapticNeuronUnit(enc_potential_48, 32, 5, _SNs,
_dropout_rate2, False)
enc_potential_48 = Concatenate()([enc_potential_48A,
enc_potential_48B]) # Skip connection
enc_potential_24 = SynapticNeuronUnit(enc_potential_48, 128, 3, _ENC,
_dropout_rate1, True) # 24
enc_potential_24A = SynapticNeuronUnit(enc_potential_24, 128, 1, _NNv,
_dropout_rate2, False)
enc_potential_24B = SynapticNeuronUnit(enc_potential_24, 128, 5, _SNs,
_dropout_rate2, False)
enc_potential_24 = Concatenate()([enc_potential_24A,
enc_potential_24B]) # Skip connection
enc_potential_12 = SynapticNeuronUnit(enc_potential_24, 512, 3, _ENC,
_dropout_rate1, True) # 12
enc_potential_12A = SynapticNeuronUnit(enc_potential_12, 512, 1, _NNv,
_dropout_rate2, False)
enc_potential_12B = SynapticNeuronUnit(enc_potential_12, 512, 5, _SNs,
_dropout_rate2, False)
enc_potential_12 = Concatenate()([enc_potential_12A,
enc_potential_12B]) # Skip connection
deep_potential = SynapticNeuronUnit(enc_potential_12, 1024, 5, _SNv,
_dropout_rate1, True) # 08
deep_potential = SynapticNeuronUnit(deep_potential, 2048, 3, _SNv,
_dropout_rate1, True) # 06
dec_potential_12 = SynapticNeuronUnit(deep_potential, 512, 3, _TUs,
_dropout_rate1, True) # 12
dec_potential_12 = Concatenate()([dec_potential_12, enc_potential_12])
dec_potential_24 = SynapticNeuronUnit(dec_potential_12, 384, 3, _TUs,
_dropout_rate1, True) # 24
dec_potential_24 = Concatenate()([dec_potential_24, enc_potential_24])
dec_potential_48 = SynapticNeuronUnit(dec_potential_24, 160, 3, _TUs,
_dropout_rate1, True) # 48
dec_potential_48 = Concatenate()([dec_potential_48, enc_potential_48])
dec_potential_96 = SynapticNeuronUnit(dec_potential_48, 56, 3, _TUs,
_dropout_rate1, False) # 96
dec_potential_96 = Concatenate()([dec_potential_96, enc_potential_96])
axon_potential = SynapticNeuronUnit(dec_potential_96, 24, 3, _TUs,
_dropout_rate1, False) # 192
axon_potential = Concatenate()([axon_potential, stimulation])
# The vision from synaptic neurons
vision = Conv2DTranspose(filters=32,
kernel_size=7,
kernel_initializer='he_uniform',
use_bias=False)(axon_potential)
vision = BatchNormalization(momentum=0.95)(vision)
vision = ParametricSwish()(vision)
vision = GaussianNoise(stddev=_stddev_g_noise)(vision)
vision = Conv2DTranspose(filters=16,
kernel_size=3,
kernel_initializer='he_uniform',
use_bias=False)(vision)
vision = BatchNormalization(momentum=0.95)(vision)
vision = ParametricSwish()(vision)
vision = GaussianNoise(stddev=_stddev_g_noise)(vision)
vision = Concatenate()([vision, inputs])
# Do not change "name='output'", because the name is used to identify the output layer in A.I.Segmentation.
outputs = Conv2D(filters=_num_classes,
kernel_size=1,
kernel_initializer='glorot_uniform')(vision)
outputs = SynapticTransmissionRegulator()(outputs)
outputs = Activation('sigmoid', name='output')(outputs)
# Generate model
return Model(inputs=inputs, outputs=outputs)
def getModel(args, input_length, vocab_size, embd, feature_length=0):
rnn_opt = mem_opt_dict[args.rnn_opt]
rnn_dim = args.rnn_dim
rnn_dropout = args.rnn_dropout
dense_dropout = args.dense_dropout
# if args.activation == 'sigmoid':
# final_init = 'he_normal'
# else:
# final_init = 'he_uniform'
if args.fix_embd:
trainable = False
else:
trainable = True
with device('/cpu:0'):
if embd is None:
embd_layer = Embedding(vocab_size, (args.embd_dim + args.ft_dim),
mask_zero=args.use_mask,
trainable=trainable)
else:
embd_layer = Embedding(vocab_size, (args.embd_dim + args.ft_dim),
mask_zero=args.use_mask,
weights=[embd],
trainable=trainable)
if args.bidirectional:
rnn_layer = Bidirectional(
LSTM(rnn_dim,
return_sequences=False,
implementation=rnn_opt,
dropout=args.input_dropout,
recurrent_dropout=rnn_dropout))
else:
rnn_layer = LSTM(rnn_dim,
return_sequences=False,
implementation=rnn_opt,
dropout=args.input_dropout,
recurrent_dropout=rnn_dropout)
if args.mot_layer:
if args.use_mask:
w2v_dense_layer = TimeDistributed(
DenseWithMasking(args.embd_dim * 2 // 3))
else:
w2v_dense_layer = TimeDistributed(Dense(args.embd_dim * 2 // 3))
meanpool = MeanOverTime()
if args.cnn_dim:
if args.use_mask:
conv1dw2 = Conv1DWithMasking(filters=args.cnn_dim,
kernel_size=2,
padding='valid',
strides=1)
conv1dw3 = Conv1DWithMasking(filters=args.cnn_dim,
kernel_size=3,
padding='valid',
strides=1)
else:
conv1dw2 = Convolution1D(filters=args.cnn_dim,
kernel_size=2,
padding='valid',
strides=1)
conv1dw3 = Convolution1D(filters=args.cnn_dim,
kernel_size=3,
padding='valid',
strides=1)
maxpool = MaxOverTime()
# maxpool = MaxPooling1DWithMasking(pool_size=input_length-1, padding='valid')
# # hidden rnn layer
# from keras.models import Sequential
# rnn_model = Sequential()
# for _ in range(args.rnn_layer-1):
# if args.bidirectional:
# rnn_model.add(Bidirectional(LSTM(rnn_dim, return_sequences=True, implementation=rnn_opt,
# dropout=args.input_dropout, recurrent_dropout=rnn_dropout)))
# else:
# rnn_model.add(LSTM(rnn_dim, return_sequences=True, implementation=rnn_opt,
# dropout=args.input_dropout, recurrent_dropout=rnn_dropout))
#
# # output rnn layer
# if args.attention:
# from util.attention_wrapper import Attention
# rnn_model.add(Attention(LSTM(rnn_dim, return_sequences=False, implementation=rnn_opt,
# dropout=args.input_dropout, recurrent_dropout=rnn_dropout)))
# else:
# if args.bidirectional:
# rnn_model.add(Bidirectional(LSTM(rnn_dim, return_sequences=False, implementation=rnn_opt,
# dropout=args.input_dropout, recurrent_dropout=rnn_dropout)))
# else:
# rnn_model.add(LSTM(rnn_dim, return_sequences=False, implementation=rnn_opt,
# dropout=args.input_dropout, recurrent_dropout=rnn_dropout))
sequence_1_input = Input(shape=(input_length, ), dtype='int32')
sequence_2_input = Input(shape=(input_length, ), dtype='int32')
vec1 = embd_layer(sequence_1_input)
vec2 = embd_layer(sequence_2_input)
merged = []
if args.mot_layer:
vec1_w2v = w2v_dense_layer(vec1)
vec2_w2v = w2v_dense_layer(vec2)
vec1_w2v = meanpool(vec1_w2v)
vec2_w2v = meanpool(vec2_w2v)
merged += [vec1_w2v, vec2_w2v]
if args.rnn_layer:
vec1_rnn = rnn_layer(vec1)
vec2_rnn = rnn_layer(vec2)
merged += [vec1_rnn, vec2_rnn]
# Conv Layer
if args.cnn_dim:
vec1_cnnw2 = conv1dw2(vec1)
vec2_cnnw2 = conv1dw2(vec2)
vec1_cnnw3 = conv1dw3(vec1)
vec2_cnnw3 = conv1dw3(vec2)
vec1_cnnw2 = maxpool(vec1_cnnw2)
vec2_cnnw2 = maxpool(vec2_cnnw2)
vec1_cnnw3 = maxpool(vec1_cnnw3)
vec2_cnnw3 = maxpool(vec2_cnnw3)
merged += [vec1_cnnw2, vec2_cnnw2, vec1_cnnw3, vec2_cnnw3]
if feature_length:
feature_input = Input(shape=(feature_length, ), dtype='float32')
if args.use_mask:
featured = DenseWithMasking(args.dense_dim // 2,
kernel_initializer='he_uniform',
activation='relu')(feature_input)
else:
featured = Dense(args.dense_dim // 2,
kernel_initializer='he_uniform',
activation='relu')(feature_input)
merged += [featured]
merged = Concatenate()(merged)
merged = BatchNormalization()(merged)
merged = Dropout(dense_dropout)(merged)
if args.use_mask:
merged = DenseWithMasking(args.dense_dim,
kernel_initializer='he_uniform',
activation='relu')(merged)
else:
merged = Dense(args.dense_dim,
kernel_initializer='he_uniform',
activation='relu')(merged)
merged = BatchNormalization()(merged)
merged = Dropout(dense_dropout)(merged)
if args.use_mask:
preds = DenseWithMasking(1,
kernel_initializer='he_normal',
activation='sigmoid')(merged)
else:
preds = Dense(1, kernel_initializer='he_normal',
activation='sigmoid')(merged)
if feature_length:
model = Model(
inputs=[sequence_1_input, sequence_2_input, feature_input],
outputs=preds)
else:
model = Model(inputs=[sequence_1_input, sequence_2_input],
outputs=preds)
return model
name="embedding")(model_input)
z = Dropout(dropout_prob[0])(z)
# Convolutional block
conv_blocks = []
for sz in filter_sizes:
conv = Convolution1D(filters=num_filters,
kernel_size=sz,
padding="valid",
activation="relu",
strides=1)(z)
conv = MaxPooling1D(pool_size=2)(conv)
conv = Flatten()(conv)
conv_blocks.append(conv)
z = Concatenate()(conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]
z = Dropout(dropout_prob[1])(z)
z = Dense(hidden_dims, activation="relu")(z)
model_output = Dense(1, activation="sigmoid")(z)
model = Model(model_input, model_output)
model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
# Initialize weights with word2vec
if model_type == "CNN-non-static":
weights = np.array([v for v in embedding_weights.values()])
print("Initializing embedding layer with word2vec weights, shape",
weights.shape)
def build_model_FCN_model_api(batch_size,
optimizer,
patch_size=(128, 128),
base_weight_decay=0.0005,
output_ROI_mask=True):
print('Using build_model_FCN_model_api')
# define the shared model:
net_name = 'Multi-view_FCN'
scale_number = 3
##################### input ###############################################
input_shape0 = (batch_size, patch_size[0], patch_size[1], 1)
input_shape1 = (batch_size, patch_size[0] / 2, patch_size[1] / 2, 1)
input_shape2 = (batch_size, patch_size[0] / 4, patch_size[1] / 4, 1)
input_shape3 = (1, patch_size[0] / 4, patch_size[1] / 4, 1)
input_patches1_s0 = Input(batch_shape=input_shape0, name='patches1_s0')
input_patches1_s1 = Input(batch_shape=input_shape1, name='patches1_s1')
input_patches1_s2 = Input(batch_shape=input_shape2, name='patches1_s2')
input_patches2_s0 = Input(batch_shape=input_shape0, name='patches2_s0')
input_patches2_s1 = Input(batch_shape=input_shape1, name='patches2_s1')
input_patches2_s2 = Input(batch_shape=input_shape2, name='patches2_s2')
input_patches3_s0 = Input(batch_shape=input_shape0, name='patches3_s0')
input_patches3_s1 = Input(batch_shape=input_shape1, name='patches3_s1')
input_patches3_s2 = Input(batch_shape=input_shape2, name='patches3_s2')
input_patches4_s0 = Input(batch_shape=input_shape0, name='patches4_s0')
input_patches4_s1 = Input(batch_shape=input_shape1, name='patches4_s1')
input_patches4_s2 = Input(batch_shape=input_shape2, name='patches4_s2')
input_depth_maps_v1 = Input(batch_shape=input_shape3,
name='depth_ratio_v1')
input_depth_maps_v2 = Input(batch_shape=input_shape3,
name='depth_ratio_v2')
input_depth_maps_v3 = Input(batch_shape=input_shape3,
name='depth_ratio_v3')
input_depth_maps_v4 = Input(batch_shape=input_shape3,
name='depth_ratio_v4')
if output_ROI_mask:
# the output density patch/map is down-sampled by a factor of 4
output_masks = Input(batch_shape=(batch_size, patch_size[0],
patch_size[1], 1),
name='output_masks')
train_flag = False
####################### view 1 #############################################
# image pyramids:
x1_s0_output = feature_extraction_view1(base_weight_decay,
input_patches1_s0, train_flag)
x1_s1_output = feature_extraction_view1(base_weight_decay,
input_patches1_s1, train_flag)
x1_s2_output = feature_extraction_view1(base_weight_decay,
input_patches1_s2, train_flag)
# view 1 decoder
# x1_7 = view1_decoder(base_weight_decay, x1_s0_output)
# conv block 5
x1_5 = Conv2D(data_format='channels_last',
trainable=True,
filters=64,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_5')(x1_s0_output)
x1_5 = Activation('relu', name='conv_block_5_act')(x1_5)
# conv block 6
x1_6 = Conv2D(data_format='channels_last',
trainable=True,
filters=32,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_6')(x1_5)
x1_6 = Activation('relu', name='conv_block_6_act')(x1_6)
# conv block 7
x1_7 = Conv2D(data_format='channels_last',
trainable=True,
filters=1,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_7')(x1_6)
x1_7 = Activation('relu', name='conv_block_7_act')(x1_7)
####################### view 2 #############################################
# image pyramids:
x2_s0_output = feature_extraction_view2(base_weight_decay,
input_patches2_s0, train_flag)
x2_s1_output = feature_extraction_view2(base_weight_decay,
input_patches2_s1, train_flag)
x2_s2_output = feature_extraction_view2(base_weight_decay,
input_patches2_s2, train_flag)
# view 2 decoder
# x2_7 = view2_decoder(base_weight_decay, x2_s0_output)
# dmap
# conv block 5
x2_5 = Conv2D(data_format='channels_last',
trainable=True,
filters=64,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_5_2')(x2_s0_output)
x2_5 = Activation('relu', name='conv_block_5_2_act')(x2_5)
# conv block 6
x2_6 = Conv2D(data_format='channels_last',
trainable=True,
filters=32,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_6_2')(x2_5)
x2_6 = Activation('relu', name='conv_block_6_2_act')(x2_6)
# conv block 7
x2_7 = Conv2D(data_format='channels_last',
trainable=True,
filters=1,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_7_2')(x2_6)
x2_7 = Activation('relu', name='conv_block_7_2_act')(x2_7)
####################### view 3 #############################################
# image pyramids:
x3_s0_output = feature_extraction_view3(base_weight_decay,
input_patches3_s0, train_flag)
x3_s1_output = feature_extraction_view3(base_weight_decay,
input_patches3_s1, train_flag)
x3_s2_output = feature_extraction_view3(base_weight_decay,
input_patches3_s2, train_flag)
# view 3 decoder
# x3_7 = view3_decoder(base_weight_decay, x3_s0_output)
# conv block 5
x3_5 = Conv2D(data_format='channels_last',
trainable=True,
filters=64,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_5_3')(x3_s0_output)
x3_5 = Activation('relu', name='conv_block_5_3_act')(x3_5)
# conv block 6
x3_6 = Conv2D(data_format='channels_last',
trainable=True,
filters=32,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_6_3')(x3_5)
x3_6 = Activation('relu', name='conv_block_6_3_act')(x3_6)
# conv block 7
x3_7 = Conv2D(data_format='channels_last',
trainable=True,
filters=1,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_7_3')(x3_6)
x3_7 = Activation('relu', name='conv_block_7_3_act')(x3_7)
####################### view 4 #############################################
# image pyramids:
x4_s0_output = feature_extraction_view4(base_weight_decay,
input_patches4_s0, train_flag)
x4_s1_output = feature_extraction_view4(base_weight_decay,
input_patches4_s1, train_flag)
x4_s2_output = feature_extraction_view4(base_weight_decay,
input_patches4_s2, train_flag)
# view 4 decoder
# x4_7 = view4_decoder(base_weight_decay, x4_s0_output)
# conv block 5
x4_5 = Conv2D(data_format='channels_last',
trainable=True,
filters=64,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_5_4')(x4_s0_output)
x4_5 = Activation('relu', name='conv_block_5_4_act')(x4_5)
# conv block 6
x4_6 = Conv2D(data_format='channels_last',
trainable=True,
filters=32,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_6_4')(x4_5)
x4_6 = Activation('relu', name='conv_block_6_4_act')(x4_6)
# conv block 7
x4_7 = Conv2D(data_format='channels_last',
trainable=True,
filters=1,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_7_4')(x4_6)
x4_7 = Activation('relu', name='conv_block_7_4_act')(x4_7)
#################################### fusion #############################################
################# get the scale-selection mask #####################
# view depth of image
batch_size = x1_s0_output.shape[0].value
height = x1_s0_output.shape[1].value
width = x1_s0_output.shape[2].value
num_channels = x1_s0_output.shape[3].value
output_shape = [1, height, width, 1]
# view1_depth = feature_scale_fusion_layer_mask(scale_number=scale_number,
# view = 1, output_shape=output_shape)
# view2_depth = feature_scale_fusion_layer_mask(scale_number=scale_number,
# view = 2, output_shape=output_shape)
# view3_depth = feature_scale_fusion_layer_mask(scale_number=scale_number,
# view = 3, output_shape=output_shape)
# view1_scale = scale_selection_mask(base_weight_decay, input_depth_maps_v1)
# view2_scale = scale_selection_mask(base_weight_decay, input_depth_maps_v2)
# view3_scale = scale_selection_mask(base_weight_decay, input_depth_maps_v3)
view1_scale = Conv2D(data_format='channels_last',
trainable=True,
filters=1,
kernel_size=(1, 1),
strides=(1, 1),
kernel_initializer=Constant(value=-1),
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
bias_initializer='ones',
activation=None,
name='scale_fusion1')(input_depth_maps_v1)
view2_scale = Conv2D(data_format='channels_last',
trainable=True,
filters=1,
kernel_size=(1, 1),
strides=(1, 1),
kernel_initializer=Constant(value=-1),
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
bias_initializer='ones',
activation=None,
name='scale_fusion2')(input_depth_maps_v2)
view3_scale = Conv2D(data_format='channels_last',
trainable=True,
filters=1,
kernel_size=(1, 1),
strides=(1, 1),
kernel_initializer=Constant(value=-1),
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
bias_initializer='ones',
activation=None,
name='scale_fusion3')(input_depth_maps_v3)
view4_scale = Conv2D(data_format='channels_last',
trainable=True,
filters=1,
kernel_size=(1, 1),
strides=(1, 1),
kernel_initializer=Constant(value=-1),
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
bias_initializer='ones',
activation=None,
name='scale_fusion4')(input_depth_maps_v4)
view1_scale_mask = feature_scale_fusion_layer_rbm(
scale_number=scale_number)(view1_scale)
view2_scale_mask = feature_scale_fusion_layer_rbm(
scale_number=scale_number)(view2_scale)
view3_scale_mask = feature_scale_fusion_layer_rbm(
scale_number=scale_number)(view3_scale)
view4_scale_mask = feature_scale_fusion_layer_rbm(
scale_number=scale_number)(view4_scale)
#################### fusion with mask ##################
# view 1
## conv
x1_s0_output_fusion = fusion_conv_v1(base_weight_decay, x1_s0_output)
x1_s1_output_fusion = fusion_conv_v1(base_weight_decay, x1_s1_output)
x1_s2_output_fusion = fusion_conv_v1(base_weight_decay, x1_s2_output)
## up sampling
x1_s1_output_fusion = UpSampling_layer(size=[height, width])(
[x1_s1_output_fusion])
x1_s2_output_fusion = UpSampling_layer(size=[height, width])(
[x1_s2_output_fusion])
concatenated_map_v1 = Concatenate(name='cat_map_v1')(
[x1_s0_output_fusion, x1_s1_output_fusion, x1_s2_output_fusion])
fusion_v1 = feature_scale_fusion_layer(scale_number=scale_number)(
[concatenated_map_v1, view1_scale_mask])
## proj
fusion_v1_proj = SpatialTransformer(
1, [int(480 / 4), int(640 / 4)])(fusion_v1)
# view 2
## conv
x2_s0_output_fusion = fusion_conv_v2(base_weight_decay, x2_s0_output)
x2_s1_output_fusion = fusion_conv_v2(base_weight_decay, x2_s1_output)
x2_s2_output_fusion = fusion_conv_v2(base_weight_decay, x2_s2_output)
## up sampling
x2_s1_output_fusion = UpSampling_layer(size=[height, width])(
[x2_s1_output_fusion])
x2_s2_output_fusion = UpSampling_layer(size=[height, width])(
[x2_s2_output_fusion])
concatenated_map_v2 = Concatenate(name='cat_map_v2')(
[x2_s0_output_fusion, x2_s1_output_fusion, x2_s2_output_fusion])
fusion_v2 = feature_scale_fusion_layer(scale_number=scale_number)(
[concatenated_map_v2, view2_scale_mask])
## proj
fusion_v2_proj = SpatialTransformer(
2, [int(480 / 4), int(640 / 4)])(fusion_v2)
# view 3
## conv
x3_s0_output_fusion = fusion_conv_v3(base_weight_decay, x3_s0_output)
x3_s1_output_fusion = fusion_conv_v3(base_weight_decay, x3_s1_output)
x3_s2_output_fusion = fusion_conv_v3(base_weight_decay, x3_s2_output)
## up sampling
x3_s1_output_fusion = UpSampling_layer(size=[height, width])(
[x3_s1_output_fusion])
x3_s2_output_fusion = UpSampling_layer(size=[height, width])(
[x3_s2_output_fusion])
concatenated_map_v3 = Concatenate(name='cat_map_v3')(
[x3_s0_output_fusion, x3_s1_output_fusion, x3_s2_output_fusion])
fusion_v3 = feature_scale_fusion_layer(scale_number=scale_number)(
[concatenated_map_v3, view3_scale_mask])
## proj
fusion_v3_proj = SpatialTransformer(
3, [int(480 / 4), int(640 / 4)])(fusion_v3)
# view 4
## conv
x4_s0_output_fusion = fusion_conv_v4(base_weight_decay, x4_s0_output)
x4_s1_output_fusion = fusion_conv_v4(base_weight_decay, x4_s1_output)
x4_s2_output_fusion = fusion_conv_v4(base_weight_decay, x4_s2_output)
## up sampling
x4_s1_output_fusion = UpSampling_layer(size=[height, width])(
[x4_s1_output_fusion])
x4_s2_output_fusion = UpSampling_layer(size=[height, width])(
[x4_s2_output_fusion])
concatenated_map_v4 = Concatenate(name='cat_map_v4')(
[x4_s0_output_fusion, x4_s1_output_fusion, x4_s2_output_fusion])
fusion_v4 = feature_scale_fusion_layer(scale_number=scale_number)(
[concatenated_map_v4, view4_scale_mask])
## proj
fusion_v4_proj = SpatialTransformer(
4, [int(480 / 4), int(640 / 4)])(fusion_v4)
################# concatenate ################
concatenated_map = Concatenate(name='cat_map_fusion')(
[fusion_v1_proj, fusion_v2_proj, fusion_v3_proj, fusion_v4_proj])
fusion_v123 = Conv2D(data_format='channels_last',
trainable=True,
filters=96,
kernel_size=(1, 1),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='scale_fusion')(concatenated_map)
fusion_v123 = Activation('relu', name='scale_fusion_act')(fusion_v123)
#################### fusion and decode #####################################
# conv block 9
x = Conv2D(data_format='channels_last',
trainable=True,
filters=64,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_fusion1')(fusion_v123)
x = Activation('relu', name='conv_block_fusion1_act')(x)
x = Conv2D(data_format='channels_last',
trainable=True,
filters=32,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_fusion2')(x)
x = Activation('relu', name='conv_block_fusion2_act')(x)
x = Conv2D(data_format='channels_last',
trainable=True,
filters=1,
kernel_size=(5, 5),
strides=(1, 1),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(base_weight_decay),
use_bias=True,
activation=None,
name='conv_block_fusion3')(x)
x_output = Activation('relu', name='conv_block_fusion3_act')(x)
if output_ROI_mask:
rgr_output = 'den_map_roi'
output = Multiply(name=rgr_output)([x_output, output_masks])
print('Layer name of regression output: {}'.format(rgr_output))
model = Model(inputs=[
input_patches1_s0, input_patches1_s1, input_patches1_s2,
input_patches2_s0, input_patches2_s1, input_patches2_s2,
input_patches3_s0, input_patches3_s1, input_patches3_s2,
input_patches4_s0, input_patches4_s1, input_patches4_s2,
input_depth_maps_v1, input_depth_maps_v2, input_depth_maps_v3,
input_depth_maps_v4, output_masks
],
outputs=[x_output],
name=net_name)
else:
model = Model(
inputs=[
input_patches1_s0,
input_patches1_s1,
input_patches1_s2,
input_patches2_s0,
input_patches2_s1,
input_patches2_s2,
input_patches3_s0,
input_patches3_s1,
input_patches3_s2,
input_patches4_s0,
input_patches4_s1,
input_patches4_s2,
input_depth_maps_v1,
input_depth_maps_v2,
input_depth_maps_v3,
input_depth_maps_v4,
],
outputs=[x_output], #x1_7, x2_7, x3_7, x4_7,
name=net_name + 'overall')
print('Compiling ...')
model.compile(optimizer=optimizer, loss='mse')
return model
def dr_network_segmentation_as_input():
# set image specifics
img_shape = (512, 512, 7)
n_filters = 32
n_out = 1
k = 3 # kernel size
s = 2 # stride
img_h, img_w, img_ch = img_shape[0], img_shape[1], img_shape[2]
padding = 'same'
l2_coeff = 0.0005
list_n_building_blocks = [2, 3]
blocks = []
inputs = Input((img_h, img_w, img_ch))
reduction_conv = conv_block(inputs, n_filters, (k, k), (s, s), padding,
l2_coeff)
blocks.append(
conv_block(reduction_conv, n_filters, (k, k), (s, s), padding,
l2_coeff))
for index, n_building_blocks in enumerate(list_n_building_blocks):
blocks.append(
resnet_blocks(blocks[index], n_building_blocks,
(2**index) * n_filters, (k, k), padding, l2_coeff))
blocks[index + 1] = conv_block(blocks[index + 1],
2**(index + 1) * n_filters, (k, k),
(s, s), padding, l2_coeff)
list_avg_pools = []
for i in range(2):
list_avg_pools.append(
AveragePooling2D((4 // (2**i), 4 // (2**i)))(blocks[i]))
blocks_concat = Concatenate(axis=3)(list_avg_pools + [blocks[-1]])
list_dilated_conv = []
list_dilated_conv.append(
Conv2D(16 * n_filters, (k, k),
padding=padding,
kernel_regularizer=regularizers.l2(l2_coeff),
bias_regularizer=regularizers.l2(l2_coeff))(blocks_concat))
list_dilated_conv.append(
Conv2D(16 * n_filters, (k, k),
dilation_rate=(2, 2),
padding=padding,
kernel_regularizer=regularizers.l2(l2_coeff),
bias_regularizer=regularizers.l2(l2_coeff))(blocks_concat))
list_dilated_conv.append(
Conv2D(16 * n_filters, (k, k),
dilation_rate=(4, 4),
padding=padding,
kernel_regularizer=regularizers.l2(l2_coeff),
bias_regularizer=regularizers.l2(l2_coeff))(blocks_concat))
final_block = Concatenate(axis=3)(list_dilated_conv)
final_block = conv_block(final_block, 16 * n_filters, (k, k), (s, s),
padding, l2_coeff)
gap = GlobalAveragePooling2D()(final_block)
outputs = Dense(n_out)(gap)
network = Model(inputs, outputs)
def L2(y_true, y_pred):
y_pred_clipped = K.clip(y_pred, 0, 4)
return objectives.mean_squared_error(y_true, y_pred_clipped)
network.compile(optimizer=SGD(lr=0, momentum=0.9, nesterov=True),
loss=L2,
metrics=['accuracy'])
return network
output1 = Dense(1)(dense2)
# Model 2
input2 = Input(shape=(3, ))
dense21 = Dense(256)(input2)
dense22 = Dense(8)(dense21)
dense23 = Dense(128)(dense22)
output2 = Dense(10)(dense23)
# concatenate
# from keras.layers.merge import concatenate # model을 사슬처럼 엮다.
# merge1 = concatenate([output1, output2]) # list 형식(=[]) # merge layer 역시 hidden layer! 따라서 끝 노드 수를 굳이 맞춰주지 않아도 된다.
# Concatenate(참조: https://keras.io/layers/merge/#concatenate)
from keras.layers.merge import Concatenate # from keras.layers import Concatenate
merge1 = Concatenate()([output1, output2])
# Model 3
middle1 = Dense(3)(merge1)
middle2 = Dense(256)(middle1)
output = Dense(1)(middle2)
# 함수형 model 선언
model = Model(inputs=[input1, input2],
outputs=output) # input, output이 다수일 경우 list 형식으로 삽입!
# # model.add(Dense(512, input_dim=3)) # 레이어 추가
# model.add(Dense(256, input_shape=(3,))) # input_shape = (1,), 벡터가 1개라는 뜻
# model.add(Dense(256)) # Node 값 조절
# model.add(Dense(1)) # regression(=회귀분석) 문제이므로 output은 하나여야 한다. # 열을 기준으로 볼 것!
def atrous_spatial_pyramid_pooling(inputs,
n_filters=256,
rates=[6, 12, 18],
imagelevel=True,
weight_decay=1e-4,
kernel_initializer="he_normal",
bn_epsilon=1e-3,
bn_momentum=0.99):
""" ASPP consists of one 1×1 convolution and three 3×3 convolutions with rates = (6, 12, 18)
when output stride = 16 (all with 256 filters and batch normalization), and (b) the image-level features
:param inputs: 4-D tensor, shape of (batch_size, height, width, channel).
:param n_filters: int, number of filters, default 256.
:param rates: list of dilation rates, default [6, 12, 18].
:param imagelevel: bool, default True.
:param weight_decay: float, default 1e-4.
:param kernel_initializer: string, default "he_normal".
:param bn_epsilon: float, default 1e-3.
:param bn_momentum: float, default 0.99.
:return: 4-D tensor, shape of (batch_size, height, width, channel).
"""
branch_features = []
if imagelevel:
# image level features
image_feature = AveragePooling2D(
pool_size=(int(inputs.shape[1]), int(inputs.shape[2])))(inputs)
image_feature = Conv2D(
n_filters, (1, 1),
use_bias=False,
activation=None,
kernel_regularizer=l2(weight_decay),
kernel_initializer=kernel_initializer)(image_feature)
image_feature = BatchNormalization(epsilon=bn_epsilon,
momentum=bn_momentum)(image_feature)
image_feature = Activation("relu")(image_feature)
image_feature = BilinearUpSampling(
target_size=(int(inputs.shape[1]),
int(inputs.shape[2])))(image_feature)
branch_features.append(image_feature)
# 1×1 conv
atrous_pool_block_1 = Conv2D(n_filters, (1, 1),
padding="same",
use_bias=False,
activation=None,
kernel_regularizer=l2(weight_decay),
kernel_initializer=kernel_initializer)(inputs)
atrous_pool_block_1 = BatchNormalization(
epsilon=bn_epsilon, momentum=bn_momentum)(atrous_pool_block_1)
atrous_pool_block_1 = Activation("relu")(atrous_pool_block_1)
branch_features.append(atrous_pool_block_1)
for i, rate in enumerate(rates):
atrous_pool_block_i = separable_conv_bn(
inputs,
256,
'aspp' + str(i + 1),
rate=rate,
depth_activation=True,
weight_decay=weight_decay,
kernel_initializer=kernel_initializer,
bn_epsilon=bn_epsilon,
bn_momentum=bn_momentum)
branch_features.append(atrous_pool_block_i)
# concatenate multi-scale features
aspp_features = Concatenate()(branch_features)
return aspp_features
def input_branch(input_shape=None):
size = int(input_shape[2] / 4)
branch_input = Input(shape=input_shape)
branch = GlobalAveragePooling2D()(branch_input)
branch = Dense(size, use_bias=False, kernel_initializer='uniform')(branch)
branch = BatchNormalization()(branch)
branch = Activation("relu")(branch)
return branch, branch_input
vgg19_branch, vgg19_input = input_branch(input_shape=(7, 7, 512))
resnet50_branch, resnet50_input = input_branch(input_shape=(1, 1, 2048))
concatenate_branches = Concatenate()([vgg19_branch, resnet50_branch])
net = Dropout(0.3)(concatenate_branches)
net = Dense(640, use_bias=False, kernel_initializer='uniform')(net)
net = BatchNormalization()(net)
net = Activation("relu")(net)
net = Dropout(0.3)(net)
net = Dense(120, kernel_initializer='uniform', activation="softmax")(net)
model = Model(inputs=[vgg19_input, resnet50_input], outputs=[net])
model.summary()
model.compile(loss='categorical_crossentropy', optimizer="rmsprop", metrics=['accuracy'])
checkpointer = ModelCheckpoint(filepath='saved_models/bestmodel.hdf5', verbose=1, save_best_only=True)
#keras.callbacks.TensorBoard(log_dir='./logs', histogram_freq=0, batch_size=32, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None, embeddings_data=None, update_freq='epoch')
def get_model(img_shape=(75, 75, 2), num_classes=1, f=8, h=128):
"""
This model structure is inspired and modified from the following kernel
https://www.kaggle.com/knowledgegrappler/a-keras-prototype-0-21174-on-pl
img_shape: dimension for input image
f: filters of first conv blocks and generate filters in the following
blocks acorrdingly
h: units in dense hidden layer
"""
#model
bn_model = 0
p_activation = 'elu'
#
input_img = Input(shape=img_shape, name='img_inputs')
input_img_bn = BatchNormalization(momentum=bn_model)(input_img)
#
input_meta = Input(shape=[1], name='angle')
input_meta_bn = BatchNormalization(momentum=bn_model)(input_meta)
#img_1
#img_1:block_1
img_1 = conv_block(input_img_bn, nf=f, k=3, s=1, nb=3, p_act=p_activation)
img_1 = bn_pooling(img_1, k=3, s=3, m=0)
#img_1:block_2
f *= 2
img_1 = Dropout(0.2)(img_1)
img_1 = conv_block(img_1, nf=f, k=3, s=1, nb=3, p_act=p_activation)
img_1 = bn_pooling(img_1, k=3, s=2, m=0)
#img_1:block_3
f *= 2
img_1 = Dropout(0.2)(img_1)
img_1 = conv_block(img_1, nf=f, k=3, s=1, nb=3, p_act=p_activation)
img_1 = bn_pooling(img_1, k=3, s=3, m=0)
#img_1:block_4
f *= 2
img_1 = Dropout(0.2)(img_1)
img_1 = conv_block(img_1, nf=f, k=3, s=1, nb=3, p_act=p_activation)
img_1 = Dropout(0.2)(img_1)
img_1 = BatchNormalization(momentum=bn_model)(GlobalMaxPooling2D()(img_1))
#img 2
img_2 = conv_block(input_img_bn, nf=f, k=3, s=1, nb=6, p_act=p_activation)
img_2 = Dropout(0.2)(img_2)
img_2 = BatchNormalization(momentum=bn_model)(GlobalMaxPooling2D()(img_2))
#full connect
concat = (Concatenate()([img_1, img_2, input_meta_bn]))
x = dense_block(concat, h=h)
x = dense_block(x, h=h)
output = Dense(num_classes, activation='sigmoid')(x)
model = Model([input_img, input_meta], output)
model.summary()
return model
def __init__(self,
dim,
batch_norm,
dropout,
rec_dropout,
header,
task,
target_repl=False,
deep_supervision=False,
num_classes=1,
depth=1,
input_dim=76,
size_coef=4,
**kwargs):
self.dim = dim
self.batch_norm = batch_norm
self.dropout = dropout
self.rec_dropout = rec_dropout
self.depth = depth
self.size_coef = size_coef
if task in ['decomp', 'ihm', 'ph']:
final_activation = 'sigmoid'
elif task in ['los']:
if num_classes == 1:
final_activation = 'relu'
else:
final_activation = 'softmax'
else:
raise ValueError("Wrong value for task")
print("==> not used params in network class:", kwargs.keys())
# Parse channels
channel_names = set()
for ch in header:
if ch.find("mask->") != -1:
continue
pos = ch.find("->")
if pos != -1:
channel_names.add(ch[:pos])
else:
channel_names.add(ch)
channel_names = sorted(list(channel_names))
print("==> found {} channels: {}".format(len(channel_names),
channel_names))
channels = [] # each channel is a list of columns
for ch in channel_names:
indices = range(len(header))
indices = list(filter(lambda i: header[i].find(ch) != -1, indices))
channels.append(indices)
# Input layers and masking
X = Input(shape=(None, input_dim), name='X')
inputs = [X]
mX = Masking()(X)
if deep_supervision:
M = Input(shape=(None, ), name='M')
inputs.append(M)
# Configurations
is_bidirectional = True
if deep_supervision:
is_bidirectional = False
# Preprocess each channel
cX = []
for ch in channels:
cX.append(Slice(ch)(mX))
pX = [] # LSTM processed version of cX
for x in cX:
p = x
for i in range(depth):
num_units = dim
if is_bidirectional:
num_units = num_units // 2
lstm = LSTM(units=num_units,
activation='tanh',
return_sequences=True,
dropout=dropout,
recurrent_dropout=rec_dropout)
if is_bidirectional:
p = Bidirectional(lstm)(p)
else:
p = lstm(p)
pX.append(p)
# Concatenate processed channels
Z = Concatenate(axis=2)(pX)
# Main part of the network
for i in range(depth - 1):
num_units = int(size_coef * dim)
if is_bidirectional:
num_units = num_units // 2
lstm = LSTM(units=num_units,
activation='tanh',
return_sequences=True,
dropout=dropout,
recurrent_dropout=rec_dropout)
if is_bidirectional:
Z = Bidirectional(lstm)(Z)
else:
Z = lstm(Z)
# Output module of the network
return_sequences = (target_repl or deep_supervision)
L = LSTM(units=int(size_coef * dim),
activation='tanh',
return_sequences=return_sequences,
dropout=dropout,
recurrent_dropout=rec_dropout)(Z)
if dropout > 0:
L = Dropout(dropout)(L)
if target_repl:
y = TimeDistributed(Dense(num_classes,
activation=final_activation),
name='seq')(L)
y_last = LastTimestep(name='single')(y)
outputs = [y_last, y]
elif deep_supervision:
y = TimeDistributed(Dense(num_classes,
activation=final_activation))(L)
y = ExtendMask()([y, M]) # this way we extend mask of y to M
outputs = [y]
else:
y = Dense(num_classes, activation=final_activation)(L)
outputs = [y]
super(Network, self).__init__(inputs=inputs, outputs=outputs)
return baseNetwork
baseNetwork = createSplitBaseNetworkSmall(inputDim, inputLength)
# Inputs
inputA = Input(shape=(inputLength, ))
inputB = Input(shape=(inputLength, ))
# because we re-use the same instance `base_network`,
# the weights of the network will be shared across the two branches
processedA = baseNetwork(inputA)
processedB = baseNetwork(inputB)
# Concatenate
conc = Concatenate()([processedA, processedB])
x = Conv1D(256,
3,
strides=1,
padding='valid',
activation='relu',
kernel_initializer=RandomNormal(mean=0.0, stddev=0.05),
bias_initializer=RandomNormal(mean=0.0, stddev=0.05))(conc)
x = Conv1D(256,
3,
strides=1,
padding='valid',
activation='relu',
kernel_initializer=RandomNormal(mean=0.0, stddev=0.05),
bias_initializer=RandomNormal(mean=0.0, stddev=0.05))(x)
def nn_base(input_tensor=None, trainable=False):
# Determine proper input shape
#input_shape=(image_size, image_size, image_size, num_channels=1) # l, h, w, c
if K.image_data_format() == 'channels_first':
input_shape = (1, None, None, None)
else:
input_shape = (None, None, None, 1)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if K.image_data_format() == 'channels_last':
bn_axis = 4
else:
bn_axis = 1
m = Convolution3D(32, (3, 3, 3), activation='relu', padding='same')(img_input)
m = MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2))(m)
shortcut = Convolution3D(16, (3, 3, 3), activation='relu', padding='same')(m)
#Bottleneck
mNew = Convolution3D(16, (1, 1, 1), activation='relu', padding='same')(m)
mNew = Convolution3D(32, (3, 3, 3), activation='relu', padding='same')(mNew)
mNew = Convolution3D(16, (1, 1, 1), padding='same')(mNew)
mNew = Dropout(0.7)(mNew)
#mMerge = merge([shortcut, mNew], mode='concat', concat_axis=-1)
mMerge = Concatenate(axis=-1)([shortcut, mNew])
m = Activation('relu')(mMerge)
shortcut2 = Convolution3D(16, (3, 3, 3), activation='relu', padding='same')(m)
#Bottleneck
mNew2 = Convolution3D(16, (1, 1, 1), activation='relu', padding='same')(m)
mNew2 = Convolution3D(32, (3, 3, 3), activation='relu', padding='same')(mNew2)
mNew2 = Convolution3D(16, (1, 1, 1), padding='same')(mNew2)
mNew2 = Dropout(0.7)(mNew2)
#mMerge2 = merge([shortcut2, mNew2], mode='concat', concat_axis=-1)
mMerge2 = Concatenate(axis=-1)([shortcut2, mNew2])
m = Activation('relu')(mMerge2)
shortcut3 = Convolution3D(16, (3, 3, 3), activation='relu', padding='same')(m)
#Bottleneck
mNew3 = Convolution3D(16, (1, 1, 1), activation='relu', padding='same')(m)
mNew3 = Convolution3D(32, (3, 3, 3), activation='relu', padding='same')(mNew3)
mNew3 = Convolution3D(16, (1, 1, 1), padding='same')(mNew3)
mNew3 = Dropout(0.7)(mNew3)
#mMerge3 = merge([shortcut3, mNew3], mode='concat', concat_axis=-1)
mMerge3 = Concatenate(axis=-1)([shortcut3, mNew3])
m = Activation('relu')(mMerge3)
shortcut4 = Convolution3D(16, (3, 3, 3), activation='relu', padding='same')(m)
#Bottleneck
mNew4 = Convolution3D(16, (1, 1, 1), activation='relu', padding='same')(m)
mNew4 = Convolution3D(32, (3, 3, 3), activation='relu', padding='same')(mNew4)
mNew4 = Convolution3D(16, (1, 1, 1), padding='same')(mNew4)
mNew4 = Dropout(0.7)(mNew4)
#mMerge4 = merge([shortcut4, mNew4], mode='concat', concat_axis=-1)
mMerge4 = Concatenate(axis=-1)([shortcut4, mNew4])
m = Activation('relu')(mMerge4)
return m
def get_training_model(weight_decay):
stages = 6
np_branch1 = 38
np_branch2 = 19
img_input_shape = (None, None, 3)
vec_input_shape = (None, None, 38)
heat_input_shape = (None, None, 19)
inputs = []
outputs = []
img_input = Input(shape=img_input_shape)
vec_weight_input = Input(shape=vec_input_shape)
heat_weight_input = Input(shape=heat_input_shape)
inputs.append(img_input)
inputs.append(vec_weight_input)
inputs.append(heat_weight_input)
img_normalized = Lambda(lambda x: x / 256 - 0.5)(img_input) # [-0.5, 0.5]
# VGG
stage0_out = vgg_block(img_normalized, weight_decay)
# stage 1 - branch 1 (PAF)
stage1_branch1_out = stage1_block(stage0_out, np_branch1, 1, weight_decay)
w1 = apply_mask(stage1_branch1_out, vec_weight_input, heat_weight_input,
np_branch1, 1, 1)
# stage 1 - branch 2 (confidence maps)
stage1_branch2_out = stage1_block(stage0_out, np_branch2, 2, weight_decay)
w2 = apply_mask(stage1_branch2_out, vec_weight_input, heat_weight_input,
np_branch2, 1, 2)
x = Concatenate()([stage1_branch1_out, stage1_branch2_out, stage0_out])
outputs.append(w1)
outputs.append(w2)
# stage sn >= 2
for sn in range(2, stages + 1):
# stage SN - branch 1 (PAF)
stageT_branch1_out = stageT_block(x, np_branch1, sn, 1, weight_decay)
w1 = apply_mask(stageT_branch1_out, vec_weight_input,
heat_weight_input, np_branch1, sn, 1)
# stage SN - branch 2 (confidence maps)
stageT_branch2_out = stageT_block(x, np_branch2, sn, 2, weight_decay)
w2 = apply_mask(stageT_branch2_out, vec_weight_input,
heat_weight_input, np_branch2, sn, 2)
outputs.append(w1)
outputs.append(w2)
if (sn < stages):
x = Concatenate()(
[stageT_branch1_out, stageT_branch2_out, stage0_out])
model = Model(inputs=inputs, outputs=outputs)
return model
def loadModel(x_train, x_val, vocabulary_inv):
# Prepare embedding layer weights and convert inputs for static model
print("Model type is", model_type)
if model_type in ["CNN-non-static", "CNN-static"]:
embedding_weights = train_word2vec(np.vstack((x_train, x_val)),
vocabulary_inv,
num_features=embedding_dim,
min_word_count=min_word_count,
context=context)
if model_type == "CNN-static":
x_train = np.stack([
np.stack([embedding_weights[word] for word in sentence])
for sentence in x_train
])
x_val = np.stack([
np.stack([embedding_weights[word] for word in sentence])
for sentence in x_val
])
print("x_train static shape:", x_train.shape)
print("x_val static shape:", x_val.shape)
elif model_type == "CNN-rand":
embedding_weights = None
else:
raise ValueError("Unknown model type")
# Build model
if model_type == "CNN-static":
input_shape = (sequence_length, embedding_dim)
else:
input_shape = (sequence_length, )
model_input = Input(shape=input_shape)
# Static model does not have embedding layer
if model_type == "CNN-static":
z = model_input
else:
z = Embedding(len(vocabulary_inv),
embedding_dim,
input_length=sequence_length,
name="embedding")(model_input)
z = Dropout(dropout_prob[0])(z)
# Convolutional block
conv_blocks = []
for sz in filter_sizes:
conv = Convolution1D(filters=num_filters,
kernel_size=sz,
padding="valid",
activation="relu",
strides=1)(z)
conv = MaxPooling1D(pool_size=2)(conv)
conv = Flatten()(conv)
conv_blocks.append(conv)
z = Concatenate()(conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]
z = Dropout(dropout_prob[1])(z)
z = Dense(hidden_dims, activation="relu")(z)
model_output = Dense(1, activation="sigmoid")(z)
model = Model(model_input, model_output)
model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
model.summary()
# Initialize weights with word2vec
if model_type == "CNN-non-static":
weights = np.array([v for v in embedding_weights.values()])
print("Initializing embedding layer with word2vec weights, shape",
weights.shape)
embedding_layer = model.get_layer("embedding")
embedding_layer.set_weights([weights])
return model
baseNetwork.add(Conv1D(128, 5, strides=1, padding='valid', activation='relu', kernel_initializer=RandomNormal(
mean=0.0, stddev=0.05), bias_initializer=RandomNormal(mean=0.0, stddev=0.05)))
baseNetwork.add(MaxPooling1D(pool_size=2, strides=2))
baseNetwork.add(Flatten())
baseNetwork.add(Dense(128, activation='relu'))
baseNetwork.add(Dropout(0.9))
baseNetwork.add(BatchNormalization())
baseNetwork.add(Dense(128, activation='relu'))
# because we re-use the same instance `base_network`,
# the weights of the network will be shared across the two branches
processed1 = baseNetwork(embedded1)
processed2 = baseNetwork(embedded2)
# Concatenate
conc = Concatenate()([processed1, processed2])
x = Dropout(0.9)(conc)
x = BatchNormalization()(x)
# Dense
x = Dense(128, activation='relu')(x)
x = Dropout(0.9)(x)
x = BatchNormalization()(x)
predictions = Dense(1, activation='sigmoid')(x)
# def euclidean_distance(vects):
# x, y = vects
# return K.sqrt(K.maximum(K.sum(K.square(x - y), axis=1, keepdims=True), K.epsilon()))
def dr_classifier(EX_segmentor, HE_segmentor, MA_segmentor, SE_segmentor,
loss_type):
s = 2
k = 3
padding = "same"
max_n_filters = 512
n_filters = 32
depth = 3
# build a convolution branch
conv_input = Input((None, None, depth))
conv_branch = basic_block(conv_input, 6, n_filters, max_n_filters, k, s,
padding)
# change names and fix layers
for layer in EX_segmentor.layers:
layer.name = "EX_segmentor_" + layer.name
layer.trainable = False
for layer in HE_segmentor.layers:
layer.name = "HE_segmentor_" + layer.name
layer.trainable = False
for layer in MA_segmentor.layers:
layer.name = "MA_segmentor_" + layer.name
layer.trainable = False
for layer in SE_segmentor.layers:
layer.name = "SE_segmentor_" + layer.name
layer.trainable = False
# set inputs
ex_input = EX_segmentor.get_layer("EX_segmentor_input_1").input
he_input = HE_segmentor.get_layer("HE_segmentor_input_1").input
ma_input = MA_segmentor.get_layer("MA_segmentor_input_1").input
se_input = SE_segmentor.get_layer("SE_segmentor_input_1").input
# get first or second bottleneck layers
ex_bottleneck_layer = EX_segmentor.get_layer(
"EX_segmentor_activation_14").output
he_bottleneck_layer = HE_segmentor.get_layer(
"HE_segmentor_activation_14").output
ma_bottleneck_layer = MA_segmentor.get_layer(
"MA_segmentor_activation_6").output
ma_bottleneck_layer_down = basic_block(ma_bottleneck_layer, 4,
n_filters * (2**2), max_n_filters,
k, s, padding)
se_bottleneck_layer = SE_segmentor.get_layer(
"SE_segmentor_activation_10").output
se_bottleneck_layer_down = basic_block(se_bottleneck_layer, 2,
n_filters * (2**4), max_n_filters,
k, s, padding)
concat = Concatenate(axis=3)([
ex_bottleneck_layer, he_bottleneck_layer, se_bottleneck_layer_down,
ma_bottleneck_layer_down, conv_branch
])
final_conv = conv_blocks(2,
concat,
max_n_filters, (k, k),
"relu",
padding=padding)
gap = GlobalAveragePooling2D()(final_conv)
output = Dense(1)(gap)
network = Model([ex_input, he_input, ma_input, se_input, conv_input],
output)
def smooth_L1(y_true, y_pred):
x = K.abs(y_true - y_pred)
HUBER_DELTA = 1
if K._BACKEND == 'tensorflow':
import tensorflow as tf
x = tf.where(x < HUBER_DELTA, 0.5 * x**2,
HUBER_DELTA * (x - 0.5 * HUBER_DELTA))
return K.sum(x)
if loss_type == "smooth_L1":
network.compile(optimizer=SGD(lr=0, momentum=0.9, nesterov=True),
loss=smooth_L1,
metrics=['accuracy'])
else:
network.compile(optimizer=SGD(lr=0, momentum=0.9, nesterov=True),
loss=objectives.mean_squared_error,
metrics=['accuracy'])
return network
conv7 = Conv2D(32, (3, 3),
padding='same',
activation='elu',
kernel_initializer='glorot_normal',
kernel_regularizer=l2(weight_decay))(pool3)
conv8 = Conv2D(32, (3, 3),
padding='same',
activation='elu',
kernel_initializer='glorot_normal',
kernel_regularizer=l2(weight_decay))(conv7)
pool4 = AveragePooling2D((2, 2), strides=(2, 2))(conv8)
flat = Flatten()(pool4)
print(inp_2)
comb_feats = (Concatenate()([flat, BatchNormalization()(inp_2)]))
dense1 = Dense(64,
activation='relu',
kernel_initializer='glorot_normal',
kernel_regularizer=l2(weight_decay))(comb_feats)
drop1 = Dropout(0.8)(dense1)
dense2 = Dense(64,
activation='relu',
kernel_initializer='glorot_normal',
kernel_regularizer=l2(weight_decay))(drop1)
drop2 = Dropout(0.8)(dense2)
out = Dense(2, activation='softmax')(drop2)
# Train the model and make the prediction
print("setting up the model...")
def dr_network(loss_type):
# set image specifics
img_shape = (512, 512, 3)
n_filters = 32
n_out = 1
k = 3 # kernel size
s = 2 # stride
img_h, img_w, img_ch = img_shape[0], img_shape[1], img_shape[2]
padding = 'same'
l2_coeff = 0.0005
list_n_building_blocks = [2, 3, 4]
blocks = []
inputs = Input((img_h, img_w, img_ch))
blocks.append(
conv_block(inputs, n_filters, (k, k), (s, s), padding, l2_coeff))
for index, n_building_blocks in enumerate(list_n_building_blocks):
blocks.append(
resnet_blocks(blocks[index], n_building_blocks,
(2**index) * n_filters, (k, k), padding, l2_coeff))
blocks[index + 1] = conv_block(blocks[index + 1],
2**(index + 1) * n_filters, (k, k),
(s, s), padding, l2_coeff)
list_avg_pools = []
for i in range(3):
list_avg_pools.append(
AveragePooling2D((8 // (2**i), 8 // (2**i)))(blocks[i]))
blocks_concat = Concatenate(axis=3)(list_avg_pools + [blocks[3]])
list_dilated_conv = []
list_dilated_conv.append(
Conv2D(16 * n_filters, (k, k),
padding=padding,
kernel_regularizer=regularizers.l2(l2_coeff),
bias_regularizer=regularizers.l2(l2_coeff))(blocks_concat))
list_dilated_conv.append(
Conv2D(16 * n_filters, (k, k),
dilation_rate=(2, 2),
padding=padding,
kernel_regularizer=regularizers.l2(l2_coeff),
bias_regularizer=regularizers.l2(l2_coeff))(blocks_concat))
list_dilated_conv.append(
Conv2D(16 * n_filters, (k, k),
dilation_rate=(4, 4),
padding=padding,
kernel_regularizer=regularizers.l2(l2_coeff),
bias_regularizer=regularizers.l2(l2_coeff))(blocks_concat))
final_block = Concatenate(axis=3)(list_dilated_conv)
final_block = conv_block(final_block, 16 * n_filters, (k, k), (s, s),
padding, l2_coeff)
gap = GlobalAveragePooling2D()(final_block)
outputs = Dense(n_out)(gap)
network = Model(inputs, outputs)
def smooth_L1(y_true, y_pred):
y_pred_clipped = K.clip(y_pred, 0, 4)
x = K.abs(y_true - y_pred_clipped)
HUBER_DELTA = 1
if K._BACKEND == 'tensorflow':
import tensorflow as tf
x = tf.where(x < HUBER_DELTA, 0.5 * x**2,
HUBER_DELTA * (x - 0.5 * HUBER_DELTA))
return x
def L2(y_true, y_pred):
y_pred_clipped = K.clip(y_pred, 0, 4)
return objectives.mean_squared_error(y_true, y_pred_clipped)
if loss_type == "smooth_L1":
network.compile(optimizer=SGD(lr=0, momentum=0.9, nesterov=True),
loss=smooth_L1,
metrics=['accuracy'])
elif loss_type == "L2":
network.compile(optimizer=SGD(lr=0, momentum=0.9, nesterov=True),
loss=L2,
metrics=['accuracy'])
return network
def __call__(self, input_shape):
"""Design embedding
Parameters
----------
input_shape : (n_frames, n_features) tuple
Shape of input sequence.
Returns
-------
model : Keras model
"""
inputs = Input(shape=input_shape,
name="input")
masking = Masking(mask_value=0.)
x = masking(inputs)
# stack (bidirectional) recurrent layers
for i, output_dim in enumerate(self.recurrent):
sole_layer = not self.mlp and len(self.recurrent) == 1
last_internal_layer = not self.mlp and i + 1 == len(self.recurrent)
params = {
'name': 'rnn_{i:d}'.format(i=i),
'return_sequences': True,
# 'go_backwards': False,
# 'stateful': False,
# 'unroll': False,
# 'implementation': 0,
'activation': 'tanh',
# 'recurrent_activation': 'hard_sigmoid',
# 'use_bias': True,
# 'kernel_initializer': 'glorot_uniform',
# 'recurrent_initializer': 'orthogonal',
# 'bias_initializer': 'zeros',
# 'unit_forget_bias': True,
# 'kernel_regularizer': None,
# 'recurrent_regularizer': None,
# 'bias_regularizer': None,
# 'activity_regularizer': None,
# 'kernel_constraint': None,
# 'recurrent_constraint': None,
# 'bias_constraint': None,
# 'dropout': 0.0,
# 'recurrent_dropout': 0.0,
}
# first RNN needs to be given the input shape
if i == 0:
params['input_shape'] = input_shape
if sole_layer and not self.bidirectional:
params['name'] = 'internal'
recurrent = self.RNN_(output_dim, **params)
# bi-directional RNN
if self.bidirectional:
recurrent = Bidirectional(recurrent,
merge_mode=self.bidirectional,
name='internal' if sole_layer else None)
# (actually) stack RNN
x = recurrent(x)
# concatenate output of all levels
if i > 0:
name = 'internal' if last_internal_layer else None
concat_x = Concatenate(axis=-1, name=name)([concat_x, x])
else:
# corner case for 1st level (i=0)
# as concat_x does not yet exist
concat_x = x
# just rename the concatenated output variable to x
x = concat_x
# (optionally) stack dense MLP layers
for i, output_dim in enumerate(self.mlp):
last_internal_layer = i + 1 == len(self.mlp)
activation = 'tanh'
if last_internal_layer and self.linear:
activation = 'linear'
mlp = Dense(output_dim,
name='mlp_{i:d}'.format(i=i),
activation=activation)
name = 'internal' if last_internal_layer else None
x = TimeDistributed(mlp, name=name)(x)
# average pooling and L2 normalization
pooling = EmbeddingAveragePooling(name='pooling')
embeddings = pooling(x)
return Model(inputs=[inputs], outputs=embeddings)
def _set_model(self):
"""
Setup model (method should only be called in self.fit())
"""
print("Setting up model...")
# Encoder
inputs = Input(batch_shape=(self.batch_size, ) + self.input_shape)
# Instantiate encoder layers
Q_0 = Conv2D(self.input_shape[2], (2, 2),
padding='same',
activation='relu')
Q_1 = Conv2D(self.filters[0], (2, 2),
padding='same',
strides=(2, 2),
activation='relu')
Q_2 = Conv2D(self.filters[1], (3, 3),
padding='same',
strides=(1, 1),
activation='relu')
Q_3 = Conv2D(self.filters[2], (3, 3),
padding='same',
strides=(1, 1),
activation='relu')
Q_4 = Flatten()
Q_5 = Dense(self.hidden_dim, activation='relu')
Q_z_mean = Dense(self.latent_cont_dim)
Q_z_log_var = Dense(self.latent_cont_dim)
# Set up encoder
x = Q_0(inputs)
x = Q_1(x)
x = Q_2(x)
x = Q_3(x)
flat = Q_4(x)
hidden = Q_5(flat)
# Parameters for continous latent distribution
z_mean = Q_z_mean(hidden)
z_log_var = Q_z_log_var(hidden)
# Parameters for concrete latent distribution
if self.latent_disc_dim:
Q_c = Dense(self.latent_disc_dim, activation='softmax')
alpha = Q_c(hidden)
# Sample from latent distributions
if self.latent_disc_dim:
z = Lambda(self._sampling_normal)([z_mean, z_log_var])
c = Lambda(self._sampling_concrete)(alpha)
encoding = Concatenate()([z, c])
else:
encoding = Lambda(self._sampling_normal)([z_mean, z_log_var])
# Generator
# Instantiate generator layers to be able to sample from latent
# distribution later
out_shape = (self.input_shape[0] / 2, self.input_shape[1] / 2,
self.filters[2])
G_0 = Dense(self.hidden_dim, activation='relu')
G_1 = Dense(np.prod(out_shape), activation='relu')
G_2 = Reshape(out_shape)
G_3 = Conv2DTranspose(self.filters[2], (3, 3),
padding='same',
strides=(1, 1),
activation='relu')
G_4 = Conv2DTranspose(self.filters[1], (3, 3),
padding='same',
strides=(1, 1),
activation='relu')
G_5 = Conv2DTranspose(self.filters[0], (2, 2),
padding='valid',
strides=(2, 2),
activation='relu')
G_6 = Conv2D(self.input_shape[2], (2, 2),
padding='same',
strides=(1, 1),
activation='sigmoid',
name='generated')
# Apply generator layers
x = G_0(encoding)
x = G_1(x)
x = G_2(x)
x = G_3(x)
x = G_4(x)
x = G_5(x)
generated = G_6(x)
self.model = Model(inputs, generated)
# Set up generator
inputs_G = Input(batch_shape=(self.batch_size, self.latent_dim))
x = G_0(inputs_G)
x = G_1(x)
x = G_2(x)
x = G_3(x)
x = G_4(x)
x = G_5(x)
generated_G = G_6(x)
self.generator = Model(inputs_G, generated_G)
# Store latent distribution parameters
self.z_mean = z_mean
self.z_log_var = z_log_var
if self.latent_disc_dim:
self.alpha = alpha
# Compile models
self.opt = RMSprop()
self.model.compile(optimizer=self.opt, loss=self._vae_loss)
# Loss and optimizer do not matter here as we do not train these models
self.generator.compile(optimizer=self.opt, loss='mse')
print("Completed model setup.")
def main():
####
start_time = time.time()
###
data_size = 25000
####
SMILES_CHARS = [
' ', 'C', 'N', 'O', '#', '=', '1', '(', ')', 'c', '[', 'n', 'H', ']',
'o', '3', '+', '-', '2', 'F', '4', '5'
]
def smiles_encoder(smiles, maxlen=34):
#print(smiles)
#smiles = Chem.MolToSmiles(Chem.MolFromSmiles(smiles))
#print(smiles)
X = np.zeros((maxlen, 22))
for i, c in enumerate(smiles):
#print(i)
#print(c)
X[i, smi2index[c]] = 1
return X
def smiles_decoder(X):
smi = ''
X = X.argmax(axis=-1)
for i in X:
smi += index2smi[i]
return smi
smi2index = dict((c, i) for i, c in enumerate(SMILES_CHARS))
index2smi = dict((i, c) for i, c in enumerate(SMILES_CHARS))
out_raw = pd.read_csv('QM9_smiles.csv', header=None)[0]
out = add_space(out_raw)
QM9_hot = []
for i in out:
#print(i)
QM9_hot.append(smiles_encoder(i))
#read properties data
QM9_properties = pd.read_csv('QM9_properties.csv')
QM9_properties.columns = [
'tag', 'index', 'A', 'B', 'C', 'mu', 'alpha', 'h**o', 'lumo', 'gap',
'r2', 'zpve', 'U0', 'U', 'H', 'G', 'Cv'
]
###vae
latent_dim = 156
input_ = Input(shape=(34, 22), name='input')
x = Conv1D(16, 8, activation='tanh', padding='same', name='e1')(input_)
x = BatchNormalization()(x)
x = Conv1D(32, 10, activation='tanh', padding='same')(x)
x = BatchNormalization()(x)
x = Conv1D(64, 12, activation='tanh', padding='same')(x)
x = BatchNormalization()(x)
x = Flatten()(x)
##hidden layers
z_mean = Dense(latent_dim)(x)
z_log_var = Dense(latent_dim)(x)
z_mean_log_var_output = Concatenate(name='kl_loss')([z_mean, z_log_var])
z = Lambda(sampling, name='z')([z_mean, z_log_var])
encoder = Model(input_, [z_mean_log_var_output, z])
encoder_output = [z_mean_log_var_output, z]
#edecoder
input_d = Input(shape=(latent_dim, ))
x_dec = Dense(156)(input_d)
x_dec = BatchNormalization(axis=-1, name='decoder_dense_norm')(x_dec)
###
x_dec = RepeatVector(34)(x_dec)
x_dec = GRU(500,
return_sequences=True,
activation='tanh',
name="decoder_gru1")(x_dec)
x_dec = GRU(500,
return_sequences=True,
activation='tanh',
name="decoder_gru2")(x_dec)
#x_dec = GRU(50, return_sequences=True, activation='tanh', name="decoder_gru3")(x_dec)
x_dec = GRU(22,
return_sequences=True,
activation='softmax',
name='decoder_gru3')(x_dec)
decoder = Model(input_d, x_dec)
#properties
input_p = Input(shape=(latent_dim, ))
#xp = Dense(67, activation='tanh')(input_p)
#xp = Dropout(0.15)(xp)
#xp = Dense(67, activation='tanh')(xp)
#xp = Dropout(0.15)(xp)
#xp = Dense(67, activation='tanh')(xp)
#xp = Dropout(0.15)(xp)
xp = Dense(1000, activation='linear')(input_p)
xp = Dropout(0.2)(xp)
xp = Dense(1000, activation='linear')(xp)
xp = Dropout(0.2)(xp)
xp = Dense(1, activation='linear')(xp)
pp_homo = Model(input_p, xp)
pp_lumo = Model(input_p, xp)
#####data aggregate
vae_outputs = decoder(encoder(input_)[1])
vae_outputs = Lambda(identity, name='x_pred')(vae_outputs)
homo_outputs = pp_homo(encoder(input_)[1])
lumo_outputs = pp_lumo(encoder(input_)[1])
homo_outputs = Lambda(identity, name='homo_loss')(homo_outputs)
lumo_outputs = Lambda(identity, name='lumo_loss')(lumo_outputs)
properties_homo = Model(input_, homo_outputs, name='properties_homo')
properties_lumo = Model(input_, lumo_outputs, name='properties_lumo')
total_ouput = encoder_output
total_ouput.append(vae_outputs)
total_ouput.append(homo_outputs)
total_ouput.append(lumo_outputs)
total_model = Model(input_, total_ouput)
####
#data collecting
x_train = QM9_hot
x_train = np.reshape(x_train, (len(x_train), 34, 22))
#y_homo = QM9_properties['h**o']
#y_lumo = QM9_properties['lumo']
#y_homo = np.reshape(y_homo, (len(y_homo),))
#y_lumo = np.reshape(y_lumo, (len(y_lumo),))
x_raw = np.reshape(x_train, (len(x_train), 34, 22))
y_raw = QM9_properties[['h**o', 'lumo']]
x_train, x_test, y_train, y_test = train_test_split(x_raw,
y_raw,
test_size=0.2,
random_state=42)
#x_val, x_test, y_val, y_test = train_test_split(x_vt, y_vt, test_size=0.5, random_state=42)
y_homo = np.array(y_train['h**o'])
y_lumo = np.array(y_train['lumo'])
y_homo_test = np.array(y_test['h**o'])
y_lumo_test = np.array(y_test['lumo'])
def kl_loss(truth_dummy, z_mean_log_var_output):
z_mean, z_log_var = tf.split(z_mean_log_var_output, 2, axis=1)
#print('x_mean shape in kl_loss: ', x_mean.get_shape())
kl = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
kl *= -0.5
return (kl)
'''
def kl_mse_loss(true, pred):
# Reconstruction loss
mse_loss = tf.keras.losses.MSE(K.flatten(true), K.flatten(pred))
#mse_loss = tf.keras.losses.MSE(true, pred)
#mse_loss = tf.keras.losses.MeanSquaredError(K.flatten(true), K.flatten(pred))
#reconstruction_loss = categorical_crossentropy(K.flatten(true), K.flatten(pred))
#reconstruction_loss *= 34*22
#* img_width * img_height
# KL divergence loss
kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
#kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
# Total loss = 50% rec + 50% KL divergence loss
return K.mean(mse_loss + kl_loss)
def kl_loss(true, pred):
kl = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
kl *= -0.5
return(kl)
'''
def mse_loss(true, pred):
#mse_loss = tf.keras.losses.MSE(true, pred)
mse_loss = tf.keras.losses.MSE(K.flatten(true), K.flatten(pred))
#mse_loss = tf.keras.losses.MeanSquaredError(K.flatten(true), K.flatten(pred), reduction=tf.keras.losses.Reduction.SUM)
#mse = tf.keras.losses.MeanSquaredError(reduction=tf.keras.losses.Reduction.SUM)
#mse(y_true, y_pred).numpy()
mse_loss *= 250
return (mse_loss)
def reconstruct_error(true, pred):
reconstruct_error = binary_crossentropy(K.flatten(true),
K.flatten(pred))
reconstruct_error *= 34 * 22
return (reconstruct_error)
opt = keras.optimizers.Adam(lr=0.00002)
####add three losses together, target also: kl loss, reconstructure, properties mse
model_targets = {
'x_pred': x_train,
'kl_loss': np.ones((np.shape(x_train)[0], latent_dim * 2)),
'homo_loss': y_homo,
'lumo_loss': y_lumo
}
'''
test_targets = {
'x_pred': x_test,
'kl_loss': np.ones((np.shape(x_test)[0], latent_dim * 2)),
'properties_loss': y_test
}
'''
total_loss = {
'x_pred': reconstruct_error,
'kl_loss': kl_loss,
'homo_loss': mse_loss,
'lumo_loss': mse_loss
}
total_loss_weight = {
'x_pred': 0.5 * 0.5,
'kl_loss': 0.5 * 0.5,
'homo_loss': 0.5,
'lumo_loss': 0.5
}
#train
total_model.compile(optimizer=opt,
loss=total_loss,
loss_weights=total_loss_weight)
#autoencoder.fit(train_images, train_images)
'''
vae.fit(x_train, x_train,
epochs=120,
batch_size=250,
shuffle=True)
#validation_data=(x_test, x_test))
'''
his = total_model.fit(
x_train,
model_targets,
epochs=1,
batch_size=250,
#epochs= 1,
#batch_size=1000,
validation_split=0.2,
shuffle=True,
verbose=1)
#validation_data=(x_test, test_targets))
his_data = pd.DataFrame.from_dict(his.history)
his_data.to_csv('test.csv')
#callbacks=callbacks_list)
total_model.save("test_totel.h5")
encoder.save("test_encoder.h5")
decoder.save("test_decoder.h5")
pp_homo.save("test_pp_homo.h5")
pp_lumo.save("test_pp_lumo.h5")
predict_raw = decoder.predict(encoder.predict(x_train)[1])
predict_st = []
for i in predict_raw:
predict_st.append(smiles_decoder(i))
###invalid cal
invalid = 0
oo = []
for i, x in enumerate(predict_st):
#print(x)
m = Chem.MolFromSmiles(x, sanitize=False)
if m is None:
invalid += 1
else:
try:
Chem.SanitizeMol(m)
oo.append(i)
except:
invalid += 1
print('invalid num =' + str(invalid))
print('invalid rate =' + str(invalid / len(predict_st)))
accuracy = 0
for i in range(len(predict_st)):
if predict_st[i] == out[i]:
accuracy += 1
print('accuracy rate = ' + str(accuracy / len(predict_st)))
#print(oo[0])
#Draw.MolToFile(Chem.MolFromSmiles(out[0]), 'raw.png')
#Draw.MolToFile(Chem.MolFromSmiles(predict_st[0]), 'pre.png')
liss = []
for i in oo:
if predict_st[i] not in liss:
liss.append(predict_st[i])
print(len(liss))
#np.mean((y_homo_test - pp_homo.predict(encoder.predict(x_test)[1]))**2)
print('test H**O MSE = ' + str(
np.mean((y_homo_test -
pp_homo.predict(encoder.predict(x_test)[1]))**2)))
print('test LUMO MSE = ' + str(
np.mean((y_lumo_test -
pp_lumo.predict(encoder.predict(x_test)[1]))**2)))
print('total time:' + str(time.time() - start_time) + 'sec')
def color_net(num_classes):
# placeholder for input image
input_image = Input(shape=(224, 224, 3))
# ============================================= TOP BRANCH ===================================================
# first top convolution layer
top_conv1 = Convolution2D(filters=48,
kernel_size=(11, 11),
strides=(4, 4),
input_shape=(224, 224, 3),
activation='relu')(input_image)
top_conv1 = BatchNormalization()(top_conv1)
top_conv1 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(top_conv1)
# second top convolution layer
# split feature map by half
top_top_conv2 = Lambda(lambda x: x[:, :, :, :24])(top_conv1)
top_bot_conv2 = Lambda(lambda x: x[:, :, :, 24:])(top_conv1)
top_top_conv2 = Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(top_top_conv2)
top_top_conv2 = BatchNormalization()(top_top_conv2)
top_top_conv2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2))(top_top_conv2)
top_bot_conv2 = Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(top_bot_conv2)
top_bot_conv2 = BatchNormalization()(top_bot_conv2)
top_bot_conv2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2))(top_bot_conv2)
# third top convolution layer
# concat 2 feature map
top_conv3 = Concatenate()([top_top_conv2, top_bot_conv2])
top_conv3 = Convolution2D(filters=192,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(top_conv3)
# fourth top convolution layer
# split feature map by half
top_top_conv4 = Lambda(lambda x: x[:, :, :, :96])(top_conv3)
top_bot_conv4 = Lambda(lambda x: x[:, :, :, 96:])(top_conv3)
top_top_conv4 = Convolution2D(filters=96,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(top_top_conv4)
top_bot_conv4 = Convolution2D(filters=96,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(top_bot_conv4)
# fifth top convolution layer
top_top_conv5 = Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(top_top_conv4)
top_top_conv5 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2))(top_top_conv5)
top_bot_conv5 = Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(top_bot_conv4)
top_bot_conv5 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2))(top_bot_conv5)
# ============================================= TOP BOTTOM ===================================================
# first bottom convolution layer
bottom_conv1 = Convolution2D(filters=48,
kernel_size=(11, 11),
strides=(4, 4),
input_shape=(224, 224, 3),
activation='relu')(input_image)
bottom_conv1 = BatchNormalization()(bottom_conv1)
bottom_conv1 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(bottom_conv1)
# second bottom convolution layer
# split feature map by half
bottom_top_conv2 = Lambda(lambda x: x[:, :, :, :24])(bottom_conv1)
bottom_bot_conv2 = Lambda(lambda x: x[:, :, :, 24:])(bottom_conv1)
bottom_top_conv2 = Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(bottom_top_conv2)
bottom_top_conv2 = BatchNormalization()(bottom_top_conv2)
bottom_top_conv2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2))(bottom_top_conv2)
bottom_bot_conv2 = Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(bottom_bot_conv2)
bottom_bot_conv2 = BatchNormalization()(bottom_bot_conv2)
bottom_bot_conv2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2))(bottom_bot_conv2)
# third bottom convolution layer
# concat 2 feature map
bottom_conv3 = Concatenate()([bottom_top_conv2, bottom_bot_conv2])
bottom_conv3 = Convolution2D(filters=192,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(bottom_conv3)
# fourth bottom convolution layer
# split feature map by half
bottom_top_conv4 = Lambda(lambda x: x[:, :, :, :96])(bottom_conv3)
bottom_bot_conv4 = Lambda(lambda x: x[:, :, :, 96:])(bottom_conv3)
bottom_top_conv4 = Convolution2D(filters=96,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(bottom_top_conv4)
bottom_bot_conv4 = Convolution2D(filters=96,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(bottom_bot_conv4)
# fifth bottom convolution layer
bottom_top_conv5 = Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(bottom_top_conv4)
bottom_top_conv5 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2))(bottom_top_conv5)
bottom_bot_conv5 = Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(bottom_bot_conv4)
bottom_bot_conv5 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2))(bottom_bot_conv5)
# ======================================== CONCATENATE TOP AND BOTTOM BRANCH =================================
conv_output = Concatenate()(
[top_top_conv5, top_bot_conv5, bottom_top_conv5, bottom_bot_conv5])
# Flatten
flatten = Flatten()(conv_output)
# Fully-connected layer
FC_1 = Dense(units=4096, activation='relu')(flatten)
FC_1 = Dropout(0.6)(FC_1)
FC_2 = Dense(units=4096, activation='relu')(FC_1)
FC_2 = Dropout(0.6)(FC_2)
output = Dense(units=num_classes, activation='softmax')(FC_2)
model = Model(inputs=input_image, outputs=output)
sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
# sgd = SGD(lr=0.01, momentum=0.9, decay=0.0005, nesterov=True)
model.compile(optimizer=sgd,
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def __init__(self,
hid_dim,
batch_norm,
dropout,
recurrent_dropout,
channels,
num_classes=1,
depth=1,
input_dim=76,
model_size=4,
**kwargs):
self.hid_dim = hid_dim
self.batch_norm = batch_norm
self.dropout = dropout
self.recurrent_dropout = recurrent_dropout
self.depth = depth
self.model_size = model_size
# Input layers and masking
X = Input(shape=(None, input_dim), name='X')
inputs = [X]
mask_X = Masking()(X)
#MSK = Input(shape=(None,), name='MSK')
#inputs.append(MSK)
# Preprocess each channel
channels_X = []
for ch in channels:
channels_X.append(Slice(ch)(mask_X))
channel_lstm__X = [] # LSTM processed version of channels_X
for cx in channels_X:
lstm_cx = cx
for i in range(depth):
units = hid_dim // 2
lstm = LSTM(units=units,
activation='tanh',
return_sequences=True,
dropout=dropout,
recurrent_dropout=recurrent_dropout)
lstm_cx = Bidirectional(lstm)(lstm_cx)
channel_lstm__X.append(lstm_cx)
# Concatenate processed channels
CWX = Concatenate(axis=2)(channel_lstm__X)
# LSTM for time series lstm processed data
for i in range(depth -
1): # last layer is left for manipulation of output.
units = int(model_size * hid_dim) // 2
lstm = LSTM(units=units,
activation='tanh',
return_sequences=True,
dropout=dropout,
recurrent_dropout=recurrent_dropout)
CWX = Bidirectional(lstm)(CWX)
# Output module of the network
last_layer = LSTM(units=int(model_size * hid_dim),
activation='tanh',
return_sequences=True,
dropout=dropout,
recurrent_dropout=recurrent_dropout)(CWX)
if dropout > 0:
last_layer = Dropout(dropout)(last_layer)
y = TimeDistributed(Dense(num_classes, activation='sigmoid'),
name='seq')(last_layer)
y_last = LastTimestep(name='single')(y)
outputs = [y_last, y]
super(ds_channel_wise_lstms, self).__init__(inputs=inputs,
outputs=outputs)
def _init_make_dataparallel(self, gdev_list, *args, **kwargs):
'''Uses data-parallelism to convert a serial model to multi-gpu. Refer
to make_parallel doc.
'''
gpucopy_ops = []
def slice_batch(x, ngpus, part, dev):
'''Divide the input batch into [ngpus] slices, and obtain slice
no. [part]. i.e. if len(x)=10, then slice_batch(x, 2, 1) will
return x[5:].
'''
sh = KB.shape(x)
L = sh[0] // ngpus
if part == ngpus - 1:
xslice = x[part * L:]
else:
xslice = x[part * L:(part + 1) * L]
# tf.split fails if batch size is not divisible by ngpus. Error:
# InvalidArgumentError (see above for traceback): Number of
# ways to split should evenly divide the split dimension
# xslice = tf.split(x, ngpus)[part]
if not self._enqueue:
return xslice
# Did not see any benefit.
with tf.device(dev):
# if self._stager is None:
stager = data_flow_ops.StagingArea(
dtypes=[xslice.dtype], shapes=[xslice.shape])
stage = stager.put([xslice])
gpucopy_ops.append(stage)
# xslice_stage = stager.get()
return stager.get()
ngpus = len(gdev_list)
if ngpus < 2:
raise RuntimeError('Number of gpus < 2. Require two or more GPUs '
'for multi-gpu model parallelization.')
model = self._smodel
noutputs = len(self._smodel.outputs)
global_scope = tf.get_variable_scope()
towers = [[] for _ in range(noutputs)]
for idev, dev in enumerate(gdev_list):
# TODO: The last slice could cause a gradient calculation outlier
# when averaging gradients. Maybe insure ahead of time that the
# batch_size is evenly divisible by number of GPUs, or maybe don't
# use the last slice.
with tf.device(self._ps_device):
slices = [] # multi-input case
for ix, x in enumerate(model.inputs):
slice_g = Lambda(
slice_batch, # lambda shape: shape,
# lambda shape: x.shape.as_list(),
name='stage_cpuSliceIn{}_Dev{}'.format(ix, idev),
arguments={'ngpus': ngpus, 'part': idev,
'dev': dev})(x)
slices.append(slice_g)
# print('SLICE_G: {}'.format(slice_g)) # DEBUG
# print('SLICES: {}'.format(slices)) # DEBUG
# with tf.variable_scope('GPU_%i' % idev), \
# tf.variable_scope(global_scope, reuse=idev > 0), \
# tf.variable_scope('GPU_{}'.format(idev),
# reuse=idev > 0) as var_scope, \
with tf.device(dev), \
tf.variable_scope(global_scope, reuse=idev > 0), \
tf.name_scope('tower_%i' % idev):
# NOTE: Currently not using model_creator. Did not observe
# any benefit in such an implementation.
# Instantiate model under device context. More complicated.
# Need to use optimizer synchronization in this scenario.
# model_ = model_creator()
# If using NCCL without re-instantiating the model then must
# set the colocate_gradients_with_ops to False in optimizer.
# if idev == 0:
# # SET STATE: Instance of serial model for checkpointing
# self._smodel = model_ # for ability to checkpoint
# Handle multi-output case
modeltower = model(slices)
if not isinstance(modeltower, list):
modeltower = [modeltower]
for imt, mt in enumerate(modeltower):
towers[imt].append(mt)
params = mt.graph._collections['trainable_variables']
# params = model_.trainable_weights
# params = tf.get_collection(
# tf.GraphKeys.TRAINABLE_VARIABLES, scope=var_scope.name)
# params = modeltower.graph._collections['trainable_variables']
# print('PARAMS: {}'.format(params)) # DEBUG
self._tower_params.append(params)
with tf.device(self._ps_device):
# merged = Concatenate(axis=0)(towers)
merged = [Concatenate(axis=0)(tw) for tw in towers]
# self._enqueue_ops.append(tf.group(*gpucopy_ops))
self._enqueue_ops += gpucopy_ops
kwargs['inputs'] = model.inputs
kwargs['outputs'] = merged
super(ModelMGPU, self).__init__(*args, **kwargs)
def build_model(baseline_cnn=False):
#Based on kernel https://www.kaggle.com/devm2024/keras-model-for-beginners-0-210-on-lb-eda-r-d
image_input = Input(shape=(75, 75, 3), name='images')
angle_input = Input(shape=[1], name='angle')
activation = 'elu'
bn_momentum = 0.99
# Simple CNN as baseline model
if baseline_cnn:
model = Sequential()
model.add(
Conv2D(16,
kernel_size=(3, 3),
activation='relu',
input_shape=(75, 75, 3)))
model.add(BatchNormalization(momentum=bn_momentum))
model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2)))
model.add(Dropout(0.2))
model.add(Conv2D(32, kernel_size=(3, 3), activation='relu'))
model.add(BatchNormalization(momentum=bn_momentum))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Dropout(0.2))
model.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))
model.add(BatchNormalization(momentum=bn_momentum))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Dropout(0.2))
model.add(Conv2D(128, kernel_size=(3, 3), activation='relu'))
model.add(BatchNormalization(momentum=bn_momentum))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Dropout(0.2))
model.add(Flatten())
model.add(Dense(256, activation='relu'))
model.add(BatchNormalization(momentum=bn_momentum))
model.add(Dropout(0.3))
model.add(Dense(128, activation='relu'))
model.add(BatchNormalization(momentum=bn_momentum))
model.add(Dropout(0.3))
model.add(Dense(1, activation='sigmoid'))
opt = Adam(lr=1e-3, beta_1=.9, beta_2=.999, decay=1e-3)
model.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=['accuracy'])
model.summary()
else:
img_1 = Conv2D(
32, kernel_size=(3, 3), activation=activation, padding='same')(
(BatchNormalization(momentum=bn_momentum))(image_input))
img_1 = MaxPooling2D((2, 2))(img_1)
img_1 = Dropout(0.2)(img_1)
img_1 = Conv2D(64,
kernel_size=(3, 3),
activation=activation,
padding='same')(
(BatchNormalization(momentum=bn_momentum))(img_1))
img_1 = MaxPooling2D((2, 2))(img_1)
img_1 = Dropout(0.2)(img_1)
# Residual block
img_2 = Conv2D(128,
kernel_size=(3, 3),
activation=activation,
padding='same')(
(BatchNormalization(momentum=bn_momentum))(img_1))
img_2 = Dropout(0.2)(img_2)
img_2 = Conv2D(64,
kernel_size=(3, 3),
activation=activation,
padding='same')(
(BatchNormalization(momentum=bn_momentum))(img_2))
img_2 = Dropout(0.2)(img_2)
img_res_1 = add([img_1, img_2])
#Residual block
img_3 = Conv2D(
128, kernel_size=(3, 3), activation=activation, padding='same')(
(BatchNormalization(momentum=bn_momentum))(img_res_1))
img_3 = Dropout(0.2)(img_3)
img_3 = Conv2D(64,
kernel_size=(3, 3),
activation=activation,
padding='same')(
(BatchNormalization(momentum=bn_momentum))(img_3))
img_3 = Dropout(0.2)(img_3)
img_res_2 = add([img_res_1, img_3])
# Filter resudial output
img_res_2 = Conv2D(128, kernel_size=(2, 2), activation=activation)(
(BatchNormalization(momentum=bn_momentum))(img_res_2))
img_res_2 = MaxPooling2D((2, 2))(img_res_2)
img_res_2 = Dropout(0.2)(img_res_2)
img_res_2 = GlobalAveragePooling2D()(img_res_2)
cnn_out = (Concatenate()(
[img_res_2,
BatchNormalization(momentum=bn_momentum)(angle_input)]))
dense_layer = Dropout(0.5)(BatchNormalization(momentum=bn_momentum)(
PReLU()(Dense(256, activation=None)(cnn_out))))
dense_layer = Dropout(0.5)(BatchNormalization(momentum=bn_momentum)(
PReLU()(Dense(64, activation=None)(dense_layer))))
output = Dense(1, activation='sigmoid')(dense_layer)
model = Model([image_input, angle_input], output)
opt = Adam(lr=1e-3, beta_1=.9, beta_2=.999, decay=1e-3)
model.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=['accuracy'])
model.summary()
return model
def _unet_from_tensor(tensor, filt, kern, acti, output_nb_feats=1):
"""
UNet used in LOUPE
TODO: this is quite rigid and poorly written right now
as well as hardcoded (# layers, features, etc)
- use for loops and such crazy things
- use a richer library for this, perhaps neuron
"""
# start first convolution of UNet
conv1 = Conv2D(filt,
kern,
activation=acti,
padding='same',
name='conv_1_1')(tensor)
conv1 = LeakyReLU()(conv1)
conv1 = BatchNormalization(name='batch_norm_1_1')(conv1)
conv1 = Conv2D(filt,
kern,
activation=acti,
padding='same',
name='conv_1_2')(conv1)
conv1 = LeakyReLU()(conv1)
conv1 = BatchNormalization(name='batch_norm_1_2')(conv1)
pool1 = AveragePooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(filt * 2,
kern,
activation=acti,
padding='same',
name='conv_2_1')(pool1)
conv2 = LeakyReLU()(conv2)
conv2 = BatchNormalization(name='batch_norm_2_1')(conv2)
conv2 = Conv2D(filt * 2,
kern,
activation=acti,
padding='same',
name='conv_2_2')(conv2)
conv2 = LeakyReLU()(conv2)
conv2 = BatchNormalization(name='batch_norm_2_2')(conv2)
pool2 = AveragePooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(filt * 4,
kern,
activation=acti,
padding='same',
name='conv_3_1')(pool2)
conv3 = LeakyReLU()(conv3)
conv3 = BatchNormalization(name='batch_norm_3_1')(conv3)
conv3 = Conv2D(filt * 4,
kern,
activation=acti,
padding='same',
name='conv_3_2')(conv3)
conv3 = LeakyReLU()(conv3)
conv3 = BatchNormalization(name='batch_norm_3_2')(conv3)
pool3 = AveragePooling2D(pool_size=(2, 2))(conv3)
conv4 = Conv2D(filt * 8,
kern,
activation=acti,
padding='same',
name='conv_4_1')(pool3)
conv4 = LeakyReLU()(conv4)
conv4 = BatchNormalization(name='batch_norm_4_1')(conv4)
conv4 = Conv2D(filt * 8,
kern,
activation=acti,
padding='same',
name='conv_4_2')(conv4)
conv4 = LeakyReLU()(conv4)
conv4 = BatchNormalization(name='batch_norm_4_2')(conv4)
pool4 = AveragePooling2D(pool_size=(2, 2))(conv4)
conv5 = Conv2D(filt * 16,
kern,
activation=acti,
padding='same',
name='conv_5_1')(pool4)
conv5 = LeakyReLU()(conv5)
conv5 = BatchNormalization(name='batch_norm_5_1')(conv5)
conv5 = Conv2D(filt * 16,
kern,
activation=acti,
padding='same',
name='conv_5_2')(conv5)
conv5 = LeakyReLU()(conv5)
conv5 = BatchNormalization(name='batch_norm_5_2')(conv5)
sub1 = UpSampling2D(size=(2, 2))(conv5)
concat1 = Concatenate(axis=-1)([conv4, sub1])
conv6 = Conv2D(filt * 8,
kern,
activation=acti,
padding='same',
name='conv_6_1')(concat1)
conv6 = LeakyReLU()(conv6)
conv6 = BatchNormalization(name='batch_norm_6_1')(conv6)
conv6 = Conv2D(filt * 8,
kern,
activation=acti,
padding='same',
name='conv_6_2')(conv6)
conv6 = LeakyReLU()(conv6)
conv6 = BatchNormalization(name='batch_norm_6_2')(conv6)
sub2 = UpSampling2D(size=(2, 2))(conv6)
concat2 = Concatenate(axis=-1)([conv3, sub2])
conv7 = Conv2D(filt * 4,
kern,
activation=acti,
padding='same',
name='conv_7_1')(concat2)
conv7 = LeakyReLU()(conv7)
conv7 = BatchNormalization(name='batch_norm_7_1')(conv7)
conv7 = Conv2D(filt * 4,
kern,
activation=acti,
padding='same',
name='conv_7_2')(conv7)
conv7 = LeakyReLU()(conv7)
conv7 = BatchNormalization(name='batch_norm_7_2')(conv7)
sub3 = UpSampling2D(size=(2, 2))(conv7)
concat3 = Concatenate(axis=-1)([conv2, sub3])
conv8 = Conv2D(filt * 2,
kern,
activation=acti,
padding='same',
name='conv_8_1')(concat3)
conv8 = LeakyReLU()(conv8)
conv8 = BatchNormalization(name='batch_norm_8_1')(conv8)
conv8 = Conv2D(filt * 2,
kern,
activation=acti,
padding='same',
name='conv_8_2')(conv8)
conv8 = LeakyReLU()(conv8)
conv8 = BatchNormalization(name='batch_norm_8_2')(conv8)
sub4 = UpSampling2D(size=(2, 2))(conv8)
concat4 = Concatenate(axis=-1)([conv1, sub4])
conv9 = Conv2D(filt,
kern,
activation=acti,
padding='same',
name='conv_9_1')(concat4)
conv9 = LeakyReLU()(conv9)
conv9 = BatchNormalization(name='batch_norm_9_1')(conv9)
conv9 = Conv2D(filt,
kern,
activation=acti,
padding='same',
name='conv_9_2')(conv9)
conv9 = LeakyReLU()(conv9)
conv9 = BatchNormalization(name='batch_norm_9_2')(conv9)
conv9 = Conv2D(output_nb_feats, 1, padding='same', name='conv_9_3')(conv9)
return conv9
print(x_sam.shape) #(504,5) print(y_sam.shape) #(504,) x_hit = hite[5:510, :] print(x_hit.shape) x_sam = x_sam.reshape(504, 5, 1) x_hit = x_hit.reshape(504, 5, 1) #2.모델 구성 input1 = Input(shape=(5, 1)) x1 = LSTM(100)(input1) x1 = Dense(10000)(x1) input2 = Input(shape=(5, 1)) x2 = LSTM(100)(input2) x2 = Dense(10000)(x2) merge = Concatenate(axis=-1)([x1, x2]) output1 = Dense(10000)(merge) output2 = Dense(1)(output1) model = Model(inputs=[input1, input2], outputs=output2) model.summary() #3.컴파일. 훈련 model.compile(optimizer='adam', loss='mse', metrics=['mse']) model.fit([x_sam, x_hit], y_sam, epochs=5)
def heartnet(load_path,
activation_function='relu',
bn_momentum=0.99,
bias=False,
dropout_rate=0.5,
dropout_rate_dense=0.0,
eps=1.1e-5,
kernel_size=5,
l2_reg=0.0,
l2_reg_dense=0.0,
lr=0.0012843784,
lr_decay=0.0001132885,
maxnorm=10000.,
padding='valid',
random_seed=1,
subsam=2,
num_filt=(8, 4),
num_dense=20,
FIR_train=False,
trainable=True,
type=1,
num_class=2,
num_class_domain=1,
hp_lambda=0,
batch_size=1024,
optim='SGD',
segments='0101'):
#num_dense = 20 default
input = Input(shape=(2500, 1))
coeff_path = '../data/filterbankcoeff60.mat'
coeff = tables.open_file(coeff_path)
b1 = coeff.root.b1[:]
b1 = np.hstack(b1)
b1 = np.reshape(b1, [b1.shape[0], 1, 1])
b2 = coeff.root.b2[:]
b2 = np.hstack(b2)
b2 = np.reshape(b2, [b2.shape[0], 1, 1])
b3 = coeff.root.b3[:]
b3 = np.hstack(b3)
b3 = np.reshape(b3, [b3.shape[0], 1, 1])
b4 = coeff.root.b4[:]
b4 = np.hstack(b4)
b4 = np.reshape(b4, [b4.shape[0], 1, 1])
## Conv1D_linearphase
# input1 = Conv1D_linearphase(1 ,61, use_bias=False,
# # kernel_initializer=initializers.he_normal(random_seed),
# weights=[b1[30:]],
# padding='same',trainable=FIR_train)(input)
# input2 = Conv1D_linearphase(1, 61, use_bias=False,
# # kernel_initializer=initializers.he_normal(random_seed),
# weights=[b2[30:]],
# padding='same',trainable=FIR_train)(input)
# input3 = Conv1D_linearphase(1, 61, use_bias=False,
# # kernel_initializer=initializers.he_normal(random_seed),
# weights=[b3[30:]],
# padding='same',trainable=FIR_train)(input)
# input4 = Conv1D_linearphase(1, 61, use_bias=False,
# # kernel_initializer=initializers.he_normal(random_seed),
# weights=[b4[30:]],
# padding='same',trainable=FIR_train)(input)
## Conv1D_linearphase Anti-Symmetric
#
input1 = Conv1D_linearphaseType(
1,
60,
use_bias=False,
# kernel_initializer=initializers.he_normal(random_seed),
weights=[b1[31:]],
padding='same',
trainable=FIR_train,
type=type)(input)
input2 = Conv1D_linearphaseType(
1,
60,
use_bias=False,
# kernel_initializer=initializers.he_normal(random_seed),
weights=[b2[31:]],
padding='same',
trainable=FIR_train,
type=type)(input)
input3 = Conv1D_linearphaseType(
1,
60,
use_bias=False,
# kernel_initializer=initializers.he_normal(random_seed),
weights=[b3[31:]],
padding='same',
trainable=FIR_train,
type=type)(input)
input4 = Conv1D_linearphaseType(
1,
60,
use_bias=False,
# kernel_initializer=initializers.he_normal(random_seed),
weights=[b4[31:]],
padding='same',
trainable=FIR_train,
type=type)(input)
t1 = branch(input1, num_filt, kernel_size, random_seed, padding, bias,
maxnorm, l2_reg, eps, bn_momentum, activation_function,
dropout_rate, subsam, trainable)
t2 = branch(input2, num_filt, kernel_size, random_seed, padding, bias,
maxnorm, l2_reg, eps, bn_momentum, activation_function,
dropout_rate, subsam, trainable)
t3 = branch(input3, num_filt, kernel_size, random_seed, padding, bias,
maxnorm, l2_reg, eps, bn_momentum, activation_function,
dropout_rate, subsam, trainable)
t4 = branch(input4, num_filt, kernel_size, random_seed, padding, bias,
maxnorm, l2_reg, eps, bn_momentum, activation_function,
dropout_rate, subsam, trainable)
#Conv1D_gammatone
# input1 = Conv1D_gammatone(kernel_size=81,filters=1,fsHz=1000,use_bias=False,padding='same')(input)
# input2 = Conv1D_gammatone(kernel_size=81,filters=1,fsHz=1000,use_bias=False,padding='same')(input)
# input3 = Conv1D_gammatone(kernel_size=81,filters=1,fsHz=1000,use_bias=False,padding='same')(input)
# input4 = Conv1D_gammatone(kernel_size=81,filters=1,fsHz=1000,use_bias=False,padding='same')(input)
xx = Concatenate(axis=-1)([t1, t2, t3, t4])
xx = res_block(xx, 64, kernel_size, 2, 'same', random_seed, bias, maxnorm,
l2_reg, eps, bn_momentum, activation_function, dropout_rate,
subsam, trainable)
xx = res_block(xx, 64, kernel_size, 1, 'same', random_seed, bias, maxnorm,
l2_reg, eps, bn_momentum, activation_function, dropout_rate,
subsam, trainable)
xx = res_block(xx, 128, kernel_size, 3, 'same', random_seed, bias, maxnorm,
l2_reg, eps, bn_momentum, activation_function, dropout_rate,
subsam, trainable)
xx = res_block(xx, 128, kernel_size, 1, 'same', random_seed, bias, maxnorm,
l2_reg, eps, bn_momentum, activation_function, dropout_rate,
subsam, trainable)
xx = res_block(xx,
128,
kernel_size,
2,
'same',
random_seed,
bias,
maxnorm,
l2_reg,
eps,
bn_momentum,
activation_function,
dropout_rate,
subsam,
trainable,
cat=False)
xx = res_block(xx,
128,
kernel_size,
2,
'same',
random_seed,
bias,
maxnorm,
l2_reg,
eps,
bn_momentum,
activation_function,
dropout_rate,
subsam,
trainable,
cat=False)
xx = Conv1D(128,
kernel_size=kernel_size,
kernel_initializer=initializers.he_normal(seed=random_seed),
padding=padding,
strides=2,
use_bias=bias,
kernel_constraint=max_norm(maxnorm),
trainable=trainable,
kernel_regularizer=l2(l2_reg))(xx)
merged = Flatten()(xx)
dann_in = GradientReversal(hp_lambda=hp_lambda, name='grl')(merged)
dsc = Dense(50,
activation=activation_function,
kernel_initializer=initializers.he_normal(seed=random_seed),
use_bias=bias,
kernel_constraint=max_norm(maxnorm),
kernel_regularizer=l2(l2_reg_dense),
name='domain_dense')(dann_in)
dsc = Dense(num_class_domain, activation='softmax', name="domain")(dsc)
merged = Dense(num_dense,
activation=activation_function,
kernel_initializer=initializers.he_normal(seed=random_seed),
use_bias=bias,
kernel_constraint=max_norm(maxnorm),
kernel_regularizer=l2(l2_reg_dense),
name='class_dense')(merged)
merged = Dense(num_class, activation='softmax', name="class")(merged)
model = Model(inputs=input, outputs=[merged, dsc])
if load_path:
model.load_weights(filepath=load_path, by_name=False)
#if load_path: # If path for loading model was specified
#model.load_weights(filepath='../../models_dbt_dann/fold_a_gt 2019-09-09 16:53:52.063276/weights.0041-0.6907.hdf5', by_name=True)
# models/fold_a_gt 2019-09-04 17:36:52.860817/weights.0200-0.7135.hdf5
if optim == 'Adam':
opt = Adam(lr=lr, decay=lr_decay)
else:
opt = SGD(lr=lr, decay=lr_decay)
if (num_class_domain > 1):
domain_loss_function = 'categorical_crossentropy'
else:
domain_loss_function = 'binary_crossentropy'
model.compile(optimizer=opt,
loss={
'class': 'categorical_crossentropy',
'domain': domain_loss_function
},
metrics=['accuracy'])
return model
def get_global_net(self):
embed_net = self.base_net()
# inputs
inputs = []
for i_input in range(self.num_patches):
input_name = 'input_{0}'.format(i_input + 1)
inputs.append(
Input((self.num_chns, self.patch_size, self.patch_size,
self.patch_size),
name=input_name))
#+++++++++++++++++++++++++++++#
## patch-level processing #
#+++++++++++++++++++++++++++++#
patch_features_list, patch_probs_list = [], []
for i_input in range(self.num_patches):
feature_map, class_prob = embed_net(inputs[i_input])
patch_features_list.append(feature_map)
patch_probs_list.append(class_prob)
patch_outputs = Concatenate(name='patch_outputs',
axis=1)(patch_probs_list)
#+++++++++++++++++++++++++++++++#
## region-level processing ##
#+++++++++++++++++++++++++++++++#
region_features_list, region_probs_list = [], []
for i_region in range(self.num_regions):
nn_idxs = self.nn_mat[i_region][0][0].tolist()
nn_features, nn_probs = [], []
num_neighbors = len(nn_idxs)
for i_neighbor in nn_idxs:
nn_features.append(patch_features_list[i_neighbor])
nn_probs.append(patch_probs_list[i_neighbor])
region_input_features = Concatenate(axis=1)(nn_features)
region_input_probs = Concatenate(axis=1)(nn_probs)
region_input_probs = Reshape([num_neighbors,
1])(region_input_probs)
in_name = 'region_input_{0}'.format(i_region + 1)
region_input = Concatenate(name=in_name, axis=-1)(
[region_input_probs, region_input_features])
conv_name = 'region_conv_{0}'.format(i_region + 1)
region_feature = Conv1D(filters=self.num_region_features,
kernel_size=num_neighbors,
kernel_regularizer=StructuredSparse(
self.region_sparse),
padding='valid',
name=conv_name)(region_input)
if self.with_bn:
region_feature = BatchNormalization(axis=-1)(region_feature)
region_feature = Activation('relu')(region_feature)
if self.with_dropout:
region_feature = Dropout(self.drop_prob)(region_feature)
ot_name = 'region_prob_{0}'.format(i_region + 1)
region_prob = Dense(units=1, activation='sigmoid',
name=ot_name)(Flatten()(region_feature))
region_features_list.append(region_feature)
region_probs_list.append(region_prob)
region_outputs = Concatenate(name='region_outputs',
axis=1)(region_probs_list)
region_features = Concatenate(name='region_features',
axis=1)(region_features_list)
region_probs = Reshape([self.num_regions, 1],
name='region_probs')(region_outputs)
region_feat_prob = Concatenate(name='region_features_probs', axis=-1)(
[region_probs, region_features])
#+++++++++++++++++++++++++++++++#
## subject-level processing #
#+++++++++++++++++++++++++++++++#
subject_feature = Conv1D(filters=self.num_subject_features,
kernel_size=self.num_regions,
kernel_regularizer=StructuredSparse(
self.subject_sparse),
padding='valid',
name='subject_conv')(region_feat_prob)
if self.with_bn:
subject_feature = BatchNormalization(axis=-1)(subject_feature)
subject_feature = Activation('relu')(subject_feature)
if self.with_dropout:
subject_feature = Dropout(self.drop_prob)(subject_feature)
# subject-level units
subject_units = Flatten(name='subject_level_units')(subject_feature)
#+++++++++++#
# MODEL #
#+++++++++++#
subject_outputs = Dense(units=1,
activation='sigmoid',
name='subject_outputs')(subject_units)
outputs = [patch_outputs, region_outputs, subject_outputs]
model = Model(inputs=inputs, outputs=outputs)
return model
