def create_nn_model_v2(max_size, embedding_matrix, word_vector_dim, drop_rate,
my_optimizer, shape_dist):
'''
Create a Siamese neural network based on pretrained word2vec embeddings and tokens and all the other features
'''
# we leave the 2nd argument of shape blank because the Embedding layer cannot accept an input_shape argument
input_1 = Input(shape=(max_size, ))
input_2 = Input(shape=(max_size, ))
embedding_1 = Embedding(
input_dim=embedding_matrix.
shape[0], # vocab size, including the 0-th word used for padding
output_dim=word_vector_dim,
weights=[embedding_matrix], # we pass our pre-trained embeddings
input_length=max_size,
trainable=True,
)(input_1)
embedding_2 = Embedding(
input_dim=embedding_matrix.
shape[0], # vocab size, including the 0-th word used for padding
output_dim=word_vector_dim,
weights=[embedding_matrix], # we pass our pre-trained embeddings
input_length=max_size,
trainable=True,
)(input_2)
embedding_dropped_1 = Dropout(drop_rate)(embedding_1)
embedding_dropped_2 = Dropout(drop_rate)(embedding_2)
### Defining CNN
conv1 = Conv1D(
filters=100,
kernel_size=3,
activation='relu',
)
conv2 = Conv1D(
filters=128,
kernel_size=4,
activation='relu',
)
#conv3 = Conv1D(filters = 128,
# kernel_size = 5,
# activation = 'relu',
# )
## First conv layer
conv1_1 = conv1(embedding_dropped_1)
pooled_conv1_1 = GlobalMaxPooling1D()(conv1_1)
pooled_conv1_dropped_1 = Dropout(drop_rate)(pooled_conv1_1)
conv1_2 = conv1(embedding_dropped_2)
pooled_conv1_2 = GlobalMaxPooling1D()(conv1_2)
pooled_conv1_dropped_2 = Dropout(drop_rate)(pooled_conv1_2)
## Second conv layer
conv2_1 = conv2(embedding_dropped_1)
pooled_conv2_1 = GlobalMaxPooling1D()(conv2_1)
pooled_conv2_dropped_1 = Dropout(drop_rate)(pooled_conv2_1)
conv2_2 = conv2(embedding_dropped_2)
pooled_conv2_2 = GlobalMaxPooling1D()(conv2_2)
pooled_conv2_dropped_2 = Dropout(drop_rate)(pooled_conv2_2)
merged_1 = keras.layers.concatenate(
[pooled_conv1_dropped_1, pooled_conv2_dropped_1])
merged_2 = keras.layers.concatenate(
[pooled_conv1_dropped_2, pooled_conv2_dropped_2])
diff = Subtract()([merged_1, merged_2])
mul = Multiply()([merged_1, merged_2])
maxi = Maximum()([merged_1, merged_2])
#dense_1 = Dense(20,activation='relu')(merged_1)
#dense_2 = Dense(20,activation='relu')(merged_2)
distance_input = Input(shape=(shape_dist, ))
distance_dense = BatchNormalization()(distance_input)
distance_dense = Dense(128, activation='relu')(distance_dense)
merge = keras.layers.concatenate([diff, mul, maxi, distance_dense],
axis=-1)
#prob = Dense(units = 1, # dimensionality of the output space
# activation = 'sigmoid'#,
# ) (merge)
prob = Dropout(0.2)(merge)
prob = BatchNormalization()(prob)
prob = Dense(300, activation='relu')(prob)
prob = Dropout(0.2)(prob)
prob = BatchNormalization()(prob)
prob = Dense(1, activation='sigmoid')(prob)
model = Model([input_1, input_2, distance_input], prob)
model.compile(loss='binary_crossentropy',
optimizer=my_optimizer,
metrics=['accuracy'])
return model
include_top=False,
input_shape=(224, 224, 3))
EfficientNetB0.load_weights(
'../input/efficientnet-keras-weights-b0b5/efficientnet-b0_imagenet_1000_notop.h5'
)
inputs = Input(shape=(224, 224, 3))
x = EfficientNetB0(inputs)
residual = Conv2D(2048, (1, 1), strides=(2, 2), padding='same')(x)
x = SeparableConv2D(2048, (3, 3),
padding='same',
strides=1,
depth_multiplier=1,
depthwise_regularizer=l2(1e-15),
pointwise_regularizer=l2(1e-15))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = SeparableConv2D(2048, (3, 3),
padding='same',
strides=1,
depth_multiplier=1,
depthwise_regularizer=l2(1e-15),
pointwise_regularizer=l2(1e-15))(x)
x = BatchNormalization()(x)
x = MaxPooling2D((3, 3), strides=(2, 2), padding="same")(x)
x = Add()([x, residual])
residual1 = Conv2D(2304, (1, 1), strides=(2, 2), padding='same')(x)
x = SeparableConv2D(2304, (3, 3),
padding='same',
strides=1,
depth_multiplier=1,
valid_data,
color_mode='grayscale',
target_size=(img_rows, img_colms),
batch_size=size,
class_mode='categorical',
shuffle=True)
model = Sequential()
#block 1
model.add(
Conv2D(32, (3, 3),
padding='same',
kernel_initializer='he_normal',
input_shape=(img_rows, img_colms, 1)))
model.add(Activation('elu'))
model.add(BatchNormalization())
model.add(
Conv2D(32, (3, 3),
padding='same',
kernel_initializer='he_normal',
input_shape=(img_rows, img_colms, 1)))
model.add(Activation('elu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=2 * 2))
model.add(Dropout(0.2))
#block2
model.add(
Conv2D(64, (3, 3),
padding='same',
kernel_initializer='he_normal',
def ResNet50(include_top=True, weights='imagenet',
input_tensor=None,trainable=[True]*4,changePool=False):
'''Instantiate the ResNet50 architecture,
optionally loading weights pre-trained
on ImageNet. Note that when using TensorFlow,
for best performance you should set
`image_dim_ordering="tf"` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The dimension ordering
convention used by the model is the one
specified in your Keras config file.
# Arguments
include_top: whether to include the 3 fully-connected
layers at the top of the network.
weights: one of `None` (random initialization)
or "imagenet" (pre-training on ImageNet).
input_tensor: optional Keras tensor (i.e. xput of `layers.Input()`)
to use as image input for the model.
# Returns
A Keras model instance.
'''
t1,t2,t3,t4=trainable
if weights not in {'imagenet', None}:
raise ValueError('The `weights` argument should be either '
'`None` (random initialization) or `imagenet` '
'(pre-training on ImageNet).')
# Determine proper input shape
if K.image_dim_ordering() == 'th':
if include_top:
input_shape = (3, 224, 224)
else:
input_shape = (3, None, None)
else:
if include_top:
input_shape = (224, 224, 3)
else:
input_shape = (None, None, 3)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if K.image_dim_ordering() == 'tf':
bn_axis = 3
else:
bn_axis = 1
x = ZeroPadding2D((3, 3))(img_input)
x = Convolution2D(64, 7, 7, subsample=(2, 2), name='conv1')(x)
x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x = Activation('relu')(x)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1),trainable=t1)
x = identity_block(x, 3, [64, 64, 256], stage=2, block='b',trainable=t1)
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c',trainable=t1)
x = conv_block(x, 3, [128, 128, 512], stage=3, block='a',trainable=t2)
x = identity_block(x, 3, [128, 128, 512], stage=3, block='b',trainable=t2)
x = identity_block(x, 3, [128, 128, 512], stage=3, block='c',trainable=t2)
x = identity_block(x, 3, [128, 128, 512], stage=3, block='d',trainable=t2)
x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a',trainable=t3)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b',trainable=t3)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c',trainable=t3)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d',trainable=t3)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e',trainable=t3)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f',trainable=t3)
x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a',trainable=t4)
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b',trainable=t4)
xlast = identity_block(x, 3, [512, 512, 2048], stage=5, block='c',trainable=t4)
if changePool=="max":
x = MaxPooling2D((7, 7), name='avg_pool')(xlast)
else:
x = AveragePooling2D((7, 7), name='avg_pool')(xlast)
if include_top:
xflat = Flatten()(x)
x = Dense(1000, activation='softmax', name='fc1000')(xflat)
model = Model(img_input, x)
# load weights
model.load_weights(WEIGHTS_PATH)
if K.backend() == 'theano':
convert_all_kernels_in_model(model)
if changePool== "none":
xflatNone = Flatten()(xlast)
modelout = Model(img_input, xflatNone, name='resnet50')
else:
modelout = Model(img_input, xflat, name='resnet50')
return modelout
def rn_encoder(input_size=(224, 224, 3),
include_top=False,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000):
input_shape = input_size
"""Instantiates the ResNet50 architecture.
Optionally loads weights pre-trained
on ImageNet. Note that when using TensorFlow,
for best performance you should set
`image_data_format="channels_last"` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The data format
convention used by the model is the one
specified in your Keras config file.
# Arguments
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization)
or "imagenet" (pre-training on ImageNet).
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(224, 224, 3)` (with `channels_last` data format)
or `(3, 224, 244)` (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 197.
E.g. `(200, 200, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if weights not in {'imagenet', None}:
raise ValueError('The `weights` argument should be either '
'`None` (random initialization) or `imagenet` '
'(pre-training on ImageNet).')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_size=224,
min_size=197,
data_format=K.image_data_format(),
require_flatten=include_top)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if K.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
x = ZeroPadding2D((3, 3))(img_input)
x = Conv2D(64, (7, 7), strides=(2, 2), name='conv1')(x)
x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x = Activation('relu')(x)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='b')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='d')
x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f')
"""
x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')
"""
x = AveragePooling2D((2, 2), name='avg_poolX')(x)
if include_top:
x = Flatten()(x)
x = Dense(classes, activation='softmax', name='fc1000')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='resnet50')
# load weights
if weights == 'imagenet':
if include_top:
weights_path = get_file(
'resnet50_weights_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH,
cache_subdir='models',
md5_hash='a7b3fe01876f51b976af0dea6bc144eb')
else:
weights_path = get_file(
'resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
md5_hash='a268eb855778b3df3c7506639542a6af')
model.load_weights(weights_path, by_name=True)
print("BBBBBBBBBBBBBBBB")
print(len(model.layers))
print(model.layers)
for l in model.layers[:-1]:
l.trainable = False
print("AAAAAAAAAAAAAAAA")
if K.backend() == 'theano':
layer_utils.convert_all_kernels_in_model(model)
if K.image_data_format() == 'channels_first':
if include_top:
maxpool = model.get_layer(name='avg_pool')
shape = maxpool.output_shape[1:]
dense = model.get_layer(name='fc1000')
layer_utils.convert_dense_weights_data_format(
dense, shape, 'channels_first')
if K.backend() == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image data format convention '
'(`image_data_format="channels_first"`). '
'For best performance, set '
'`image_data_format="channels_last"` in '
'your Keras config '
'at ~/.keras/keras.json.')
return model
def build_Model(input_shape, n_states=2, n_speaker=40):
'''
architecture:
1. 4 layers CNN (dilated convolution)
2. multi head attention(head num: 2)
3. FC
'''
# Input layer
inputs = Input(name='the_input', shape=input_shape, dtype='float32')
# Convolution layer (VGG)
inner = Conv2D(32, (3, 3),
padding='same',
name='conv1',
dilation_rate=2,
kernel_initializer='he_normal')(inputs)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = MaxPooling2D(pool_size=(2, 2), name='max1')(inner)
inner = Conv2D(64, (3, 3),
padding='same',
name='conv2',
dilation_rate=2,
kernel_initializer='he_normal')(inner)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = MaxPooling2D(pool_size=(2, 2), name='max2')(inner)
inner = Conv2D(256, (3, 3),
padding='same',
name='conv3',
dilation_rate=2,
kernel_initializer='he_normal')(inner)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
inner = Conv2D(256, (3, 3),
padding='same',
name='conv4',
dilation_rate=2,
kernel_initializer='he_normal')(inner)
inner = BatchNormalization()(inner)
inner = Activation('relu')(inner)
# CNN reshape
inner = Reshape(target_shape=((125, 2560)), name='reshape')(inner)
inner = Dense(256,
activation='relu',
kernel_initializer='he_normal',
name='dense1')(inner)
# Multi-head attention layer
inner = MultiHeadAttention(head_num=2, name='Multi-Head')(inner)
inner = Lambda(lambda xin: K.sum(xin, axis=1))(inner)
# Gradient Reversal Layer
Flip = GradientReversal(hp_lambda=0.31)
dann_in = Flip(inner)
dann_out = Dense(units=n_speaker,
activation='softmax',
name='gradient_reversal')(dann_in)
# transforms RNN output to character activations:
predictions = Dense(units=n_states,
activation='softmax',
name='output_layer')(inner) # (None, 3)
model = Model(inputs=inputs, outputs=[predictions, dann_out])
adam = optimizers.Adam(lr=0.00001)
model.compile(optimizer=adam,
loss={
'output_layer': 'categorical_crossentropy',
'gradient_reversal': 'categorical_crossentropy'
},
loss_weights={
'output_layer': 0.997,
'gradient_reversal': 0.003
},
metrics=['accuracy'])
model.summary()
return model
def SENET50(include_top=True, weights='vggface',
input_tensor=None, input_shape=None,
pooling=None,
classes=8631):
input_shape = _obtain_input_shape(input_shape,
default_size=224,
min_size=197,
data_format=K.image_data_format(),
require_flatten=include_top,
weights=weights)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if K.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
bn_eps = 0.0001
x = Conv2D(
64, (7, 7), use_bias=False, strides=(2, 2), padding='same',
name='conv1/7x7_s2')(img_input)
x = BatchNormalization(axis=bn_axis, name='conv1/7x7_s2/bn',epsilon=bn_eps)(x)
x = Activation('relu')(x)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = senet_conv_block(x, 3, [64, 64, 256], stage=2, block=1, strides=(1, 1))
x = senet_identity_block(x, 3, [64, 64, 256], stage=2, block=2)
x = senet_identity_block(x, 3, [64, 64, 256], stage=2, block=3)
x = senet_conv_block(x, 3, [128, 128, 512], stage=3, block=1)
x = senet_identity_block(x, 3, [128, 128, 512], stage=3, block=2)
x = senet_identity_block(x, 3, [128, 128, 512], stage=3, block=3)
x = senet_identity_block(x, 3, [128, 128, 512], stage=3, block=4)
x = senet_conv_block(x, 3, [256, 256, 1024], stage=4, block=1)
x = senet_identity_block(x, 3, [256, 256, 1024], stage=4, block=2)
x = senet_identity_block(x, 3, [256, 256, 1024], stage=4, block=3)
x = senet_identity_block(x, 3, [256, 256, 1024], stage=4, block=4)
x = senet_identity_block(x, 3, [256, 256, 1024], stage=4, block=5)
x = senet_identity_block(x, 3, [256, 256, 1024], stage=4, block=6)
x = senet_conv_block(x, 3, [512, 512, 2048], stage=5, block=1)
x = senet_identity_block(x, 3, [512, 512, 2048], stage=5, block=2)
x = senet_identity_block(x, 3, [512, 512, 2048], stage=5, block=3)
x = AveragePooling2D((7, 7), name='avg_pool')(x)
if include_top:
x = Flatten()(x)
x = Dense(classes, activation='softmax', name='classifier')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='vggface_senet50')
# load weights
if weights == 'vggface':
if include_top:
weights_path = get_file('rcmalli_vggface_tf_senet50.h5',
utils.SENET50_WEIGHTS_PATH,
cache_subdir=utils.VGGFACE_DIR)
else:
weights_path = get_file('rcmalli_vggface_tf_notop_senet50.h5',
utils.SENET50_WEIGHTS_PATH_NO_TOP,
cache_subdir=utils.VGGFACE_DIR)
model.load_weights(weights_path)
if K.backend() == 'theano':
layer_utils.convert_all_kernels_in_model(model)
if include_top:
maxpool = model.get_layer(name='avg_pool')
shape = maxpool.output_shape[1:]
dense = model.get_layer(name='classifier')
layer_utils.convert_dense_weights_data_format(dense, shape,
'channels_first')
if K.image_data_format() == 'channels_first' and K.backend() == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image data format convention '
'(`image_data_format="channels_first"`). '
'For best performance, set '
'`image_data_format="channels_last"` in '
'your Keras config '
'at ~/.keras/keras.json.')
elif weights is not None:
model.load_weights(weights)
return model
def discriminator(input_shape_fundus=(512, 512, 3),
input_shape_angio=(512, 512, 1),
ndf=32,
n_layers=3,
activation='tanh',
n_downsampling=1,
name='Discriminator'):
X_input_fundus = Input(shape=input_shape_fundus, name="input_fundus")
X_input_angio = Input(shape=input_shape_angio, name="input_angio")
features = []
X = Concatenate(axis=-1, name="concat")([X_input_fundus, X_input_angio])
for i in range(n_downsampling):
X = AveragePooling2D((3, 3), strides=(2, 2), padding='same')(X)
X = Conv2D(ndf,
kernel_size=(4, 4),
strides=(2, 2),
padding='same',
kernel_initializer=RandomNormal(stddev=0.02))(X)
X = LeakyReLU(alpha=0.2)(X)
features.append(X)
X = SeparableConv2D(ndf,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
kernel_initializer=RandomNormal(stddev=0.02))(X)
X = BatchNormalization()(X)
X = LeakyReLU(alpha=0.2)(X)
features.append(X)
for i in range(1, n_layers):
down_filters = min(ndf * 2, 512)
X = Conv2D(down_filters,
kernel_size=(4, 4),
strides=(2, 2),
padding='same',
kernel_initializer=RandomNormal(stddev=0.02))(X)
X = BatchNormalization()(X)
X = LeakyReLU(alpha=0.2)(X)
X_skip = X
X = SeparableConv2D(down_filters,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
kernel_initializer=RandomNormal(stddev=0.02))(X)
X = BatchNormalization()(X)
X = LeakyReLU(alpha=0.2)(X)
features.append(X)
X = Add(name=name + "/add_" + str(i))([X, X_skip])
X = Conv2D(1,
kernel_size=(4, 4),
strides=(1, 1),
padding='same',
kernel_initializer=RandomNormal(stddev=0.02))(X)
X = Activation(activation)(X)
model = Model(inputs=[X_input_fundus, X_input_angio],
outputs=[X] + features,
name=name)
model.summary()
loss = ['mse', None, None, None, None]
if n_layers == 2:
loss = loss[:-1]
elif n_layers == 1:
loss = loss[:len(loss) - 2]
model.compile(loss=loss,
optimizer=Adam(lr=0.0002, beta_1=0.5, beta_2=0.999))
return model
def get_model(x_tr, y_tr, x_val, y_val):
inp = Input(shape=(x_tr.shape[1], ))
# x = Dense(2048, input_dim=X.shape[1], activation='elu')(inp)
# x = Dropout(0.5)(x)
# x = BatchNormalization()(x)
# x = Dense(1024, activation='elu')(x)
x = Dense(1024, input_dim=X.shape[1], activation='elu')(inp)
x = Dropout(0.5)(x)
x = BatchNormalization()(x)
x = Dense(512, activation='elu')(x)
x = Dropout(0.5)(x)
x = BatchNormalization()(x)
x = Dense(256, activation='elu')(x)
x = Dropout(0.5)(x)
x = BatchNormalization()(x)
if classify_type < 128:
x = Dense(256, activation='elu')(x)
x = Dropout(0.5)(x)
x = BatchNormalization()(x)
# if classify_type < 64:
# x = Dense(64, activation='elu')(x)
# x = Dropout(0.5)(x)
# x = BatchNormalization()(x)
out = Dense(classify_type, activation='softmax')(x)
model = Model(inp, out)
optadam = Adam(lr=0.001)
model.compile(optimizer=optadam,
loss='categorical_crossentropy',
metrics=[])
es = EarlyStopping(monitor='CRPS_score_val',
mode='min',
restore_best_weights=True,
verbose=False,
patience=80)
mc = ModelCheckpoint('best_model.h5',
monitor='CRPS_score_val',
mode='min',
save_best_only=True,
verbose=False,
save_weights_only=True)
bsz = 1024
steps = x_tr.shape[0] / bsz
model.fit(x_tr,
y_tr,
callbacks=[CRPSCallback(validation=(x_val, y_val)), es, mc],
epochs=100,
batch_size=bsz,
verbose=False)
model.load_weights("best_model.h5")
y_pred = model.predict(x_val)
y_valid = y_val
y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
val_s = (
(y_true - y_pred)**2).sum(axis=1).sum(axis=0) / (199 * x_val.shape[0])
crps = np.round(val_s, 8)
gc.collect()
return model, crps
def Xception(model_input):
## Entry flow
x = conv2d_bn(model_input, 16, (3, 3))
# x = conv2d_bn(x, 16, (3, 3), strides=2)
x = conv2d_bn(x, 32, (3, 3))
for fliters in [64, 128, 256]:
if fliters in [64, 128]:
residual = conv2d_bn(x,
fliters, (1, 1),
strides=2,
activation=None)
else:
residual = conv2d_bn(x, fliters, (1, 1), activation=None)
x = Activation(activation='relu')(x)
x = sepconv2d_bn(x, fliters, (3, 3))
x = sepconv2d_bn(x, fliters, (3, 3), activation=None)
if fliters == 64:
x1 = sepconv2d_bn(x, fliters, (3, 3), activation=None)
x = sepconv2d_bn(x, fliters, (3, 3), activation=None)
x = MaxPooling2D((3, 3), padding='same', strides=2)(x)
elif fliters == 128:
x2 = sepconv2d_bn(x, fliters, (3, 3), activation=None)
x = sepconv2d_bn(x, fliters, (3, 3), activation=None)
x = MaxPooling2D((3, 3), padding='same', strides=2)(x)
else:
x3 = sepconv2d_bn(x, fliters, (3, 3), activation=None)
x = sepconv2d_bn(x, fliters, (3, 3), activation=None)
x = Add()([x, residual])
## Middle flow
for i in range(8):
residual = x
x = sepconv2d_bn(x, 256, (3, 3))
x = sepconv2d_bn(x, 256, (3, 3))
x = sepconv2d_bn(x, 256, (3, 3), activation=None)
x = Add()([x, residual])
## Exit flow
residual = conv2d_bn(x, 384, (1, 1), activation=None)
x = Activation(activation='relu')(x)
x = sepconv2d_bn(x, 256, (3, 3))
x = sepconv2d_bn(x, 384, (3, 3), activation=None)
x4 = sepconv2d_bn(x, 384, (3, 3), activation=None)
x = Add()([x, residual])
x = sepconv2d_bn(x, 512, (3, 3))
x = sepconv2d_bn(x, 640, (3, 3))
deconv3 = Conv2DTranspose(384, (3, 3), strides=(2, 2), padding="same")(x)
x = concatenate([deconv3, x2])
x = Dropout(0.25)(x)
x = Conv2D(384, (3, 3), activation="relu", padding="same")(x)
x = BatchNormalization()(x)
deconv2 = Conv2DTranspose(128, (3, 3), strides=(2, 2), padding="same")(x)
x = concatenate([deconv2, x1])
x = Dropout(0.25)(x)
x = Conv2D(128, (3, 3), activation="relu", padding="same")(x)
x = BatchNormalization()(x)
output_layer = Conv2D(1, (1, 1), padding="same", activation='relu')(x)
model = Model(model_input, output_layer)
return model
def fine_generator(x_coarse_shape=(256, 256, 64),
input_shape=(512, 512, 3),
nff=64,
n_blocks=3,
n_coarse_gen=1,
n_channels=1):
X_input = Input(shape=input_shape, name="input")
X_coarse = Input(shape=x_coarse_shape, name="x_input")
print("X_coarse", X_coarse.shape)
for i in range(1, n_coarse_gen + 1):
# Downsampling layers
down_filters = nff * (2**(n_coarse_gen - i))
X = ReflectionPadding2D((3, 3), name="rf_" + str(i))(X_input)
X = Conv2D(down_filters,
kernel_size=(7, 7),
strides=(1, 1),
padding='valid',
kernel_initializer=RandomNormal(stddev=0.02),
name="conv_" + str(i))(X)
X = BatchNormalization(name="in_" + str(i))(X)
X_pre_down = LeakyReLU(alpha=0.2, name="leakyRelu_" + str(i))(X)
X_down1 = encoder_block(X, down_filters, i - 1)
# Connection from Coarse Generator
X = Add(name="add_X_coarse")([X_coarse, X_down1])
X = SeparableConv2D(down_filters * 2,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
kernel_initializer=RandomNormal(stddev=0.02),
name="sepconv_" + str(i))(X)
X = BatchNormalization(name="sep_in_" + str(i))(X)
X = LeakyReLU(alpha=0.2, name="sep_leakyRelu_" + str(i))(X)
for j in range(n_blocks - 1):
res_filters = nff * (2**(n_coarse_gen - i)) * 2
X = novel_residual_block(X,
res_filters,
base="block_" + str(j + 1))
# Upsampling layers
up_filters = nff * (2**(n_coarse_gen - i))
X_up1 = decoder_block(X, up_filters, i - 1)
X_up1_att = Attention(X_pre_down, up_filters, i - 1)
X_up1_add = Add(name="skip_" + str(i))([X_up1_att, X_up1])
X = ReflectionPadding2D((3, 3), name="final/rf")(X_up1_add)
X = Conv2D(n_channels,
kernel_size=(7, 7),
strides=(1, 1),
padding='valid',
name="final/conv")(X)
X = Activation('tanh', name="tanh")(X)
model = Model(inputs=[X_input, X_coarse], outputs=X, name='G_Fine')
model.compile(loss='mse',
optimizer=Adam(lr=0.0002, beta_1=0.5, beta_2=0.999))
model.summary()
return model
#print(label_index.shape)
y_train = y_train.values
#print(y_train.shape)
y_train = np.array(y_train)
#print('y_train is:', y_train.shape)
del labels, fnames
gc.collect()
from keras import optimizers, losses, activations, models
from keras.layers import Convolution2D, Dense, Input, Flatten, Dropout, MaxPooling2D, BatchNormalization
from sklearn.model_selection import train_test_split
input_shape = (99, 161, 1)
nclass = 12
inp = Input(shape=input_shape)
norm_inp = BatchNormalization()(inp)
img_1 = Convolution2D(16, kernel_size=2, activation=activations.relu)(norm_inp)
img_1 = Convolution2D(16, kernel_size=2, activation=activations.relu)(img_1)
img_1 = MaxPooling2D(pool_size=(2, 2))(img_1)
img_1 = Dropout(rate=0.2)(img_1)
img_1 = Convolution2D(32, kernel_size=3, activation=activations.relu)(img_1)
img_1 = Convolution2D(32, kernel_size=3, activation=activations.relu)(img_1)
img_1 = MaxPooling2D(pool_size=(2, 2))(img_1)
img_1 = Dropout(rate=0.2)(img_1)
img_1 = Convolution2D(64, kernel_size=3, activation=activations.relu)(img_1)
img_1 = MaxPooling2D(pool_size=(2, 2))(img_1)
img_1 = Dropout(rate=0.2)(img_1)
img_1 = Flatten()(img_1)
dense_1 = BatchNormalization()(Dense(256, activation=activations.relu)(img_1))
dense_1 = BatchNormalization()(Dense(256,
def build_stage2_discriminator():
"""
Create Stage-II discriminator network
"""
input_layer = Input(shape=(256, 256, 3))
x = Conv2D(64, (4, 4),
padding='same',
strides=2,
input_shape=(256, 256, 3),
use_bias=False)(input_layer)
x = LeakyReLU(alpha=0.2)(x)
x = Conv2D(128, (4, 4), padding='same', strides=2, use_bias=False)(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.2)(x)
x = Conv2D(256, (4, 4), padding='same', strides=2, use_bias=False)(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.2)(x)
x = Conv2D(512, (4, 4), padding='same', strides=2, use_bias=False)(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.2)(x)
x = Conv2D(1024, (4, 4), padding='same', strides=2, use_bias=False)(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.2)(x)
x = Conv2D(2048, (4, 4), padding='same', strides=2, use_bias=False)(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.2)(x)
x = Conv2D(1024, (1, 1), padding='same', strides=1, use_bias=False)(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.2)(x)
x = Conv2D(512, (1, 1), padding='same', strides=1, use_bias=False)(x)
x = BatchNormalization()(x)
x2 = Conv2D(128, (1, 1), padding='same', strides=1, use_bias=False)(x)
x2 = BatchNormalization()(x2)
x2 = LeakyReLU(alpha=0.2)(x2)
x2 = Conv2D(128, (3, 3), padding='same', strides=1, use_bias=False)(x2)
x2 = BatchNormalization()(x2)
x2 = LeakyReLU(alpha=0.2)(x2)
x2 = Conv2D(512, (3, 3), padding='same', strides=1, use_bias=False)(x2)
x2 = BatchNormalization()(x2)
added_x = add([x, x2])
added_x = LeakyReLU(alpha=0.2)(added_x)
input_layer2 = Input(shape=(4, 4, 128))
merged_input = concatenate([added_x, input_layer2])
x3 = Conv2D(64 * 8, kernel_size=1, padding="same", strides=1)(merged_input)
x3 = BatchNormalization()(x3)
x3 = LeakyReLU(alpha=0.2)(x3)
x3 = Flatten()(x3)
x3 = Dense(1)(x3)
x3 = Activation('sigmoid')(x3)
stage2_dis = Model(inputs=[input_layer, input_layer2], outputs=[x3])
return stage2_dis
def build_stage2_generator():
"""
Create Stage-II generator containing the CA Augmentation Network,
the image encoder and the generator network
"""
# 1. CA Augmentation Network
input_layer = Input(shape=(1024, ))
input_lr_images = Input(shape=(64, 64, 3))
ca = Dense(256)(input_layer)
mean_logsigma = LeakyReLU(alpha=0.2)(ca)
c = Lambda(generate_c)(mean_logsigma)
# 2. Image Encoder
x = ZeroPadding2D(padding=(1, 1))(input_lr_images)
x = Conv2D(128, kernel_size=(3, 3), strides=1, use_bias=False)(x)
x = ReLU()(x)
x = ZeroPadding2D(padding=(1, 1))(x)
x = Conv2D(256, kernel_size=(4, 4), strides=2, use_bias=False)(x)
x = BatchNormalization()(x)
x = ReLU()(x)
x = ZeroPadding2D(padding=(1, 1))(x)
x = Conv2D(512, kernel_size=(4, 4), strides=2, use_bias=False)(x)
x = BatchNormalization()(x)
x = ReLU()(x)
# 3. Joint
c_code = Lambda(joint_block)([c, x])
x = ZeroPadding2D(padding=(1, 1))(c_code)
x = Conv2D(512, kernel_size=(3, 3), strides=1, use_bias=False)(x)
x = BatchNormalization()(x)
x = ReLU()(x)
# 4. Residual blocks
x = residual_block(x)
x = residual_block(x)
x = residual_block(x)
x = residual_block(x)
# 5. Upsampling blocks
x = UpSampling2D(size=(2, 2))(x)
x = Conv2D(512, kernel_size=3, padding="same", strides=1,
use_bias=False)(x)
x = BatchNormalization()(x)
x = ReLU()(x)
x = UpSampling2D(size=(2, 2))(x)
x = Conv2D(256, kernel_size=3, padding="same", strides=1,
use_bias=False)(x)
x = BatchNormalization()(x)
x = ReLU()(x)
x = UpSampling2D(size=(2, 2))(x)
x = Conv2D(128, kernel_size=3, padding="same", strides=1,
use_bias=False)(x)
x = BatchNormalization()(x)
x = ReLU()(x)
x = UpSampling2D(size=(2, 2))(x)
x = Conv2D(64, kernel_size=3, padding="same", strides=1, use_bias=False)(x)
x = BatchNormalization()(x)
x = ReLU()(x)
x = Conv2D(3, kernel_size=3, padding="same", strides=1, use_bias=False)(x)
x = Activation('tanh')(x)
model = Model(inputs=[input_layer, input_lr_images],
outputs=[x, mean_logsigma])
return model
def unet_model_3d(input_shape,
pool_size=(2, 2, 2),
n_labels=n_labels,
deconvolution=False,
depth=3,
n_base_filters=32,
include_label_wise_dice_coefficients=False,
batch_normalization=True,
activation_name="sigmoid"):
"""
Builds the 3D UNet Keras model.f
:param metrics: List metrics to be calculated during model training (default is dice coefficient).
:param include_label_wise_dice_coefficients: If True and n_labels is greater than 1, model will report the dice
coefficient for each label as metric.
:param n_base_filters: The number of filters that the first layer in the convolution network will have. Following
layers will contain a multiple of this number. Lowering this number will likely reduce the amount of memory required
to train the model.
:param depth: indicates the depth of the U-shape for the model. The greater the depth, the more max pooling
layers will be added to the model. Lowering the depth may reduce the amount of memory required for training.
:param input_shape: Shape of the input data (n_chanels, x_size, y_size, z_size). The x, y, and z sizes must be
divisible by the pool size to the power of the depth of the UNet, that is pool_size^depth.
:param pool_size: Pool size for the max pooling operations.
:param n_labels: Number of binary labels that the model is learning.
:param initial_learning_rate: Initial learning rate for the model. This will be decayed during training.
:param deconvolution: If set to True, will use transpose convolution(deconvolution) instead of up-sampling. This
increases the amount memory required during training.
:return: Untrained 3D UNet Model
"""
inputs = Input(input_shape)
# set image specifics
kernel = (3, 3, 3) # kernel size
s = 2 # stride
#img_height, img_width = img_size[0], img_size[1]
padding = 'same'
conv1 = create_convolution_block(
input_layer=inputs,
n_filters=n_base_filters * 4, # 256 * 256 *8
batch_normalization=batch_normalization)
conv1 = create_convolution_block(input_layer=conv1,
n_filters=n_base_filters * 4,
batch_normalization=batch_normalization)
pool1 = Conv3D(n_base_filters, (3, 3, 3),
padding=padding,
strides=(2, 2, 1))(conv1)
conv2 = create_convolution_block(
input_layer=pool1,
n_filters=n_base_filters * 4, # 128 * 128 *8
batch_normalization=batch_normalization)
conv2 = create_convolution_block(input_layer=conv2,
n_filters=n_base_filters * 4,
batch_normalization=batch_normalization)
pool2 = Conv3D(n_base_filters * 2, (3, 3, 3),
padding=padding,
strides=(1, 1, 2))(conv2)
conv3 = create_convolution_block(
input_layer=pool2,
n_filters=n_base_filters * 2, # 128 * 128 *4
batch_normalization=batch_normalization)
conv3 = create_convolution_block(input_layer=conv3,
n_filters=n_base_filters,
batch_normalization=batch_normalization)
pool3 = Conv3D(n_base_filters * 4, (3, 3, 3),
padding=padding,
strides=(2, 2, 1))(conv3)
conv4 = create_convolution_block(
input_layer=pool3,
n_filters=n_base_filters * 2, # 64 * 64 *4
batch_normalization=batch_normalization)
conv4 = create_convolution_block(input_layer=conv4,
n_filters=n_base_filters,
batch_normalization=batch_normalization)
pool4 = Conv3D(n_base_filters * 2, (3, 3, 3),
padding=padding,
strides=(2, 2, 1))(conv4)
conv5 = create_convolution_block(
input_layer=pool4,
n_filters=n_base_filters, # 32 * 32 *4
batch_normalization=batch_normalization)
conv5 = create_convolution_block(input_layer=conv5,
n_filters=n_base_filters,
batch_normalization=batch_normalization)
pool5 = Conv3D(n_base_filters * 4, (3, 3, 3),
padding=padding,
strides=(2, 2, 2))(conv5)
conv6 = create_convolution_block(
input_layer=pool5,
n_filters=n_base_filters * 2, # 16 * 16 *2
batch_normalization=batch_normalization)
conv6 = create_convolution_block(input_layer=conv6,
n_filters=n_base_filters * 2,
batch_normalization=batch_normalization)
convskip0 = Conv3D(n_base_filters * 8, (1, 1, 1),
padding=padding,
strides=(1, 1, 1))(inputs)
convskip0 = BatchNormalization(axis=-1)(convskip0)
convskip0 = Activation('relu')(convskip0)
convskip0 = Conv3D(n_base_filters * 4, (2, 2, 1),
padding=padding,
strides=(1, 1, 1))(convskip0)
convskip0 = BatchNormalization(axis=-1)(convskip0)
convskip0 = Activation('relu')(convskip0)
convskip0 = Conv3D(n_base_filters * 4, (2, 2, 1),
padding=padding,
strides=(1, 1, 1))(convskip0)
convskip0 = BatchNormalization(axis=-1)(convskip0)
convskip0 = Activation('relu')(convskip0)
convskip0 = Conv3D(n_base_filters * 4, (1, 1, 2),
padding=padding,
strides=(1, 1, 1))(convskip0)
convskip0 = BatchNormalization(axis=-1)(convskip0)
convskip0 = Activation('relu')(convskip0)
convskip1 = Conv3D(n_base_filters * 4, (1, 1, 1),
padding=padding,
strides=(1, 1, 1))(conv1)
convskip1 = BatchNormalization(axis=-1)(convskip1)
convskip1 = Activation('relu')(convskip1)
convskip1 = Conv3D(n_base_filters * 4, (1, 1, 2),
padding=padding,
strides=(1, 1, 1))(convskip1)
convskip1 = BatchNormalization(axis=-1)(convskip1)
convskip1 = Activation('relu')(convskip1)
convskip1 = Conv3D(n_base_filters * 4, (1, 1, 1),
padding=padding,
strides=(1, 1, 1))(convskip1)
convskip1 = BatchNormalization(axis=-1)(convskip1)
convskip1 = Activation('relu')(convskip1)
convskip1 = Conv3D(n_base_filters * 4, (1, 1, 2),
padding=padding,
strides=(1, 1, 1))(convskip1)
convskip1 = BatchNormalization(axis=-1)(convskip1)
convskip1 = Activation('relu')(convskip1)
convskip2 = Conv3D(n_base_filters * 2, (1, 1, 1),
padding=padding,
strides=(1, 1, 1))(conv2)
convskip2 = BatchNormalization(axis=-1)(convskip2)
convskip2 = Activation('relu')(convskip2)
convskip2 = Conv3D(n_base_filters * 2, (1, 1, 2),
padding=padding,
strides=(1, 1, 1))(convskip2)
convskip2 = BatchNormalization(axis=-1)(convskip2)
convskip2 = Activation('relu')(convskip2)
convskip2 = Conv3D(n_base_filters * 2, (1, 1, 1),
padding=padding,
strides=(1, 1, 1))(convskip2)
convskip2 = BatchNormalization(axis=-1)(convskip2)
convskip2 = Activation('relu')(convskip2)
convskip2 = Conv3D(n_base_filters * 2, (1, 1, 2),
padding=padding,
strides=(1, 1, 1))(convskip2)
convskip2 = BatchNormalization(axis=-1)(convskip2)
convskip2 = Activation('relu')(convskip2)
up_convolution7 = UpSampling3D(size=(2, 2, 2))(conv6) # 32 * 32 *4
concat7 = concatenate([up_convolution7, conv5], axis=-1)
conv7 = create_convolution_block(input_layer=concat7,
n_filters=n_base_filters * 2,
batch_normalization=batch_normalization)
conv7 = create_convolution_block(input_layer=conv7,
n_filters=n_base_filters * 2,
batch_normalization=batch_normalization)
up_convolution8 = UpSampling3D(size=(2, 2, 1))(conv7) # 64 * 64 *4
concat8 = concatenate([up_convolution8, conv4], axis=-1)
conv8 = create_convolution_block(input_layer=concat8,
n_filters=n_base_filters,
batch_normalization=batch_normalization)
conv8 = create_convolution_block(input_layer=conv8,
n_filters=n_base_filters,
batch_normalization=batch_normalization)
up_convolution8 = UpSampling3D(size=(2, 2, 1))(conv8) # 128 * 128 *4
concat9 = concatenate([up_convolution8, conv3], axis=-1)
conv9 = create_convolution_block(input_layer=concat9,
n_filters=n_base_filters,
batch_normalization=batch_normalization)
conv9 = create_convolution_block(input_layer=conv9,
n_filters=n_base_filters,
batch_normalization=batch_normalization)
up_convolution10 = UpSampling3D(size=(1, 1, 2))(conv9) # 128 * 128 *8
concat10 = concatenate([up_convolution10, conv2, convskip2], axis=-1)
conv10 = create_convolution_block(input_layer=concat10,
n_filters=n_base_filters,
batch_normalization=batch_normalization)
conv10 = create_convolution_block(input_layer=conv10,
n_filters=n_base_filters,
batch_normalization=batch_normalization)
up_convolution11 = UpSampling3D(size=(2, 2, 1))(conv10) # 256*256 *8
concat11 = concatenate([up_convolution11, conv1, convskip0], axis=-1)
conv11 = create_convolution_block(input_layer=concat11,
n_filters=n_base_filters * 4,
batch_normalization=batch_normalization)
conv11 = create_convolution_block(input_layer=conv11,
n_filters=n_base_filters * 4,
batch_normalization=batch_normalization)
concat12 = concatenate([conv11, convskip1], axis=-1)
concat12 = create_convolution_block(
input_layer=conv11,
n_filters=n_base_filters * 4,
batch_normalization=batch_normalization)
concat12 = create_convolution_block(
input_layer=concat12,
n_filters=n_base_filters * 4,
batch_normalization=batch_normalization)
final_convolution = Conv3D(n_labels, (1, 1, 1))(conv11)
act = Activation(activation_name)(final_convolution)
model = Model(inputs=inputs, outputs=act)
return model
dopt = Adam(lr=1e-3) #Discriminator optimizer
# Build Generative model ...
g_input = Input(shape=(input_shape,))
x = g_input
for i in range(len(args.gen_layers_shape)):
x = Dense(args.gen_layers_shape[i],
init=args.init,
activation=args.gen_activation,
bias=args.bias)(x)
print("dense[" + str(i) + "] -> (" + str(args.gen_layers_shape[i]) + ")")
if args.dropout:
x = Dropout(args.drop_rate)(x)
if args.batch_norm:
x = BatchNormalization(mode=1)(x)
g_V = Dense(input_shape, activation='sigmoid')(x)
generator = Model(g_input, g_V)
generator.compile(loss='binary_crossentropy', optimizer=opt)
generator.summary()
# Build Discriminative model ...
d_input = Input(shape=(input_shape,))
z = d_input
for i in range(len(args.disc_layers_shape)):
z = Dense(args.disc_layers_shape[i],
init=args.init,
activation=args.disc_activation,
bias=args.bias)(z)
print("dense[" + str(i) + "] -> (" + str(args.disc_layers_shape[i]) + ")")
if args.dropout:
def finalNetwork(batch_size, initial_filter_value, time): # INITIAL FACE CHANNEL input_x = Input(shape=(time,256,256,3)) x = Conv3D(filters=initial_filter_value, kernel_size=3, padding='same', activation='relu')(input_x) x = BatchNormalization()(x) x = MaxPooling3D(pool_size=(2,2,2))(x) x = Dropout(0.2)(x) x = Conv3D(filters=initial_filter_value, kernel_size=3, padding='same', activation='relu')(x) x = BatchNormalization()(x) x = MaxPooling3D(pool_size=(4,4,4))(x) x = Dropout(0.2)(x) x = Conv3D(filters=initial_filter_value*2, kernel_size=3, padding='same', activation='relu')(x) x = BatchNormalization()(x) # END INITIAL FACE CHANNEL # INITIAL AUDIO CHANNEL input_y = Input(shape=(256,256,3)) y = Conv2D(filters=initial_filter_value, kernel_size=3, padding='same', activation='relu')(input_y) # END INITIAL AUDIO CHANNEL # CREATE CHANNEL FOR IMAGE # cross_channel_x = Flatten()(x) # cross_channel_x = Dense((25*256*256*8)//128)(cross_channel_x) # END CHANNEL # CREATE CHANNEL FOR AUDIO # cross_channel_y = Flatten()(y) # cross_channel_y = Dense((25*256*256*8)//128)(cross_channel_y) # END CHANNEL FOR AUDIO #CHANNEL MERGE # merged_channel = concatenate([cross_channel_x, cross_channel_y]) # merged_channel = Reshape((2,25,2048,2), input_shape=(1,204800))(merged_channel) # merged_channel = Conv3D(filters=8, kernel_size=5, padding='same')(merged_channel) # merge_input_x = Flatten()(merged_channel) # merge_input_x = Dense(2048)(merge_input_x) # merged_channel_output_x = Reshape((1,16,16,8), input_shape=(1,2048))(merge_input_x) # merge_input_x = None # merge_input_y = Flatten()(merged_channel) # merge_input_y = Dense(256*256*3)(merge_input_y) # merged_channel_output_y = Reshape((256,256,3), input_shape=(1,256*256*3))(merge_input_y) # merge_input_y = None # merged_channel = None # END CHANNEL MERGE # START END OF VISUAL # x = concatenate([x, merged_channel_output_x]) # merged_channel_output_x = None x = Flatten()(x) x = Dense(100, activation='relu')(x) # x = Dropout(0.5)(x) #x = Dense(150, activation='relu')(x) #x = Dropout(0.5)(x) visual_channel = Dense(time, activation='linear')(x) # FINISH VISUAL # START END OF AUDIO # y = concatenate([y, merged_channel_output_y]) # merged_channel_output_y = None y = Conv2D(filters=initial_filter_value*2, kernel_size=3, padding='same', activation='relu')(y) y = Flatten()(y) y = Dense(100, activation='relu')(y) # y = Dropout(0.5)(y) audio_channel = Dense(time, activation='linear')(y) # FINISH AUDIO model = Model(inputs=[input_x,input_y], outputs=[visual_channel,audio_channel]) model.summary() return multi_gpu_model(model)
def build_model(max_features,
continue_cols,
K=8,
solver='adam',
l2=0.0,
l2_fm=0.0,
is_self=False):
np.random.seed(2018)
inputs = []
flatten_layers = []
columns = range(len(max_features))
###------second order term-------###
for c in columns:
#print (c,max_features[c])
inputs_c = Input(shape=(1, ), dtype='int32', name='input_%s' % (c))
num_c = max_features[c]
inputs.append(inputs_c)
#print (num_c,K,c)
embed_c = Embedding(num_c,
K,
input_length=1,
name='embed_%s' % (c),
W_regularizer=l2_reg(l2_fm))(inputs_c)
#print (embed_c.get_shape(),'---')
#flatten_c = Flatten()(embed_c)
flatten_c = Reshape((K, ))(embed_c)
flatten_layers.append(flatten_c)
inputs_dict = []
continue_cols_columns = range(len(continue_cols))
for col in continue_cols_columns:
#print (col,continue_cols[col])
inputs_c = Input(shape=(1, ),
dtype='float',
name='input_sec_%s' % (col))
inputs.append(inputs_c)
inputs_c = BatchNormalization(name='BN_%s' % (col))(inputs_c)
inputs_dict.append(inputs_c)
inputs_cK = MyLayer(output_dim=K)(inputs_c)
flatten_layers.append(inputs_cK) #### F * None * K
summed_features_emb = add(flatten_layers) #### None * K
summed_features_emb_square = multiply(
[summed_features_emb, summed_features_emb]) ##### None * K
squared_features_emb = []
for layer in flatten_layers:
squared_features_emb.append(multiply([layer, layer]))
squared_sum_features_emb = add(squared_features_emb) ###### None * K
subtract_layer = Lambda(lambda inputs: inputs[0] - inputs[1],
output_shape=lambda shapes: shapes[0])
y_second_order = subtract_layer(
[summed_features_emb_square, squared_sum_features_emb])
y_second_order = Lambda(lambda x: x * 0.5)(y_second_order)
y_second_order = Dropout(0.9, seed=2018)(y_second_order)
###----first order------######
fm_layers = []
for c in columns:
num_c = max_features[c]
embed_c = Embedding(num_c,
1,
input_length=1,
name='linear_%s' % (c),
W_regularizer=l2_reg(l2))(inputs[c])
flatten_c = Flatten()(embed_c)
fm_layers.append(flatten_c)
for col in continue_cols_columns:
inputs_c = MyLayer(output_dim=1)(inputs_dict[col])
#layer.build(inputs_c.get_shape().as_list())
#inputs_c = RepeatVector(K)(inputs_c)
#inputs_c = layer.call(inputs_c)
fm_layers.append(inputs_c) #####---- None * 1
y_first_order = add(fm_layers)
y_first_order = BatchNormalization()(y_first_order)
y_first_order = Dropout(0.8, seed=2018)(y_first_order)
##deep
y_deep = concatenate(flatten_layers) ##### None * (F*K)
y_deep = Dense(32)(y_deep)
y_deep = Activation('relu', name='output_1')(y_deep)
y_deep = Dropout(rate=0.5, seed=2012)(y_deep)
y_deep = Dense(32)(y_deep)
y_deep = Activation('relu', name='output_2')(y_deep)
y_deep = Dropout(rate=0.5, seed=2012)(y_deep)
concat_input = concatenate([y_first_order, y_second_order, y_deep], axis=1)
# concat_input=Dense(16)(concat_input)
# concat_input = Activation('relu',name='concat')(concat_input)
# #y_deep = Dropout(rate=0.5,seed=2012)(y_deep)
# concat_input = Dropout(rate=0.5,seed=2012)(concat_input)
outputs = Dense(1, activation='sigmoid', name='main_output')(concat_input)
model = Model(inputs=inputs, outputs=outputs, name='model')
solver = Adam(lr=0.01, decay=0.1)
if (is_self == True):
model.compile(optimizer=solver,
loss=binary_crossentropy_with_ranking,
metrics=[auc, log_loss])
else:
model.compile(optimizer=solver,
loss='binary_crossentropy',
metrics=[auc, log_loss])
#model.fit(X,y,batch_size=batch_size,validation_data=(vali_X,vali_y),epochs=epochs)
return model
def set_model(train_target): # 0:x,y, 1:m, 2:v
activation = 'elu'
padding = 'same'
model = Sequential()
nf = 32
fs = (3,1)
model.add(Conv2D(nf,fs, padding=padding, activation=activation,input_shape=(375,5,1)))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 1)))
model.add(Conv2D(nf*2,fs, padding=padding, activation=activation))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 1)))
model.add(Conv2D(nf*4,fs, padding=padding, activation=activation))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 1)))
model.add(Conv2D(nf*8,fs, padding=padding, activation=activation))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 1)))
model.add(Conv2D(nf*16,fs, padding=padding, activation=activation))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 1)))
model.add(Conv2D(nf*32,fs, padding=padding, activation=activation))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 1)))
model.add(Flatten())
model.add(Dense(256, activation ='elu'))
model.add(Dense(128, activation ='elu'))
model.add(Dense(64, activation ='elu'))
# model.add(Dense(32, activation ='elu'))
model.add(Dense(32, activation ='elu'))
model.add(Dense(8, activation ='elu'))
model.add(Dense(4))
optimizer = keras.optimizers.Adam()
global weight2
if train_target == 1: # only for M
weight2 = np.array([0,0,1,0])
else: # only for V
weight2 = np.array([0,0,0,1])
if train_target==0:
model.compile(loss=my_loss_E1,
optimizer=optimizer,
)
else:
model.compile(loss=my_loss_E2,
optimizer=optimizer,
)
model.summary()
return model
def _depthwise_conv_block(inputs,
pointwise_conv_filters,
alpha,
depth_multiplier=1,
strides=(1, 1),
block_id=1):
"""Adds a depthwise convolution block.
A depthwise convolution block consists of a depthwise conv,
batch normalization, relu6, pointwise convolution,
batch normalization and relu6 activation.
# Arguments
inputs: Input tensor of shape `(rows, cols, channels)`
(with `channels_last` data format) or
(channels, rows, cols) (with `channels_first` data format).
pointwise_conv_filters: Integer, the dimensionality of the output space
(i.e. the number output of filters in the pointwise convolution).
alpha: controls the width of the network.
- If `alpha` < 1.0, proportionally decreases the number
of filters in each layer.
- If `alpha` > 1.0, proportionally increases the number
of filters in each layer.
- If `alpha` = 1, default number of filters from the paper
are used at each layer.
depth_multiplier: The number of depthwise convolution output channels
for each input channel.
The total number of depthwise convolution output
channels will be equal to `filters_in * depth_multiplier`.
strides: An integer or tuple/list of 2 integers,
specifying the strides of the convolution along the width and height.
Can be a single integer to specify the same value for
all spatial dimensions.
Specifying any stride value != 1 is incompatible with specifying
any `dilation_rate` value != 1.
block_id: Integer, a unique identification designating the block number.
# Input shape
4D tensor with shape:
`(batch, channels, rows, cols)` if data_format='channels_first'
or 4D tensor with shape:
`(batch, rows, cols, channels)` if data_format='channels_last'.
# Output shape
4D tensor with shape:
`(batch, filters, new_rows, new_cols)` if data_format='channels_first'
or 4D tensor with shape:
`(batch, new_rows, new_cols, filters)` if data_format='channels_last'.
`rows` and `cols` values might have changed due to stride.
# Returns
Output tensor of block.
"""
channel_axis = 1 if K.image_data_format() == 'channels_first' else -1
pointwise_conv_filters = int(pointwise_conv_filters * alpha)
x = DepthwiseConv2D((3, 3),
padding='same',
depth_multiplier=depth_multiplier,
strides=strides,
use_bias=False,
name='conv_dw_%d' % block_id)(inputs)
x = BatchNormalization(axis=channel_axis,
name='conv_dw_%d_bn' % block_id)(x)
x = Activation(relu6, name='conv_dw_%d_relu' % block_id)(x)
x = Conv2D(pointwise_conv_filters, (1, 1),
padding='same',
use_bias=False,
strides=(1, 1),
name='conv_pw_%d' % block_id)(x)
x = BatchNormalization(axis=channel_axis,
name='conv_pw_%d_bn' % block_id)(x)
return Activation(relu6, name='conv_pw_%d_relu' % block_id)(x)
for i in range(train_data_size):
idx = np.where(class_category == train_edge_class[i])[0]
y_train[i, idx] = 1.
# 4. Model
# Lexical level feature
lexical_inp = Input(shape = [word2vec_dim*6])
# Word level feature
word_position_inp = Input(shape = (100, word2vec_dim*3+2, 1))
conv1 = Conv2D(filters = 256, kernel_size = (5, word2vec_dim*3+2), strides = (1, 1),
padding = 'same', activation = 'relu')(word_position_inp)
pool1 = MaxPooling2D(pool_size = (2, 2))(conv1)
bn1 = BatchNormalization()(pool1)
conv2 = Conv2D(filters = 256, kernel_size = (3, 1), strides = (1, 1),
padding = 'same', activation = 'relu')(bn1)
pool2 = MaxPooling2D(pool_size = (2, 2))(conv2)
drop1 = Dropout(0.2)(pool2)
conv3 = Conv2D(filters = 128, kernel_size = (3, 1), strides = (1, 1),
padding = 'same', activation = 'relu')(drop1)
pool3 = MaxPooling2D(pool_size = (2, 2))(conv3)
bn3 = BatchNormalization()(pool3)
flatten = Flatten()(bn3)
word_position_feat_embed = Dense(units = 256, activation = 'softmax')(flatten)
X_train = x_train[train_index]
Y_train = y_train[train_index]
X_valid = x_train[test_index]
Y_valid = y_train[test_index]
num_fold += 1
print('Start KFold number {} from {}'.format(num_fold, nfolds))
print('Split train: ', len(X_train), len(Y_train))
print('Split valid: ', len(X_valid), len(Y_valid))
kfold_weights_path = os.path.join('',
'weights_kfold_' + str(num_fold) + '.h5')
model = Sequential()
model.add(BatchNormalization(input_shape=(64, 64, 3)))
model.add(
Conv2D(32,
kernel_size=(3, 3),
strides=(1, 1),
data_format='channels_last',
padding='same',
activation='relu'))
model.add(Conv2D(32, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(
Conv2D(64,
kernel_size=(3, 3),
strides=(1, 1),
sc = StandardScaler()
x_train = sc.fit_transform(x_train)
len_x = int(x_train.shape[1])
print("len_x is:", len_x)
# Neural Network
print("\nSetting up neural network model...")
nn = Sequential()
nn.add(Dense(units=400, kernel_initializer='normal', input_dim=len_x))
nn.add(PReLU())
nn.add(Dropout(.4))
nn.add(Dense(units=160, kernel_initializer='normal'))
nn.add(PReLU())
nn.add(BatchNormalization())
nn.add(Dropout(.63))
nn.add(Dense(units=64, kernel_initializer='normal'))
nn.add(PReLU())
nn.add(BatchNormalization())
nn.add(Dropout(.45))
nn.add(Dense(units=28, kernel_initializer='normal'))
nn.add(PReLU())
nn.add(BatchNormalization())
nn.add(Dropout(.5))
nn.add(Dense(1, kernel_initializer='normal'))
nn.compile(loss='mae', optimizer=Adam(lr=4e-3, decay=1e-4))
print("\nFitting neural network model...")
nn.fit(np.array(x_train),
np.array(y_train),
def create_posenet(weights_path=None, tune=False):
# creates Posenet from GoogLeNet a.k.a. Inception v1 (Szegedy, 2015)
with tf.device('/cpu:0'):
input = Input(shape=(224, 224, 3))
conv1 = Convolution2D(64,
7,
7,
subsample=(2, 2),
border_mode='same',
activation='relu',
name='conv1')(input)
pool1 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='same',
name='pool1')(conv1)
norm1 = BatchNormalization(axis=3, name='norm1')(pool1)
reduction2 = Convolution2D(64,
1,
1,
border_mode='same',
activation='relu',
name='reduction2')(norm1)
conv2 = Convolution2D(192,
3,
3,
border_mode='same',
activation='relu',
name='conv2')(reduction2)
norm2 = BatchNormalization(axis=3, name='norm2')(conv2)
pool2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='valid',
name='pool2')(norm2)
icp1_reduction1 = Convolution2D(96,
1,
1,
border_mode='same',
activation='relu',
name='icp1_reduction1')(pool2)
icp1_out1 = Convolution2D(128,
3,
3,
border_mode='same',
activation='relu',
name='icp1_out1')(icp1_reduction1)
icp1_reduction2 = Convolution2D(16,
1,
1,
border_mode='same',
activation='relu',
name='icp1_reduction2')(pool2)
icp1_out2 = Convolution2D(32,
5,
5,
border_mode='same',
activation='relu',
name='icp1_out2')(icp1_reduction2)
icp1_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='icp1_pool')(pool2)
icp1_out3 = Convolution2D(32,
1,
1,
border_mode='same',
activation='relu',
name='icp1_out3')(icp1_pool)
icp1_out0 = Convolution2D(64,
1,
1,
border_mode='same',
activation='relu',
name='icp1_out0')(pool2)
icp2_in = merge([icp1_out0, icp1_out1, icp1_out2, icp1_out3],
mode='concat',
concat_axis=3,
name='icp2_in')
icp2_reduction1 = Convolution2D(128,
1,
1,
border_mode='same',
activation='relu',
name='icp2_reduction1')(icp2_in)
icp2_out1 = Convolution2D(192,
3,
3,
border_mode='same',
activation='relu',
name='icp2_out1')(icp2_reduction1)
icp2_reduction2 = Convolution2D(32,
1,
1,
border_mode='same',
activation='relu',
name='icp2_reduction2')(icp2_in)
icp2_out2 = Convolution2D(96,
5,
5,
border_mode='same',
activation='relu',
name='icp2_out2')(icp2_reduction2)
icp2_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='icp2_pool')(icp2_in)
icp2_out3 = Convolution2D(64,
1,
1,
border_mode='same',
activation='relu',
name='icp2_out3')(icp2_pool)
icp2_out0 = Convolution2D(128,
1,
1,
border_mode='same',
activation='relu',
name='icp2_out0')(icp2_in)
icp2_out = merge([icp2_out0, icp2_out1, icp2_out2, icp2_out3],
mode='concat',
concat_axis=3,
name='icp2_out')
icp3_in = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='same',
name='icp3_in')(icp2_out)
icp3_reduction1 = Convolution2D(96,
1,
1,
border_mode='same',
activation='relu',
name='icp3_reduction1')(icp3_in)
icp3_out1 = Convolution2D(208,
3,
3,
border_mode='same',
activation='relu',
name='icp3_out1')(icp3_reduction1)
icp3_reduction2 = Convolution2D(16,
1,
1,
border_mode='same',
activation='relu',
name='icp3_reduction2')(icp3_in)
icp3_out2 = Convolution2D(48,
5,
5,
border_mode='same',
activation='relu',
name='icp3_out2')(icp3_reduction2)
icp3_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='icp3_pool')(icp3_in)
icp3_out3 = Convolution2D(64,
1,
1,
border_mode='same',
activation='relu',
name='icp3_out3')(icp3_pool)
icp3_out0 = Convolution2D(192,
1,
1,
border_mode='same',
activation='relu',
name='icp3_out0')(icp3_in)
icp3_out = merge([icp3_out0, icp3_out1, icp3_out2, icp3_out3],
mode='concat',
concat_axis=3,
name='icp3_out')
cls1_pool = AveragePooling2D(pool_size=(5, 5),
strides=(3, 3),
border_mode='valid',
name='cls1_pool')(icp3_out)
cls1_reduction_pose = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='cls1_reduction_pose')(cls1_pool)
cls1_fc1_flat = Flatten()(cls1_reduction_pose)
cls1_fc1_pose = Dense(1024, activation='relu',
name='cls1_fc1_pose')(cls1_fc1_flat)
cls1_fc_pose_xyz = Dense(3, name='cls1_fc_pose_xyz')(cls1_fc1_pose)
cls1_fc_pose_wpqr = Dense(4, name='cls1_fc_pose_wpqr')(cls1_fc1_pose)
icp4_reduction1 = Convolution2D(112,
1,
1,
border_mode='same',
activation='relu',
name='icp4_reduction1')(icp3_out)
icp4_out1 = Convolution2D(224,
3,
3,
border_mode='same',
activation='relu',
name='icp4_out1')(icp4_reduction1)
icp4_reduction2 = Convolution2D(24,
1,
1,
border_mode='same',
activation='relu',
name='icp4_reduction2')(icp3_out)
icp4_out2 = Convolution2D(64,
5,
5,
border_mode='same',
activation='relu',
name='icp4_out2')(icp4_reduction2)
icp4_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='icp4_pool')(icp3_out)
icp4_out3 = Convolution2D(64,
1,
1,
border_mode='same',
activation='relu',
name='icp4_out3')(icp4_pool)
icp4_out0 = Convolution2D(160,
1,
1,
border_mode='same',
activation='relu',
name='icp4_out0')(icp3_out)
icp4_out = merge([icp4_out0, icp4_out1, icp4_out2, icp4_out3],
mode='concat',
concat_axis=3,
name='icp4_out')
icp5_reduction1 = Convolution2D(128,
1,
1,
border_mode='same',
activation='relu',
name='icp5_reduction1')(icp4_out)
icp5_out1 = Convolution2D(256,
3,
3,
border_mode='same',
activation='relu',
name='icp5_out1')(icp5_reduction1)
icp5_reduction2 = Convolution2D(24,
1,
1,
border_mode='same',
activation='relu',
name='icp5_reduction2')(icp4_out)
icp5_out2 = Convolution2D(64,
5,
5,
border_mode='same',
activation='relu',
name='icp5_out2')(icp5_reduction2)
icp5_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='icp5_pool')(icp4_out)
icp5_out3 = Convolution2D(64,
1,
1,
border_mode='same',
activation='relu',
name='icp5_out3')(icp5_pool)
icp5_out0 = Convolution2D(128,
1,
1,
border_mode='same',
activation='relu',
name='icp5_out0')(icp4_out)
icp5_out = merge([icp5_out0, icp5_out1, icp5_out2, icp5_out3],
mode='concat',
concat_axis=3,
name='icp5_out')
icp6_reduction1 = Convolution2D(144,
1,
1,
border_mode='same',
activation='relu',
name='icp6_reduction1')(icp5_out)
icp6_out1 = Convolution2D(288,
3,
3,
border_mode='same',
activation='relu',
name='icp6_out1')(icp6_reduction1)
icp6_reduction2 = Convolution2D(32,
1,
1,
border_mode='same',
activation='relu',
name='icp6_reduction2')(icp5_out)
icp6_out2 = Convolution2D(64,
5,
5,
border_mode='same',
activation='relu',
name='icp6_out2')(icp6_reduction2)
icp6_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='icp6_pool')(icp5_out)
icp6_out3 = Convolution2D(64,
1,
1,
border_mode='same',
activation='relu',
name='icp6_out3')(icp6_pool)
icp6_out0 = Convolution2D(112,
1,
1,
border_mode='same',
activation='relu',
name='icp6_out0')(icp5_out)
icp6_out = merge([icp6_out0, icp6_out1, icp6_out2, icp6_out3],
mode='concat',
concat_axis=3,
name='icp6_out')
cls2_pool = AveragePooling2D(pool_size=(5, 5),
strides=(3, 3),
border_mode='valid',
name='cls2_pool')(icp6_out)
cls2_reduction_pose = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='cls2_reduction_pose')(cls2_pool)
cls2_fc1_flat = Flatten()(cls2_reduction_pose)
cls2_fc1 = Dense(1024, activation='relu',
name='cls2_fc1')(cls2_fc1_flat)
cls2_fc_pose_xyz = Dense(3, name='cls2_fc_pose_xyz')(cls2_fc1)
cls2_fc_pose_wpqr = Dense(4, name='cls2_fc_pose_wpqr')(cls2_fc1)
icp7_reduction1 = Convolution2D(160,
1,
1,
border_mode='same',
activation='relu',
name='icp7_reduction1')(icp6_out)
icp7_out1 = Convolution2D(320,
3,
3,
border_mode='same',
activation='relu',
name='icp7_out1')(icp7_reduction1)
icp7_reduction2 = Convolution2D(32,
1,
1,
border_mode='same',
activation='relu',
name='icp7_reduction2')(icp6_out)
icp7_out2 = Convolution2D(128,
5,
5,
border_mode='same',
activation='relu',
name='icp7_out2')(icp7_reduction2)
icp7_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='icp7_pool')(icp6_out)
icp7_out3 = Convolution2D(128,
1,
1,
border_mode='same',
activation='relu',
name='icp7_out3')(icp7_pool)
icp7_out0 = Convolution2D(256,
1,
1,
border_mode='same',
activation='relu',
name='icp7_out0')(icp6_out)
icp7_out = merge([icp7_out0, icp7_out1, icp7_out2, icp7_out3],
mode='concat',
concat_axis=3,
name='icp7_out')
icp8_in = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='same',
name='icp8_in')(icp7_out)
icp8_reduction1 = Convolution2D(160,
1,
1,
border_mode='same',
activation='relu',
name='icp8_reduction1')(icp8_in)
icp8_out1 = Convolution2D(320,
3,
3,
border_mode='same',
activation='relu',
name='icp8_out1')(icp8_reduction1)
icp8_reduction2 = Convolution2D(32,
1,
1,
border_mode='same',
activation='relu',
name='icp8_reduction2')(icp8_in)
icp8_out2 = Convolution2D(128,
5,
5,
border_mode='same',
activation='relu',
name='icp8_out2')(icp8_reduction2)
icp8_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='icp8_pool')(icp8_in)
icp8_out3 = Convolution2D(128,
1,
1,
border_mode='same',
activation='relu',
name='icp8_out3')(icp8_pool)
icp8_out0 = Convolution2D(256,
1,
1,
border_mode='same',
activation='relu',
name='icp8_out0')(icp8_in)
icp8_out = merge([icp8_out0, icp8_out1, icp8_out2, icp8_out3],
mode='concat',
concat_axis=3,
name='icp8_out')
icp9_reduction1 = Convolution2D(192,
1,
1,
border_mode='same',
activation='relu',
name='icp9_reduction1')(icp8_out)
icp9_out1 = Convolution2D(384,
3,
3,
border_mode='same',
activation='relu',
name='icp9_out1')(icp9_reduction1)
icp9_reduction2 = Convolution2D(48,
1,
1,
border_mode='same',
activation='relu',
name='icp9_reduction2')(icp8_out)
icp9_out2 = Convolution2D(128,
5,
5,
border_mode='same',
activation='relu',
name='icp9_out2')(icp9_reduction2)
icp9_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='icp9_pool')(icp8_out)
icp9_out3 = Convolution2D(128,
1,
1,
border_mode='same',
activation='relu',
name='icp9_out3')(icp9_pool)
icp9_out0 = Convolution2D(384,
1,
1,
border_mode='same',
activation='relu',
name='icp9_out0')(icp8_out)
icp9_out = merge([icp9_out0, icp9_out1, icp9_out2, icp9_out3],
mode='concat',
concat_axis=3,
name='icp9_out')
cls3_pool = AveragePooling2D(pool_size=(7, 7),
strides=(1, 1),
border_mode='valid',
name='cls3_pool')(icp9_out)
cls3_fc1_flat = Flatten()(cls3_pool)
cls3_fc1_pose = Dense(2048, activation='relu',
name='cls3_fc1_pose')(cls3_fc1_flat)
cls3_fc_pose_xyz = Dense(3, name='cls3_fc_pose_xyz')(cls3_fc1_pose)
cls3_fc_pose_wpqr = Dense(4, name='cls3_fc_pose_wpqr')(cls3_fc1_pose)
posenet = Model(input=input,
output=[
cls1_fc_pose_xyz, cls1_fc_pose_wpqr,
cls2_fc_pose_xyz, cls2_fc_pose_wpqr,
cls3_fc_pose_xyz, cls3_fc_pose_wpqr
])
if tune:
if weights_path:
weights_data = np.load(weights_path).item()
for layer in posenet.layers:
if layer.name in weights_data.keys():
layer_weights = weights_data[layer.name]
layer.set_weights(
(layer_weights['weights'], layer_weights['biases']))
print("FINISHED SETTING THE WEIGHTS!")
return posenet
num_epoch = 200
numExamples = (X_train.shape)[0]
numBatches = int(numExamples/float(batchSize))
print('Number of examples: ', numExamples)
print('Number of Batches: ', numBatches)
print('Number of epochs: ', num_epoch)
# Generator
generator = Sequential([
# generates images in (28,28,1)
# FC 1: 7,7,16
Dense(784, input_shape=(100,)),
Reshape(target_shape=(7, 7, 16)),
BatchNormalization(),
LeakyReLU(alpha=0.02),
# Conv 1: 14,14,32
Conv2DTranspose(32, kernel_size=5, strides=2, padding='same'),
BatchNormalization(),
LeakyReLU(alpha=0.02),
# Conv 2: 28,28,1
Conv2DTranspose(1, kernel_size=5, strides=2, padding='same'),
Activation('tanh')
])
generator.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
# Discriminator
def _build_network(self, network_input, network_output, additional_network_outputs):
cluster_counts = list(self.data_provider.get_cluster_counts())
# The simple loss cluster NN requires a specific output: a list of softmax distributions
# First in this list are all softmax distributions for k=k_min for each object, then for k=k_min+1 for each
# object etc. At the end, there is the cluster count output.
# First we get an embedding for the network inputs
embeddings = self._get_embedding(network_input)
# Reshape all embeddings to 1d vectors
# embedding_shape = self._embedding_nn.model.layers[-1].output_shape
# embedding_size = np.prod(embedding_shape[1:])
embedding_shape = embeddings[0].shape
embedding_size = int(str(np.prod(embedding_shape[1:])))
embedding_reshaper = self._s_layer('embedding_reshape', lambda name: Reshape((1, embedding_size), name=name))
embeddings_reshaped = [embedding_reshaper(embedding) for embedding in embeddings]
# Merge all embeddings to one tensor
embeddings_merged = self._s_layer('embeddings_merge', lambda name: Concatenate(axis=1, name=name))(embeddings_reshaped)
self._add_additional_prediction_output(embeddings_merged, '0_Embeddings')
# The value range of the embeddings must be [-1, 1] to allow an efficient execution of the cluster evaluations,
# so be sure the embeddings are processed by tanh or some similar activation
# Use some LSTMs for some kind of preprocessing. After these layers a regularisation is applied
processed = embeddings_merged
for i in range(self.__pre_lstm_layers):
processed = self._s_layer(
'PRE_LSTM_proc_{}'.format(i), lambda name: Bidirectional(LSTM(self.__lstm_units, return_sequences=True), name=name)
)(processed)
processed = self._s_layer(
'PRE_LSTM_proc_{}_batch'.format(i), lambda name: BatchNormalization(name=name)
)(processed)
# Reshape the data now to a embeddings sized representation representation
processed = TimeDistributed(Dense(embedding_size, activation='tanh'))(processed)
# # Store these processed embeddings
# preprocessed_embeddings = [slice_layer(processed, i) for i in range(len(network_input))]
self._add_additional_prediction_output(processed, "1_LSTM_Preprocessed_Embeddings")
# Use now some LSTM-layer to process all embeddings
# processed = embeddings_merged
for i in range(self.__lstm_layers):
processed = self._s_layer(
'LSTM_proc_{}'.format(i), lambda name: Bidirectional(LSTM(self.__lstm_units, return_sequences=True), name=name)
)(processed)
processed = self._s_layer(
'LSTM_proc_{}_batch'.format(i), lambda name: BatchNormalization(name=name)
)(processed)
# Split the tensor to seperate layers
embeddings_processed = [self._s_layer('slice_{}'.format(i), lambda name: slice_layer(processed, i, name)) for i in range(len(network_input))]
# Create now two outputs: The cluster count and for each cluster count / object combination a softmax distribution.
# These outputs are independent of each other, therefore it doesn't matter which is calculated first. Let us start
# with the cluster count / object combinations.
# First prepare some generally required layers
layers = []
for i in range(self.__output_dense_layers):
layers += [
self._s_layer('output_dense{}'.format(i), lambda name: Dense(self.__output_dense_units, name=name)),
self._s_layer('output_batch'.format(i), lambda name: BatchNormalization(name=name)),
# self._s_layer('output_relu'.format(i), lambda name: Activation('relu', name=name))
LeakyReLU()
]
cluster_softmax = {
k: self._s_layer('softmax_cluster_{}'.format(k), lambda name: Dense(k, activation='softmax', name=name)) for k in cluster_counts
}
# Create now the outputs
clusters_output = additional_network_outputs['clusters'] = {}
cluster_classifiers = {k: [] for k in cluster_counts}
for i in range(len(embeddings_processed)):
embedding_proc = embeddings_processed[i]
# Add the required layers
for layer in layers:
embedding_proc = layer(embedding_proc)
input_clusters_output = clusters_output['input{}'.format(i)] = {}
for k in cluster_counts:
# Create now the required softmax distributions
output_classifier = cluster_softmax[k](embedding_proc)
input_clusters_output['cluster{}'.format(k)] = output_classifier
network_output.append(output_classifier)
cluster_classifiers[k].append(output_classifier)
# clustering_quality = 0
# sum_cohesion = 0
# sum_separation = 0
# alpha = 0.5
# beta = 0.25
# self._add_debug_output(Concatenate(axis=1)(preprocessed_embeddings), 'eval_embeddings')
# # Squared euclidean distance
# distance_f = lambda x, y: K.sum(K.square(x - y), axis=2)
# # Euclidean distance
# distance_f = lambda x, y: K.sqrt(K.sum(K.square(x - y), axis=2))
# for k in cluster_counts:
#
# # Create evaluation metrics
# self._add_debug_output(Concatenate(axis=1)(cluster_classifiers[k]), 'eval_classifications_k{}'.format(k))
# cluster_centers = get_cluster_centers(preprocessed_embeddings, cluster_classifiers[k])
# self._add_debug_output(Concatenate(axis=1)(cluster_centers), 'eval_cluster_centers_k{}'.format(k))
# cohesion = get_cluster_cohesion(cluster_centers, preprocessed_embeddings, cluster_classifiers[k])
# self._add_debug_output(cohesion, 'eval_cohesion_k{}'.format(k))
# separation = get_cluster_separation(cluster_centers, cluster_classifiers[k])
# self._add_debug_output(separation, 'eval_separation_k{}'.format(k))
#
# self._add_additional_prediction_output(
# Concatenate(axis=1, name='2_cluster_centers_k{}'.format(k))(cluster_centers),
# 'cluster_centers_k{}'.format(k)
# )
#
# sum_cohesion = Lambda(lambda cohesion: sum_cohesion + cohesion)(cohesion)
# sum_separation = Lambda(lambda separation: sum_separation + separation)(separation)
# # clustering_quality = Lambda(lambda cohesion:
# #
# # # Update the loss
# # clustering_quality + (alpha * cohesion - beta * separation)
# # )(cohesion)
# # Add alpha and beta to the cohesion and the separation
# sum_cohesion = Lambda(lambda sum_cohesion: alpha * sum_cohesion)(sum_cohesion)
# sum_separation = Lambda(lambda sum_separation: beta * sum_separation)(sum_separation)
#
# # Normalize the cohesion and the separation by the cluster_counts
# sum_cohesion = Lambda(lambda sum_cohesion: sum_cohesion / len(cluster_counts))(sum_cohesion)
# sum_separation = Lambda(lambda sum_separation: sum_separation / len(cluster_counts))(sum_separation)
#
# # Add the losses for the cohesion and the separation. Use for both the same loss function
# cluster_quality_loss = lambda x: lambda similiarty_loss, x=x: K.exp(- similiarty_loss * similiarty_loss * 4) * x
# self._register_additional_grouping_similarity_loss(
# 'cluster_cohesion',
# cluster_quality_loss(sum_cohesion)
# )
# self._register_additional_grouping_similarity_loss(
# 'cluster_separation',
# cluster_quality_loss(- K.log(1 + 2 * sum_separation))
# )
# # Normalize the cluster quality
# clustering_quality = Lambda(lambda x: x / len(cluster_counts))(clustering_quality)
#
# # What to do with the cluster quality? We use it for an additional loss, this loss should optimize
# # the cluster quality as soon as the clustering works relatively well.
# self._register_additional_grouping_similarity_loss(
# 'cluster_quality',
# lambda similiarty_loss: K.exp(- similiarty_loss * similiarty_loss) * clustering_quality
# )
# Calculate the real cluster count
cluster_count = self._s_layer('cluster_count_LSTM_merge', lambda name: Bidirectional(LSTM(self.__lstm_units), name=name)(embeddings_merged))
cluster_count = self._s_layer('cluster_count_LSTM_merge_batch', lambda name: BatchNormalization(name=name))(cluster_count)
for i in range(self.__cluster_count_dense_layers):
cluster_count = self._s_layer('cluster_count_dense{}'.format(i), lambda name: Dense(self.__cluster_count_dense_units, name=name))(cluster_count)
cluster_count = self._s_layer('cluster_count_batch{}'.format(i), lambda name: BatchNormalization(name=name))(cluster_count)
# cluster_count = self._s_layer('cluster_count_relu{}'.format(i), lambda name: Activation('relu', name=name))(cluster_count)
cluster_count = LeakyReLU()(cluster_count)
# The next layer is an output-layer, therefore the name must not be formatted
cluster_count = self._s_layer(
'cluster_count_output',
lambda name: Dense(len(cluster_counts), activation='softmax', name=name),
format_name=False
)(cluster_count)
additional_network_outputs['cluster_count_output'] = cluster_count
network_output.append(cluster_count)
return True
cnn1 = GlobalMaxPooling1D()(relu1)
# filter_size =4
cnn2 = Conv1D(num_filters, 4, strides=1, padding='same')(embed)
relu2 = Activation(activation='relu')(cnn2)
cnn2 = GlobalMaxPooling1D()(relu2)
# filter_size = 5
cnn3 = Conv1D(num_filters, 5, strides=1, padding='same')(embed)
relu3 = Activation(activation='relu')(cnn3)
cnn3 = GlobalMaxPooling1D()(relu3)
# filter_size = 6
cnn4 = Conv1D(num_filters, 6, strides=1, padding='same')(embed)
relu4 = Activation(activation='relu')(cnn4)
cnn4 = GlobalMaxPooling1D()(relu4)
# 合并三个模型的输出向量
cnn = concatenate([cnn1, cnn2, cnn3, cnn4], axis=-1)
bn = BatchNormalization()(cnn)
drop1 = Dropout(dropout_rate)(bn)
dense = Dense(num_hidden, activation="relu")(drop1)
drop2 = Dropout(dropout_rate)(dense)
main_output = Dense(num_classes, activation='sigmoid')(drop2)
model = Model(inputs=input, outputs=main_output)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
plot_model(model, to_file="./pics/textcnn_filter345.png", show_shapes=True)
#########################################################################
for epoch in range(num_epochs):
model.fit(x=train_fact_pad_seq,
# partition the data into training and testing splits
(trainX, testX, trainY, testY) = train_test_split(data,labels, test_size=0.20, random_state=42)
testY = to_categorical(testY, num_classes=2) #num_classes may give issue for >2 classes, better not to use this feature
# convert into a matrix with one-hot-encoded values
trainY = to_categorical(trainY, num_classes=2)
# --------------- The NN layers ---------------------
# initialize the model
model = Sequential()
model.add(Conv2D(filters=8, kernel_size=(3,3), strides=1, input_shape=(28,28,3),name="Layer1Conv2D") ) #change shape of images
model.add(BatchNormalization(axis=-1,name="Layer2BN")) #-1 for channels_last
model.add(Activation('relu',name="Layer3ActRelu"))
model.add(Conv2D(filters=16, kernel_size=(3,3), strides=1,name="Layer4Conv2D"))
model.add(BatchNormalization(axis=-1,name="Layer5BN"))
model.add(Activation('relu',name="Layer6ActRelu"))
model.add(Conv2D(filters=32, kernel_size=(3,3), strides=1,name="Layer7Conv2D" ))
model.add(BatchNormalization(axis=-1,name="Layer8BN"))
model.add(Activation('relu',name="Layer9ActRelu"))
model.add(Conv2D(filters=64, kernel_size=(3,3), strides=2,name="Layer10Conv2D" ))
model.add(BatchNormalization(axis=-1,name="Layer11BN"))
model.add(Activation('relu',name="Layer12ActRelu"))
model.add(Flatten(name="Layer13Flat"))
def build_model_speed(args , Compute_Time = False):
#steer model
Steer_In =Input(shape=(227,227,3),name='model1_in')
#conv1
Steer = Conv2D(filters = 96, activation='relu',kernel_size = (11, 11), strides=(4,4))(Steer_In) #assumed that stride step is 1x1
Steer = BatchNormalization()(Steer)
Steer = MaxPooling2D(pool_size=(3,3),strides=(2,2))(Steer)#assumed that stride step is 3x3
#conv2
Steer = Conv2D(filters = 256, kernel_size = (5,5), activation='relu', strides=(1,1))(Steer)
Steer = BatchNormalization()(Steer)
Steer = MaxPooling2D(pool_size=(3,3),strides=(2,2))(Steer)#assumed that stride step is 3x3
#conv3
Steer = Conv2D(filters =384,kernel_size =(3, 3), activation='relu', strides=(1,1))(Steer)
#conv4
Steer = Conv2D(filters =384,kernel_size =(3,3), activation='relu', strides=(1,1))(Steer)
#conv5
Steer = Conv2D(filters = 256,kernel_size =(3, 3), activation='relu',strides=(1,1))(Steer)
Steer = Flatten()(Steer)
#FC1
Steer = Dense(1024, activation='relu')(Steer)
Steer = Dropout(args.keep_prob , name='Dropout1')(Steer)
#FC2
Steer = Dense(50, activation='relu')(Steer)
Steer_out = Dropout(args.keep_prob, name='Dropout2')(Steer)
Steer_Model1 = Model(inputs=Steer_In, outputs=Steer_out)
################################################################################
##############################speed model2 before concate_model#################
Speed_In = Input(shape=(10,1),name='Speed_In')
Speed =LSTM(128,return_sequences=True)(Speed_In)
Speed = Flatten()(Speed)
#L2_out = (LSTM(300, dropout_W = 0.2, dropout_U = 0.2)(L2_out)
#L2_out = Flatten()(L2_out)
#FC1
Speed = Dense(50,activation='elu')(Speed)
Speed = Dropout(args.keep_prob, name='Dropout3')(Speed)
#FC2
Speed = Dense(50, activation='elu' )(Speed)
Speed = Dropout(args.keep_prob, name='Dropout4')(Speed)
Speed_Model1 = Model(inputs=Speed_In, outputs=Speed)
################ here is our concate_model#########################
#concatenate(inputs, axis=-1, **kwargs):
First_Merge = concatenate([Speed_Model1.output, Steer_Model1.output], name='Concatenate')
#print(merged_layers)
#out = BatchNormalization()(merged_layers)
Speed_Continue = Dense(50, activation='elu')(First_Merge)
Speed_Continue = Dropout(args.keep_prob, name='Dropout5')(Speed_Continue)
Speed_Out = Dense(1, name='Speed')(Speed_Continue)
Speed_Model = Model(inputs = [Steer_In,Speed_In],outputs = [Speed_Out])
#continue Steering model
Steer_Continue = Dense(50, activation='relu')(Steer_out) #node 1 output to FC3
Steer_Continue = Dropout(args.keep_prob, name='Dropout6')(Steer_Continue) #steer output
Steer_Continue = Dense(1 , name = 'Steer')(Steer_Continue)
Steer_Model = Model(inputs = [Steer_In],outputs = [Steer_Continue])
Final_Model = Model(inputs=[Steer_In , Speed_In], outputs=[Speed_Model.output , Steer_Model.output])
########################################################################
#Steer_Model.compile(loss='mean_squared_error', optimizer=Adam(lr=args.learning_rate))
if Compute_Time == True :
return Steer_Model1 , Speed_Model1 , Speed_Model ,Steer_Model ,Final_Model #, Steer_Model
else:
Final_Model.summary()
return Final_Model
num_of_classes = 27
angvelinput = Input(shape=(65, features, 1))
angvelmodel = Conv2D(32, (3, 3),
kernel_regularizer=l2(0.001),
kernel_initializer='glorot_normal',
activation='relu')(angvelinput)
angvelmodel = Conv2D(32, (3, 3),
kernel_regularizer=l2(0.001),
kernel_initializer='glorot_normal',
activation="relu")(angvelmodel)
angvelmodel = MaxPooling2D(pool_size=(2, 2), strides=(2, 2),
padding='same')(angvelmodel)
angvelmodel = BatchNormalization()(angvelmodel)
angvelmodel = Conv2D(64, (3, 3),
kernel_regularizer=l2(0.001),
kernel_initializer='glorot_normal',
activation="relu")(angvelmodel)
angvelmodel = Conv2D(64, (3, 3),
kernel_regularizer=l2(0.001),
kernel_initializer='glorot_normal',
activation="relu")(angvelmodel)
angvelmodel = MaxPooling2D(pool_size=(2, 2), strides=(2, 2),
padding='same')(angvelmodel)
angvelmodel = BatchNormalization()(angvelmodel)
angvelmodel = Conv2D(128, (3, 3),