def build_model():
# epoch, dropout = best_model()
epoch, dropout = 20, 0.2
print('EPOCH = ', epoch)
print('DROPOUT = ', dropout)
model = Sequential()
model.add(Embedding(input_dim=length_word_index, output_dim=length_tag_index, input_length=MAX_LENGTH))
# model.add(Conv1D(filters=MAX_LENGTH, kernel_size=4, activation='relu'))
# model.add(GlobalMaxPooling1D())
# model.add(InputLayer(input_shape=(MAX_LENGTH,)))
# model.add(Embedding(input_dim=length_word_index, output_dim=128, input_length=MAX_LENGTH))
model.add(Conv1D(filters=MAX_LENGTH, kernel_size=4, padding='same', activation='relu'))
model.add(Dropout(dropout))
model.add(Dense(length_tag_index))
model.add(Activation('softmax'))
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc'])
# print(model.summary())
# One hot encode the tags
def onehot_encode_tags(sequences, categories):
cat_sequences = []
for seq in sequences:
cats = []
for item in seq:
cats.append(np.zeros(categories))
cats[-1][item] = 1.0
cat_sequences.append(cats)
return np.array(cat_sequences)
cat_train_tags_y = onehot_encode_tags(y_train, length_tag_index)
print(cat_train_tags_y[0])
cat_train_tags_y = onehot_encode_tags(y_train, length_tag_index)
print(cat_train_tags_y[0])
print('y_train shape: ', y_train.shape)
print('MAX_LENGTH: ', MAX_LENGTH)
print('length_word_index: ', length_word_index)
print('length_tag_index: ', length_tag_index)
# Train the model
history = model.fit(X_train, onehot_encode_tags(y_train, length_tag_index), batch_size=128, epochs=epoch, validation_split=0.2)
# Evaluate the model
loss, accuracy = model.evaluate(X_test, onehot_encode_tags(y_test, length_tag_index))
print('Accuracy: %f' % (accuracy * 100))
def display():
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train','test'], loc = 'upper left')
plt.show()
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train','test'], loc = 'upper left')
plt.show()
display()
def MobileNet(input_shape=None,
alpha=1.0,
depth_multiplier=1,
dropout=1e-3,
include_top=True,
weights='imagenet',
input_tensor=None,
pooling=None,
classes=1000):
"""Instantiates the MobileNet architecture.
Arguments:
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(224, 224, 3)` (with `channels_last` data format)
or (3, 224, 224) (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 32.
E.g. `(200, 200, 3)` would be one valid value.
alpha: controls the width of the network.
- If `alpha` < 1.0, proportionally decreases the number
of filters in each layer.
- If `alpha` > 1.0, proportionally increases the number
of filters in each layer.
- If `alpha` = 1, default number of filters from the paper
are used at each layer.
depth_multiplier: depth multiplier for depthwise convolution
(also called the resolution multiplier)
dropout: dropout rate
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of
`layers.Input()`)
to use as image input for the model.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model
will be the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a
2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
Returns:
A Keras model instance.
Raises:
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
RuntimeError: If attempting to run this model with a
backend that does not support separable convolutions.
"""
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as ImageNet with `include_top` '
'as true, `classes` should be 1000')
# Determine proper input shape and default size.
if input_shape is None:
default_size = 224
else:
if K.image_data_format() == 'channels_first':
rows = input_shape[1]
cols = input_shape[2]
else:
rows = input_shape[0]
cols = input_shape[1]
if rows == cols and rows in [128, 160, 192, 224]:
default_size = rows
else:
default_size = 224
input_shape = _obtain_input_shape(input_shape,
default_size=default_size,
min_size=32,
data_format=K.image_data_format(),
require_flatten=include_top,
weights=weights)
if K.image_data_format() == 'channels_last':
row_axis, col_axis = (0, 1)
else:
row_axis, col_axis = (1, 2)
rows = input_shape[row_axis]
cols = input_shape[col_axis]
if weights == 'imagenet':
if depth_multiplier != 1:
raise ValueError('If imagenet weights are being loaded, '
'depth multiplier must be 1')
if alpha not in [0.25, 0.50, 0.75, 1.0]:
raise ValueError('If imagenet weights are being loaded, '
'alpha can be one of'
'`0.25`, `0.50`, `0.75` or `1.0` only.')
if rows != cols or rows not in [128, 160, 192, 224]:
if rows is None:
rows = 224
logging.warning(
'MobileNet shape is undefined.'
' Weights for input shape (224, 224) will be loaded.')
else:
raise ValueError(
'If imagenet weights are being loaded, '
'input must have a static square shape (one of '
'(128, 128), (160, 160), (192, 192), or (224, 224)).'
' Input shape provided = %s' % (input_shape, ))
if K.image_data_format() != 'channels_last':
logging.warning(
'The MobileNet family of models is only available '
'for the input data format "channels_last" '
'(width, height, channels). '
'However your settings specify the default '
'data format "channels_first" (channels, width, height).'
' You should set `image_data_format="channels_last"` '
'in your Keras config located at ~/.keras/keras.json. '
'The model being returned right now will expect inputs '
'to follow the "channels_last" data format.')
K.set_image_data_format('channels_last')
old_data_format = 'channels_first'
else:
old_data_format = None
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
x = _conv_block(img_input, 32, alpha, strides=(2, 2))
x = _depthwise_conv_block(x, 64, alpha, depth_multiplier, block_id=1)
x = _depthwise_conv_block(x,
128,
alpha,
depth_multiplier,
strides=(2, 2),
block_id=2)
x = _depthwise_conv_block(x, 128, alpha, depth_multiplier, block_id=3)
x = _depthwise_conv_block(x,
256,
alpha,
depth_multiplier,
strides=(2, 2),
block_id=4)
x = _depthwise_conv_block(x, 256, alpha, depth_multiplier, block_id=5)
x = _depthwise_conv_block(x,
512,
alpha,
depth_multiplier,
strides=(2, 2),
block_id=6)
x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=7)
x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=8)
x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=9)
x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=10)
x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=11)
x = _depthwise_conv_block(x,
1024,
alpha,
depth_multiplier,
strides=(2, 2),
block_id=12)
x = _depthwise_conv_block(x, 1024, alpha, depth_multiplier, block_id=13)
if include_top:
if K.image_data_format() == 'channels_first':
shape = (int(1024 * alpha), 1, 1)
else:
shape = (1, 1, int(1024 * alpha))
x = GlobalAveragePooling2D()(x)
x = Reshape(shape, name='reshape_1')(x)
x = Dropout(dropout, name='dropout')(x)
x = Conv2D(classes, (1, 1), padding='same', name='conv_preds')(x)
x = Activation('softmax', name='act_softmax')(x)
x = Reshape((classes, ), name='reshape_2')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
x = Flatten(name='custom')(x) ##DB
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = layer_utils.get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='mobilenet_%0.2f_%s' % (alpha, rows))
# load weights
if weights == 'imagenet':
if K.image_data_format() == 'channels_first':
raise ValueError('Weights for "channels_first" format '
'are not available.')
if alpha == 1.0:
alpha_text = '1_0'
elif alpha == 0.75:
alpha_text = '7_5'
elif alpha == 0.50:
alpha_text = '5_0'
else:
alpha_text = '2_5'
if include_top:
model_name = 'mobilenet_%s_%d_tf.h5' % (alpha_text, rows)
weigh_path = BASE_WEIGHT_PATH + model_name
weights_path = get_file(model_name,
weigh_path,
cache_subdir='models')
else:
model_name = 'mobilenet_%s_%d_tf_no_top.h5' % (alpha_text, rows)
weigh_path = BASE_WEIGHT_PATH + model_name
weights_path = get_file(model_name,
weigh_path,
cache_subdir='models')
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
if old_data_format:
K.set_image_data_format(old_data_format)
return model
def Deeplabv3(weights='pascal_voc',
input_tensor=None,
input_shape=(512, 512, 3),
classes=21,
backbone='mobilenetv2',
OS=16,
alpha=1.,
activation=None):
""" Instantiates the Deeplabv3+ architecture
Optionally loads weights pre-trained
on PASCAL VOC or Cityscapes. This model is available for TensorFlow only.
# Arguments
weights: one of 'pascal_voc' (pre-trained on pascal voc),
'cityscapes' (pre-trained on cityscape) or None (random initialization)
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: shape of input image. format HxWxC
PASCAL VOC model was trained on (512,512,3) images. None is allowed as shape/width
classes: number of desired classes. PASCAL VOC has 21 classes, Cityscapes has 19 classes.
If number of classes not aligned with the weights used, last layer is initialized randomly
backbone: backbone to use. one of {'xception','mobilenetv2'}
activation: optional activation to add to the top of the network.
One of 'softmax', 'sigmoid' or None
OS: determines input_shape/feature_extractor_output ratio. One of {8,16}.
Used only for xception backbone.
alpha: controls the width of the MobileNetV2 network. This is known as the
width multiplier in the MobileNetV2 paper.
- If `alpha` < 1.0, proportionally decreases the number
of filters in each layer.
- If `alpha` > 1.0, proportionally increases the number
of filters in each layer.
- If `alpha` = 1, default number of filters from the paper
are used at each layer.
Used only for mobilenetv2 backbone. Pretrained is only available for alpha=1.
# Returns
A Keras model instance.
# Raises
RuntimeError: If attempting to run this model with a
backend that does not support separable convolutions.
ValueError: in case of invalid argument for `weights` or `backbone`
"""
if not (weights in {'pascal_voc', 'cityscapes', None}):
raise ValueError(
'The `weights` argument should be either '
'`None` (random initialization), `pascal_voc`, or `cityscapes` '
'(pre-trained on PASCAL VOC)')
if not (backbone in {'xception', 'mobilenetv2'}):
raise ValueError('The `backbone` argument should be either '
'`xception` or `mobilenetv2` ')
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
img_input = input_tensor
if backbone == 'xception':
if OS == 8:
entry_block3_stride = 1
middle_block_rate = 2 # ! Not mentioned in paper, but required
exit_block_rates = (2, 4)
atrous_rates = (12, 24, 36)
else:
entry_block3_stride = 2
middle_block_rate = 1
exit_block_rates = (1, 2)
atrous_rates = (6, 12, 18)
x = Conv2D(32, (3, 3),
strides=(2, 2),
name='entry_flow_conv1_1',
use_bias=False,
padding='same')(img_input)
x = BatchNormalization(name='entry_flow_conv1_1_BN')(x)
x = Activation(tf.nn.relu)(x)
x = _conv2d_same(x, 64, 'entry_flow_conv1_2', kernel_size=3, stride=1)
x = BatchNormalization(name='entry_flow_conv1_2_BN')(x)
x = Activation(tf.nn.relu)(x)
x = _xception_block(x, [128, 128, 128],
'entry_flow_block1',
skip_connection_type='conv',
stride=2,
depth_activation=False)
x, skip1 = _xception_block(x, [256, 256, 256],
'entry_flow_block2',
skip_connection_type='conv',
stride=2,
depth_activation=False,
return_skip=True)
x = _xception_block(x, [728, 728, 728],
'entry_flow_block3',
skip_connection_type='conv',
stride=entry_block3_stride,
depth_activation=False)
for i in range(16):
x = _xception_block(x, [728, 728, 728],
'middle_flow_unit_{}'.format(i + 1),
skip_connection_type='sum',
stride=1,
rate=middle_block_rate,
depth_activation=False)
x = _xception_block(x, [728, 1024, 1024],
'exit_flow_block1',
skip_connection_type='conv',
stride=1,
rate=exit_block_rates[0],
depth_activation=False)
x = _xception_block(x, [1536, 1536, 2048],
'exit_flow_block2',
skip_connection_type='none',
stride=1,
rate=exit_block_rates[1],
depth_activation=True)
else:
OS = 8
first_block_filters = _make_divisible(32 * alpha, 8)
x = Conv2D(first_block_filters,
kernel_size=3,
strides=(2, 2),
padding='same',
use_bias=False,
name='Conv')(img_input)
x = BatchNormalization(epsilon=1e-3, momentum=0.999, name='Conv_BN')(x)
x = Activation(tf.nn.relu6, name='Conv_Relu6')(x)
x = _inverted_res_block(x,
filters=16,
alpha=alpha,
stride=1,
expansion=1,
block_id=0,
skip_connection=False)
x = _inverted_res_block(x,
filters=24,
alpha=alpha,
stride=2,
expansion=6,
block_id=1,
skip_connection=False)
x = _inverted_res_block(x,
filters=24,
alpha=alpha,
stride=1,
expansion=6,
block_id=2,
skip_connection=True)
x = _inverted_res_block(x,
filters=32,
alpha=alpha,
stride=2,
expansion=6,
block_id=3,
skip_connection=False)
x = _inverted_res_block(x,
filters=32,
alpha=alpha,
stride=1,
expansion=6,
block_id=4,
skip_connection=True)
x = _inverted_res_block(x,
filters=32,
alpha=alpha,
stride=1,
expansion=6,
block_id=5,
skip_connection=True)
# stride in block 6 changed from 2 -> 1, so we need to use rate = 2
x = _inverted_res_block(
x,
filters=64,
alpha=alpha,
stride=1, # 1!
expansion=6,
block_id=6,
skip_connection=False)
x = _inverted_res_block(x,
filters=64,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=7,
skip_connection=True)
x = _inverted_res_block(x,
filters=64,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=8,
skip_connection=True)
x = _inverted_res_block(x,
filters=64,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=9,
skip_connection=True)
x = _inverted_res_block(x,
filters=96,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=10,
skip_connection=False)
x = _inverted_res_block(x,
filters=96,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=11,
skip_connection=True)
x = _inverted_res_block(x,
filters=96,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=12,
skip_connection=True)
x = _inverted_res_block(
x,
filters=160,
alpha=alpha,
stride=1,
rate=2, # 1!
expansion=6,
block_id=13,
skip_connection=False)
x = _inverted_res_block(x,
filters=160,
alpha=alpha,
stride=1,
rate=4,
expansion=6,
block_id=14,
skip_connection=True)
x = _inverted_res_block(x,
filters=160,
alpha=alpha,
stride=1,
rate=4,
expansion=6,
block_id=15,
skip_connection=True)
x = _inverted_res_block(x,
filters=320,
alpha=alpha,
stride=1,
rate=4,
expansion=6,
block_id=16,
skip_connection=False)
# end of feature extractor
# branching for Atrous Spatial Pyramid Pooling
# Image Feature branch
shape_before = tf.shape(x)
b4 = GlobalAveragePooling2D()(x)
# from (b_size, channels)->(b_size, 1, 1, channels)
b4 = Lambda(lambda x: K.expand_dims(x, 1))(b4)
b4 = Lambda(lambda x: K.expand_dims(x, 1))(b4)
b4 = Conv2D(256, (1, 1),
padding='same',
use_bias=False,
name='image_pooling')(b4)
b4 = BatchNormalization(name='image_pooling_BN', epsilon=1e-5)(b4)
b4 = Activation(tf.nn.relu)(b4)
# upsample. have to use compat because of the option align_corners
size_before = tf.keras.backend.int_shape(x)
b4 = Lambda(lambda x: tf.compat.v1.image.resize(
x, size_before[1:3], method='bilinear', align_corners=True))(b4)
# simple 1x1
b0 = Conv2D(256, (1, 1), padding='same', use_bias=False, name='aspp0')(x)
b0 = BatchNormalization(name='aspp0_BN', epsilon=1e-5)(b0)
b0 = Activation(tf.nn.relu, name='aspp0_activation')(b0)
# there are only 2 branches in mobilenetV2. not sure why
if backbone == 'xception':
# rate = 6 (12)
b1 = SepConv_BN(x,
256,
'aspp1',
rate=atrous_rates[0],
depth_activation=True,
epsilon=1e-5)
# rate = 12 (24)
b2 = SepConv_BN(x,
256,
'aspp2',
rate=atrous_rates[1],
depth_activation=True,
epsilon=1e-5)
# rate = 18 (36)
b3 = SepConv_BN(x,
256,
'aspp3',
rate=atrous_rates[2],
depth_activation=True,
epsilon=1e-5)
# concatenate ASPP branches & project
x = Concatenate()([b4, b0, b1, b2, b3])
else:
x = Concatenate()([b4, b0])
x = Conv2D(256, (1, 1),
padding='same',
use_bias=False,
name='concat_projection')(x)
x = BatchNormalization(name='concat_projection_BN', epsilon=1e-5)(x)
x = Activation(tf.nn.relu)(x)
x = Dropout(0.1)(x)
# DeepLab v.3+ decoder
if backbone == 'xception':
# Feature projection
# x4 (x2) block
skip_size = tf.keras.backend.int_shape(skip1)
x = Lambda(lambda xx: tf.compat.v1.image.resize(
xx, skip_size[1:3], method='bilinear', align_corners=True))(x)
dec_skip1 = Conv2D(48, (1, 1),
padding='same',
use_bias=False,
name='feature_projection0')(skip1)
dec_skip1 = BatchNormalization(name='feature_projection0_BN',
epsilon=1e-5)(dec_skip1)
dec_skip1 = Activation(tf.nn.relu)(dec_skip1)
x = Concatenate()([x, dec_skip1])
x = SepConv_BN(x,
256,
'decoder_conv0',
depth_activation=True,
epsilon=1e-5)
x = SepConv_BN(x,
256,
'decoder_conv1',
depth_activation=True,
epsilon=1e-5)
# you can use it with arbitary number of classes
if (weights == 'pascal_voc' and classes == 21) or (weights == 'cityscapes'
and classes == 19):
last_layer_name = 'logits_semantic'
else:
last_layer_name = 'custom_logits_semantic'
x = Conv2D(classes, (1, 1), padding='same', name=last_layer_name)(x)
size_before3 = tf.keras.backend.int_shape(img_input)
x = Lambda(lambda xx: tf.compat.v1.image.resize(
xx, size_before3[1:3], method='bilinear', align_corners=True))(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
if activation in {'softmax', 'sigmoid'}:
x = tf.keras.layers.Activation(activation)(x)
model = Model(inputs, x, name='deeplabv3plus')
# load weights
if weights == 'pascal_voc':
if backbone == 'xception':
weights_path = get_file(
'deeplabv3_xception_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH_X,
cache_subdir='models')
else:
weights_path = get_file(
'deeplabv3_mobilenetv2_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH_MOBILE,
cache_subdir='models')
model.load_weights(weights_path, by_name=True)
elif weights == 'cityscapes':
if backbone == 'xception':
weights_path = get_file(
'deeplabv3_xception_tf_dim_ordering_tf_kernels_cityscapes.h5',
WEIGHTS_PATH_X_CS,
cache_subdir='models')
else:
weights_path = get_file(
'deeplabv3_mobilenetv2_tf_dim_ordering_tf_kernels_cityscapes.h5',
WEIGHTS_PATH_MOBILE_CS,
cache_subdir='models')
model.load_weights(weights_path, by_name=True)
return model
from tensorflow.python.keras.layers import (Conv3D, BatchNormalization, AveragePooling3D, concatenate, Lambda, SpatialDropout3D,
Activation, Input, GlobalAvgPool3D, Dense, Conv3DTranspose, add)
from tensorflow.python.keras.regularizers import l2 as l2_penalty
from tensorflow.python.keras.models import Model
from mylib.models.metrics import precision, recall, fmeasure
from mylib.models.losses import DiceLoss
PARAMS = {
'activation': lambda: Activation('relu'), # the activation functions
'bn_scale': True, # whether to use the scale function in BN
'weight_decay': 0, # l2 weight decay
'kernel_initializer': 'he_uniform', # initialization
'first_scale': lambda x: x / 128. - 1., # the first pre-processing function
'dhw': [32, 32, 32], # the input shape
'k': 24, # the `growth rate` in DenseNet
'bottleneck': 4, # the `bottleneck` in DenseNet
'compression': 2, # the `compression` in DenseNet
'first_layer': 48, # the channel of the first layer
'down_structure': [4, 4, 4], # the down-sample structure
'output_size': 2, # the output number of the classification head
'dropout_rate': 0.3 # whether to use dropout, and how much to use
}
def _conv_block(x, filters):
bn_scale = PARAMS['bn_scale']
activation = PARAMS['activation']
kernel_initializer = PARAMS['kernel_initializer']
weight_decay = PARAMS['weight_decay']
bottleneck = PARAMS['bottleneck']
def create_model(num_frames=30, num_classes=27, batch_size=10):
inputs = Input(shape=(num_frames, 112, 112, 3),
batch_size=batch_size) # Not quite sure this line
# conv layer1
conv1 = Convolution3D(64,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding="same")(inputs)
norm1 = BatchNormalization()(conv1)
act1 = Activation('relu')(norm1)
pol1 = MaxPool3D(pool_size=(1, 2, 2), strides=(1, 2, 2))(act1)
# conv layer2
conv2 = Convolution3D(128,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same')(pol1)
norm2 = BatchNormalization()(conv2)
act2 = Activation('relu')(norm2)
pol2 = MaxPool3D(pool_size=(2, 2, 2), strides=(2, 2, 2))(act2)
# conv layer3
conv3 = Convolution3D(256,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same')(pol2)
# conv layer4
conv4 = Convolution3D(256,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same')(conv3)
norm3 = BatchNormalization()(conv4)
act3 = Activation('relu')(norm3)
pre_output_temp = ConvLSTM2D(256,
kernel_size=3,
strides=(1, 1),
padding='same',
batch_input_shape=(batch_size, num_frames / 2,
1),
return_sequences=True,
stateful=True)(act3)
pre_output = ConvLSTM2D(256,
kernel_size=3,
strides=(1, 1),
padding='same',
batch_input_shape=(batch_size, num_frames / 2, 1),
stateful=True)(pre_output_temp)
# SPP Layer
spp1 = MaxPool2D(pool_size=(28, 28), strides=(28, 28))(pre_output)
spp1 = Flatten()(spp1)
spp2 = MaxPool2D(pool_size=(14, 14), strides=(14, 14))(pre_output)
spp2 = Flatten()(spp2)
spp4 = MaxPool2D(pool_size=(7, 7), strides=(7, 4))(pre_output)
spp4 = Flatten()(spp4)
spp7 = MaxPool2D(pool_size=(4, 4), strides=(4, 4))(pre_output)
spp7 = Flatten()(spp7)
merge = concatenate([spp1, spp2, spp4, spp7], name="Concat")
# final_model = Sequential()
# final_model.add(merge)
# FC Layer
# merge = Dense(1024, activation='relu')(merge)
classes = Dense(num_classes, activation='softmax')(merge)
final_model = Model(inputs=[
inputs,
], outputs=classes)
# final_model.add(Dense(classes, activation='softmax'))
return final_model
Y_val = np_utils.to_categorical(label_val, num_classes)
# Model
model = Sequential()
# Stage 1
model.add(
Conv2D(kernel_size=(3, 4),
strides=1,
filters=32,
padding='same',
data_format='channels_last',
name='layer_conv1',
input_shape=(img_rows, img_cols, 1)))
model.add(Activation('relu'))
# Stage 2
model.add(
Conv2D(kernel_size=(3, 3),
strides=1,
filters=32,
padding='same',
name='layer_conv2'))
model.add(Activation('relu'))
model.add(Dropout(0.15))
model.add(MaxPooling2D(pool_size=2, strides=1))
# Stage 3
model.add(
Conv2D(kernel_size=(2, 1),
def UNet3D(input_size=(9, 512, 512, 1),
clip=(1024, 2048),
dropout=0.5,
nchannels=(32, 64, 128, 256),
nclasses=1,
bn=None,
activation=lambda x: Activation('relu')(x),
upsampling='copy',
dilation_rates=(1, 1, 1, 1),
residual=False):
"""Creates a U-Net model.
This 3D U-Net model is composed of len(nchannels) "contracting blocks" and len(nchannels) "expansive blocks".
Args:
input_size (tuple, optional): Shape of input image. Defaults to (9, 512, 512, 1).
clip: If not None, clips input values between clip[0] and clip[1]
dropout: If None, applies no dropout; Otherwise, applies dropout of probability equal
to the parameter value (0-1 only)
nchannels: Number of channels for each conv block; len(nchannels) decides number of blocks
nclasses: Number of target classes for segmentation
bn: [None, before, after] adds batchnorm layers across every convolution,
before indicates adding BN before activation function is applied
after indicates adding BN after activation function is applied
Check https://github.com/ducha-aiki/caffenet-benchmark/blob/master/batchnorm.md for related ablations!
activation: Standard Keras activation functions
upsampling: (copy, conv) Use copy for interpolation upsampling \
Use conv for transposed convolution based upsampling (learnable)
dilation_rates: Add dilation to the encoder block conv layers [len(dilation_rates) == len(nchannels)]
residual: False = no residual connections, True = residual connections in every layer
NOTE: This particular model squashes down k 3D frames (batch * k * m * m * 1) into
1 output frame (batch * 1 * m * m * nclasses).
If different behavior is desired, please change the CNN model as necessary.
Returns:
'Model' object: U-Net model.
"""
assert dropout is None or 0 <= dropout <= 1, "Invalid value for dropout parameter (None or 0 to 1 only)"
assert bn in [None, "before", "after"], "Invalid bn parameter value"
assert len(nchannels) >= 2, "At least 2 channels necessary for UNet"
assert len(nchannels) == len(
dilation_rates
), "len(nchannels) should be the same as len(dilation_rates)"
assert isinstance(residual, bool), 'Residual argument can be boolean only'
# Handle callable activations as well
if isinstance(activation, str):
act = lambda x: Activation(activation)(x)
else:
act = activation
inputs = Input(input_size)
inp = inputs
if clip:
inputs = _clamp(inputs, clip[0], clip[1], min_max_scaling=True)
levels = []
# Contracting blocks
for idx, nc in enumerate(nchannels):
if idx == 0:
W = (3, 3, 3)
else:
W = (1, 3, 3)
C = _conv3D_block(inputs,
nc,
W,
nblocks=4,
dropout=dropout if dropout else 0,
bn="before",
prefix='3d_unet_conv_%d' % (idx),
activation=act,
dilation_rate=dilation_rates[idx],
residual=residual)
if idx != len(nchannels) - 1:
C_pooled = _pooling_combo3D(C,
window=(1, 2, 2),
prefix='3d_unet_pooling_%d' % (idx))
else:
C_pooled = None
levels.append((C, C_pooled))
if C_pooled is not None:
inputs = C_pooled
# Expanding blocks
inp1, inp2 = levels[-1][0], levels[-2][0]
for idx, nc in enumerate(reversed(nchannels[1:])):
if idx == 0:
d = dropout
dilation = dilation_rates[-1]
else:
d = 0
dilation = 1
D = _up_concat(inp1,
inp2,
2, (nchannels[-1 - idx], nchannels[-2 - idx]),
3,
dropout=d,
bn=bn,
activation=act,
upsampling=upsampling,
dilation_rate=dilation,
idx=idx,
residual=residual)
if idx != len(nchannels) - 2:
inp1, inp2 = D, levels[-3 - idx][0]
C_end1 = _conv3D_block(D,
64, (1, 3, 3),
dropout=dropout,
bn=bn,
activation=act,
residual=residual)
C_end2 = Conv3D(2, (1, 3, 3),
kernel_initializer='he_normal',
padding='same')(C_end1)
if bn:
C_end2 = BatchNormalization()(C_end2)
C_end2 = act(C_end2)
y_dist = Conv3D(nclasses,
1,
activation='sigmoid',
kernel_initializer='he_normal',
use_bias=True)(C_end2)
model = Model(inputs=inp, outputs=y_dist)
return model
def convolutional_block(X, f, filters, stage, block, s=2):
"""
Implementation of the convolutional block as defined in Figure 4
Arguments:
X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev)
f -- integer, specifying the shape of the middle CONV's window for the main path
filters -- python list of integers, defining the number of filters in the CONV layers of the main path
stage -- integer, used to name the layers, depending on their position in the network
block -- string/character, used to name the layers, depending on their position in the network
s -- Integer, specifying the stride to be used
Returns:
X -- output of the convolutional block, tensor of shape (n_H, n_W, n_C)
"""
# defining name basis
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
# Retrieve Filters
F1, F2, F3 = filters
# Save the input value
X_shortcut = X
##### MAIN PATH #####
# First component of main path
X = Conv2D(F1, (1, 1),
strides=(s, s),
name=conv_name_base + '2a',
kernel_initializer=glorot_uniform(seed=0))(X)
X = BatchNormalization(axis=3, name=bn_name_base + '2a')(X)
X = Activation('relu')(X)
# Second component of main path
X = Conv2D(filters=F2,
kernel_size=(f, f),
strides=(1, 1),
padding='same',
name=conv_name_base + '2b',
kernel_initializer=glorot_uniform(seed=0))(X)
X = BatchNormalization(axis=3, name=bn_name_base + '2b')(X)
X = Activation('relu')(X)
# Third component of main path
X = Conv2D(filters=F3,
kernel_size=(1, 1),
strides=(1, 1),
padding='valid',
name=conv_name_base + '2c',
kernel_initializer=glorot_uniform(seed=0))(X)
X = BatchNormalization(axis=3, name=bn_name_base + '2c')(X)
##### SHORTCUT PATH ####
X_shortcut = Conv2D(F3, (1, 1),
strides=(s, s),
padding='valid',
name=conv_name_base + '1',
kernel_initializer=glorot_uniform(seed=0))(X_shortcut)
X_shortcut = BatchNormalization(axis=3,
name=bn_name_base + '1')(X_shortcut)
# Final step: Add shortcut value to main path, and pass it through a RELU activation
X = Add()([X_shortcut, X])
X = Activation('relu')(X)
return X
def assemble_layers(self):
from tensorflow.python.keras.layers import Activation, Dense
import tensorflow as tf
return Activation('softmax', name='softmax')(Dense(2 + 1)(
tf.keras.layers.Flatten()(self.inputs)))
def __init__(self, input_dim, dr_rate=0.2, opt_name='sgd', lr=0.001, initializer='he_uniform', batchnorm=False):
self.input_dim = input_dim
self.dr_rate = dr_rate
self.opt_name = opt_name
self.initializer = initializer
self.lr = lr
# layers = [1000, 1000, 500, 250, 125, 60, 30]
inputs = Input(shape=(self.input_dim,), name='inputs')
# inputs = Input(shape=(input_dim,))
#x = Lambda(lambda x: x, output_shape=(1000,))(inputs)
# attn_lin = Dense(1000, activation='relu', name='attn_lin')(inputs)
# attn_probs = Dense(1000, activation='softmax', name='attn_probs')(inputs)
# x = keras.layers.multiply( [attn_lin, attn_probs], name='attn')
# ------------------------------------------
# New attention layer (Rick, Austin)
"""
a = Dense(1000)(inputs)
a = BatchNormalization()(a)
a = Activation('relu')(a)
b = Attention(1000)(a)
x = keras.layers.multiply([b, a])
"""
# Old attention layer
a = Dense(1000, activation='relu')(inputs)
b = Dense(1000, activation='softmax')(inputs)
x = keras.layers.multiply( [a, b] )
x = Dense(1000)(x)
# x = BatchNormalization()(x)
if batchnorm: x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(dr_rate)(x)
x = Dense(500)(x)
# x = BatchNormalization()(x)
if batchnorm: x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(dr_rate)(x)
x = Dense(250)(x)
# x = BatchNormalization()(x)
if batchnorm: x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(dr_rate)(x)
x = Dense(125)(x)
# x = BatchNormalization()(x)
if batchnorm: x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(dr_rate)(x)
x = Dense(60)(x)
# x = BatchNormalization()(x)
if batchnorm: x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(dr_rate)(x)
x = Dense(30)(x)
# x = BatchNormalization()(x)
if batchnorm: x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(dr_rate)(x)
# ------------------------------------------
outputs = Dense(1, activation='relu')(x)
model = Model(inputs=inputs, outputs=outputs)
# if opt_name == 'sgd':
# opt = SGD(lr=1e-4, momentum=0.9)
# elif opt_name == 'adam':
# opt = Adam(lr=1e-3, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
# else:
# opt = SGD(lr=1e-4, momentum=0.9) # for clr
opt = self.get_optimizer()
model.compile(loss='mean_squared_error', optimizer=opt, metrics=['mae']) # r2_krs
self.model = model
def network_model(input_shape, input_name, num_classes):
X_input = Input(shape=input_shape, name=input_name)
# Stage 1
X = Conv2D(64, (7, 7),
strides=(1, 1),
padding='same',
name='conv1',
kernel_initializer=glorot_uniform(seed=0))(X_input)
X = BatchNormalization(axis=3, name='bn_conv1')(X)
X = Activation('relu')(X)
X = MaxPooling2D((3, 3), strides=(1, 1), padding='same')(X)
# Stage 2
X = convolutional_block(X,
f=3,
filters=[64, 64, 256],
stage=2,
block='a',
s=1)
X = identity_block(X, 3, [64, 64, 256], stage=2, block='b')
X = identity_block(X, 3, [64, 64, 256], stage=2, block='c')
# Stage 3
X = convolutional_block(X,
f=3,
filters=[128, 128, 512],
stage=3,
block='a',
s=2)
X = identity_block(X, 3, [128, 128, 512], stage=3, block='b')
X = identity_block(X, 3, [128, 128, 512], stage=3, block='c')
X = identity_block(X, 3, [128, 128, 512], stage=3, block='d')
# Stage 4
X = convolutional_block(X,
f=3,
filters=[256, 256, 1024],
stage=4,
block='a',
s=2)
X = identity_block(X, 3, [256, 256, 1024], stage=4, block='b')
X = identity_block(X, 3, [256, 256, 1024], stage=4, block='c')
X = identity_block(X, 3, [256, 256, 1024], stage=4, block='d')
X = identity_block(X, 3, [256, 256, 1024], stage=4, block='e')
X = identity_block(X, 3, [256, 256, 1024], stage=4, block='f')
# Stage 5
X = convolutional_block(X,
f=3,
filters=[512, 512, 2048],
stage=5,
block='a',
s=2)
X = identity_block(X, 3, [512, 512, 2048], stage=5, block='b')
X = identity_block(X, 3, [512, 512, 2048], stage=5, block='c')
# Average pooling
X = AveragePooling2D(pool_size=(7, 7), strides=(1, 1), name='avg_pool')(X)
# output layer
X = Flatten()(X)
X = Dense(num_classes,
activation='softmax',
name='fc' + str(num_classes),
kernel_initializer=glorot_uniform(seed=0))(X)
model = Model(inputs=X_input, outputs=X, name='Resnet50')
return model
def creat_net(train_generator, validation_generator, batch_size, image_lengh,
image_width):
model = Sequential()
model.add(
Conv2D(filters=32,
kernel_size=(5, 5),
padding='same',
input_shape=(image_lengh, image_width, 3),
data_format='channels_last',
kernel_initializer=initializers.he_normal()))
model.add(BatchNormalization(axis=-1, momentum=0.9, epsilon=1e-6))
model.add(Activation(advanced_activations.LeakyReLU(alpha=0.2)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(
Conv2D(filters=32,
kernel_size=(5, 5),
padding='same',
data_format='channels_last',
kernel_initializer=initializers.he_normal()))
model.add(BatchNormalization(axis=-1, momentum=0.9, epsilon=1e-6))
model.add(Activation(advanced_activations.LeakyReLU(alpha=0.2)))
model.add(AveragePooling2D(pool_size=(2, 2)))
model.add(
Conv2D(filters=64,
kernel_size=(5, 5),
padding='same',
data_format='channels_last',
kernel_initializer=initializers.he_normal()))
model.add(BatchNormalization(axis=-1, momentum=0.9, epsilon=1e-6))
model.add(Activation(advanced_activations.LeakyReLU(alpha=0.2)))
model.add(AveragePooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(1024, kernel_initializer=initializers.he_normal()))
model.add(BatchNormalization(epsilon=1e-6))
model.add(Activation(advanced_activations.LeakyReLU(alpha=0.2)))
model.add(
Dense(256,
activation=advanced_activations.LeakyReLU(alpha=0.1),
kernel_initializer=initializers.he_normal()))
model.add(
Dense(4,
activation='softmax',
kernel_initializer=initializers.he_normal()))
adam = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8)
Reduce = ReduceLROnPlateau(monitor='val_accuracy',
factor=0.1,
patience=2,
verbose=1,
mode='auto',
epsilon=0.0001,
cooldown=0,
min_lr=0)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
#保存最优模型
filepath = './模型/resnet_v2_weights-improvement-{epoch:02d}-{val_accuracy:.2f}.hdf5'
checkpoint = callbacks.ModelCheckpoint(filepath,
monitor='val_accuracy',
verbose=1,
save_best_only=True,
mode='max')
model.fit_generator(train_generator,
epochs=20,
steps_per_epoch=1707 // batch_size,
validation_data=validation_generator,
validation_steps=264 // batch_size,
callbacks=[checkpoint, Reduce])
#绘制误差和准确率曲线
loss = model.history.history['loss']
val_loss = model.history.history['val_loss']
epoches = range(1, len(loss) + 1)
acc = model.history.history['accuracy']
val_acc = model.history.history['val_accuracy']
plt.subplot(121)
plt.plot(epoches, loss, 'bo', label='training_loss')
plt.plot(epoches, val_loss, 'r', label='validation_loss')
plt.xlabel('epoches')
plt.ylabel('loss')
plt.title('losses of train and val')
plt.legend()
plt.subplot(122)
plt.plot(epoches, acc, 'bo', label='training_acc')
plt.plot(epoches, val_acc, 'r', label='validation_acc')
plt.xlabel('epoches')
plt.ylabel('acc')
plt.title('accuracy of train and val')
plt.legend()
plt.show()
def build_model():
# epoch, dropout = best_model()
epoch, dropout = 20, 0.1
print('EPOCH = ', epoch)
print('DROPOUT = ', dropout)
model = Sequential()
model.add(InputLayer(input_shape=(MAX_LENGTH, )))
model.add(Embedding(length_word_index, 128))
model.add(Bidirectional(LSTM(256, return_sequences=True)))
model.add(Dropout(dropout))
model.add(TimeDistributed(Dense(length_tag_index)))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=Adam(0.001),
metrics=['acc'])
# metrics=['accuracy', ignore_class_accuracy(0)])
model.summary()
# One hot encode the tags
def onehot_encode_tags(sequences, categories):
cat_sequences = []
for seq in sequences:
cats = []
for item in seq:
cats.append(np.zeros(categories))
cats[-1][item] = 1.0
cat_sequences.append(cats)
return np.array(cat_sequences)
cat_train_tags_y = onehot_encode_tags(y_train, length_tag_index)
print(cat_train_tags_y[0])
# Train the model
history = model.fit(X_train, onehot_encode_tags(y_train, length_tag_index), batch_size=228, epochs=epoch, validation_split=0.2)
# Evaluate the model
loss, accuracy = model.evaluate(X_test, onehot_encode_tags(y_test, length_tag_index))
print('Accuracy: %f' % (accuracy * 100))
def display():
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train','test'], loc = 'upper left')
plt.show()
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train','test'], loc = 'upper left')
plt.show()
display()
def best_model():
epochs = [5, 10, 15, 20]
dropout_rate = [0.1, 0.2, 0.3]
list_of_all_scores = list()
list_of_scores = list()
list_of_dropout = list()
list_of_all_dropouts = list()
list_of_epochs = list()
for i in dropout_rate:
model = Sequential()
model.add(InputLayer(input_shape=(MAX_LENGTH, )))
model.add(Embedding(length_word_index, 128))
model.add(Bidirectional(LSTM(256, return_sequences=True)))
model.add(Dropout(i))
model.add(TimeDistributed(Dense(length_tag_index)))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=Adam(0.001),
metrics=['accuracy'])
# metrics=['accuracy', ignore_class_accuracy(0)])
# model.summary()
list_of_dropout.append(i)
# One hot encode the tags
def onehot_encode_tags(sequences, categories):
cat_sequences = []
for seq in sequences:
cats = []
for item in seq:
cats.append(np.zeros(categories))
cats[-1][item] = 1.0
cat_sequences.append(cats)
return np.array(cat_sequences)
cat_train_tags_y = onehot_encode_tags(y_train, length_tag_index)
# print(cat_train_tags_y[0])
for e in epochs:
list_of_all_dropouts.append(i)
list_of_epochs.append(e)
model.fit(X_train, onehot_encode_tags(y_train, length_tag_index), batch_size=128, epochs=e, validation_split=0.2)
score = model.evaluate(X_test, onehot_encode_tags(y_test, length_tag_index))
list_of_all_scores.append(score)
if score not in list_of_scores:
list_of_scores.append(score)
#print('Dropout:', i, '\n', 'Epoch:', e, '\n', 'Score:', float(score))
lowest = min(list_of_all_scores)
num = list_of_scores.index(lowest)
epoch = list_of_epochs[num]
dropout = list_of_all_dropouts[num]
print('Lowest score:', lowest, 'Epoch:', epoch, 'Dropout', dropout)
return epoch, dropout
return wav
def pre_emphasis(wav):
pre_emphasis = np.random.uniform(0.95, 0.97)
ret_wav = np.append(wav[0], wav[1:] - pre_emphasis * wav[:-1])
wav_max = max(abs(ret_wav))
ret_wav = ret_wav / wav_max
return ret_wav
def swish(x):
return (K.sigmoid(x) * x)
get_custom_objects().update({'swish': Activation(swish)})
silence_data = np.concatenate(
[read_wav_file(x) for x in silence_files.wav_file.values])
from scipy.signal import stft
def process_wav_file(fname, phase, dim='1D'):
wav = read_wav_file(fname)
# time streching
if phase == 'TRAIN':
time_strech_flag = np.random.randint(2)
if time_strech_flag == 1:
ts_ratio = np.random.uniform(0.8, 1.2)
def resblock(x,
kernel_size,
resample,
nfilters,
name,
norm=BatchNormalization,
is_first=False,
conv_layer=Conv2D):
assert resample in ["UP", "SAME", "DOWN"]
feature_axis = 1 if K.image_data_format() == 'channels_first' else -1
identity = lambda x: x
if norm is None:
norm = lambda axis, name: identity
if resample == "UP":
resample_op = UpSampling2D(size=(2, 2), name=name + '_up')
elif resample == "DOWN":
resample_op = AveragePooling2D(pool_size=(2, 2), name=name + '_pool')
else:
resample_op = identity
in_filters = K.int_shape(x)[feature_axis]
if resample == "SAME" and in_filters == nfilters:
shortcut_layer = identity
else:
shortcut_layer = conv_layer(kernel_size=(1, 1),
filters=int(nfilters),
kernel_initializer=he_init,
name=name + 'shortcut')
### SHORTCUT PAHT
if is_first:
shortcut = resample_op(x)
shortcut = shortcut_layer(shortcut)
else:
shortcut = shortcut_layer(x)
shortcut = resample_op(shortcut)
### CONV PATH
convpath = x
if not is_first:
convpath = norm(axis=feature_axis, name=name + '_bn1')(convpath)
convpath = Activation('relu')(convpath)
if resample == "UP":
convpath = resample_op(convpath)
convpath = conv_layer(filters=int(nfilters),
kernel_size=kernel_size,
kernel_initializer=he_init,
use_bias=True,
padding='same',
name=name + '_conv1')(convpath)
convpath = norm(axis=feature_axis, name=name + '_bn2')(convpath)
convpath = Activation('relu')(convpath)
convpath = conv_layer(filters=int(nfilters),
kernel_size=kernel_size,
kernel_initializer=he_init,
use_bias=True,
padding='same',
name=name + '_conv2')(convpath)
if resample == "DOWN":
convpath = resample_op(convpath)
y = Add()([shortcut, convpath])
return y
def construct_model():
output_parameters()
model = Sequential()
# mask_zero 在 MaxPooling 层中不能支持
model.add(Embedding(creative_id_window, embedding_size, input_length=max_len))
if model_type == 'MLP':
model.add(Flatten())
model.add(Dense(8, activation='relu', kernel_regularizer=l2(0.001)))
model.add(Dropout(0.5))
model.add(Dense(4, activation='relu', kernel_regularizer=l2(0.001)))
model.add(Dropout(0.5))
elif model_type == 'Conv1D':
model.add(Conv1D(32, 7, activation='relu', kernel_regularizer=l2(0.001)))
model.add(Conv1D(32, 7, activation='relu', kernel_regularizer=l2(0.001)))
model.add(GlobalMaxPooling1D())
elif model_type == 'GlobalMaxPooling1D':
model.add(GlobalMaxPooling1D())
elif model_type == 'GlobalMaxPooling1D+MLP':
model.add(GlobalMaxPooling1D())
model.add(Dropout(0.5))
model.add(Dense(embedding_size, kernel_regularizer=l2(0.001)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(embedding_size, kernel_regularizer=l2(0.001)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(embedding_size // 2, kernel_regularizer=l2(0.001)))
model.add(BatchNormalization())
model.add(Activation('softmax'))
model.add(Dropout(0.5))
model.add(Dense(embedding_size // 2, kernel_regularizer=l2(0.001)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(embedding_size // 2, kernel_regularizer=l2(0.001)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(embedding_size // 4, kernel_regularizer=l2(0.001)))
model.add(BatchNormalization())
model.add(Activation('softmax'))
model.add(Dropout(0.5))
model.add(Dense(embedding_size // 4, kernel_regularizer=l2(0.001)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(embedding_size // 4, kernel_regularizer=l2(0.001)))
model.add(BatchNormalization())
model.add(Activation('relu'))
elif model_type == 'GRU+MLP':
model.add(GRU(embedding_size, dropout=0.5, recurrent_dropout=0.5))
model.add(Dropout(0.5))
model.add(Dense(embedding_size, kernel_regularizer=l2(0.001)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(embedding_size, kernel_regularizer=l2(0.001)))
model.add(BatchNormalization())
model.add(Activation('relu'))
elif model_type == 'GRU':
model.add(GRU(embedding_size, dropout=0.2, recurrent_dropout=0.2))
# model.add(LSTM(128, dropout = 0.5, recurrent_dropout = 0.5))
elif model_type == 'Conv1D+LSTM':
model.add(Conv1D(32, 5, activation='relu', kernel_regularizer=l2(0.001)))
model.add(Conv1D(32, 5, activation='relu', kernel_regularizer=l2(0.001)))
model.add(LSTM(16, dropout=0.5, recurrent_dropout=0.5))
elif model_type == 'Bidirectional-LSTM':
model.add(Bidirectional(LSTM(embedding_size, dropout=0.2, recurrent_dropout=0.2)))
else:
raise Exception("错误的网络模型类型")
if label_name == "age":
model.add(Dropout(0.5))
model.add(Dense(10, kernel_regularizer=l2(0.001)))
model.add(BatchNormalization())
model.add(Activation('softmax'))
# model.add(Dense(10, activation = 'softmax', kernel_regularizer = l2(0.001)))
print("%s——模型构建完成！" % model_type)
print("* 编译模型")
# Keras 好像不能支持 report_tensor_allocations_upon_oom
# 运行时会 Python 会报错：Process finished with exit code -1073741819 (0xC0000005)
model.compile(optimizer=optimizers.RMSprop(lr=RMSProp_lr),
loss=losses.sparse_categorical_crossentropy,
metrics=[metrics.sparse_categorical_accuracy])
elif label_name == 'gender':
# model.add(Dropout(0.5))
# model.add(Dense(1, kernel_regularizer = l2(0.001)))
# model.add(BatchNormalization())
# model.add(Activation('sigmoid'))
model.add(Dense(1, activation='sigmoid', kernel_regularizer=l2(0.001)))
print("%s——模型构建完成！" % model_type)
print("* 编译模型")
model.compile(optimizer=optimizers.RMSprop(lr=RMSProp_lr),
loss=losses.binary_crossentropy,
metrics=[metrics.binary_accuracy])
else:
raise Exception("错误的标签类型！")
return model
num_classes = len(train_generator.class_indices)
train_classes = train_generator.classes
train_labels = utils.to_categorical(train_classes, num_classes=num_classes)
num_classes = len(validate_generator.class_indices)
validate_classes = validate_generator.classes
validate_labels = utils.to_categorical(validate_classes,
num_classes=num_classes)
print("Create top layer model...")
# Builds linear stack of layers for model
model = tf.keras.Sequential()
# FCL 1 - Flatten to 1D -> Hidden FCL -> Relu -> Dropout prob 0.5
model.add(Flatten(input_shape=bottleneck_train_features.shape[1:]))
model.add(Dense(FCL_SIZE))
model.add(Activation('relu'))
model.add(Dropout(DROPOUT_2))
# FCL 2 - Flatten to 1D -> Final Fully Connected Layer -> softmax
model.add(Dense(CLASSES))
model.add(Activation('softmax')) # produces error as a probability
# Configure optimizer for gradient descent
# Compile training process for Multi-class classification
optimizer = optimizers.RMSprop(lr=LEARNING_RATE, decay=DECAY_RATE)
model.compile(
optimizer=optimizer, # Adam optimizer or rmsprop
loss=
'categorical_crossentropy', # use cros-entropy loss function to minimise loss
metrics=['accuracy']) # report on accuracy
print(" Train top layer model on bottleneck features...")
def buil_bcnn(all_trainable=False,
size_height=448,
size_width=448,
no_class=200,
no_last_layer_backbone=17,
name_optimizer='sgd',
learning_rate=1.0,
decay_learning_rate=0.0,
decay_weight_rate=0.0,
name_initializer='glorot_normal',
name_activation='softmax',
name_loss='categorical_crossentropy'):
'''Build Bilinear CNN.
Detector and extractor are both VGG16.
Args:
all_trainable: fix or unfix VGG16 layers.
size_height: default 448.
size_width: default 448.
no_class: number of prediction classes.
no_last_layer_backbone: number of VGG16 backbone layer.
name_optimizer: optimizer method.
learning_rate: learning rate.
decay_learning_rate: learning rate decay.
decay_weight_rate: l2 normalization decay rate.
name_initializer: initializer method.
name_activation: activation method.
name_loss: loss function.
Returns:
Bilinear CNN model.
'''
##########################
# Load pre-trained model #
##########################
# Load model
input_tensor = Input(shape=[size_height, size_width, 3])
pre_train_model = VGG16(input_tensor=input_tensor,
include_top=False,
weights='imagenet')
# Pre-trained weights
for layer in pre_train_model.layers:
layer.trainable = all_trainable
######################
# Combine two models #
######################
# Extract features form detecotr
model_detector = pre_train_model
output_detector = model_detector.layers[no_last_layer_backbone].output
shape_detector = model_detector.layers[no_last_layer_backbone].output_shape
# Extract features from extractor
model_extractor = pre_train_model
output_extractor = model_extractor.layers[no_last_layer_backbone].output
shape_extractor = model_extractor.layers[
no_last_layer_backbone].output_shape
# Reshape tensor to (minibatch_size, total_pixels, filter_size)
output_detector = Reshape(
[shape_detector[1] * shape_detector[2],
shape_detector[-1]])(output_detector)
output_extractor = Reshape(
[shape_extractor[1] * shape_extractor[2],
shape_extractor[-1]])(output_extractor)
# Outer-products
x = Lambda(_outer_product)([output_detector, output_extractor])
# Reshape tensor to (minibatch_size, filter_size_detector*filter_size_extractor)
x = Reshape([shape_detector[-1] * shape_extractor[-1]])(x)
# Signed square-root
x = Lambda(_signed_sqrt)(x)
# L2 normalization
x = Lambda(_l2_normalize)(x)
###############################
# Attach full-connected layer #
###############################
if name_initializer is not None:
name_initializer = eval(name_initializer + '()')
# FC layer
x = Dense(units=no_class,
kernel_initializer=name_initializer,
kernel_regularizer=l2(decay_weight_rate))(x)
output_tensor = Activation(name_activation)(x)
#################
# Compile model #
#################
model_bcnn = Model(inputs=[input_tensor], outputs=[output_tensor])
# Optimizer
if name_optimizer == 'adam':
optimizer = adam(lr=learning_rate, decay=decay_learning_rate)
elif name_optimizer == 'rmsprop':
optimizer = RMSprop(lr=learning_rate, decay=decay_learning_rate)
elif name_optimizer == 'sgd':
optimizer = SGD(lr=learning_rate,
decay=decay_learning_rate,
momentum=0.9,
nesterov=None)
else:
raise RuntimeError('Optimizer should be one of Adam, RMSprop and SGD.')
# Compile
model_bcnn.compile(loss=name_loss,
optimizer=optimizer,
metrics=['accuracy'])
# print('-------- Mode summary --------')
# print(model_bcnn.summary())
# print('------------------------------')
return model_bcnn
x = x/255.0
#the best parameter setting from model4
dense_layers = [1]
layer_sizes = [32]
conv_layers = [3]
for dense_layer in dense_layers:
for layer_size in layer_sizes:
for conv_layer in conv_layers:
name = "{}-conv-{}-nodes-{}-dense-{}".format(conv_layer, layer_size, dense_layer, int(time.time()))
print(name)
model = Sequential()
model.add(Conv2D(layer_size, (3,3), input_shape = x.shape[1:]))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.2))
for l in range(conv_layer-1):
model.add(Conv2D(layer_size, (3,3)))
model.add(Activation("relu"))
model.add(Conv2D(layer_size, (3,3)))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.2))
model.add(Flatten())
for l in range(dense_layer):
model.add(Dense(layer_size))
model.add(Activation("relu"))
test = all_data[ntrain:]
print("after spilt", train.shape, test.shape)
X_train, X_validation, y_train, y_validation = train_test_split(
train, label, test_size=0.2, random_state=42)
input_dim = X_train.shape[1]
regularize_rate = 5.0e-5
# model = Sequential()
model = Sequential()
# input
# model.add(Convolution1D(nb_filter=512, filter_length=1, input_shape=(input_dim, )))
# model.add(BatchNormalization())
model.add(
Dense(512, kernel_initializer="lecun_normal", input_shape=(input_dim, )))
# model.add(BatchNormalization())
model.add(Activation('selu'))
model.add(Dropout(0.4))
model.add(
Dense(256,
kernel_initializer="lecun_normal",
kernel_regularizer=regularizers.l2(regularize_rate)))
# model.add(BatchNormalization())
model.add(Activation('selu'))
model.add(Dropout(0.2))
model.add(
Dense(512,
kernel_initializer="lecun_normal",
kernel_regularizer=regularizers.l2(regularize_rate)))
# model.add(BatchNormalization())
model.add(Activation('selu'))
y = pickle.load(pickle_in)
X = X / 255.0
layer_sizes = [128]
conv_layers = [1]
model = Sequential()
for layer_size in layer_sizes:
for conv_layer in conv_layers:
NAME = "{}-conv-{}-nodes-{}".format(conv_layer, layer_size,
int(time.time()))
print(NAME)
model.add(Conv2D(layer_size, (4, 4), input_shape=X.shape[1:]))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
for l in range(conv_layer - 1):
model.add(Conv2D(layer_size, (4, 4)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
model.add(Flatten())
model.add(Dense(5))
model.add(Activation('sigmoid'))
def call(self, x, **kwargs):
x = Activation(self.swish)(x)
return x
def inception_resnet_block(x, scale, block_type, block_idx, activation='relu'):
"""Adds a Inception-ResNet block.
This function builds 3 types of Inception-ResNet blocks mentioned
in the paper, controlled by the `block_type` argument (which is the
block name used in the official TF-slim implementation):
- Inception-ResNet-A: `block_type='block35'`
- Inception-ResNet-B: `block_type='block17'`
- Inception-ResNet-C: `block_type='block8'`
Arguments:
x: input tensor.
scale: scaling factor to scale the residuals (i.e., the output of
passing `x` through an inception module) before adding them
to the shortcut branch. Let `r` be the output from the residual
branch,
the output of this block will be `x + scale * r`.
block_type: `'block35'`, `'block17'` or `'block8'`, determines
the network structure in the residual branch.
block_idx: an `int` used for generating layer names. The Inception-ResNet
blocks
are repeated many times in this network. We use `block_idx` to
identify
each of the repetitions. For example, the first Inception-ResNet-A
block
will have `block_type='block35', block_idx=0`, ane the layer names
will have
a common prefix `'block35_0'`.
activation: activation function to use at the end of the block.
When `activation=None`, no activation is applied
(i.e., "linear" activation: `a(x) = x`).
Returns:
Output tensor for the block.
Raises:
ValueError: if `block_type` is not one of `'block35'`,
`'block17'` or `'block8'`.
"""
if block_type == 'block35':
branch_0 = conv2d_bn(x, 32, 1)
branch_1 = conv2d_bn(x, 32, 1)
branch_1 = conv2d_bn(branch_1, 32, 3)
branch_2 = conv2d_bn(x, 32, 1)
branch_2 = conv2d_bn(branch_2, 48, 3)
branch_2 = conv2d_bn(branch_2, 64, 3)
branches = [branch_0, branch_1, branch_2]
elif block_type == 'block17':
branch_0 = conv2d_bn(x, 192, 1)
branch_1 = conv2d_bn(x, 128, 1)
branch_1 = conv2d_bn(branch_1, 160, [1, 7])
branch_1 = conv2d_bn(branch_1, 192, [7, 1])
branches = [branch_0, branch_1]
elif block_type == 'block8':
branch_0 = conv2d_bn(x, 192, 1)
branch_1 = conv2d_bn(x, 192, 1)
branch_1 = conv2d_bn(branch_1, 224, [1, 3])
branch_1 = conv2d_bn(branch_1, 256, [3, 1])
branches = [branch_0, branch_1]
else:
raise ValueError('Unknown Inception-ResNet block type. '
'Expects "block35", "block17" or "block8", '
'but got: ' + str(block_type))
block_name = block_type + '_' + str(block_idx)
channel_axis = 1 if K.image_data_format() == 'channels_first' else 3
mixed = Concatenate(axis=channel_axis, name=block_name + '_mixed')(branches)
up = conv2d_bn(
mixed,
K.int_shape(x)[channel_axis],
1,
activation=None,
use_bias=True,
name=block_name + '_conv')
x = Lambda(
lambda inputs, scale: inputs[0] + inputs[1] * scale,
output_shape=K.int_shape(x)[1:],
arguments={'scale': scale},
name=block_name)([x, up])
if activation is not None:
x = Activation(activation, name=block_name + '_ac')(x)
return x
def build_3d_cnn(w, h, d, s, num_outputs):
#Credit: https://github.com/jessecha/DNRacing/blob/master/3D_CNN_Model/model.py
'''
w : width
h : height
d : depth
s : n_stacked
'''
input_shape = (s, h, w, d)
model = Sequential()
#First layer
#model.add(Cropping3D(cropping=((0,0), (50,10), (0,0)), input_shape=input_shape) ) #trim pixels off top
# Second layer
model.add(
Conv3D(filters=16,
kernel_size=(3, 3, 3),
strides=(1, 3, 3),
data_format='channels_last',
padding='same',
input_shape=input_shape))
model.add(Activation('relu'))
model.add(
MaxPooling3D(pool_size=(1, 2, 2),
strides=(1, 2, 2),
padding='valid',
data_format=None))
# Third layer
model.add(
Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
data_format='channels_last',
padding='same'))
model.add(Activation('relu'))
model.add(
MaxPooling3D(pool_size=(1, 2, 2),
strides=(1, 2, 2),
padding='valid',
data_format=None))
# Fourth layer
model.add(
Conv3D(filters=64,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
data_format='channels_last',
padding='same'))
model.add(Activation('relu'))
model.add(
MaxPooling3D(pool_size=(1, 2, 2),
strides=(1, 2, 2),
padding='valid',
data_format=None))
# Fifth layer
model.add(
Conv3D(filters=128,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
data_format='channels_last',
padding='same'))
model.add(Activation('relu'))
model.add(
MaxPooling3D(pool_size=(1, 2, 2),
strides=(1, 2, 2),
padding='valid',
data_format=None))
# Fully connected layer
model.add(Flatten())
model.add(Dense(256))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(256))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(num_outputs))
#model.add(Activation('tanh'))
return model
def add_model(self, input_data, target_data=None):
"""Implements core of model that transforms input_data into predictions.
The core transformation for this model which transforms a batch of input
data into a batch of predictions.
Args:
input_data: A tensor of shape (batch_size, num_steps, time_stamps).
target_data: A tensor of shape (batch_size, num_steps, time_stamps).
Returns:
predict: A tensor of shape (batch_size, num_steps, time_stamps)
"""
# Consider signal matrix as an image with channels.
height = self.config.num_steps
width = self.config.time_stamps
channels = self.config.channels
batch_size = self.config.batch_size
# input_data: (-1, height, width, channels)
input_data = tf.reshape(input_data, [-1, channels, height, width])
input_data = tf.transpose(input_data, perm=[0, 2, 3, 1])
# module-1
# conv/BN/ReLU-1
x = layer_conv_bn_relu(input_data, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# conv/BN/ReLU-2
x = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# conv/BN/ReLU-3
x_1 = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# module-2
# conv/BN/ReLU-1
x = layer_conv_bn_relu(x_1, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# conv/BN/ReLU-2
x = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# conv/BN/ReLU-3
x_2 = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# module-3
# conv/BN/ReLU-1
x = layer_conv_bn_relu(x_2, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# conv/BN/ReLU-2
x = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# conv/BN/ReLU-3
x_3 = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# module-4
# conv/BN/ReLU-1
x = layer_conv_bn_relu(x_3, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# conv/BN/ReLU-2
x = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# conv/BN/ReLU-3
x_4 = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# module-5
# conv/BN/ReLU-1
x = layer_conv_bn_relu(x_4, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# conv/BN/ReLU-2
x = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# conv/BN/ReLU-3
x = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# module-6
# conv/BN/ReLU-1
x = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# concatenate
x = concatenate([x, x_4], axis=-1)
# conv/BN/ReLU-2
x = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# conv/BN/ReLU-3
x = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# module-7
# conv/BN/ReLU-1
x = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# concatenate
x = concatenate([x, x_3], axis=-1)
# conv/BN/ReLU-2
x = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# conv/BN/ReLU-3
x = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# module-8
# conv/BN/ReLU-1
x = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# concatenate
x = concatenate([x, x_2], axis=-1)
# conv/BN/ReLU-2
x = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# conv/BN/ReLU-3
x = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# module-9
# conv/BN/ReLU-1
x = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# concatenate
x = concatenate([x, x_1], axis=-1)
# conv/BN/ReLU-2
x = layer_conv_bn_relu(x, self.config.n1, self.config.f1,
self.config.f1, self.config.regularizer)
# conv/BN/ReLU-3
x = Convolution2D(
self.config.n1,
(self.config.f1, self.config.f1),
activation='relu',
padding='same',
kernel_regularizer=self.config.regularizer,
bias_regularizer=self.config.regularizer,
data_format='channels_last',
)(x)
x = BatchNormalization(axis=-1,
epsilon=1e-7,
gamma_initializer='one',
beta_initializer='zero',
weights=None)(x)
x = Activation('tanh')(x)
# output x: (-1, height, width, channels)
output = Convolution2D(channels, (self.config.f2, self.config.f2),
activation='linear',
padding='same',
name='output',
kernel_regularizer=self.config.regularizer,
bias_regularizer=self.config.regularizer,
data_format='channels_last')(x)
prediction = tf.transpose(output, perm=[0, 3, 1, 2])
prediction = tf.reshape(prediction, [batch_size, height, width])
return prediction
def ResNet50(include_top=True,
weights='imagenet',
input_shape=None,
pooling=None,
classes=1000):
"""Instantiates the ResNet50 architecture.
Optionally loads weights pre-trained
on ImageNet. Note that when using TensorFlow,
for best performance you should set
`image_data_format="channels_last"` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The data format
convention used by the model is the one
specified in your Keras config file.
# Arguments
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization)
or "imagenet" (pre-training on ImageNet).
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(224, 224, 3)` (with `channels_last` data format)
or `(3, 224, 244)` (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 197.
E.g. `(200, 200, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if weights not in {'imagenet', None}:
raise ValueError('The `weights` argument should be either '
'`None` (random initialization) or `imagenet` '
'(pre-training on ImageNet).')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
img_input = Input(shape=input_shape)
if K.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
x = ZeroPadding2D((3, 3))(img_input)
x = Conv2D(64, (7, 7), strides=(2, 2), name='conv1')(x)
x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x = Activation('relu')(x)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='b')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='d')
x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f')
x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')
x = AveragePooling2D((7, 7), name='avg_pool')(x)
if include_top:
x = Flatten()(x)
x = Dense(classes, activation='softmax', name='fc1000')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
inputs = img_input
# Create model.
model = Model(inputs, x, name='resnet50')
# load weights
if weights == 'imagenet':
if include_top:
weights_path = get_file(
'resnet50_weights_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH,
cache_subdir='models',
md5_hash='a7b3fe01876f51b976af0dea6bc144eb')
else:
weights_path = get_file(
'resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
md5_hash='a268eb855778b3df3c7506639542a6af')
model.load_weights(weights_path)
if K.backend() == 'theano':
layer_utils.convert_all_kernels_in_model(model)
if K.image_data_format() == 'channels_first':
if include_top:
maxpool = model.get_layer(name='avg_pool')
shape = maxpool.output_shape[1:]
dense = model.get_layer(name='fc1000')
layer_utils.convert_dense_weights_data_format(
dense, shape, 'channels_first')
if K.backend() == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image data format convention '
'(`image_data_format="channels_first"`). '
'For best performance, set '
'`image_data_format="channels_last"` in '
'your Keras config '
'at ~/.keras/keras.json.')
return model
def multibox_head(source_layers,
num_priors,
normalizations=None,
softmax=True):
num_classes = 2
class_activation = 'softmax' if softmax else 'sigmoid'
mbox_conf = []
mbox_loc = []
mbox_quad = []
mbox_rbox = []
for i in range(len(source_layers)):
x = source_layers[i]
name = x.name.split('/')[0]
# normalize
if normalizations is not None and normalizations[i] > 0:
name = name + '_norm'
x = Normalize(normalizations[i], name=name)(x)
# confidence
name1 = name + '_mbox_conf'
x1 = Conv2D(num_priors[i] * num_classes, (3, 5),
padding='same',
name=name1)(x)
x1 = Flatten(name=name1 + '_flat')(x1)
mbox_conf.append(x1)
# location, Delta(x,y,w,h)
name2 = name + '_mbox_loc'
x2 = Conv2D(num_priors[i] * 4, (3, 5), padding='same', name=name2)(x)
x2 = Flatten(name=name2 + '_flat')(x2)
mbox_loc.append(x2)
# quadrilateral, Delta(x1,y1,x2,y2,x3,y3,x4,y4)
name3 = name + '_mbox_quad'
x3 = Conv2D(num_priors[i] * 8, (3, 5), padding='same', name=name3)(x)
x3 = Flatten(name=name3 + '_flat')(x3)
mbox_quad.append(x3)
# rotated rectangle, Delta(x1,y1,x2,y2,h)
name4 = name + '_mbox_rbox'
x4 = Conv2D(num_priors[i] * 5, (3, 5), padding='same', name=name4)(x)
x4 = Flatten(name=name4 + '_flat')(x4)
mbox_rbox.append(x4)
mbox_conf = concatenate(mbox_conf, axis=1, name='mbox_conf')
mbox_conf = Reshape((-1, num_classes), name='mbox_conf_logits')(mbox_conf)
mbox_conf = Activation(class_activation, name='mbox_conf_final')(mbox_conf)
mbox_loc = concatenate(mbox_loc, axis=1, name='mbox_loc')
mbox_loc = Reshape((-1, 4), name='mbox_loc_final')(mbox_loc)
mbox_quad = concatenate(mbox_quad, axis=1, name='mbox_quad')
mbox_quad = Reshape((-1, 8), name='mbox_quad_final')(mbox_quad)
mbox_rbox = concatenate(mbox_rbox, axis=1, name='mbox_rbox')
mbox_rbox = Reshape((-1, 5), name='mbox_rbox_final')(mbox_rbox)
predictions = concatenate([mbox_loc, mbox_quad, mbox_rbox, mbox_conf],
axis=2,
name='predictions')
return predictions
def Deeplab_xcep_ade20k(weights=None,
input_tensor=None,
input_shape=(512, 512, 3),
num_classes=151,
backbone='xception',
OS=8,
model_path="",
activation=None):
if not (weights in {'ade20k', None}):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `ade20k` '
'(pre-trained on ADE20K)')
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
img_input = input_tensor
if OS == 8:
entry_block3_stride = 1
middle_block_rate = 2 # ! Not mentioned in paper, but required
exit_block_rates = (2, 4)
atrous_rates = (12, 24, 36)
x = Conv2D(32, (3, 3),
strides=(2, 2),
name='entry_flow_conv1_1',
use_bias=False,
padding='same')(img_input)
x = BatchNormalization(name='entry_flow_conv1_1_BN')(x)
x = Activation(tf.nn.relu)(x)
x = _conv2d_same(x, 64, 'entry_flow_conv1_2', kernel_size=3, stride=1)
x = BatchNormalization(name='entry_flow_conv1_2_BN')(x)
x = Activation(tf.nn.relu)(x)
x = _xception_block(x, [128, 128, 128],
'entry_flow_block1',
skip_connection_type='conv',
stride=2,
depth_activation=False)
x, skip1 = _xception_block(x, [256, 256, 256],
'entry_flow_block2',
skip_connection_type='conv',
stride=2,
depth_activation=False,
return_skip=True)
x = _xception_block(x, [728, 728, 728],
'entry_flow_block3',
skip_connection_type='conv',
stride=entry_block3_stride,
depth_activation=False)
for i in range(16):
x = _xception_block(x, [728, 728, 728],
'middle_flow_unit_{}'.format(i + 1),
skip_connection_type='sum',
stride=1,
rate=middle_block_rate,
depth_activation=False)
x = _xception_block(x, [728, 1024, 1024],
'exit_flow_block1',
skip_connection_type='conv',
stride=1,
rate=exit_block_rates[0],
depth_activation=False)
x = _xception_block(x, [1536, 1536, 2048],
'exit_flow_block2',
skip_connection_type='none',
stride=1,
rate=exit_block_rates[1],
depth_activation=True)
else:
entry_block3_stride = 2
middle_block_rate = 1
exit_block_rates = (1, 2)
atrous_rates = (6, 12, 18)
x = Conv2D(32, (3, 3),
strides=(2, 2),
name='entry_flow_conv1_1',
use_bias=False,
padding='same')(img_input)
x = BatchNormalization(name='entry_flow_conv1_1_BN')(x)
x = Activation(tf.nn.relu)(x)
x = _conv2d_same(x, 64, 'entry_flow_conv1_2', kernel_size=3, stride=1)
x = BatchNormalization(name='entry_flow_conv1_2_BN')(x)
x = Activation(tf.nn.relu)(x)
x = _xception_block(x, [128, 128, 128],
'entry_flow_block1',
skip_connection_type='conv',
stride=2,
depth_activation=False)
x, skip1 = _xception_block(x, [256, 256, 256],
'entry_flow_block2',
skip_connection_type='conv',
stride=2,
depth_activation=False,
return_skip=True)
x = _xception_block(x, [728, 728, 728],
'entry_flow_block3',
skip_connection_type='conv',
stride=entry_block3_stride,
depth_activation=False)
for i in range(16):
x = _xception_block(x, [728, 728, 728],
'middle_flow_unit_{}'.format(i + 1),
skip_connection_type='sum',
stride=1,
rate=middle_block_rate,
depth_activation=False)
x = _xception_block(x, [728, 1024, 1024],
'exit_flow_block1',
skip_connection_type='conv',
stride=1,
rate=exit_block_rates[0],
depth_activation=False)
x = _xception_block(x, [1536, 1536, 2048],
'exit_flow_block2',
skip_connection_type='none',
stride=1,
rate=exit_block_rates[1],
depth_activation=True)
shape_before = tf.shape(x)
b4 = GlobalAveragePooling2D()(x)
# from (b_size, channels)->(b_size, 1, 1, channels)
b4 = Lambda(lambda x: K.expand_dims(x, 1))(b4)
b4 = Lambda(lambda x: K.expand_dims(x, 1))(b4)
b4 = Conv2D(256, (1, 1),
padding='same',
use_bias=False,
name='image_pooling')(b4)
b4 = BatchNormalization(name='image_pooling_BN', epsilon=1e-5)(b4)
b4 = Activation(tf.nn.relu)(b4)
# upsample. have to use compat because of the option align_corners
size_before = tf.keras.backend.int_shape(x)
b4 = Lambda(lambda x: tf.compat.v1.image.resize(
x, size_before[1:3], method='bilinear', align_corners=True))(b4)
# simple 1x1
b0 = Conv2D(256, (1, 1), padding='same', use_bias=False, name='aspp0')(x)
b0 = BatchNormalization(name='aspp0_BN', epsilon=1e-5)(b0)
b0 = Activation(tf.nn.relu, name='aspp0_activation')(b0)
if backbone == 'xception':
# rate = 6 (12)
b1 = SepConv_BN(x,
256,
'aspp1',
rate=atrous_rates[0],
depth_activation=True,
epsilon=1e-5)
# rate = 12 (24)
b2 = SepConv_BN(x,
256,
'aspp2',
rate=atrous_rates[1],
depth_activation=True,
epsilon=1e-5)
# rate = 18 (36)
b3 = SepConv_BN(x,
256,
'aspp3',
rate=atrous_rates[2],
depth_activation=True,
epsilon=1e-5)
# concatenate ASPP branches & project
x = Concatenate()([b4, b0, b1, b2, b3])
x = Conv2D(256, (1, 1),
padding='same',
use_bias=False,
name='concat_projection')(x)
x = BatchNormalization(name='concat_projection_BN', epsilon=1e-5)(x)
x = Activation(tf.nn.relu)(x)
x = Dropout(0.1)(x)
# DeepLab v.3+ decoder
if backbone == 'xception':
# Feature projection
# x4 (x2) block
size_before2 = tf.keras.backend.int_shape(x)
x = Lambda(lambda xx: tf.compat.v1.image.resize(
xx, skip1.shape[1:3], method='bilinear', align_corners=True))(x)
dec_skip1 = Conv2D(48, (1, 1),
padding='same',
use_bias=False,
name='feature_projection0')(skip1)
dec_skip1 = BatchNormalization(name='feature_projection0_BN',
epsilon=1e-5)(dec_skip1)
dec_skip1 = Activation(tf.nn.relu)(dec_skip1)
x = Concatenate()([x, dec_skip1])
x = SepConv_BN(x,
256,
'decoder_conv0',
depth_activation=True,
epsilon=1e-5)
x = SepConv_BN(x,
256,
'decoder_conv1',
depth_activation=True,
epsilon=1e-5)
#if (weights == 'ade20k' and num_classes == 150):
last_layer_name = 'logits_semantic'
#else:
#last_layer_name = 'custom_logits_semantic'
x = Conv2D(num_classes,
kernel_size=(1, 1),
padding='same',
name=last_layer_name)(x)
size_before3 = tf.keras.backend.int_shape(img_input)
x = Lambda(lambda xx: tf.compat.v1.image.resize(
xx, size_before3[1:3], method='bilinear', align_corners=True))(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
if activation in {'softmax', 'sigmoid'}:
x = tf.keras.layers.Activation(activation)(x)
model = Model(inputs, x, name='deeplabv3plus')
if weights == 'ade20k':
weights_path = model_path
model.load_weights(weights_path, by_name=True)
return model
def test_tf_keras_mnist_cnn():
""" This is the basic mnist cnn example from keras.
"""
try:
import tensorflow as tf
from tensorflow.python import keras
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Dense, Dropout, Flatten, Activation
from tensorflow.python.keras.layers import Conv2D, MaxPooling2D
from tensorflow.python.keras import backend as K
except Exception as e:
print("Skipping test_tf_keras_mnist_cnn!")
return
import shap
batch_size = 128
num_classes = 10
epochs = 1
# input image dimensions
img_rows, img_cols = 28, 28
# the data, split between train and test sets
(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
if K.image_data_format() == 'channels_first':
x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)
input_shape = (1, img_rows, img_cols)
else:
x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)
input_shape = (img_rows, img_cols, 1)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
model = Sequential()
model.add(
Conv2D(32,
kernel_size=(3, 3),
activation='relu',
input_shape=input_shape))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(32, activation='relu')) # 128
model.add(Dropout(0.5))
model.add(Dense(num_classes))
model.add(Activation('softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'])
model.fit(x_train[:1000, :],
y_train[:1000, :],
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_test[:1000, :], y_test[:1000, :]))
# explain by passing the tensorflow inputs and outputs
np.random.seed(0)
inds = np.random.choice(x_train.shape[0], 20, replace=False)
e = shap.GradientExplainer((model.layers[0].input, model.layers[-1].input),
x_train[inds, :, :])
shap_values = e.shap_values(x_test[:1], nsamples=2000)
sess = tf.keras.backend.get_session()
diff = sess.run(model.layers[-1].input, feed_dict={model.layers[0].input: x_test[:1]}) - \
sess.run(model.layers[-1].input, feed_dict={model.layers[0].input: x_train[inds,:,:]}).mean(0)
sums = np.array([shap_values[i].sum() for i in range(len(shap_values))])
d = np.abs(sums - diff).sum()
assert d / np.abs(diff).sum(
) < 0.05, "Sum of SHAP values does not match difference! %f" % (
d / np.abs(diff).sum())
