from keras.datasets import cifar10
from keras.utils import np_utils
from keras.layers.normalization import BatchNormalization
input_1 = Input(shape=(32, 32, 3), name="head")
x1 = Conv2D(16, (3, 3), padding='same', name='2-1')(input_1)
x2 = Conv2D(16, (3, 3), padding='same', name='2-2')(input_1)
x3 = Conv2D(16, (3, 3), padding='same', name='2-3')(input_1)
x4 = Conv2D(10, (3, 3), padding='same', name='3-1')(x1)
x5 = Concatenate(axis=3, name="3-2")([x1, x2])
x6 = Concatenate(axis=3, name="3-3")([x2, x3])
x7 = Conv2D(10, (3, 3), padding='same', name='3-4')(x3)
out1 = GlobalAveragePooling2D(name="4-1")(x4)
out1 = Activation('softmax', name='r_hand')(out1)
x10 = Concatenate(axis=3, name="4-2")([x5, x6])
out4 = GlobalAveragePooling2D(name="4-3")(x7)
out4 = Activation('softmax', name='l_hand')(out4)
x11 = MaxPooling2D(pool_size=(2, 2), name="5-1")(x10)
out2 = Flatten(name="6-1")(x11)
out2 = Dense(10, activation='relu', name="7-1")(out2)
out2 = Activation('softmax', name="r_foot")(out2)
out3 = Flatten(name="6-2")(x11)
out3 = Dense(10, activation='relu', name="7-2")(out3)
def __create_mobilenet(classes, img_input, include_top, alpha,
depth_multiplier, dropout, pooling):
''' Creates a MobileNet model with specified parameters
Args:
classes: Number of output classes
img_input: Input tensor or layer
include_top: Flag to include the last dense layer
alpha: width multiplier of the MobileNet.
depth_multiplier: depth multiplier for depthwise convolution
(also called the resolution multiplier)
dropout: dropout rate
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
Returns: a Keras Model
'''
x = __conv_block(img_input, 32, alpha, strides=(2, 2))
x = __depthwise_conv_block(x, 64, alpha, depth_multiplier, id=1)
x = __depthwise_conv_block(x,
128,
alpha,
depth_multiplier,
strides=(2, 2),
id=2)
x = __depthwise_conv_block(x, 128, alpha, depth_multiplier, id=3)
x = __depthwise_conv_block(x,
256,
alpha,
depth_multiplier,
strides=(2, 2),
id=4)
x = __depthwise_conv_block(x, 256, alpha, depth_multiplier, id=5)
x = __depthwise_conv_block(x,
512,
alpha,
depth_multiplier,
strides=(2, 2),
id=6)
x = __depthwise_conv_block(x, 512, alpha, depth_multiplier, id=7)
x = __depthwise_conv_block(x, 512, alpha, depth_multiplier, id=8)
x = __depthwise_conv_block(x, 512, alpha, depth_multiplier, id=9)
x = __depthwise_conv_block(x, 512, alpha, depth_multiplier, id=10)
x = __depthwise_conv_block(x, 512, alpha, depth_multiplier, id=11)
x = __depthwise_conv_block(x,
1024,
alpha,
depth_multiplier,
strides=(2, 2),
id=12)
x = __depthwise_conv_block(x, 1024, alpha, depth_multiplier, id=13)
if include_top:
if K.image_data_format() == 'channels_first':
shape = (int(1024 * alpha), 1, 1)
else:
shape = (1, 1, int(1024 * alpha))
x = GlobalAveragePooling2D()(x)
x = Reshape(shape, name='reshape_1')(x)
x = Dropout(dropout, name='dropout')(x)
x = Convolution2D(classes, (1, 1), padding='same',
name='conv_preds')(x)
x = Activation('softmax', name='act_softmax')(x)
x = Reshape((classes, ), name='reshape_2')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
return x
def disc1(img_dim,
bn_mode,
model,
wd,
monsterClass,
inject_noise,
n_classes,
model_name="discriminator",
use_mbd=False):
if K.image_dim_ordering() == "th":
bn_axis = 1
min_s = min(img_dim[1:])
else:
bn_axis = -1
min_s = min(img_dim[:-1])
disc_input = Input(shape=img_dim, name="discriminator_input")
# Get the list of number of conv filters
# (first layer starts with 64), filters are subsequently doubled
nb_conv = int(np.floor(np.log(min_s // 4) / np.log(2)))
list_f = [64 * min(8, (2**i)) for i in range(nb_conv)]
# First conv with 2x2 strides
x = Convolution2D(list_f[0],
3,
3,
name="disc_conv2d_1",
border_mode="same",
bias=False,
kernel_initializer="he_normal",
W_regularizer=l2(wd))(disc_input)
x = BatchNormalization(mode=bn_mode, axis=bn_axis)(x)
x = LeakyReLU(0.2)(x)
# Conv blocks: Conv2D(2x2 strides)->BN->LReLU
for i, f in enumerate(list_f[1:]):
name = "disc_conv2d_%s" % (i + 2)
x1 = Convolution2D(64,
3,
3,
border_mode='same',
kernel_initializer="he_normal",
W_regularizer=l2(wd))(x)
x1 = BatchNormalization(mode=bn_mode, axis=bn_axis)(x1)
x1 = LeakyReLU(0.2)(x1)
x1 = Convolution2D(64,
3,
3,
border_mode='same',
kernel_initializer="he_normal",
W_regularizer=l2(wd))(x1)
x1 = BatchNormalization(mode=bn_mode, axis=1)(x1)
x = merge([x1, x], mode='sum')
x = LeakyReLU(0.2)(x)
# Last convolution
aux_feats = Convolution2D(n_classes * 2,
3,
3,
name="aux_conv",
border_mode="same",
bias=False,
kernel_initializer="he_normal",
W_regularizer=l2(wd))(x)
aux_feats = GlobalAveragePooling2D()(aux_feats)
aux_feats = LeakyReLU(0.2)(aux_feats)
x = Convolution2D(1,
3,
3,
name="final_conv",
border_mode="same",
bias=False,
kernel_initializer="he_normal",
W_regularizer=l2(wd))(x)
# Average pooling, it serves as traditional GAN single number true/fake
x = GlobalAveragePooling2D()(x)
x = Dropout(0.5)(x)
if monsterClass: #2*nClasses (nClasses True, nClasses False) and no true/fake output
aux = Dense(n_classes * 2,
activation='softmax',
name='auxiliary',
W_regularizer=l2(wd))(aux_feats)
discriminator_model = Model(input=[disc_input],
output=aux,
name=model_name)
else:
aux = Dense(n_classes,
activation='softmax',
name='auxiliary',
W_regularizer=l2(wd))(x)
discriminator_model = Model(input=[disc_input],
output=[x, aux],
name=model_name)
visualize_model(discriminator_model)
return discriminator_model
def __create_dense_net(classes,
img_input,
nb_dense_block=3,
growth_rate=12,
nb_filter=-1,
nb_layers_per_block=4,
reduction=0.5,
dropout_rate=0.0,
weight_decay=1e-3,
subsample_initial_block=False):
concat_axis = 1 if K.image_data_format() == 'channels_first' else -1
if nb_filter <= 0:
nb_filter = 2 * growth_rate
compression = 1 - reduction
if subsample_initial_block:
initial_kernel = (7, 7)
initial_strides = (3, 3)
else:
initial_kernel = (3, 3)
initial_strides = (1, 1)
x = Conv2D(nb_filter,
initial_kernel,
kernel_initializer='he_normal',
padding='same',
strides=initial_strides,
kernel_regularizer=l2(weight_decay))(img_input)
if subsample_initial_block:
x = BatchNormalization(axis=concat_axis, epsilon=1e-5)(x)
x = LeakyReLU(0.3)(x)
x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
for block_idx in range(nb_dense_block - 1):
x, nb_filter = __dense_block(x,
nb_filter,
nb_layers_per_block,
dropout_rate=dropout_rate,
weight_decay=weight_decay,
growth_rate=growth_rate)
#add transition block
x = __transition_block(x,
nb_filter,
compression=compression,
weight_decay=weight_decay)
nb_filter += int(nb_filter * compression)
#
x, nb_filter = __dense_block(x,
nb_filter,
nb_layers_per_block,
dropout_rate=dropout_rate,
weight_decay=weight_decay,
growth_rate=growth_rate)
x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(x)
x = LeakyReLU(0.3)(x)
x = GlobalAveragePooling2D()(x)
x = Dense(classes, activation='softmax')(x)
return x
def DenseNet(nb_classes, img_dim, nb_dense_block, stages, growth_rate,
nb_filter, dropout_rate=None, weight_decay=1E-4):
""" Build the DenseNet model
:param nb_classes: int -- number of classes
:param img_dim: tuple -- (channels, rows, columns)
:param nb_dense_block: int -- number of dense blocks to add to end
:param stages: list -- number of layers within each dense block
:param growth_rate: int -- number of filters to add
:param nb_filter: int -- number of filters
:param dropout_rate: float -- dropout rate
:param weight_decay: float -- weight decay
:returns: keras model with nb_layers of conv_factory appended
:rtype: keras model
"""
if K.image_data_format() == "channels_first":
concat_axis = 1
elif K.image_data_format() == "channels_last":
concat_axis = -1
model_input = Input(shape=img_dim)
message = "number of dense blocks must be the same with size of layer list"
assert len(stages) == nb_dense_block, message
# Initial convolution
x = Conv2D(nb_filter, (3, 3),
kernel_initializer="he_uniform",
padding="same",
name="initial_conv2D",
use_bias=False,
kernel_regularizer=l2(weight_decay))(model_input)
# Add dense blocks
for block_idx in range(nb_dense_block - 1):
nb_layers = stages[block_idx]
x, nb_filter = denseblock_altern(x, concat_axis, nb_layers,
nb_filter, growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# add transition
x = transition(x, concat_axis, nb_filter,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# The last denseblock does not have a transition
nb_layers = stages[-1]
x, nb_filter = denseblock_altern(x, concat_axis, nb_layers,
nb_filter, growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
x = BatchNormalization(axis=concat_axis,
gamma_regularizer=l2(weight_decay),
beta_regularizer=l2(weight_decay))(x)
x = Activation('relu')(x)
x = GlobalAveragePooling2D(data_format=K.image_data_format())(x)
x = Dense(nb_classes,
activation='softmax',
kernel_regularizer=l2(weight_decay),
bias_regularizer=l2(weight_decay))(x)
densenet = Model(inputs=[model_input], outputs=[x], name="DenseNet")
return densenet
def _create_res_next_imagenet(nb_classes, img_input, include_top, depth, cardinality=32, width=4,
weight_decay=5e-4, pooling=None):
''' Creates a ResNeXt model with specified parameters
Args:
nb_classes: Number of output classes
img_input: Input tensor or layer
include_top: Flag to include the last dense layer
depth: Depth of the network. List of integers.
Increasing cardinality improves classification accuracy,
width: Width of the network.
weight_decay: weight_decay (l2 norm)
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
Returns: a Keras Model
'''
if type(depth) is list or type(depth) is tuple:
# If a list is provided, defer to user how many blocks are present
N = list(depth)
else:
# Otherwise, default to 3 blocks each of default number of group convolution blocks
N = [(depth - 2) // 9 for _ in range(3)]
filters = cardinality * width
filters_list = []
for i in range(len(N)):
filters_list.append(filters)
filters *= 2 # double the size of the filters
x = __initial_conv_block_inception(img_input, weight_decay)
# block 1 (no pooling)
for i in range(N[0]):
x = __bottleneck_block(x, filters_list[0], cardinality, strides=1, weight_decay=weight_decay)
N = N[1:] # remove the first block from block definition list
filters_list = filters_list[1:] # remove the first filter from the filter list
# block 2 to N
for block_idx, n_i in enumerate(N):
for i in range(n_i):
if i == 0:
x = __bottleneck_block(x, filters_list[block_idx], cardinality, strides=2,
weight_decay=weight_decay)
else:
x = __bottleneck_block(x, filters_list[block_idx], cardinality, strides=1,
weight_decay=weight_decay)
if include_top:
x = GlobalAveragePooling2D()(x)
x = Dense(nb_classes, use_bias=False, kernel_regularizer=l2(weight_decay),
kernel_initializer='he_normal', activation='softmax')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
return x
def create_model(nb_classes, input_shape, config=None):
"""Create a VGG-16 like model."""
if len(input_shape) != 3:
raise Exception("Input shape should be a tuple (nb_channels, nb_rows, "
"nb_cols) or (nb_rows, nb_cols, nb_channels), "
"depending on your backend.")
if config is None:
config = {'model': {}}
min_feature_map_dimension = min(input_shape[:2])
if min_feature_map_dimension < 32:
print("ERROR: Please upsample the feature maps to have at least "
"a size of 32 x 32. Currently, it has {}".format(input_shape))
nb_filter = 32
# Network definition
# input_shape = (None, None, 3) # for fcn
input_ = Input(shape=input_shape)
x = input_
# Scale feature maps down to [63, 32] x [63, 32]
tmp = min_feature_map_dimension / 32.
if tmp >= 2:
while tmp >= 2.:
for _ in range(2):
x = Convolution2D(nb_filter, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(x)
x = BatchNormalization()(x)
x = PReLU()(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
nb_filter *= 2
tmp /= 2
# 32x32
x = Convolution2D(nb_filter, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(x)
x = BatchNormalization()(x)
x = PReLU()(x)
x = Convolution2D(nb_filter, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(x)
x = BatchNormalization()(x)
x = PReLU()(x)
# 16x16
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Convolution2D(2 * nb_filter, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(x)
x = BatchNormalization()(x)
x = PReLU()(x)
x = Convolution2D(2 * nb_filter, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(x)
x = BatchNormalization()(x)
x = PReLU()(x)
# 8x8
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Convolution2D(2 * nb_filter, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(x)
x = BatchNormalization()(x)
x = PReLU()(x)
# 4x4
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Convolution2D(512, (4, 4),
padding='valid',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(x)
x = BatchNormalization()(x)
x = PReLU()(x)
x = Dropout(0.5)(x)
# 1x1
x = Convolution2D(512, (1, 1),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(x)
x = BatchNormalization()(x)
x = PReLU()(x)
x = Dropout(0.5)(x)
x = Convolution2D(nb_classes, (1, 1),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(x)
x = GlobalAveragePooling2D()(x) # Adjust for FCN
x = BatchNormalization()(x)
x = Activation('softmax')(x)
model = Model(inputs=input_, outputs=x)
return model
def DenseNet(nb_classes,
img_dim,
depth,
nb_dense_block,
growth_rate,
nb_filter,
dropout_rate=None,
weight_decay=1E-4):
""" Build the DenseNet model
:param nb_classes: int -- number of classes
:param img_dim: tuple -- (channels, rows, columns)
:param depth: int -- how many layers
:param nb_dense_block: int -- number of dense blocks to add to end
:param growth_rate: int -- number of filters to add
:param nb_filter: int -- number of filters
:param dropout_rate: float -- dropout rate
:param weight_decay: float -- weight decay
:returns: keras model with nb_layers of conv_factory appended
:rtype: keras model
"""
if K.common.image_dim_ordering() == "th":
concat_axis = 1
elif K.common.image_dim_ordering() == "tf":
concat_axis = -1
model_input = Input(shape=img_dim)
assert (depth - 4) % 3 == 0, "Depth must be 3 N + 4"
# layers in each dense block
nb_layers = int((depth - 4) / 3)
# Initial convolution
x = Conv2D(nb_filter, (3, 3),
kernel_initializer="he_uniform",
padding="same",
name="initial_conv2D",
use_bias=False,
kernel_regularizer=l2(weight_decay))(model_input)
# Add dense blocks
for block_idx in range(nb_dense_block - 1):
x, nb_filter = denseblock(x,
concat_axis,
nb_layers,
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# add transition
x = transition(x,
nb_filter,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# The last denseblock does not have a transition
x, nb_filter = denseblock(x,
concat_axis,
nb_layers,
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
x = BatchNormalization(axis=concat_axis,
gamma_regularizer=l2(weight_decay),
beta_regularizer=l2(weight_decay))(x)
x = Activation('relu')(x)
x = GlobalAveragePooling2D(data_format=K.image_data_format())(x)
x = Dense(nb_classes,
activation='softmax',
kernel_regularizer=l2(weight_decay),
bias_regularizer=l2(weight_decay))(x)
densenet = Model(inputs=[model_input], outputs=[x], name="DenseNet")
return densenet
def __init__(self):
img_input = Input(shape=(ROW, COL, CHANNEL), name='batch_train_image')
#size_input= Input(shape=(3,), name='size_list')
x = Convolution2D(32,
3,
3,
subsample=(2, 2),
bias=False,
name='block1_conv1',
init='he_normal')(img_input)
x = BatchNormalization(name='block1_conv1_bn')(x)
x = Activation('relu', name='block1_conv1_act')(x)
x = Convolution2D(64,
3,
3,
bias=False,
name='block1_conv2',
init='he_normal')(x)
x = BatchNormalization(name='block1_conv2_bn')(x)
x = Activation('relu', name='block1_conv2_act')(x)
residual = Convolution2D(128,
1,
1,
subsample=(2, 2),
border_mode='same',
bias=False,
init='he_normal')(x)
residual = BatchNormalization()(residual)
x = SeparableConv2D(128,
3,
3,
border_mode='same',
bias=False,
name='block2_sepconv1',
init='he_normal')(x)
x = BatchNormalization(name='block2_sepconv1_bn')(x)
x = Activation('relu', name='block2_sepconv2_act')(x)
x = SeparableConv2D(128,
3,
3,
border_mode='same',
bias=False,
name='block2_sepconv2',
init='he_normal')(x)
x = BatchNormalization(name='block2_sepconv2_bn')(x)
x = MaxPooling2D((3, 3),
strides=(2, 2),
border_mode='same',
name='block2_pool')(x)
x = merge([x, residual], mode='sum')
residual = Convolution2D(256,
1,
1,
subsample=(2, 2),
border_mode='same',
bias=False,
init='he_normal')(x)
residual = BatchNormalization()(residual)
x = Activation('relu', name='block3_sepconv1_act')(x)
x = SeparableConv2D(256,
3,
3,
border_mode='same',
bias=False,
name='block3_sepconv1',
init='he_normal')(x)
x = BatchNormalization(name='block3_sepconv1_bn')(x)
x = Activation('relu', name='block3_sepconv2_act')(x)
x = SeparableConv2D(256,
3,
3,
border_mode='same',
bias=False,
name='block3_sepconv2',
init='he_normal')(x)
x = BatchNormalization(name='block3_sepconv2_bn')(x)
x = MaxPooling2D((3, 3),
strides=(2, 2),
border_mode='same',
name='block3_pool')(x)
#x = layers.add([x, residual])
x = merge([x, residual], mode='sum')
residual = Convolution2D(1024,
1,
1,
subsample=(2, 2),
border_mode='same',
bias=False,
init='he_normal')(x)
residual = BatchNormalization()(residual)
x = Activation('relu', name='block4_sepconv1_act')(x)
x = SeparableConv2D(1024,
3,
3,
border_mode='same',
bias=False,
name='block4_sepconv1',
init='he_normal')(x)
x = BatchNormalization(name='block4_sepconv1_bn')(x)
x = Activation('relu', name='block4_sepconv2_act')(x)
x = SeparableConv2D(1024,
3,
3,
border_mode='same',
bias=False,
name='block4_sepconv2',
init='he_normal')(x)
x = BatchNormalization(name='block4_sepconv2_bn')(x)
x = MaxPooling2D((3, 3),
strides=(2, 2),
border_mode='same',
name='block4_pool')(x)
x = merge([x, residual], mode='sum')
for i in range(8):
residual = x
prefix = 'block' + str(i + 5)
x = Activation('relu', name=prefix + '_sepconv1_act')(x)
x = SeparableConv2D(1024,
3,
3,
border_mode='same',
bias=False,
name=prefix + '_sepconv1',
init='he_normal')(x)
x = BatchNormalization(name=prefix + '_sepconv1_bn')(x)
x = Activation('relu', name=prefix + '_sepconv2_act')(x)
x = SeparableConv2D(1024,
3,
3,
border_mode='same',
bias=False,
name=prefix + '_sepconv2',
init='he_normal')(x)
x = BatchNormalization(name=prefix + '_sepconv2_bn')(x)
x = Activation('relu', name=prefix + '_sepconv3_act')(x)
x = SeparableConv2D(1024,
3,
3,
border_mode='same',
bias=False,
name=prefix + '_sepconv3',
init='he_normal')(x)
x = BatchNormalization(name=prefix + '_sepconv3_bn')(x)
x = merge([x, residual], mode='sum')
residual = Convolution2D(1536,
1,
1,
subsample=(2, 2),
border_mode='same',
bias=False,
init='he_normal')(x)
residual = BatchNormalization()(residual)
x = Activation('relu', name='block13_sepconv1_act')(x)
x = SeparableConv2D(1024,
3,
3,
border_mode='same',
bias=False,
name='block13_sepconv1',
init='he_normal')(x)
x = BatchNormalization(name='block13_sepconv1_bn')(x)
x = Activation('relu', name='block13_sepconv2_act')(x)
x = SeparableConv2D(1536,
3,
3,
border_mode='same',
bias=False,
name='block13_sepconv2',
init='he_normal')(x)
x = BatchNormalization(name='block13_sepconv2_bn')(x)
x = MaxPooling2D((3, 3),
strides=(2, 2),
border_mode='same',
name='block13_pool')(x)
x = merge([x, residual], mode='sum')
x = SeparableConv2D(2048,
3,
3,
border_mode='same',
bias=False,
name='block14_sepconv1',
init='he_normal')(x)
x = BatchNormalization(name='block14_sepconv1_bn')(x)
x = Activation('relu', name='block14_sepconv1_act')(x)
x = SeparableConv2D(4096,
3,
3,
border_mode='same',
bias=False,
name='block14_sepconv2',
init='he_normal')(x)
x = BatchNormalization(name='block14_sepconv2_bn')(x)
x = Activation('relu', name='block14_sepconv2_act')(x)
x = GlobalAveragePooling2D(name='avg_pool')(x)
x = Dense(2048, activation='relu', init='he_normal')(x)
x = BatchNormalization()(x)
x = Dropout(0.5)(x)
x = Dense(1024, activation='relu', init='he_normal')(x)
x = BatchNormalization()(x)
x = Dropout(0.5)(x)
x = Dense(512, activation='relu', init='he_normal')(x)
x = BatchNormalization()(x)
x = Dropout(0.5)(x)
nodules = Dense(2, activation='softmax', name="nodule")(x)
#malignancy_feature = merge([x, size_input], mode='concat')
# malignancy_feature = merge([batch_feature_label, size_input], mode='concat')
# batch_cate_label = Dense(1, activation='linear', name="malignancy")(malignancy_feature)
self.model = Model(input=[img_input], output=[nodules])
#self.model = Model(input=[img_input, size_input], output=[batch_feature_label])
self.model.compile(optimizer=Adam(lr=0.0005,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0),
loss='binary_crossentropy',
metrics=['accuracy'])
def X4(inputa,
inputb,
inputc,
demographics=None,
dropout=0.8,
num_cls=1000,
is_train=True,
scope='X4',
supermd=False):
with tf.variable_scope(scope, 'X4', [inputa, inputb, inputc]):
xa, auxa = Branch(inputa,
dropout_keep_prob=dropout,
num_classes=num_cls,
is_training=is_train)
xb, auxb = Branch(inputb,
dropout_keep_prob=dropout,
num_classes=num_cls,
is_training=is_train)
xc, auxc = Branch(inputc,
dropout_keep_prob=dropout,
num_classes=num_cls,
is_training=is_train)
x = concatenate([xa, xb, xc], axis=3) # Output: 8 * 8 * 2688
x = Conv2D(2688, (1, 1),
kernel_regularizer=l2(0.0002),
activation="relu",
padding="same")(x)
net = x
loss2_classifier = tf.add(auxa, tf.add(auxb, auxc))
# Average Pooling
x = GlobalAveragePooling2D(name='avg_pool')(x) # Output: 2688
pool5_drop_10x10_s1 = Dropout(dropout)(x, training=is_train)
if supermd:
demographics = Dense(2,
name='demographic_fc1',
activation="relu",
kernel_regularizer=l2(0.0002))(demographics)
merged = concatenate([pool5_drop_10x10_s1, demographics])
else:
merged = pool5_drop_10x10_s1
loss3_classifier_w = Dense(num_cls,
name='loss3/classifier',
kernel_regularizer=l2(0.0002))
loss3_classifier = loss3_classifier_w(merged)
w_variables = loss3_classifier_w.get_weights()
w_variables = w_variables[0]
logits = tf.cond(
tf.equal(is_train, tf.constant(True)),
lambda: tf.add(loss3_classifier,
tf.scalar_mul(tf.constant(0.1), loss2_classifier)),
lambda: loss3_classifier)
return logits, net, tf.convert_to_tensor(w_variables)
def ResNet(input_shape=None,
classes=10,
block='bottleneck',
residual_unit='v2',
repetitions=None,
initial_filters=64,
activation='softmax',
include_top=True,
input_tensor=None,
dropout=None,
transition_dilation_rate=(1, 1),
initial_strides=(2, 2),
initial_kernel_size=(7, 7),
initial_pooling='max',
final_pooling=None,
top='classification'):
"""Builds a custom ResNet like architecture. Defaults to ResNet50 v2.
Args:
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(224, 224, 3)` (with `channels_last` dim ordering)
or `(3, 224, 224)` (with `channels_first` dim ordering).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 8.
E.g. `(224, 224, 3)` would be one valid value.
classes: The number of outputs at final softmax layer
block: The block function to use. This is either `'basic'` or `'bottleneck'`.
The original paper used `basic` for layers < 50.
repetitions: Number of repetitions of various block units.
At each block unit, the number of filters are doubled and the input size is halved.
Default of None implies the ResNet50v2 values of [3, 4, 6, 3].
transition_dilation_rate: Used for pixel-wise prediction tasks such as image segmentation.
residual_unit: the basic residual unit, 'v1' for conv bn relu, 'v2' for bn relu conv.
See [Identity Mappings in Deep Residual Networks](https://arxiv.org/abs/1603.05027)
for details.
dropout: None for no dropout, otherwise rate of dropout from 0 to 1.
Based on [Wide Residual Networks.(https://arxiv.org/pdf/1605.07146) paper.
transition_dilation_rate: Dilation rate for transition layers. For semantic
segmentation of images use a dilation rate of (2, 2).
initial_strides: Stride of the very first residual unit and MaxPooling2D call,
with default (2, 2), set to (1, 1) for small images like cifar.
initial_kernel_size: kernel size of the very first convolution, (7, 7) for imagenet
and (3, 3) for small image datasets like tiny imagenet and cifar.
See [ResNeXt](https://arxiv.org/abs/1611.05431) paper for details.
initial_pooling: Determine if there will be an initial pooling layer,
'max' for imagenet and None for small image datasets.
See [ResNeXt](https://arxiv.org/abs/1611.05431) paper for details.
final_pooling: Optional pooling mode for feature extraction at the final model layer
when `include_top` is `False`.
- `None` means that the output of the model
will be the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a
2D tensor.
- `max` means that global max pooling will
be applied.
top: Defines final layers to evaluate based on a specific problem type. Options are
'classification' for ImageNet style problems, 'segmentation' for problems like
the Pascal VOC dataset, and None to exclude these layers entirely.
Returns:
The keras `Model`.
"""
if activation not in ['softmax', 'sigmoid', None]:
raise ValueError(
'activation must be one of "softmax", "sigmoid", or None')
if activation == 'sigmoid' and classes != 1:
raise ValueError(
'sigmoid activation can only be used when classes = 1')
if repetitions is None:
repetitions = [3, 4, 6, 3]
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_size=32,
min_size=8,
data_format=K.image_data_format(),
require_flatten=include_top)
_handle_dim_ordering()
if len(input_shape) != 3:
raise Exception(
"Input shape should be a tuple (nb_channels, nb_rows, nb_cols)")
if block == 'basic':
block_fn = basic_block
elif block == 'bottleneck':
block_fn = bottleneck
elif isinstance(block, six.string_types):
block_fn = _string_to_function(block)
else:
block_fn = block
if residual_unit == 'v2':
residual_unit = _bn_relu_conv
elif residual_unit == 'v1':
residual_unit = _conv_bn_relu
elif isinstance(residual_unit, six.string_types):
residual_unit = _string_to_function(residual_unit)
else:
residual_unit = residual_unit
# Permute dimension order if necessary
if K.image_data_format() == 'channels_first':
input_shape = (input_shape[1], input_shape[2], input_shape[0])
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_size=32,
min_size=8,
data_format=K.image_data_format(),
require_flatten=include_top)
img_input = Input(shape=input_shape, tensor=input_tensor)
x = _conv_bn_relu(filters=initial_filters,
kernel_size=initial_kernel_size,
strides=initial_strides)(img_input)
if initial_pooling == 'max':
x = MaxPooling2D(pool_size=(3, 3),
strides=initial_strides,
padding="same")(x)
block = x
filters = initial_filters
for i, r in enumerate(repetitions):
transition_dilation_rates = [transition_dilation_rate] * r
transition_strides = [(1, 1)] * r
if transition_dilation_rate == (1, 1):
transition_strides[0] = (2, 2)
block = _residual_block(
block_fn,
filters=filters,
stage=i,
blocks=r,
is_first_layer=(i == 0),
dropout=dropout,
transition_dilation_rates=transition_dilation_rates,
transition_strides=transition_strides,
residual_unit=residual_unit)(block)
filters *= 2
# Last activation
x = _bn_relu(block)
# Classifier block
if include_top and top is 'classification':
x = GlobalAveragePooling2D()(x)
x = Dense(units=classes,
activation=activation,
kernel_initializer="he_normal")(x)
elif include_top and top is 'segmentation':
x = Conv2D(classes, (1, 1), activation='linear', padding='same')(x)
if K.image_data_format() == 'channels_first':
channel, row, col = input_shape
else:
row, col, channel = input_shape
x = Reshape((row * col, classes))(x)
x = Activation(activation)(x)
x = Reshape((row, col, classes))(x)
elif final_pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif final_pooling == 'max':
x = GlobalMaxPooling2D()(x)
model = Model(inputs=img_input, outputs=x)
return model
_inputBN = BatchNormalization(axis=-1)(_input)
C1 = ConvSN2D(15, (5, 5), padding='valid',
data_format='channels_last')(_inputBN)
C1 = LeakyReLU()(C1)
C2 = ConvSN2D(25, (7, 7), padding='valid', data_format='channels_last')(C1)
C2 = LeakyReLU()(C2)
C3 = ConvSN2D(40, (9, 9), padding='valid', data_format='channels_last')(C2)
C3 = LeakyReLU()(C3)
C4 = ConvSN2D(50, (11, 11), padding='valid', data_format='channels_last')(C3)
C4 = LeakyReLU()(C4)
Average_score = GlobalAveragePooling2D(name='Average_score')(
C4) #(batch_size, channels)
D1 = DenseSN(50)(Average_score)
D1 = LeakyReLU()(D1)
D2 = DenseSN(10)(D1)
D2 = LeakyReLU()(D2)
Score = DenseSN(1)(D2)
Discriminator = Model(outputs=Score, inputs=_input)
Discriminator.trainable = True
Discriminator.compile(loss='mse', optimizer='adam')
#### Combine the two networks to become MetricGAN
def create_model(nb_classes, input_shape, config=None):
"""Create a VGG-16 like model."""
if len(input_shape) != 3:
raise Exception("Input shape should be a tuple (nb_channels, nb_rows, "
"nb_cols) or (nb_rows, nb_cols, nb_channels), "
"depending on your backend.")
if config is None:
config = {'model': {}}
# input_shape = (None, None, 3) # for fcn
input_ = Input(shape=input_shape)
x = input_
min_feature_map_dimension = min(input_shape[:2])
if min_feature_map_dimension < 32:
print("ERROR: Please upsample the feature maps to have at least "
"a size of 32 x 32. Currently, it has {}".format(input_shape))
tmp = min_feature_map_dimension / 32.
nb_filter = 32
if tmp >= 2:
while tmp >= 2.:
y = x
for _ in range(2):
x = Convolution2D(nb_filter / 2, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(x)
x = BatchNormalization()(x)
x = Activation('elu')(x)
y = Convolution2D(nb_filter / 2, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(y)
y = BatchNormalization()(y)
y = Activation('relu')(y)
x = concatenate([x, y], axis=-1)
x = MaxPooling2D(pool_size=(2, 2))(x)
# nb_filter *= 2
tmp /= 2
if 'c1' in config['model']:
c1_nb_filter = config['model']['c1']['nb_filter']
else:
c1_nb_filter = nb_filter
if 'c8' in config['model']:
c8_nb_filter = config['model']['c8']['nb_filter']
else:
c8_nb_filter = 512
y = x
x = Convolution2D(c1_nb_filter / 2, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(x)
x = BatchNormalization()(x)
x = Activation('elu')(x)
y = Convolution2D(c1_nb_filter / 2, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(y)
y = BatchNormalization()(y)
y = Activation('relu')(y)
x = concatenate([x, y], axis=-1)
#
y = x
x = Convolution2D(nb_filter / 2, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(x)
x = BatchNormalization()(x)
x = Activation('elu')(x)
y = Convolution2D(nb_filter / 2, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(y)
y = BatchNormalization()(y)
y = Activation('relu')(y)
x = concatenate([x, y], axis=-1)
#
x = MaxPooling2D(pool_size=(2, 2))(x)
y = x
x = Convolution2D(nb_filter, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(x)
x = BatchNormalization()(x)
x = Activation('elu')(x)
y = Convolution2D(nb_filter, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(y)
y = BatchNormalization()(y)
y = Activation('relu')(y)
x = concatenate([x, y], axis=-1)
#
y = x
x = Convolution2D(nb_filter, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(x)
x = BatchNormalization()(x)
x = Activation('elu')(x)
y = Convolution2D(nb_filter, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(y)
y = BatchNormalization()(y)
y = Activation('relu')(y)
x = concatenate([x, y], axis=-1)
#
x = MaxPooling2D(pool_size=(2, 2))(x)
y = x
x = Convolution2D(nb_filter, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(x)
x = BatchNormalization()(x)
x = Activation('elu')(x)
y = Convolution2D(nb_filter, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(y)
y = BatchNormalization()(y)
y = Activation('relu')(y)
x = concatenate([x, y], axis=-1)
#
x = MaxPooling2D(pool_size=(2, 2))(x)
y = x
x = Convolution2D(256, (4, 4),
padding='valid',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(x)
x = BatchNormalization()(x)
x = Activation('elu')(x)
x = Dropout(0.5)(x)
y = Convolution2D(256, (4, 4),
padding='valid',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(y)
y = BatchNormalization()(y)
y = Activation('relu')(y)
y = Dropout(0.5)(y)
x = concatenate([x, y], axis=-1)
#
y = x
x = Convolution2D(c8_nb_filter / 2, (1, 1),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(x)
x = BatchNormalization()(x)
x = Activation('elu')(x)
x = Dropout(0.5)(x)
y = Convolution2D(c8_nb_filter / 2, (1, 1),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(x)
y = BatchNormalization()(y)
y = Activation('relu')(y)
y = Dropout(0.5)(y)
x = concatenate([x, y], axis=-1)
#
x = Convolution2D(nb_classes, (1, 1),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(0.0001))(x)
x = GlobalAveragePooling2D()(x) # Adjust for FCN
x = BatchNormalization()(x)
x = Activation('softmax')(x)
model = Model(inputs=input_, outputs=x)
return model
nb_kernels = 2 * nb_kernels
block6 = residual_block(block5, nb_kernels, doPooling=True)
block7 = residual_block(block6, nb_kernels, doPooling=False)
block8 = residual_block(block7, nb_kernels, doPooling=False)
nb_kernels = 2 * nb_kernels
block9 = residual_block(block8, nb_kernels, doPooling=True)
block10 = residual_block(block9, nb_kernels, doPooling=False)
block11 = residual_block(block10, nb_kernels, doPooling=False)
nb_kernels = 2 * nb_kernels
block12 = residual_block(block11, nb_kernels, doPooling=True)
block13 = residual_block(block12, nb_kernels, doPooling=False)
block14 = residual_block(block13, nb_kernels, doPooling=False)
g_avep = GlobalAveragePooling2D()(block14)
g_avep = Dropout(0.3)(g_avep)
output = Dense(2,
kernel_initializer='normal',
kernel_regularizer=regularizers.l2(L2_regularizers))(g_avep)
output = Activation('softmax')(output)
model = Model(inputs=input, outputs=output)
#SGD + momentum
#model.compile provides the cross entropy loss function
sgd = SGD(lr=0.01, decay=1e-7, momentum=0.95, nesterov=True)
model.compile(optimizer=sgd, loss='binary_crossentropy', metrics=['accuracy'])
model.summary()
##################
# trainデータからvalidデータを分割
X_train, X_valid, y_train, y_valid = train_test_split(X_train,
y_train,
random_state=0,
test_size=0.2)
print(X_train.shape, y_train.shape, X_valid.shape, y_valid.shape)
"""モデルの構築
KerasのXceptionを使用する。
Argments:
include_top: ネットワーク出力層の全結合層を除去(False)。
early_stopping: 過学習を防ぐ関数。
"""
base_model = Xception(include_top=False, weights="imagenet", input_shape=None)
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(1024, activation='relu')(x)
predictions = Dense(1)(x)
datagen = ImageDataGenerator(featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=0,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
vertical_flip=False)
# EarlyStopping
early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=1)
cat_id = idx2cat[cat_idx]
categories_df.loc[cat_id]
from keras.models import Sequential
from keras.layers import Dropout, Flatten, Dense
from keras.layers.convolutional import Conv2D
from keras.layers.pooling import MaxPooling2D, GlobalAveragePooling2D
model = Sequential()
model.add(Conv2D(32, 3, padding="same", activation="relu", input_shape=(180, 180, 3)))
model.add(MaxPooling2D())
model.add(Conv2D(64, 3, padding="same", activation="relu"))
model.add(MaxPooling2D())
model.add(Conv2D(128, 3, padding="same", activation="relu"))
model.add(MaxPooling2D())
model.add(GlobalAveragePooling2D())
model.add(Dense(num_classes, activation="softmax"))
model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=["accuracy"])
model.summary()
# To train the model:
model.fit_generator(train_gen,
steps_per_epoch = 10, #num_train_images // batch_size,
epochs = 3,
validation_data = val_gen,
validation_steps = 10, #num_val_images // batch_size,
def create_model(weights_path=None, use_distribution=False, use_multigap=False,
use_semantics=False, rapid_style=False, use_comments=False, embedding_layer=None,
extra_conv_layer=False,load_weights_by_name=True,textInputMaxLength=100, embedding_dim=300):
input_image = Input(shape=(3, 224, 224))
if rapid_style:
if not use_semantics:
print("[WARN] Semantics is not enabled, rapid style parameter will be ignored")
if use_comments:
if embedding_layer is None:
print("[ERROR] Embedding layer is required for creating comments model")
return None
comment_input = Input(shape=(textInputMaxLength,), dtype='int32')
embedded_sequences = embedding_layer(comment_input)
x_text_aesthetics = GRU(embedding_dim,dropout_W = 0.3,dropout_U = 0.3,return_sequences=True, name='gru_aesthetics_1')(embedded_sequences)
x_text_aesthetics = GRU(embedding_dim,dropout_W = 0.3,dropout_U = 0.3,name='gru_aesthetics_2')(x_text_aesthetics)
if use_semantics and not rapid_style:
x_text_semantics = GRU(embedding_dim,dropout_W = 0.3,dropout_U = 0.3,return_sequences=True, name='gru_semantics_1')(embedded_sequences)
x_text_semantics = GRU(embedding_dim,dropout_W = 0.3,dropout_U = 0.3, name='gru_semantics_2')(x_text_semantics)
conv1_7x7_s2 = Convolution2D(64,7,7,subsample=(2,2),border_mode='same',activation='relu',name='conv1/7x7_s2',W_regularizer=l2(0.0002))(input_image)
conv1_zero_pad = ZeroPadding2D(padding=(1, 1))(conv1_7x7_s2)
pool1_helper = PoolHelper()(conv1_zero_pad)
pool1_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='pool1/3x3_s2')(pool1_helper)
pool1_norm1 = LRN(name='pool1/norm1')(pool1_3x3_s2)
conv2_3x3_reduce = Convolution2D(64,1,1,border_mode='same',activation='relu',name='conv2/3x3_reduce',W_regularizer=l2(0.0002))(pool1_norm1)
conv2_3x3 = Convolution2D(192,3,3,border_mode='same',activation='relu',name='conv2/3x3',W_regularizer=l2(0.0002))(conv2_3x3_reduce)
conv2_norm2 = LRN(name='conv2/norm2')(conv2_3x3)
conv2_zero_pad = ZeroPadding2D(padding=(1, 1))(conv2_norm2)
pool2_helper = PoolHelper()(conv2_zero_pad)
pool2_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='pool2/3x3_s2')(pool2_helper)
inception_3a_1x1 = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_3a/1x1',W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_3x3_reduce = Convolution2D(96,1,1,border_mode='same',activation='relu',name='inception_3a/3x3_reduce',W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_3x3 = Convolution2D(128,3,3,border_mode='same',activation='relu',name='inception_3a/3x3',W_regularizer=l2(0.0002))(inception_3a_3x3_reduce)
inception_3a_5x5_reduce = Convolution2D(16,1,1,border_mode='same',activation='relu',name='inception_3a/5x5_reduce',W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_5x5 = Convolution2D(32,5,5,border_mode='same',activation='relu',name='inception_3a/5x5',W_regularizer=l2(0.0002))(inception_3a_5x5_reduce)
inception_3a_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_3a/pool')(pool2_3x3_s2)
inception_3a_pool_proj = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_3a/pool_proj',W_regularizer=l2(0.0002))(inception_3a_pool)
inception_3a_output = merge([inception_3a_1x1,inception_3a_3x3,inception_3a_5x5,inception_3a_pool_proj],mode='concat',concat_axis=1,name='inception_3a/output')
inception_3b_1x1 = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_3b/1x1',W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_3x3_reduce = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_3b/3x3_reduce',W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_3x3 = Convolution2D(192,3,3,border_mode='same',activation='relu',name='inception_3b/3x3',W_regularizer=l2(0.0002))(inception_3b_3x3_reduce)
inception_3b_5x5_reduce = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_3b/5x5_reduce',W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_5x5 = Convolution2D(96,5,5,border_mode='same',activation='relu',name='inception_3b/5x5',W_regularizer=l2(0.0002))(inception_3b_5x5_reduce)
inception_3b_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_3b/pool')(inception_3a_output)
inception_3b_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_3b/pool_proj',W_regularizer=l2(0.0002))(inception_3b_pool)
inception_3b_output = merge([inception_3b_1x1,inception_3b_3x3,inception_3b_5x5,inception_3b_pool_proj],mode='concat',concat_axis=1,name='inception_3b/output')
inception_3b_output_zero_pad = ZeroPadding2D(padding=(1, 1))(inception_3b_output)
pool3_helper = PoolHelper()(inception_3b_output_zero_pad)
pool3_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='pool3/3x3_s2')(pool3_helper)
inception_4a_1x1 = Convolution2D(192,1,1,border_mode='same',activation='relu',name='inception_4a/1x1',W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_3x3_reduce = Convolution2D(96,1,1,border_mode='same',activation='relu',name='inception_4a/3x3_reduce',W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_3x3 = Convolution2D(208,3,3,border_mode='same',activation='relu',name='inception_4a/3x3',W_regularizer=l2(0.0002))(inception_4a_3x3_reduce)
inception_4a_5x5_reduce = Convolution2D(16,1,1,border_mode='same',activation='relu',name='inception_4a/5x5_reduce',W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_5x5 = Convolution2D(48,5,5,border_mode='same',activation='relu',name='inception_4a/5x5',W_regularizer=l2(0.0002))(inception_4a_5x5_reduce)
inception_4a_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4a/pool')(pool3_3x3_s2)
inception_4a_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_4a/pool_proj',W_regularizer=l2(0.0002))(inception_4a_pool)
inception_4a_output = merge([inception_4a_1x1,inception_4a_3x3,inception_4a_5x5,inception_4a_pool_proj],mode='concat',concat_axis=1,name='inception_4a/output')
if extra_conv_layer:
conv_4a_output = Convolution2D(624, 3, 3, activation='relu',name='conv_4a',border_mode = 'same',W_regularizer=l2(0.0002))(inception_4a_output)
inception_4a_gap = GlobalAveragePooling2D()(conv_4a_output)
inception_4b_1x1 = Convolution2D(160,1,1,border_mode='same',activation='relu',name='inception_4b/1x1',W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_3x3_reduce = Convolution2D(112,1,1,border_mode='same',activation='relu',name='inception_4b/3x3_reduce',W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_3x3 = Convolution2D(224,3,3,border_mode='same',activation='relu',name='inception_4b/3x3',W_regularizer=l2(0.0002))(inception_4b_3x3_reduce)
inception_4b_5x5_reduce = Convolution2D(24,1,1,border_mode='same',activation='relu',name='inception_4b/5x5_reduce',W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_5x5 = Convolution2D(64,5,5,border_mode='same',activation='relu',name='inception_4b/5x5',W_regularizer=l2(0.0002))(inception_4b_5x5_reduce)
inception_4b_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4b/pool')(inception_4a_output)
inception_4b_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_4b/pool_proj',W_regularizer=l2(0.0002))(inception_4b_pool)
inception_4b_output = merge([inception_4b_1x1,inception_4b_3x3,inception_4b_5x5,inception_4b_pool_proj],mode='concat',concat_axis=1,name='inception_4b_output')
if extra_conv_layer:
conv_4b_output = Convolution2D(648, 3, 3, activation='relu',name='conv_4b',border_mode = 'same',W_regularizer=l2(0.0002))(inception_4b_output)
inception_4b_gap = GlobalAveragePooling2D()(conv_4b_output)
inception_4c_1x1 = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_4c/1x1',W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_3x3_reduce = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_4c/3x3_reduce',W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_3x3 = Convolution2D(256,3,3,border_mode='same',activation='relu',name='inception_4c/3x3',W_regularizer=l2(0.0002))(inception_4c_3x3_reduce)
inception_4c_5x5_reduce = Convolution2D(24,1,1,border_mode='same',activation='relu',name='inception_4c/5x5_reduce',W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_5x5 = Convolution2D(64,5,5,border_mode='same',activation='relu',name='inception_4c/5x5',W_regularizer=l2(0.0002))(inception_4c_5x5_reduce)
inception_4c_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4c/pool')(inception_4b_output)
inception_4c_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_4c/pool_proj',W_regularizer=l2(0.0002))(inception_4c_pool)
inception_4c_output = merge([inception_4c_1x1,inception_4c_3x3,inception_4c_5x5,inception_4c_pool_proj],mode='concat',concat_axis=1,name='inception_4c/output')
if extra_conv_layer:
conv_4c_output = Convolution2D(663, 3, 3, activation='relu',name='conv_4c',border_mode = 'same',W_regularizer=l2(0.0002))(inception_4c_output)
inception_4c_gap = GlobalAveragePooling2D()(conv_4c_output)
inception_4d_1x1 = Convolution2D(112,1,1,border_mode='same',activation='relu',name='inception_4d/1x1',W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_3x3_reduce = Convolution2D(144,1,1,border_mode='same',activation='relu',name='inception_4d/3x3_reduce',W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_3x3 = Convolution2D(288,3,3,border_mode='same',activation='relu',name='inception_4d/3x3',W_regularizer=l2(0.0002))(inception_4d_3x3_reduce)
inception_4d_5x5_reduce = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_4d/5x5_reduce',W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_5x5 = Convolution2D(64,5,5,border_mode='same',activation='relu',name='inception_4d/5x5',W_regularizer=l2(0.0002))(inception_4d_5x5_reduce)
inception_4d_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4d/pool')(inception_4c_output)
inception_4d_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_4d/pool_proj',W_regularizer=l2(0.0002))(inception_4d_pool)
inception_4d_output = merge([inception_4d_1x1,inception_4d_3x3,inception_4d_5x5,inception_4d_pool_proj],mode='concat',concat_axis=1,name='inception_4d/output')
if extra_conv_layer:
conv_4d_output = Convolution2D(704, 3, 3, activation='relu',name='conv_4d',border_mode = 'same',W_regularizer=l2(0.0002))(inception_4d_output)
inception_4d_gap = GlobalAveragePooling2D()(conv_4d_output)
if use_semantics:
inception_4e_1x1_aesthetics = Convolution2D(256,1,1,border_mode='same',activation='relu',name='inception_4e/1x1_aesthetics',W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3_reduce_aesthetics = Convolution2D(160,1,1,border_mode='same',activation='relu',name='inception_4e/3x3_reduce_aesthetics',W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3_aesthetics= Convolution2D(320,3,3,border_mode='same',activation='relu',name='inception_4e/3x3_aesthetics',W_regularizer=l2(0.0002))(inception_4e_3x3_reduce_aesthetics)
inception_4e_5x5_reduce_aesthetics = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_4e/5x5_reduce_aesthetics',W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_5x5_aesthetics = Convolution2D(128,5,5,border_mode='same',activation='relu',name='inception_4e/5x5_aesthetics',W_regularizer=l2(0.0002))(inception_4e_5x5_reduce_aesthetics)
inception_4e_pool_aesthetics = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4e/pool_aesthetics')(inception_4d_output)
inception_4e_pool_proj_aesthetics = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_4e/pool_proj_aesthetics',W_regularizer=l2(0.0002))(inception_4e_pool_aesthetics)
inception_4e_output_aesthetics = merge([inception_4e_1x1_aesthetics,inception_4e_3x3_aesthetics,inception_4e_5x5_aesthetics,inception_4e_pool_proj_aesthetics],mode='concat',concat_axis=1,name='inception_4e/output_aesthetics')
conv_output_aesthetics = Convolution2D(1024, 3, 3, activation='relu',name='conv_6_1',border_mode = 'same',W_regularizer=l2(0.0002))(inception_4e_output_aesthetics)
if rapid_style:
conv1_7x7_s2 = Convolution2D(64,7,7,subsample=(2,2),border_mode='same',activation='relu',name='semantic_conv1/7x7_s2',W_regularizer=l2(0.0002))(input_image)
conv1_zero_pad = ZeroPadding2D(padding=(1, 1))(conv1_7x7_s2)
pool1_helper = PoolHelper()(conv1_zero_pad)
pool1_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='semantic_pool1/3x3_s2')(pool1_helper)
pool1_norm1 = LRN(name='semantic_pool1/norm1')(pool1_3x3_s2)
conv2_3x3_reduce = Convolution2D(64,1,1,border_mode='same',activation='relu',name='semantic_conv2/3x3_reduce',W_regularizer=l2(0.0002))(pool1_norm1)
conv2_3x3 = Convolution2D(192,3,3,border_mode='same',activation='relu',name='semantic_conv2/3x3',W_regularizer=l2(0.0002))(conv2_3x3_reduce)
conv2_norm2 = LRN(name='semantic_conv2/norm2')(conv2_3x3)
conv2_zero_pad = ZeroPadding2D(padding=(1, 1))(conv2_norm2)
pool2_helper = PoolHelper()(conv2_zero_pad)
pool2_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='semantic_pool2/3x3_s2')(pool2_helper)
inception_3a_1x1 = Convolution2D(64,1,1,border_mode='same',activation='relu',name='semantic_inception_3a/1x1',W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_3x3_reduce = Convolution2D(96,1,1,border_mode='same',activation='relu',name='semantic_inception_3a/3x3_reduce',W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_3x3 = Convolution2D(128,3,3,border_mode='same',activation='relu',name='semantic_inception_3a/3x3',W_regularizer=l2(0.0002))(inception_3a_3x3_reduce)
inception_3a_5x5_reduce = Convolution2D(16,1,1,border_mode='same',activation='relu',name='semantic_inception_3a/5x5_reduce',W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_5x5 = Convolution2D(32,5,5,border_mode='same',activation='relu',name='semantic_inception_3a/5x5',W_regularizer=l2(0.0002))(inception_3a_5x5_reduce)
inception_3a_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='semantic_inception_3a/pool')(pool2_3x3_s2)
inception_3a_pool_proj = Convolution2D(32,1,1,border_mode='same',activation='relu',name='semantic_inception_3a/pool_proj',W_regularizer=l2(0.0002))(inception_3a_pool)
inception_3a_output = merge([inception_3a_1x1,inception_3a_3x3,inception_3a_5x5,inception_3a_pool_proj],mode='concat',concat_axis=1,name='semantic_inception_3a/output')
inception_3b_1x1 = Convolution2D(128,1,1,border_mode='same',activation='relu',name='semantic_inception_3b/1x1',W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_3x3_reduce = Convolution2D(128,1,1,border_mode='same',activation='relu',name='semantic_inception_3b/3x3_reduce',W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_3x3 = Convolution2D(192,3,3,border_mode='same',activation='relu',name='semantic_inception_3b/3x3',W_regularizer=l2(0.0002))(inception_3b_3x3_reduce)
inception_3b_5x5_reduce = Convolution2D(32,1,1,border_mode='same',activation='relu',name='semantic_inception_3b/5x5_reduce',W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_5x5 = Convolution2D(96,5,5,border_mode='same',activation='relu',name='semantic_inception_3b/5x5',W_regularizer=l2(0.0002))(inception_3b_5x5_reduce)
inception_3b_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='semantic_inception_3b/pool')(inception_3a_output)
inception_3b_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='semantic_inception_3b/pool_proj',W_regularizer=l2(0.0002))(inception_3b_pool)
inception_3b_output = merge([inception_3b_1x1,inception_3b_3x3,inception_3b_5x5,inception_3b_pool_proj],mode='concat',concat_axis=1,name='semantic_inception_3b/output')
inception_3b_output_zero_pad = ZeroPadding2D(padding=(1, 1))(inception_3b_output)
pool3_helper = PoolHelper()(inception_3b_output_zero_pad)
pool3_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='semantic_pool3/3x3_s2')(pool3_helper)
inception_4a_1x1 = Convolution2D(192,1,1,border_mode='same',activation='relu',name='semantic_inception_4a/1x1',W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_3x3_reduce = Convolution2D(96,1,1,border_mode='same',activation='relu',name='semantic_inception_4a/3x3_reduce',W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_3x3 = Convolution2D(208,3,3,border_mode='same',activation='relu',name='semantic_inception_4a/3x3',W_regularizer=l2(0.0002))(inception_4a_3x3_reduce)
inception_4a_5x5_reduce = Convolution2D(16,1,1,border_mode='same',activation='relu',name='semantic_inception_4a/5x5_reduce',W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_5x5 = Convolution2D(48,5,5,border_mode='same',activation='relu',name='semantic_inception_4a/5x5',W_regularizer=l2(0.0002))(inception_4a_5x5_reduce)
inception_4a_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='semantic_inception_4a/pool')(pool3_3x3_s2)
inception_4a_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='semantic_inception_4a/pool_proj',W_regularizer=l2(0.0002))(inception_4a_pool)
inception_4a_output = merge([inception_4a_1x1,inception_4a_3x3,inception_4a_5x5,inception_4a_pool_proj],mode='concat',concat_axis=1,name='semantic_inception_4a/output')
inception_4b_1x1 = Convolution2D(160,1,1,border_mode='same',activation='relu',name='semantic_inception_4b/1x1',W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_3x3_reduce = Convolution2D(112,1,1,border_mode='same',activation='relu',name='semantic_inception_4b/3x3_reduce',W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_3x3 = Convolution2D(224,3,3,border_mode='same',activation='relu',name='semantic_inception_4b/3x3',W_regularizer=l2(0.0002))(inception_4b_3x3_reduce)
inception_4b_5x5_reduce = Convolution2D(24,1,1,border_mode='same',activation='relu',name='semantic_inception_4b/5x5_reduce',W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_5x5 = Convolution2D(64,5,5,border_mode='same',activation='relu',name='semantic_inception_4b/5x5',W_regularizer=l2(0.0002))(inception_4b_5x5_reduce)
inception_4b_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='semantic_inception_4b/pool')(inception_4a_output)
inception_4b_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='semantic_inception_4b/pool_proj',W_regularizer=l2(0.0002))(inception_4b_pool)
inception_4b_output = merge([inception_4b_1x1,inception_4b_3x3,inception_4b_5x5,inception_4b_pool_proj],mode='concat',concat_axis=1,name='semantic_inception_4b_output')
inception_4c_1x1 = Convolution2D(128,1,1,border_mode='same',activation='relu',name='semantic_inception_4c/1x1',W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_3x3_reduce = Convolution2D(128,1,1,border_mode='same',activation='relu',name='semantic_inception_4c/3x3_reduce',W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_3x3 = Convolution2D(256,3,3,border_mode='same',activation='relu',name='semantic_inception_4c/3x3',W_regularizer=l2(0.0002))(inception_4c_3x3_reduce)
inception_4c_5x5_reduce = Convolution2D(24,1,1,border_mode='same',activation='relu',name='semantic_inception_4c/5x5_reduce',W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_5x5 = Convolution2D(64,5,5,border_mode='same',activation='relu',name='semantic_inception_4c/5x5',W_regularizer=l2(0.0002))(inception_4c_5x5_reduce)
inception_4c_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='semantic_inception_4c/pool')(inception_4b_output)
inception_4c_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='semantic_inception_4c/pool_proj',W_regularizer=l2(0.0002))(inception_4c_pool)
inception_4c_output = merge([inception_4c_1x1,inception_4c_3x3,inception_4c_5x5,inception_4c_pool_proj],mode='concat',concat_axis=1,name='semantic_inception_4c/output')
inception_4d_1x1 = Convolution2D(112,1,1,border_mode='same',activation='relu',name='semantic_inception_4d/1x1',W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_3x3_reduce = Convolution2D(144,1,1,border_mode='same',activation='relu',name='semantic_inception_4d/3x3_reduce',W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_3x3 = Convolution2D(288,3,3,border_mode='same',activation='relu',name='semantic_inception_4d/3x3',W_regularizer=l2(0.0002))(inception_4d_3x3_reduce)
inception_4d_5x5_reduce = Convolution2D(32,1,1,border_mode='same',activation='relu',name='semantic_inception_4d/5x5_reduce',W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_5x5 = Convolution2D(64,5,5,border_mode='same',activation='relu',name='semantic_inception_4d/5x5',W_regularizer=l2(0.0002))(inception_4d_5x5_reduce)
inception_4d_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='semantic_inception_4d/pool')(inception_4c_output)
inception_4d_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='semantic_inception_4d/pool_proj',W_regularizer=l2(0.0002))(inception_4d_pool)
inception_4d_output = merge([inception_4d_1x1,inception_4d_3x3,inception_4d_5x5,inception_4d_pool_proj],mode='concat',concat_axis=1,name='semantic_inception_4d/output')
inception_4e_1x1_semantics = Convolution2D(256,1,1,border_mode='same',activation='relu',name='inception_4e/1x1_semantics',W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3_reduce_semantics = Convolution2D(160,1,1,border_mode='same',activation='relu',name='inception_4e/3x3_reduce_semantics',W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3_semantics = Convolution2D(320,3,3,border_mode='same',activation='relu',name='inception_4e/3x3_semantics',W_regularizer=l2(0.0002))(inception_4e_3x3_reduce_semantics)
inception_4e_5x5_reduce_semantics = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_4e/5x5_reduce_semantics',W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_5x5_semantics = Convolution2D(128,5,5,border_mode='same',activation='relu',name='inception_4e/5x5_semantics',W_regularizer=l2(0.0002))(inception_4e_5x5_reduce_semantics)
inception_4e_pool_semantics = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4e/pool_semantics')(inception_4d_output)
inception_4e_pool_proj_semantics = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_4e/pool_proj_semantics',W_regularizer=l2(0.0002))(inception_4e_pool_semantics)
inception_4e_output_semantics = merge([inception_4e_1x1_semantics,inception_4e_3x3_semantics,inception_4e_5x5_semantics,inception_4e_pool_proj_semantics],mode='concat',concat_axis=1,name='inception_4e/output_semantics')
conv_output_semantics = Convolution2D(1024, 3, 3, activation='relu',name='conv_6_1_semantics',border_mode = 'same',W_regularizer=l2(0.0002))(inception_4e_output_semantics)
if not rapid_style:
x_semantics = GlobalAveragePooling2D()(conv_output_semantics)
if use_comments:
x_semantics = merge([x_semantics, x_text_semantics],mode='concat',concat_axis=1)
output_semantics = Dense(65, activation = 'softmax', name="output_semantics")(x_semantics)
else:
inception_4e_1x1 = Convolution2D(256,1,1,border_mode='same',activation='relu',name='inception_4e/1x1',W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3_reduce = Convolution2D(160,1,1,border_mode='same',activation='relu',name='inception_4e/3x3_reduce',W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3 = Convolution2D(320,3,3,border_mode='same',activation='relu',name='inception_4e/3x3',W_regularizer=l2(0.0002))(inception_4e_3x3_reduce)
inception_4e_5x5_reduce = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_4e/5x5_reduce',W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_5x5 = Convolution2D(128,5,5,border_mode='same',activation='relu',name='inception_4e/5x5',W_regularizer=l2(0.0002))(inception_4e_5x5_reduce)
inception_4e_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4e/pool')(inception_4d_output)
inception_4e_pool_proj = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_4e/pool_proj',W_regularizer=l2(0.0002))(inception_4e_pool)
inception_4e_output = merge([inception_4e_1x1,inception_4e_3x3,inception_4e_5x5,inception_4e_pool_proj],mode='concat',concat_axis=1,name='inception_4e/output')
conv_output_aesthetics = Convolution2D(1024, 3, 3, activation='relu',name='conv_6_1',border_mode = 'same',W_regularizer=l2(0.0002))(inception_4e_output)
if rapid_style:
merged_conv = merge([conv_output_aesthetics, conv_output_semantics], mode='concat', concat_axis=1)
x_aesthetics = GlobalAveragePooling2D()(merged_conv)
else:
x_aesthetics = GlobalAveragePooling2D()(conv_output_aesthetics)
if use_multigap:
if use_comments:
x_aesthetics = merge([x_aesthetics, x_text_aesthetics, inception_4a_gap, inception_4b_gap, inception_4c_gap, inception_4d_gap],mode='concat',concat_axis=1)
else:
x_aesthetics = merge([x_aesthetics, inception_4a_gap, inception_4b_gap, inception_4c_gap, inception_4d_gap],mode='concat',concat_axis=1)
else:
if use_comments:
x_aesthetics = merge([x_aesthetics, x_text_aesthetics],mode='concat',concat_axis=1)
if use_distribution:
if use_multigap:
output_aesthetics = Dense(10, activation = 'softmax', name="main_output__")(x_aesthetics)
else:
output_aesthetics = Dense(10, activation = 'softmax', name="main_output_")(x_aesthetics)
else:
if use_multigap:
output_aesthetics = Dense(2, activation = 'softmax', name="main_output_")(x_aesthetics)
else:
output_aesthetics = Dense(2, activation = 'softmax', name="main_output")(x_aesthetics)
if use_semantics and not rapid_style:
if use_comments:
googlenet = Model(input=[input_image, comment_input], output=[output_aesthetics,output_semantics])
else:
googlenet = Model(input=input_image, output=[output_aesthetics,output_semantics])
else:
if use_comments:
googlenet = Model(input=[input_image, comment_input], output=output_aesthetics)
else:
googlenet = Model(input=input_image, output=output_aesthetics)
if weights_path:
if use_semantics:
googlenet.load_weights('weights/named_googlenet_semantics_weights.h5', by_name=True)
googlenet.load_weights(weights_path,by_name=load_weights_by_name)
if rapid_style:
for i, layer in enumerate(googlenet.layers):
if 'semantic' in layer.name:
# print("{} - {}".format(i, layer.name))
layer.trainable = False
return googlenet
def dense_net_121(img_rows, img_cols, color_type=1, nb_dense_block=4, growth_rate=32, nb_filter=64, reduction=0.5,
dropout_rate=0.0, weight_decay=0.0, dropout_fc=0.0, weights_path=None, freeze_dense_block_num=-1):
'''
DenseNet 121 Model for Keras
Model Schema is based on
https://github.com/flyyufelix/DenseNet-Keras
ImageNet Pretrained Weights
Theano: https://drive.google.com/open?id=0Byy2AcGyEVxfMlRYb3YzV210VzQ
TensorFlow: https://drive.google.com/open?id=0Byy2AcGyEVxfSTA4SHJVOHNuTXc
# Arguments
nb_dense_block: number of dense blocks to add to end
growth_rate: number of filters to add per dense block
nb_filter: initial number of filters
reduction: reduction factor of transition blocks.
dropout_rate: dropout rate
weight_decay: weight decay factor
classes: optional number of classes to classify images
weights_path: path to pre-trained weights
# Returns
A Keras model instance.
'''
eps = 1.1e-5
# compute compression factor
compression = 1.0 - reduction
# Handle Dimension Ordering for different backends
global concat_axis
if K.image_dim_ordering() == 'tf':
concat_axis = 3
img_input = Input(shape=(img_rows, img_cols, color_type), name='data')
else:
concat_axis = 1
img_input = Input(shape=(color_type, img_rows, img_cols), name='data')
# From architecture for ImageNet (Table 1 in the paper)
nb_filter = 64
nb_layers = [6, 12, 24, 16] # For DenseNet-121
# random crop
X = img_input
X = Lambda(tf.image.resize_image_with_crop_or_pad,
output_shape=(224, 224, 3),
arguments={'target_height': 224, 'target_width': 224})(X)
# Normalize
X = Lambda(lambda x: (x - MEAN) * SCALE, output_shape=(224, 224, 3))(X)
# Initial convolution
trainable = (-1 >= freeze_dense_block_num)
x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(X)
x = Convolution2D(nb_filter, kernel_size=(7, 7), strides=(2, 2), name='conv1', bias=False,
kernel_regularizer=l2(weight_decay), trainable=trainable)(x)
x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv1_bn', beta_regularizer=l2(weight_decay),
gamma_regularizer=l2(weight_decay), trainable=trainable)(x)
x = Scale(axis=concat_axis, name='conv1_scale', trainable=trainable)(x)
x = Activation('relu', name='relu1')(x)
x = ZeroPadding2D((1, 1), name='pool1_zeropadding')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1')(x)
# Add dense blocks
for block_idx in range(nb_dense_block - 1):
stage = block_idx + 2
trainable = (block_idx >= freeze_dense_block_num)
x, nb_filter = dense_block(x, stage, nb_layers[block_idx], nb_filter, growth_rate, dropout_rate=dropout_rate,
weight_decay=weight_decay, trainable=trainable)
# Add transition_block
x = transition_block(x, stage, nb_filter, compression=compression, dropout_rate=dropout_rate,
weight_decay=weight_decay, trainable=trainable)
nb_filter = int(nb_filter * compression)
final_stage = stage + 1
trainable = (nb_dense_block - 1 >= freeze_dense_block_num)
x, nb_filter = dense_block(x, final_stage, nb_layers[-1], nb_filter, growth_rate, dropout_rate=dropout_rate,
weight_decay=weight_decay, trainable=trainable)
x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv' + str(final_stage) + '_blk_bn',
beta_regularizer=l2(weight_decay), gamma_regularizer=l2(weight_decay), trainable=trainable)(
x)
x = Scale(axis=concat_axis, name='conv' + str(final_stage) + '_blk_scale', trainable=trainable)(x)
x = Activation('relu', name='relu' + str(final_stage) + '_blk')(x)
x_fc = GlobalAveragePooling2D(name='pool' + str(final_stage))(x)
x_fc = Dense(1000, name='fc6')(x_fc)
x_fc = Activation('softmax', name='prob')(x_fc)
model = Model(img_input, x_fc, name='densenet')
model.load_weights(weights_path, by_name=True)
# Truncate and replace softmax layer for transfer learning
# Cannot use model.layers.pop() since model is not of Sequential() type
# The method below works since pre-trained weights are stored in layers but not in the model
x_newfc = GlobalAveragePooling2D(name='pool' + str(final_stage))(x)
x_newfc = Dropout(dropout_fc)(x_newfc)
x_newfc = Dense(1, name='fc6', kernel_regularizer=l2(weight_decay))(x_newfc)
x_newfc = Activation('sigmoid', name='prob')(x_newfc)
model = Model(img_input, x_newfc)
# Learning rate
model.compile(
optimizer=Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08),
loss='binary_crossentropy',
metrics=['accuracy']
)
return model
def VGG16(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling='Max',
classes=1000):
if weights not in {'imagenet', None}:
raise ValueError('The `weights` argument should be either '
'`None` (random initialization) or `imagenet` '
'(pre-training on ImageNet).')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_size=224,
min_size=48,
data_format=K.image_data_format(),
include_top=include_top)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
# Block 1
x = Conv2D(64, (3, 3),
activation='relu',
padding='same',
name='block1_conv1')(img_input)
x = Conv2D(64, (3, 3),
activation='relu',
padding='same',
name='block1_conv2')(x)
x = AveragePooling2D((2, 2), strides=(2, 2))(x)
# Block 2
x = Conv2D(128, (3, 3),
activation='relu',
padding='same',
name='block2_conv1')(x)
x = Conv2D(128, (3, 3),
activation='relu',
padding='same',
name='block2_conv2')(x)
x = AveragePooling2D((2, 2), strides=(2, 2))(x)
# Block 3
x = Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv1')(x)
x = Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv2')(x)
x = Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv3')(x)
x = AveragePooling2D((2, 2), strides=(2, 2))(x)
# Block 4
x = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv1')(x)
x = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv2')(x)
x = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv3')(x)
x = AveragePooling2D((2, 2), strides=(2, 2))(x)
# Block 5
x = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv1')(x)
x = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv2')(x)
x = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv3')(x)
x = AveragePooling2D((2, 2), strides=(2, 2))(x)
if include_top:
# Classification block
x = Flatten(name='flatten')(x)
x = Dense(4096, activation='relu', name='fc1')(x)
x = Dense(4096, activation='relu', name='fc2')(x)
x = Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'Average':
x = GlobalAveragePooling2D()(x)
elif pooling == 'Max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='vgg16')
# load weights
if weights == 'imagenet':
if include_top:
weights_path = get_file(
'vgg16_weights_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH,
cache_subdir='models')
else:
weights_path = get_file(
'vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models')
model.load_weights(weights_path)
if K.backend() == 'theano':
layer_utils.convert_all_kernels_in_model(model)
if K.image_data_format() == 'channels_first':
if include_top:
maxpool = model.get_layer(name='block5_pool')
shape = maxpool.output_shape[1:]
dense = model.get_layer(name='fc1')
layer_utils.convert_dense_weights_data_format(
dense, shape, 'channels_first')
if K.backend() == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image data format convention '
'(`image_data_format="channels_first"`). '
'For best performance, set '
'`image_data_format="channels_last"` in '
'your Keras config '
'at ~/.keras/keras.json.')
return model
def build_denseNet(img_shape=(3, 224, 224),
n_classes=1000,
depth=40,
growth_rate=32,
l2_reg=0.,
load_pretrained=False,
freeze_layers_from=None):
# Decide if load pretrained weights from imagenet
if load_pretrained:
weights = 'imagenet'
else:
weights = None
if K.image_dim_ordering() == 'tf':
concat_axis = -1
else:
concat_axis = 1
assert (depth - 4) % 3 == 0, 'Depth must be 3 N + 4'
# layers in each dense block
nb_layers = int((depth - 4) / 3)
compression = 0.5
nb_blocks = 3
# compute initial nb_filter if -1, else accept users initial nb_filter
nb_filter = 2 * growth_rate
# Generate the model
model_input = Input(shape=img_shape)
# Initial block
x = Convolution2D(nb_filter,
7,
7,
border_mode='same',
name='initial_conv',
W_regularizer=l2(l2_reg))(model_input)
# Add dense blocks
for block in range(1, nb_blocks):
x, nb_filter = dense_block(x, nb_layers, nb_filter, growth_rate,
l2_reg, block)
# add transition_block
x = transition_block(x, nb_filter, compression, l2_reg, block)
nb_filter = int(nb_filter * compression)
# The last dense_block does not have a transition_block
x, nb_filter = dense_block(x, nb_layers, nb_filter, growth_rate, l2_reg,
nb_blocks)
base_model = Model(input=model_input, output=x)
# Classification block
y = base_model.output
y = GlobalAveragePooling2D()(y)
predictions = Dense(n_classes, activation='softmax')(y)
model = Model(input=base_model.input, output=predictions)
# Freeze some layers
if freeze_layers_from is not None:
if freeze_layers_from == 'base_model':
print(' Freezing base model layers')
for layer in base_model.layers:
layer.trainable = False
else:
for i, layer in enumerate(model.layers):
print(i, layer.name)
print(' Freezing from layer 0 to ' + str(freeze_layers_from))
for layer in model.layers[:freeze_layers_from]:
layer.trainable = False
for layer in model.layers[freeze_layers_from:]:
layer.trainable = True
return model
def densenet169_model(img_dim, nb_dense_block=4, growth_rate=32, nb_filter=64, reduction=0.5, dropout_rate=0.2, weight_decay=1e-4, num_classes=17):
'''
DenseNet 169 Model for Keras
Model Schema is based on
https://github.com/flyyufelix/DenseNet-Keras
ImageNet Pretrained Weights
Theano: https://drive.google.com/open?id=0Byy2AcGyEVxfN0d3T1F1MXg0NlU
TensorFlow: https://drive.google.com/open?id=0Byy2AcGyEVxfSEc5UC1ROUFJdmM
# Arguments
nb_dense_block: number of dense blocks to add to end
growth_rate: number of filters to add per dense block
nb_filter: initial number of filters
reduction: reduction factor of transition blocks.
dropout_rate: dropout rate
weight_decay: weight decay factor
classes: optional number of classes to classify images
weights_path: path to pre-trained weights
# Returns
A Keras model instance.
'''
global concat_axis
if K.image_dim_ordering() == 'tf':
concat_axis = -1
else:
concat_axis = 1
eps = 1.1e-5
# compute compression factor
compression = 1.0 - reduction
# From architecture for ImageNet (Table 1 in the paper)
nb_filter = 64
nb_layers = [6,12,32,32] # For DenseNet-169
# Initial convolution
img_input = Input(shape=img_dim, name='data')
x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input)
x = Convolution2D(nb_filter, 7, 7, subsample=(2, 2), name='conv1', bias=False)(x)
x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv1_bn')(x)
x = Scale(axis=concat_axis)(x)
x = Activation('relu', name='relu1')(x)
x = ZeroPadding2D((1, 1), name='pool1_zeropadding')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1')(x)
# Add dense blocks
for block_idx in range(nb_dense_block - 1):
stage = block_idx+2
x, nb_filter = dense_block(x, stage, nb_layers[block_idx], nb_filter, growth_rate, dropout_rate=dropout_rate, weight_decay=weight_decay)
# Add transition_block
x = transition_block(x, stage, nb_filter, compression=compression, dropout_rate=dropout_rate, weight_decay=weight_decay)
nb_filter = int(nb_filter * compression)
final_stage = stage + 1
x, nb_filter = dense_block(x, final_stage, nb_layers[-1], nb_filter, growth_rate, dropout_rate=dropout_rate, weight_decay=weight_decay)
x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv'+str(final_stage)+'_blk_bn')(x)
x = Scale(axis=concat_axis)(x)
x = Activation('relu', name='relu'+str(final_stage)+'_blk')(x)
# Truncate and replace softmax layer for transfer learning
# Cannot use model.layers.pop() since model is not of Sequential() type
# The method below works since pre-trained weights are stored in layers but not in the model
x_newfc = GlobalAveragePooling2D(name='pool'+str(final_stage))(x)
x_newfc = Dense(num_classes, name='fc6')(x_newfc)
x_newfc = Activation('softmax', name='prob')(x_newfc)
model = Model(img_input, x_newfc, name='densenet')
# Learning rate is changed to 0.001
# sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
# model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])
return model
def __create_res_next(nb_classes,
img_input,
depth,
cardinality=32,
width=4,
weight_decay=5e-4,
first_filters=64,
first_kernal_size=7):
''' Creates a ResNeXt model with specified parameters
Args:
nb_classes: Number of output classes
img_input: Input tensor or layer
include_top: Flag to include the last dense layer
depth: Depth of the network. List of integers.
Increasing cardinality improves two_stage_classification accuracy,
width: Width of the network.
weight_decay: weight_decay (l2 norm)
Returns: a Keras Model
'''
N = list(depth)
filters = cardinality * width #width 的作用
filters_list = []
for i in range(len(N)):
filters_list.append(filters)
filters *= 2 # double the size of the filters
#开始部分 first conv (7*7) max Pooling
x = __initial_conv_block(img_input,
weight_decay,
first_filters=first_filters,
first_kernal_size=first_kernal_size)
# first block
for i in range(N[0]):
x = __bottleneck_block(x,
filters_list[0],
cardinality,
strides=1,
weight_decay=weight_decay)
N = N[1:] # remove the first block from block definition list
filters_list = filters_list[
1:] # remove the first filter from the filter list
# block 2 to N
for block_idx, n_i in enumerate(N):
for i in range(n_i):
if i == 0: #block的第一个conv stride=2
x = __bottleneck_block(x,
filters_list[block_idx],
cardinality,
strides=2,
weight_decay=weight_decay)
else: #block中的其他residual unit stride=1
x = __bottleneck_block(x,
filters_list[block_idx],
cardinality,
strides=1,
weight_decay=weight_decay)
x = GlobalAveragePooling2D()(x)
x = Dense(nb_classes,
use_bias=False,
kernel_regularizer=l2(weight_decay),
kernel_initializer='he_normal',
activation='softmax')(x)
return x
def densenet121_model(img_rows,
img_cols,
color_type=1,
nb_dense_block=4,
growth_rate=32,
nb_filter=64,
reduction=0.5,
dropout_rate=0.0,
weight_decay=1e-4,
num_classes=None):
'''
DenseNet 121 Model for Keras
Model Schema is based on
https://github.com/flyyufelix/DenseNet-Keras
ImageNet Pretrained Weights
Theano: https://drive.google.com/open?id=0Byy2AcGyEVxfMlRYb3YzV210VzQ
TensorFlow: https://drive.google.com/open?id=0Byy2AcGyEVxfSTA4SHJVOHNuTXc
# Arguments
nb_dense_block: number of dense blocks to add to end
growth_rate: number of filters to add per dense block
nb_filter: initial number of filters
reduction: reduction factor of transition blocks.
dropout_rate: dropout rate
weight_decay: weight decay factor
classes: optional number of classes to classify images
weights_path: path to pre-trained weights
# Returns
A Keras model instance.
'''
eps = 1.1e-5
# compute compression factor
compression = 1.0 - reduction
# Handle Dimension Ordering for different backends
global concat_axis
if K.image_dim_ordering() == 'tf':
concat_axis = 3
img_input = Input(shape=(img_rows, img_cols, color_type), name='data')
else:
concat_axis = 1
img_input = Input(shape=(color_type, img_rows, img_cols), name='data')
# From architecture for ImageNet (Table 1 in the paper)
nb_filter = 64
nb_layers = [6, 12, 24, 16] # For DenseNet-121
# Initial convolution
x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input)
x = Conv2D(nb_filter, (7, 7), strides=(2, 2), name='conv1',
use_bias=False)(x)
x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv1_bn')(x)
x = Scale(axis=concat_axis, name='conv1_scale')(x)
x = Activation('relu', name='relu1')(x)
x = ZeroPadding2D((1, 1), name='pool1_zeropadding')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1')(x)
# Add dense blocks
for block_idx in range(nb_dense_block - 1):
stage = block_idx + 2
x, nb_filter = dense_block(x,
stage,
nb_layers[block_idx],
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# Add transition_block
x = transition_block(x,
stage,
nb_filter,
compression=compression,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
nb_filter = int(nb_filter * compression)
final_stage = stage + 1
x, nb_filter = dense_block(x,
final_stage,
nb_layers[-1],
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
x = BatchNormalization(epsilon=eps,
axis=concat_axis,
name='conv' + str(final_stage) + '_blk_bn')(x)
x = Scale(axis=concat_axis,
name='conv' + str(final_stage) + '_blk_scale')(x)
x = Activation('relu', name='relu' + str(final_stage) + '_blk')(x)
x_fc = GlobalAveragePooling2D(name='pool' + str(final_stage))(x)
x_fc = Dense(1000, name='fc6')(x_fc)
x_fc = Activation('softmax', name='prob')(x_fc)
model = Model(img_input, x_fc, name='densenet')
if K.image_dim_ordering() == 'th':
# Use pre-trained weights for Theano backend
weights_path = '../imagenet_models/densenet121_weights_th.h5'
else:
# Use pre-trained weights for Tensorflow backend
weights_path = '../imagenet_models/densenet121_weights_tf.h5'
# model.load_weights(weights_path, by_name=True)
# Truncate and replace softmax layer for transfer learning
# Cannot use model.layers.pop() since model is not of Sequential() type
# The method below works since pre-trained weights are stored in layers but not in the model
x_newfc = GlobalAveragePooling2D(name='pool' + str(final_stage))(x)
x_newfc = Dense(num_classes, name='fc6')(x_newfc)
x_newfc = Activation('softmax', name='prob')(x_newfc)
model = Model(img_input, x_newfc)
# Learning rate is changed to 0.001
sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd,
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def DenseNet_model(input_shape,
classes,
depth=40,
nb_dense_block=3,
growth_rate=12,
include_top=True,
nb_filter=-1,
nb_layers_per_block=[6, 12, 32, 32],
bottleneck=False,
reduction=0.0,
dropout_rate=None,
weight_decay=1e-4,
subsample_initial_block=False,
activation='softmax'):
'''
Build the DenseNet model
Args:
classes: number of classes
input_shape: tuple of shape (channels, rows, columns) or (rows, columns, channels)
include_top: flag to include the final Dense layer
depth: number or layers
nb_dense_block: number of dense blocks to add to end (generally = 3)
growth_rate: number of filters to add per dense block
nb_filter: initial number of filters. Default -1 indicates initial number of filters is 2 * growth_rate
nb_layers_per_block: number of layers in each dense block.
Can be a -1, positive integer or a list.
If -1, calculates nb_layer_per_block from the depth of the network.
If positive integer, a set number of layers per dense block.
If list, nb_layer is used as provided. Note that list size must
be (nb_dense_block + 1)
bottleneck: add bottleneck blocks
reduction: reduction factor of transition blocks. Note : reduction value is inverted to compute compression
dropout_rate: dropout rate
weight_decay: weight decay rate
subsample_initial_block: Set to True to subsample the initial convolution and
add a MaxPool2D before the dense blocks are added.
subsample_initial:
activation: Type of activation at the top layer. Can be one of 'softmax' or 'sigmoid'.
Note that if sigmoid is used, classes must be 1.
Returns: keras tensor with nb_layers of conv_block appended
'''
concat_axis = 1 if K.image_data_format() == 'channel_first' else -1
if type(nb_layers_per_block) is not list:
print('nb_layers_per_block should be a list !!!')
return 0
final_nb_layer = nb_layers_per_block[-1]
nb_layers = nb_layers_per_block[:-1]
# compute initial nb_filter if -1 else accept users initial nb_filter
if nb_filter <= 0:
nb_filter = 2 * growth_rate
# compute compression factor
compression = 1.0 - reduction
# initial convolution
if subsample_initial_block:
initial_kernel = (7, 7)
initial_strides = (2, 2)
else:
initial_kernel = (3, 3)
initial_strides = (1, 1)
Inp = Input(shape=input_shape)
x = Conv2D(nb_filter,
initial_kernel,
kernel_initializer='he_normal',
padding='same',
strides=initial_strides,
use_bias=False,
kernel_regularizer=l2(weight_decay))(Inp)
if subsample_initial_block:
x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(x)
x = Activation('relu')(x)
x = Maxpooling2D((3, 3), strides=(2, 2), padding='same')(x)
# add dense blocks
for block_index in range(nb_dense_block - 1):
x, nb_filter = dense_block(x,
nb_layers[block_index],
nb_filter,
growth_rate,
bottleneck=bottleneck,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# add transition block
x = transition_block(x,
nb_filter,
compression=compression,
weight_decay=weight_decay)
nb_filter = int(nb_filter * compression)
# the last dense block does not have a transition_block
x, nb_filter = dense_block(x,
final_nb_layer,
nb_filter,
growth_rate,
bottleneck=bottleneck,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(x)
x = Activation('relu')(x)
x = GlobalAveragePooling2D()(x)
if include_top:
x = Dense(classes, activation=activation)(x)
model = Model(Inp, output=x)
model.summary()
return model
def densenet121_model(img_rows,
img_cols,
color_type=1,
nb_dense_block=4,
growth_rate=32,
nb_filter=64,
reduction=0.5,
dropout_rate=0.,
weight_decay=1e-4,
num_classes=None):
eps = 1.1e-5
# compute compression factor
compression = 1.0 - reduction
# Handle Dimension Ordering for different backends
global concat_axis
if K.image_dim_ordering() == 'tf':
concat_axis = 3
img_input = Input(shape=(img_rows, img_cols, color_type), name='data')
else:
concat_axis = 1
img_input = Input(shape=(color_type, img_rows, img_cols), name='data')
# From architecture for ImageNet (Table 1 in the paper)
nb_filter = 64
nb_layers = [6, 12, 24, 16] # For DenseNet-121
# Initial convolution
x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input)
x = Conv2D(nb_filter,
kernel_size=(7, 7),
strides=(2, 2),
name='conv1',
use_bias=False)(x)
x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv1_bn')(x)
x = Scale(axis=concat_axis, name='conv1_scale')(x)
x = Activation('relu', name='relu1')(x)
x = ZeroPadding2D((1, 1), name='pool1_zeropadding')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1')(x)
# Add dense blocks
for block_idx in range(nb_dense_block - 1):
stage = block_idx + 2
x, nb_filter = dense_block(x,
stage,
nb_layers[block_idx],
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# Add transition_block
x = transition_block(x,
stage,
nb_filter,
compression=compression,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
nb_filter = int(nb_filter * compression)
final_stage = stage + 1
x, nb_filter = dense_block(x,
final_stage,
nb_layers[-1],
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
x = BatchNormalization(epsilon=eps,
axis=concat_axis,
name='conv' + str(final_stage) + '_blk_bn')(x)
x = Scale(axis=concat_axis,
name='conv' + str(final_stage) + '_blk_scale')(x)
x = Activation('relu', name='relu' + str(final_stage) + '_blk')(x)
x_fc = GlobalAveragePooling2D(name='pool' + str(final_stage))(x)
x_fc = Dense(1000, name='fc6')(x_fc)
x_fc = Activation('softmax', name='prob')(x_fc)
model = Model(img_input, x_fc, name='densenet')
if K.image_dim_ordering() == 'th':
# Use pre-trained weights for Theano backend
weights_path = 'pre_train/imagenet_models/densenet121_weights_th.h5'
else:
# Use pre-trained weights for Tensorflow backend
weights_path = 'pre_train/imagenet_models/densenet121_weights_tf.h5'
model.load_weights(weights_path, by_name=True)
# Truncate and replace softmax layer for transfer learning
# Cannot use model.layers.pop() since model is not of Sequential() type
# The method below works since pre-trained weights are stored in layers but not in the model
x_newfc = GlobalAveragePooling2D(name='pool' + str(final_stage))(x)
x_newfc = Dropout(0.5)(x_newfc)
x_newfc = Dense(num_classes,
name='fc6',
kernel_regularizer=regularizers.l2(l2_lambda),
kernel_initializer='he_uniform')(x_newfc)
x_newfc = Activation('sigmoid', name='prob')(x_newfc)
model = Model(img_input, x_newfc)
nadam1 = keras.optimizers.Nadam(lr=0.0003,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
schedule_decay=0.0)
#sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(optimizer=nadam1,
loss='binary_crossentropy',
metrics=['accuracy'])
return model
assert os.path.isfile('dogbreed_model.hdf5') and os.path.isfile(
'dogbreed_labels.pickle')
except:
print(
"Run the Train_Dog_Breed_Model Script first to train the Dog Breed Classification Model"
)
raise
# In[3]:
inception_model = InceptionV3(weights='imagenet', include_top=False)
# In[4]:
net_input = Input(shape=(8, 8, 2048))
net = GlobalAveragePooling2D()(net_input)
net = Dense(512, use_bias=False, kernel_initializer='uniform')(net)
net = BatchNormalization()(net)
net = Activation("relu")(net)
net = Dropout(0.5)(net)
net = Dense(256, use_bias=False, kernel_initializer='uniform')(net)
net = BatchNormalization()(net)
net = Activation("relu")(net)
net = Dropout(0.5)(net)
net = Dense(133, kernel_initializer='uniform', activation="softmax")(net)
dog_breed_model = Model(inputs=[net_input], outputs=[net])
dog_breed_model.summary()
dog_breed_model.load_weights('dogbreed_model.hdf5')
# In[ ]:
def DenseNet(nb_dense_block=4,
growth_rate=32,
nb_filter=64,
reduction=0.0,
dropout_rate=0.0,
weight_decay=1e-4,
classes=1000,
weights_path=None):
'''Instantiate the DenseNet 121 architecture,
# Arguments
nb_dense_block: number of dense blocks to add to end
growth_rate: number of filters to add per dense block
nb_filter: initial number of filters
reduction: reduction factor of transition blocks.
dropout_rate: dropout rate
weight_decay: weight decay factor
classes: optional number of classes to classify images
weights_path: path to pre-trained weights
# Returns
A Keras model instance.
'''
eps = 1.1e-5
# compute compression factor
compression = 1.0 - reduction
# Handle Dimension Ordering for different backends
global concat_axis
if K.image_dim_ordering() == 'tf':
concat_axis = 3
img_input = Input(shape=(224, 224, 3), name='data')
else:
concat_axis = 1
img_input = Input(shape=(3, 224, 224), name='data')
# From architecture for ImageNet (Table 1 in the paper)
nb_filter = 64
nb_layers = [6, 12, 24, 16] # For DenseNet-121
# Initial convolution
x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input)
x = Conv2D(nb_filter, (7, 7), strides=(2, 2), name='conv1',
use_bias=False)(x)
x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv1_bn')(x)
x = Scale(axis=concat_axis, name='conv1_scale')(x)
x = Activation('relu', name='relu1')(x)
x = ZeroPadding2D((1, 1), name='pool1_zeropadding')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1')(x)
# Add dense blocks
for block_idx in range(nb_dense_block - 1):
stage = block_idx + 2
x, nb_filter = dense_block(x,
stage,
nb_layers[block_idx],
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# Add transition_block
x = transition_block(x,
stage,
nb_filter,
compression=compression,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
nb_filter = int(nb_filter * compression)
final_stage = stage + 1
x, nb_filter = dense_block(x,
final_stage,
nb_layers[-1],
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
x = BatchNormalization(epsilon=eps,
axis=concat_axis,
name='conv' + str(final_stage) + '_blk_bn')(x)
x = Scale(axis=concat_axis,
name='conv' + str(final_stage) + '_blk_scale')(x)
x = Activation('relu', name='relu' + str(final_stage) + '_blk')(x)
x = GlobalAveragePooling2D(name='pool' + str(final_stage))(x)
x = Dense(classes, name='fc6')(x)
x = Activation('softmax', name='prob')(x)
model = Model(img_input, x, name='densenet')
if weights_path is not None:
model.load_weights(weights_path)
return model
def DenseNet(nb_classes,
img_dim,
depth,
nb_dense_block,
growth_rate,
nb_filter,
dropout_rate=None,
weight_decay=1e-4):
"""
This is different from the architecture in the paper, because it's designed for cifar10 dataset (32 by 32)
The architecture in paper is designed for ImageNet (224 by 224)
:param bn_classes:
:param img_dim:
:param depth:
:param nb_dense_block:
:param growth_rate:
:param nb_filter:
:param dropout_rate:
:param weight_decay:
:return:
"""
if K.image_dim_ordering() == 'th':
concat_axis = 1
elif K.image_dim_ordering() == 'tf':
concat_axis = -1
model_input = Input(shape=img_dim)
# Depth must be 3 N + 4 ??? should be K*N+K0
assert (depth - 4) % 3 == 0
nb_layers = int((depth - 4) / 3)
#Inital convolution
x = Conv2D(nb_filter, (3, 3),
kernel_initializer='he_uniform',
padding='same',
name='initial_conv2D',
use_bias=False,
kernel_regularizer=l2(weight_decay))(model_input)
#Add dense blocks
for block_idx in range(nb_dense_block - 1):
x, nb_filter = dense_block(x,
concat_axis,
nb_layers,
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# add transition
x = transition(x,
nb_filter,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# The last denseblock does not have a transition
x, nb_filter = dense_block(x,
concat_axis,
nb_layers,
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
x = BatchNormalization(axis=concat_axis,
gamma_regularizer=l2(weight_decay),
beta_regularizer=l2(weight_decay))(x)
x = Activation('relu')(x)
x = GlobalAveragePooling2D(data_format=K.image_data_format())(x)
x = Dense(nb_classes,
activation='softmax',
kernel_regularizer=l2(weight_decay),
bias_regularizer=l2(weight_decay))(x)
densenet = Model(inputs=[model_input], outputs=[x], name='DenseNet')
return densenet
def __create_dense_net(nb_classes,
img_input,
include_top,
depth=40,
nb_dense_block=3,
growth_rate=12,
nb_filter=-1,
nb_layers_per_block=-1,
bottleneck=False,
reduction=0.0,
dropout_rate=None,
weight_decay=1e-4,
subsample_initial_block=False,
activation='softmax'):
''' Build the DenseNet model
Args:
nb_classes: number of classes
img_input: tuple of shape (channels, rows, columns) or (rows, columns, channels)
include_top: flag to include the final Dense layer
depth: number or layers
nb_dense_block: number of dense blocks to add to end (generally = 3)
growth_rate: number of filters to add per dense block
nb_filter: initial number of filters. Default -1 indicates initial number of filters is 2 * growth_rate
nb_layers_per_block: number of layers in each dense block.
Can be a -1, positive integer or a list.
If -1, calculates nb_layer_per_block from the depth of the network.
If positive integer, a set number of layers per dense block.
If list, nb_layer is used as provided. Note that list size must
be (nb_dense_block + 1)
bottleneck: add bottleneck blocks
reduction: reduction factor of transition blocks. Note : reduction value is inverted to compute compression
dropout_rate: dropout rate
weight_decay: weight decay rate
subsample_initial_block: Set to True to subsample the initial convolution and
add a MaxPool2D before the dense blocks are added.
subsample_initial:
activation: Type of activation at the top layer. Can be one of 'softmax' or 'sigmoid'.
Note that if sigmoid is used, classes must be 1.
Returns: keras tensor with nb_layers of conv_block appended
'''
concat_axis = 1 if K.image_data_format() == 'channels_first' else -1
if reduction != 0.0:
assert reduction <= 1.0 and reduction > 0.0, 'reduction value must lie between 0.0 and 1.0'
# layers in each dense block
if type(nb_layers_per_block) is list or type(nb_layers_per_block) is tuple:
nb_layers = list(nb_layers_per_block) # Convert tuple to list
assert len(nb_layers) == (nb_dense_block), 'If list, nb_layer is used as provided. ' \
'Note that list size must be (nb_dense_block)'
final_nb_layer = nb_layers[-1]
nb_layers = nb_layers[:-1]
else:
if nb_layers_per_block == -1:
assert (
depth - 4
) % 3 == 0, 'Depth must be 3 N + 4 if nb_layers_per_block == -1'
count = int((depth - 4) / 3)
if bottleneck:
count = count // 2
nb_layers = [count for _ in range(nb_dense_block)]
final_nb_layer = count
else:
final_nb_layer = nb_layers_per_block
nb_layers = [nb_layers_per_block] * nb_dense_block
# compute initial nb_filter if -1, else accept users initial nb_filter
if nb_filter <= 0:
nb_filter = 2 * growth_rate
# compute compression factor
compression = 1.0 - reduction
# Initial convolution
if subsample_initial_block:
initial_kernel = (7, 7)
initial_strides = (2, 2)
else:
initial_kernel = (3, 3)
initial_strides = (1, 1)
x = Conv2D(nb_filter,
initial_kernel,
kernel_initializer='he_normal',
padding='same',
strides=initial_strides,
use_bias=False,
kernel_regularizer=l2(weight_decay))(img_input)
if subsample_initial_block:
x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(x)
x = Activation('relu')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
# Add dense blocks
for block_idx in range(nb_dense_block - 1):
x, nb_filter = __dense_block(x,
nb_layers[block_idx],
nb_filter,
growth_rate,
bottleneck=bottleneck,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# add transition_block
x = __transition_block(x,
nb_filter,
compression=compression,
weight_decay=weight_decay)
nb_filter = int(nb_filter * compression)
# The last dense_block does not have a transition_block
x, nb_filter = __dense_block(x,
final_nb_layer,
nb_filter,
growth_rate,
bottleneck=bottleneck,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(x)
x = Activation('relu')(x)
x = GlobalAveragePooling2D()(x)
if include_top:
x = Dense(nb_classes, activation=activation)(x)
return x
def create_model(img_rows=224,
img_cols=224,
color_type=3,
nb_dense_block=4,
growth_rate=32,
nb_filter=64,
reduction=0.5,
dropout_rate=0.0,
weight_decay=1e-4):
'''
DenseNet 169 Model for Keras
Model Schema is based on
https://github.com/flyyufelix/DenseNet-Keras
ImageNet Pretrained Weights
Theano: https://drive.google.com/open?id=0Byy2AcGyEVxfN0d3T1F1MXg0NlU
TensorFlow: https://drive.google.com/open?id=0Byy2AcGyEVxfSEc5UC1ROUFJdmM
# Arguments
nb_dense_block: number of dense blocks to add to end
growth_rate: number of filters to add per dense block
nb_filter: initial number of filters
reduction: reduction factor of transition blocks.
dropout_rate: dropout rate
weight_decay: weight decay factor
classes: optional number of classes to classify images
weights_path: path to pre-trained weights
# Returns
A Keras models instance.
'''
eps = 1.1e-5
# compute compression factor
compression = 1.0 - reduction
# Handle Dimension Ordering for different backends
global concat_axis
if K.image_dim_ordering() == 'tf':
concat_axis = 3
img_input = Input(shape=(img_rows, img_cols, color_type), name='data')
else:
concat_axis = 1
img_input = Input(shape=(color_type, img_rows, img_cols), name='data')
# From architecture for ImageNet (Table 1 in the paper)
nb_filter = 64
nb_layers = [6, 12, 32, 32] # For DenseNet-169
# Initial convolution
x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input)
x = Convolution2D(nb_filter,
7,
7,
subsample=(2, 2),
name='conv1',
bias=False)(x)
x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv1_bn')(x)
x = Scale(axis=concat_axis, name='conv1_scale')(x)
x = Activation('relu', name='relu1')(x)
x = ZeroPadding2D((1, 1), name='pool1_zeropadding')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1')(x)
# Add dense blocks
for block_idx in range(nb_dense_block - 1):
stage = block_idx + 2
x, nb_filter = dense_block(x,
stage,
nb_layers[block_idx],
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# Add transition_block
x = transition_block(x,
stage,
nb_filter,
compression=compression,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
nb_filter = int(nb_filter * compression)
final_stage = stage + 1
x, nb_filter = dense_block(x,
final_stage,
nb_layers[-1],
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
x = BatchNormalization(epsilon=eps,
axis=concat_axis,
name='conv' + str(final_stage) + '_blk_bn')(x)
x = Scale(axis=concat_axis,
name='conv' + str(final_stage) + '_blk_scale')(x)
x = Activation('relu', name='relu' + str(final_stage) + '_blk')(x)
x_fc = GlobalAveragePooling2D(name='pool' + str(final_stage))(x)
x_fc = Dense(1000, name='fc6')(x_fc)
x_fc = Activation('softmax', name='prob')(x_fc)
model = Model(img_input, x_fc, name='densenet')
return model, x, final_stage, img_input
