def ___conv4_block(input, k=1, dropout=0.0):
init = input
channel_axis = 1 if K.image_dim_ordering() == "th" else -1
# Check if input number of filters is same as 64 * k, else create
# convolution2d for this input
if K.image_dim_ordering() == "th":
if init._keras_shape[1] != 64 * k:
init = Convolution2D(64 * k, (1, 1), activation='linear',
padding='same')(init)
else:
if init._keras_shape[-1] != 64 * k:
init = Convolution2D(64 * k, (1, 1), activation='linear',
padding='same')(init)
x = Convolution2D(64 * k, (3, 3), padding='same')(input)
x = BatchNormalization(axis=channel_axis)(x)
x = Activation('relu')(x)
if dropout > 0.0:
x = Dropout(dropout)(x)
x = Convolution2D(64 * k, (3, 3), padding='same')(x)
x = BatchNormalization(axis=channel_axis)(x)
x = Activation('relu')(x)
m = add([init, x])
return m
def load_mask_labels():
'''Load both target and style masks.
A mask image (nr x nc) with m labels/colors will be loaded
as a 4D boolean tensor: (1, m, nr, nc) for 'th' or (1, nr, nc, m) for 'tf'
'''
target_mask_img = load_img(target_mask_path,
target_size=(img_nrows, img_ncols))
target_mask_img = img_to_array(target_mask_img)
style_mask_img = load_img(style_mask_path,
target_size=(img_nrows, img_ncols))
style_mask_img = img_to_array(style_mask_img)
if K.image_dim_ordering() == 'th':
mask_vecs = np.vstack([style_mask_img.reshape((3, -1)).T,
target_mask_img.reshape((3, -1)).T])
else:
mask_vecs = np.vstack([style_mask_img.reshape((-1, 3)),
target_mask_img.reshape((-1, 3))])
labels = kmeans(mask_vecs, nb_labels)
style_mask_label = labels[:img_nrows *
img_ncols].reshape((img_nrows, img_ncols))
target_mask_label = labels[img_nrows *
img_ncols:].reshape((img_nrows, img_ncols))
stack_axis = 0 if K.image_dim_ordering() == 'th' else -1
style_mask = np.stack([style_mask_label == r for r in xrange(nb_labels)],
axis=stack_axis)
target_mask = np.stack([target_mask_label == r for r in xrange(nb_labels)],
axis=stack_axis)
return (np.expand_dims(style_mask, axis=0),
np.expand_dims(target_mask, axis=0))
def denseblock_altern(x, nb_layers, nb_filter, growth_rate,
dropout_rate=None, weight_decay=1E-4):
"""Build a denseblock where the output of each conv_factory
is fed to subsequent ones. (Alternative of a above)
:param x: keras model
:param nb_layers: int -- the number of layers of conv_
factory to append to the model.
:param nb_filter: int -- number of filters
:param dropout_rate: int -- dropout rate
:param weight_decay: int -- weight decay factor
:returns: keras model with nb_layers of conv_factory appended
:rtype: keras model
* The main difference between this implementation and the implementation
above is that the one above
"""
if K.image_dim_ordering() == "th":
concat_axis = 1
elif K.image_dim_ordering() == "tf":
concat_axis = -1
for i in range(nb_layers):
merge_tensor = conv_factory(x, growth_rate, dropout_rate, weight_decay)
x = merge([merge_tensor, x], mode='concat', concat_axis=concat_axis)
nb_filter += growth_rate
return x, nb_filter
def add_channels(self):
n_channels = self.n_channels
if n_channels == 1:
super().add_channels()
else:
X = self.X
if X.ndim < 4: # if X.dim == 4, no need to add a channel rank.
N, img_rows, img_cols = X.shape
if K.image_dim_ordering() == 'th':
X = X.reshape(X.shape[0], 1, img_rows, img_cols)
X = np.concatenate([X, X, X], axis=1)
input_shape = (n_channels, img_rows, img_cols)
else:
X = X.reshape(X.shape[0], img_rows, img_cols, 1)
X = np.concatenate([X, X, X], axis=3)
input_shape = (img_rows, img_cols, n_channels)
else:
if K.image_dim_ordering() == 'th':
N, Ch, img_rows, img_cols = X.shape
if Ch == 1:
X = np.concatenate([X, X, X], axis=1)
input_shape = (n_channels, img_rows, img_cols)
else:
N, img_rows, img_cols, Ch = X.shape
if Ch == 1:
X = np.concatenate([X, X, X], axis=3)
input_shape = (img_rows, img_cols, n_channels)
if self.preprocessing_flag:
X = preprocess_input(X)
self.X = X
self.input_shape = input_shape
def load_mask(mask_path, shape, return_mask_img=False):
if K.image_dim_ordering() == "th":
_, channels, width, height = shape
else:
_, width, height, channels = shape
mask = imread(mask_path, mode="L") # Grayscale mask load
mask = imresize(mask, (width, height)).astype('float32')
# Perform binarization of mask
mask[mask <= 127] = 0
mask[mask > 128] = 255
max = np.amax(mask)
mask /= max
if return_mask_img: return mask
mask_shape = shape[1:]
mask_tensor = np.empty(mask_shape)
for i in range(channels):
if K.image_dim_ordering() == "th":
mask_tensor[i, :, :] = mask
else:
mask_tensor[:, :, i] = mask
return mask_tensor
def conv3_block(input, k=1, dropout=0.0):
init = input
channel_axis = 1 if K.image_dim_ordering() == "th" else -1
# Check if input number of filters is same as 64 * k, else create convolution2d for this input
if K.image_dim_ordering() == "th":
if init._keras_shape[1] != 64 * k:
init = Convolution2D(64 * k, 1, 1, activation='linear', border_mode='same')(init)
else:
if init._keras_shape[-1] != 64 * k:
init = Convolution2D(64 * k, 1, 1, activation='linear', border_mode='same')(init)
x = Convolution2D(64 * k, 3, 3, border_mode='same')(input)
x = BatchNormalization(axis=channel_axis)(x)
x = Activation('relu')(x)
if dropout > 0.0: x = Dropout(dropout)(x)
x = Convolution2D(64 * k, 3, 3, border_mode='same')(x)
x = BatchNormalization(axis=channel_axis)(x)
x = Activation('relu')(x)
m = merge([init, x], mode='sum')
return m
def denseblock(x, nb_layers, nb_filter, growth_rate,
dropout_rate=None, weight_decay=1E-4):
"""Build a denseblock where the output of each
conv_factory is fed to subsequent ones
:param x: keras model
:param nb_layers: int -- the number of layers of conv_
factory to append to the model.
:param nb_filter: int -- number of filters
:param dropout_rate: int -- dropout rate
:param weight_decay: int -- weight decay factor
:returns: keras model with nb_layers of conv_factory appended
:rtype: keras model
"""
list_feat = [x]
if K.image_dim_ordering() == "th":
concat_axis = 1
elif K.image_dim_ordering() == "tf":
concat_axis = -1
for i in range(nb_layers):
x = conv_factory(x, growth_rate, dropout_rate, weight_decay)
list_feat.append(x)
x = merge(list_feat, mode='concat', concat_axis=concat_axis)
nb_filter += growth_rate
return x, nb_filter
def is_image_shape(self, x):
if len(x.shape) != 3 and len(x.shape) != 2:
return False
if len(x.shape) == 2:
return True
# check if it has either 1 or 3 channel
if K.image_dim_ordering() == 'th':
return (x.shape[0] == 1 or x.shape[0] == 3)
if K.image_dim_ordering() == 'tf':
return (x.shape[2] == 1 or x.shape[2] == 3)
def nn_base(input_tensor=None, trainable=False):
# Determine proper input shape
if K.image_dim_ordering() == 'th':
input_shape = (3, None, None)
else:
input_shape = (None, None, 3)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if K.image_dim_ordering() == 'tf':
bn_axis = 3
else:
bn_axis = 1
# Block 1
x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv1')(img_input)
x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv2')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
# Block 2
x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv1')(x)
x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv2')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
# Block 3
x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv1')(x)
x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv2')(x)
x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv3')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
# Block 4
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv1')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv2')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv3')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
# Block 5
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv1')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv2')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv3')(x)
# x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)
return x
def predict(self, image):
if K.image_dim_ordering() == 'th' and image.shape != (1, 3, IMAGE_SIZE, IMAGE_SIZE):
image = resize_with_pad(image)
image = image.reshape((1, 3, IMAGE_SIZE, IMAGE_SIZE))
elif K.image_dim_ordering() == 'tf' and image.shape != (1, IMAGE_SIZE, IMAGE_SIZE, 3):
image = resize_with_pad(image)
image = image.reshape((1, IMAGE_SIZE, IMAGE_SIZE, 3))
image = image.astype('float32')
image /= 255
result = self.model.predict_proba(image)
print(result)
result = self.model.predict_classes(image)
return result[0]
def get_resnet50():
"""
this function returns the 50-layer residual network model
you should load pretrained weights if you want to use it directly.
Note that since the pretrained weights is converted from caffemodel
the order of channels for input image should be 'BGR' (the channel order of caffe)
"""
if K.image_dim_ordering() == 'tf':
inp = Input(shape=(224, 224, 3))
axis = 3
else:
inp = Input(shape=(3, 224, 224))
axis = 1
dim_ordering = K.image_dim_ordering()
out = ZeroPadding2D((3, 3), dim_ordering=dim_ordering)(inp)
out = Convolution2D(64, 7, 7, subsample=(2, 2), dim_ordering=dim_ordering, name='conv1')(out)
out = BatchNormalization(axis=axis, name='bn_conv1')(out)
out = Activation('relu')(out)
out = MaxPooling2D((3, 3), strides=(2, 2), dim_ordering=dim_ordering)(out)
out = conv_block(out, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
out = identity_block(out, 3, [64, 64, 256], stage=2, block='b')
out = identity_block(out, 3, [64, 64, 256], stage=2, block='c')
out = conv_block(out, 3, [128, 128, 512], stage=3, block='a')
out = identity_block(out, 3, [128, 128, 512], stage=3, block='b')
out = identity_block(out, 3, [128, 128, 512], stage=3, block='c')
out = identity_block(out, 3, [128, 128, 512], stage=3, block='d')
out = conv_block(out, 3, [256, 256, 1024], stage=4, block='a')
out = identity_block(out, 3, [256, 256, 1024], stage=4, block='b')
out = identity_block(out, 3, [256, 256, 1024], stage=4, block='c')
out = identity_block(out, 3, [256, 256, 1024], stage=4, block='d')
out = identity_block(out, 3, [256, 256, 1024], stage=4, block='e')
out = identity_block(out, 3, [256, 256, 1024], stage=4, block='f')
out = conv_block(out, 3, [512, 512, 2048], stage=5, block='a')
out = identity_block(out, 3, [512, 512, 2048], stage=5, block='b')
out = identity_block(out, 3, [512, 512, 2048], stage=5, block='c')
out = AveragePooling2D((7, 7), dim_ordering=dim_ordering)(out)
out = Flatten()(out)
out = Dense(1000, activation='softmax', name='fc1000')(out)
model = Model(inp, out)
return model
def __init__(self, scale, **kwargs):
if K.image_dim_ordering() == 'tf':
self.axis = 3
else:
self.axis = 1
self.scale = scale
super(Normalize, self).__init__(**kwargs)
def keras_mnist_data():
"""
retrieve the MNIST database for keras
"""
# the data, shuffled and split between train and test sets
(X_train, y_train), (X_test, y_test) = mnist.load_data()
img_rows, img_cols = 28, 28 # should be cmputed from the data
if K.image_dim_ordering() == 'th':
X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)
X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)
else:
X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1)
X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255
# convert class vectors to binary class matrices
nb_classes = len(set(y_train))
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
return (X_train, Y_train), (X_test, Y_test)
def gram_matrix(x):
if K.image_dim_ordering() == "th":
features = K.batch_flatten(x)
else:
features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1)))
gram = K.dot(features, K.transpose(features))
return gram
def get_output_shape_for(self, input_shape):
if K.image_dim_ordering() == "th":
b, k, r, c = input_shape
return (b, self.channels, r * self.r, c * self.r)
else:
b, r, c, k = input_shape
return (b, r * self.r, c * self.r, self.channels)
def identity_block(input_tensor, kernel_size, filters, stage, block):
'''The identity_block is the block that has no conv layer at shortcut
# Arguments
input_tensor: input tensor
kernel_size: defualt 3, the kernel size of middle conv layer at main path
filters: list of integers, the nb_filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
'''
dim_ordering = K.image_dim_ordering()
nb_filter1, nb_filter2, nb_filter3 = filters
if dim_ordering == 'tf':
bn_axis = 3
else:
bn_axis = 1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
out = Convolution2D(nb_filter1, 1, 1, dim_ordering=dim_ordering, name=conv_name_base + '2a')(input_tensor)
out = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(out)
out = Activation('relu')(out)
out = Convolution2D(nb_filter2, kernel_size, kernel_size, border_mode='same',
dim_ordering=dim_ordering, name=conv_name_base + '2b')(out)
out = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(out)
out = Activation('relu')(out)
out = Convolution2D(nb_filter3, 1, 1, dim_ordering=dim_ordering, name=conv_name_base + '2c')(out)
out = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(out)
out = merge([out, input_tensor], mode='sum')
out = Activation('relu')(out)
return out
def block_inception_c(input):
if K.image_dim_ordering() == "th":
channel_axis = 1
else:
channel_axis = -1
branch_0 = conv2d_bn(input, 256, 1, 1)
branch_1 = conv2d_bn(input, 384, 1, 1)
branch_10 = conv2d_bn(branch_1, 256, 1, 3)
branch_11 = conv2d_bn(branch_1, 256, 3, 1)
branch_1 = merge([branch_10, branch_11], mode='concat', concat_axis=channel_axis)
branch_2 = conv2d_bn(input, 384, 1, 1)
branch_2 = conv2d_bn(branch_2, 448, 3, 1)
branch_2 = conv2d_bn(branch_2, 512, 1, 3)
branch_20 = conv2d_bn(branch_2, 256, 1, 3)
branch_21 = conv2d_bn(branch_2, 256, 3, 1)
branch_2 = merge([branch_20, branch_21], mode='concat', concat_axis=channel_axis)
branch_3 = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same')(input)
branch_3 = conv2d_bn(branch_3, 256, 1, 1)
x = merge([branch_0, branch_1, branch_2, branch_3], mode='concat', concat_axis=channel_axis)
return x
def load_vgg_weight(self, model):
# Loading VGG 16 weights
if K.image_dim_ordering() == "th":
weights = get_file('vgg16_weights_th_dim_ordering_th_kernels_notop.h5', THEANO_WEIGHTS_PATH_NO_TOP,
cache_subdir='models')
else:
weights = get_file('vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5', TF_WEIGHTS_PATH_NO_TOP,
cache_subdir='models')
f = h5py.File(weights)
layer_names = [name for name in f.attrs['layer_names']]
if self.vgg_layers is None:
self.vgg_layers = [layer for layer in model.layers
if 'vgg_' in layer.name]
for i, layer in enumerate(self.vgg_layers):
g = f[layer_names[i]]
weights = [g[name] for name in g.attrs['weight_names']]
layer.set_weights(weights)
# Freeze all VGG layers
for layer in self.vgg_layers:
layer.trainable = False
return model
def __init__(self, x, y, image_data_generator,
batch_size=32, shuffle=False, seed=None,
dim_ordering='default',
save_to_dir=None, save_prefix='', save_format='jpeg'):
if y is not None and len(x) != len(y):
raise ValueError('X (images tensor) and y (labels) '
'should have the same length. '
'Found: X.shape = %s, y.shape = %s' %
(np.asarray(x).shape, np.asarray(y).shape))
if dim_ordering == 'default':
dim_ordering = K.image_dim_ordering()
self.x = np.asarray(x)
if self.x.ndim != 4:
raise ValueError('Input data in `NumpyArrayIterator` '
'should have rank 4. You passed an array '
'with shape', self.x.shape)
channels_axis = 3 if dim_ordering == 'tf' else 1
# if self.x.shape[channels_axis] not in {1, 3, 4}:
# raise ValueError('NumpyArrayIterator is set to use the '
# 'dimension ordering convention "' + dim_ordering + '" '
# '(channels on axis ' + str(channels_axis) + '), i.e. expected '
# 'either 1, 3 or 4 channels on axis ' + str(channels_axis) + '. '
# 'However, it was passed an array with shape ' + str(self.x.shape) +
# ' (' + str(self.x.shape[channels_axis]) + ' channels).')
if y is not None:
self.y = np.asarray(y)
else:
self.y = None
self.image_data_generator = image_data_generator
self.dim_ordering = dim_ordering
self.save_to_dir = save_to_dir
self.save_prefix = save_prefix
self.save_format = save_format
super(NumpyArrayIterator, self).__init__(x.shape[0], batch_size, shuffle, seed)
def predict(self, image):
img_rows,img_cols=64,64
if K.image_dim_ordering() == 'th' and image.shape != (1, 3, img_rows, img_cols):
image =cv2.resize(image,(img_rows,img_cols))
print(image.shape)
image = image.reshape((1, 3, img_rows, img_cols))
elif K.image_dim_ordering() == 'tf' and image.shape != (1, img_rows, img_cols, 3):
image =cv2.resize(image,(img_rows,img_cols))
image = image.reshape((1, img_rows, img_cols, 3))
image = image.astype('float32')
image /= 255
result = self.model.predict_proba(image)
print(result)
result = self.model.predict_classes(image)
return result[0]
def save_sample_grid(samples, filename, activation='tanh'):
if K.image_dim_ordering() == 'tf':
num_samples, img_height, img_width, img_channels = samples.shape
else:
num_samples, img_channels, img_height, img_width = samples.shape
num_wide = int(np.sqrt(num_samples))
num_heigh = int(np.ceil(num_samples / num_wide))
width = num_wide * img_width
height = num_heigh * img_height
img_mode = {1: 'L', 3: 'RGB'}[img_channels]
output_img = Image.new(img_mode, (width, height), 'black')
for i in range(num_samples):
x = (i % num_wide) * img_width
y = (i / num_wide) * img_height
if activation == 'relu':
sample_arr = samples[i]
elif activation == 'tanh':
sample_arr = (samples[i] + 1.) * 127.5
elif activation == 'sigmoid':
sample_arr = samples[i] * 256.
sample_img = array_to_img(sample_arr, scale=False)
output_img.paste(sample_img, (x, y))
if filename is None:
filename = self.config.sample_output_filename
output_img.save(filename)
def __init__(self, nb_filter, nb_row, nb_col,
init='glorot_uniform', activation=None, weights=None,
border_mode='valid', subsample=(1, 1),
dim_ordering='default',
W_regularizer=None, b_regularizer=None, activity_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
if dim_ordering == 'default':
dim_ordering = K.image_dim_ordering()
if border_mode != 'valid':
raise ValueError('Invalid border mode for LocallyConnected2D '
'(only "valid" is supported):', border_mode)
self.nb_filter = nb_filter
self.nb_row = nb_row
self.nb_col = nb_col
self.init = initializations.get(init, dim_ordering=dim_ordering)
self.activation = activations.get(activation)
self.border_mode = border_mode
self.subsample = tuple(subsample)
if dim_ordering not in {'tf', 'th'}:
raise ValueError('`dim_ordering` must be in {tf, th}.')
self.dim_ordering = dim_ordering
self.W_regularizer = regularizers.get(W_regularizer)
self.b_regularizer = regularizers.get(b_regularizer)
self.activity_regularizer = regularizers.get(activity_regularizer)
self.W_constraint = constraints.get(W_constraint)
self.b_constraint = constraints.get(b_constraint)
self.bias = bias
self.input_spec = [InputSpec(ndim=4)]
self.initial_weights = weights
super(LocallyConnected2D, self).__init__(**kwargs)
def test_cropping_2d():
nb_samples = 2
stack_size = 2
input_len_dim1 = 8
input_len_dim2 = 8
cropping = ((2, 2), (3, 3))
dim_ordering = K.image_dim_ordering()
if dim_ordering == 'th':
input = np.random.rand(nb_samples, stack_size, input_len_dim1, input_len_dim2)
else:
input = np.random.rand(nb_samples, input_len_dim1, input_len_dim2, stack_size)
# basic test
layer_test(convolutional.Cropping2D,
kwargs={'cropping': cropping,
'dim_ordering': dim_ordering},
input_shape=input.shape)
# correctness test
layer = convolutional.Cropping2D(cropping=cropping, dim_ordering=dim_ordering)
layer.set_input(K.variable(input), shape=input.shape)
out = K.eval(layer.output)
# compare with numpy
if dim_ordering == 'th':
expected_out = input[:,
:,
cropping[0][0]:-cropping[0][1],
cropping[1][0]:-cropping[1][1]]
else:
expected_out = input[:,
cropping[0][0]:-cropping[0][1],
cropping[1][0]:-cropping[1][1],
:]
assert_allclose(out, expected_out)
def identity_block_td(input_tensor, kernel_size, filters, stage, block, trainable=True):
# identity block time distributed
nb_filter1, nb_filter2, nb_filter3 = filters
if K.image_dim_ordering() == 'tf':
bn_axis = 3
else:
bn_axis = 1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
x = TimeDistributed(Convolution2D(nb_filter1, (1, 1), trainable=trainable, kernel_initializer='normal'), name=conv_name_base + '2a')(input_tensor)
x = TimeDistributed(FixedBatchNormalization(axis=bn_axis), name=bn_name_base + '2a')(x)
x = Activation('relu')(x)
x = TimeDistributed(Convolution2D(nb_filter2, (kernel_size, kernel_size), trainable=trainable, kernel_initializer='normal',padding='same'), name=conv_name_base + '2b')(x)
x = TimeDistributed(FixedBatchNormalization(axis=bn_axis), name=bn_name_base + '2b')(x)
x = Activation('relu')(x)
x = TimeDistributed(Convolution2D(nb_filter3, (1, 1), trainable=trainable, kernel_initializer='normal'), name=conv_name_base + '2c')(x)
x = TimeDistributed(FixedBatchNormalization(axis=bn_axis), name=bn_name_base + '2c')(x)
x = Add()([x, input_tensor])
x = Activation('relu')(x)
return x
def __init__(self, nb_filter, nb_row, nb_col, init='glorot_uniform', activation='linear', weights=None,
border_mode='valid', subsample=(1, 1), dim_ordering=K.image_dim_ordering(), W_regularizer=None,
b_regularizer=None, activity_regularizer=None, W_constraint=None, b_constraint=None, bias=True,
**kwargs):
if border_mode not in {'valid', 'same','bow','1D'}:
raise Exception('Invalid border mode for Convolution2D:', border_mode)
if border_mode=='bow':
# bow convolution
self.bow_size = nb_row
self.convolution_type = 'bow'
self.bow_output_length = None
border_mode ='valid'
nb_row = 1
elif border_mode=='1D':
# 1D convolution
self.convolution_type = '1D'
border_mode ='valid'
else:
self.convolution_type = border_mode
super(Convolution2DWrapper, self).__init__(
nb_filter,
nb_row,
nb_col, init, activation, weights, border_mode,
subsample, dim_ordering, W_regularizer, b_regularizer,
activity_regularizer, W_constraint, b_constraint, bias, **kwargs)
def create_model(self, height=32, width=32, channels=3, load_weights=False, batch_size=128,
small_train_images=False):
"""
Creates a model to remove / reduce noise from upscaled images.
"""
from keras.layers.convolutional import Deconvolution2D
assert height % 4 == 0, "Height of the image must be divisible by 4"
assert width % 4 == 0, "Width of the image must be divisible by 4"
if K.image_dim_ordering() == "th":
shape = (channels, width, height)
else:
shape = (width, height, channels)
init = Input(shape=shape)
level1_1 = Convolution2D(self.n1, 3, 3, activation='relu', border_mode='same')(init)
level2_1 = Convolution2D(self.n1, 3, 3, activation='relu', border_mode='same')(level1_1)
level2_2 = Deconvolution2D(self.n1, 3, 3, activation='relu', output_shape=(None, channels, height, width), border_mode='same')(level2_1)
level2 = merge([level2_1, level2_2], mode='sum')
level1_2 = Deconvolution2D(self.n1, 3, 3, activation='relu', output_shape=(None, channels, height, width), border_mode='same')(level2)
level1 = merge([level1_1, level1_2], mode='sum')
decoded = Convolution2D(channels, 5, 5, activation='linear', border_mode='same')(level1)
model = Model(init, decoded)
model.compile(optimizer='adam', loss='mse', metrics=[PSNRLoss])
if load_weights: model.load_weights("weights/Denoising AutoEncoder.h5")
self.model = model
return model
def make_mnist_model():
"""MNIST ConvNet from keras/examples/mnist_cnn.py"""
np.random.seed(1337) # for reproducibility
nb_classes = 10
# input image dimensions
img_rows, img_cols = 28, 28
# number of convolutional filters to use
nb_filters = 32
# size of pooling area for max pooling
pool_size = (2, 2)
# convolution kernel size
kernel_size = (3, 3)
if K.image_dim_ordering() == 'th':
input_shape = (1, img_rows, img_cols)
else:
input_shape = (img_rows, img_cols, 1)
model = Sequential()
model.add(Convolution2D(nb_filters, kernel_size[0], kernel_size[1],
border_mode='valid',
input_shape=input_shape))
model.add(Activation('relu'))
model.add(Convolution2D(nb_filters, kernel_size[0], kernel_size[1]))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=pool_size))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
return model
def depth_to_scale(x, scale, output_shape, dim_ordering=K.image_dim_ordering(), name=None):
''' Uses phase shift algorithm [1] to convert channels/depth for spacial resolution '''
import theano.tensor as T
scale = int(scale)
if dim_ordering == "tf":
x = x.transpose((0, 3, 1, 2))
out_row, out_col, out_channels = output_shape
else:
out_channels, out_row, out_col = output_shape
b, k, r, c = x.shape
out_b, out_k, out_r, out_c = b, k // (scale * scale), r * scale, c * scale
out = K.reshape(x, (out_b, out_k, out_r, out_c))
for channel in range(out_channels):
channel += 1
for i in range(out_row):
for j in range(out_col):
a = i // scale #T.floor(i / scale).astype('int32')
b = j // scale #T.floor(j / scale).astype('int32')
d = channel * scale * (j % scale) + channel * (i % scale)
T.set_subtensor(out[:, channel - 1, i, j], x[:, d, a, b], inplace=True)
if dim_ordering == 'tf':
out = out.transpose((0, 2, 3, 1))
return out
def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2), trainable=True):
nb_filter1, nb_filter2, nb_filter3 = filters
if K.image_dim_ordering() == 'tf':
bn_axis = 3
else:
bn_axis = 1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
x = Convolution2D(nb_filter1, (1, 1), strides=strides, name=conv_name_base + '2a', trainable=trainable)(input_tensor)
x = FixedBatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
x = Activation('relu')(x)
x = Convolution2D(nb_filter2, (kernel_size, kernel_size), padding='same', name=conv_name_base + '2b', trainable=trainable)(x)
x = FixedBatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)
x = Activation('relu')(x)
x = Convolution2D(nb_filter3, (1, 1), name=conv_name_base + '2c', trainable=trainable)(x)
x = FixedBatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)
shortcut = Convolution2D(nb_filter3, (1, 1), strides=strides, name=conv_name_base + '1', trainable=trainable)(input_tensor)
shortcut = FixedBatchNormalization(axis=bn_axis, name=bn_name_base + '1')(shortcut)
x = Add()([x, shortcut])
x = Activation('relu')(x)
return x
def __init__(self, img_size=(300.0, 300.0), min_size=30.0, max_size=None, aspect_ratios=None,
flip=True, variances=[0.1], clip=True, **kwargs):
if K.image_dim_ordering() == 'tf':
self.waxis = 2
self.haxis = 1
else:
self.waxis = 3
self.haxis = 2
self.img_size = img_size
if min_size <= 0:
raise Exception('min_size must be positive.')
self.min_size = min_size
self.max_size = max_size
self.aspect_ratios = [1.0]
if max_size:
if max_size < min_size:
raise Exception('max_size must be greater than min_size.')
self.aspect_ratios.append(1.0)
if aspect_ratios:
for ar in aspect_ratios:
if ar in self.aspect_ratios:
continue
self.aspect_ratios.append(ar)
if flip:
self.aspect_ratios.append(1.0 / ar)
self.variances = np.array(variances)
self.clip = True
super(PriorBox, self).__init__(**kwargs)
def pil_image_reader(filepath, target_mode=None, target_size=None, dim_ordering=K.image_dim_ordering(), **kwargs):
img = load_img(filepath, target_mode=target_mode, target_size=target_size)
return img_to_array(img, dim_ordering=dim_ordering)
def SSD(input_shape, num_classes):
"""SSD512 architecture.
# Arguments
input_shape: Shape of the input image,
expected to be either (512, 512, 3) or (3, 512, 512)(not tested).
num_classes: Number of classes including background.
# References
https://arxiv.org/abs/1512.02325
"""
net = {}
# Block 1
input_tensor = Input(shape=input_shape)
input_shape = (input_shape[1], input_shape[0], 3)
img_size = (input_shape[1], input_shape[0])
vgg16_input_shape = (224, 224, 3)
net = {}
net['input'] = Input(input_shape)
vgg16 = VGG16(input_shape=vgg16_input_shape,
include_top=False,
weights='imagenet')
FeatureExtractor = Model(inputs=vgg16.input,
outputs=vgg16.get_layer('block3_pool').output)
net['pool3'] = FeatureExtractor(net['input'])
# Block 4
net['conv4_1'] = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='conv4_1')(net['pool3'])
net['conv4_2'] = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='conv4_2')(net['conv4_1'])
net['conv4_3'] = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='conv4_3')(net['conv4_2'])
net['pool4'] = MaxPooling2D((2, 2),
strides=(2, 2),
padding='same',
name='pool4')(net['conv4_3'])
# Block 5
net['conv5_1'] = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='conv5_1')(net['pool4'])
net['conv5_2'] = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='conv5_2')(net['conv5_1'])
net['conv5_3'] = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='conv5_3')(net['conv5_2'])
net['pool5'] = MaxPooling2D((3, 3),
strides=(1, 1),
padding='same',
name='pool5')(net['conv5_3'])
# FC6
net['fc6'] = Conv2D(1024, (3, 3),
dilation_rate=(6, 6),
activation='relu',
padding='same',
name='fc6')(net['pool5'])
net['drop6'] = Dropout(0.5, name='drop6')(net['fc6'])
# FC7
net['fc7'] = Conv2D(1024, (1, 1),
activation='relu',
padding='same',
name='fc7')(net['drop6'])
net['drop7'] = Dropout(0.5, name='drop7')(net['fc7'])
# Block 6
net['conv6_1'] = Conv2D(256, (1, 1),
activation='relu',
padding='same',
name='conv6_1')(net['fc7'])
net['conv6_2'] = Conv2D(512, (3, 3),
strides=(2, 2),
activation='relu',
padding='same',
name='conv6_2')(net['conv6_1'])
# Block 7
net['conv7_1'] = Conv2D(128, (1, 1),
activation='relu',
padding='same',
name='conv7_1')(net['conv6_2'])
net['conv7_2'] = ZeroPadding2D()(net['conv7_1'])
net['conv7_2'] = Conv2D(256, (3, 3),
strides=(2, 2),
activation='relu',
padding='valid',
name='conv7_2')(net['conv7_2'])
# Block 8
net['conv8_1'] = Conv2D(128, (1, 1),
activation='relu',
padding='same',
name='conv8_1')(net['conv7_2'])
net['conv8_2'] = Conv2D(256, (3, 3),
strides=(2, 2),
activation='relu',
padding='same',
name='conv8_2')(net['conv8_1'])
# Block 9
net['conv9_1'] = Conv2D(128, (1, 1),
activation='relu',
padding='same',
name='conv9_1')(net['conv8_2'])
net['conv9_2'] = Conv2D(256, (3, 3),
strides=(2, 2),
activation='relu',
padding='same',
name='conv9_2')(net['conv9_1'])
# Block 10
net['conv10_1'] = Conv2D(128, (1, 1),
activation='relu',
padding='same',
name='conv10_1')(net['conv9_2'])
net['conv10_2'] = Conv2D(256, (3, 3),
strides=(2, 2),
activation='relu',
padding='same',
name='conv10_2')(net['conv10_1'])
# Last Pool
net['pool6'] = GlobalAveragePooling2D(name='pool6')(net['conv10_2'])
# Prediction from conv4_3
net['conv4_3_norm'] = Normalize(20, name='conv4_3_norm')(net['conv4_3'])
num_priors = 4
x = Conv2D(num_priors * 4, (3, 3),
padding='same',
name='conv4_3_norm_mbox_loc')(net['conv4_3_norm'])
net['conv4_3_norm_mbox_loc'] = x
flatten = Flatten(name='conv4_3_norm_mbox_loc_flat')
net['conv4_3_norm_mbox_loc_flat'] = flatten(net['conv4_3_norm_mbox_loc'])
name = 'conv4_3_norm_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Conv2D(num_priors * num_classes, (3, 3), padding='same',
name=name)(net['conv4_3_norm'])
net['conv4_3_norm_mbox_conf'] = x
flatten = Flatten(name='conv4_3_norm_mbox_conf_flat')
net['conv4_3_norm_mbox_conf_flat'] = flatten(net['conv4_3_norm_mbox_conf'])
priorbox = PriorBox(img_size,
35.84,
max_size=76.8,
aspect_ratios=[2],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv4_3_norm_mbox_priorbox')
net['conv4_3_norm_mbox_priorbox'] = priorbox(net['conv4_3_norm'])
# Prediction from fc7
num_priors = 6
net['fc7_mbox_loc'] = Conv2D(num_priors * 4, (3, 3),
padding='same',
name='fc7_mbox_loc')(net['fc7'])
flatten = Flatten(name='fc7_mbox_loc_flat')
net['fc7_mbox_loc_flat'] = flatten(net['fc7_mbox_loc'])
name = 'fc7_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
net['fc7_mbox_conf'] = Conv2D(num_priors * num_classes, (3, 3),
padding='same',
name=name)(net['fc7'])
flatten = Flatten(name='fc7_mbox_conf_flat')
net['fc7_mbox_conf_flat'] = flatten(net['fc7_mbox_conf'])
priorbox = PriorBox(img_size,
76.8,
max_size=153.6,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='fc7_mbox_priorbox')
net['fc7_mbox_priorbox'] = priorbox(net['fc7'])
# Prediction from conv6_2
num_priors = 6
x = Conv2D(num_priors * 4, (3, 3), padding='same',
name='conv6_2_mbox_loc')(net['conv6_2'])
net['conv6_2_mbox_loc'] = x
flatten = Flatten(name='conv6_2_mbox_loc_flat')
net['conv6_2_mbox_loc_flat'] = flatten(net['conv6_2_mbox_loc'])
name = 'conv6_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Conv2D(num_priors * num_classes, (3, 3), padding='same',
name=name)(net['conv6_2'])
net['conv6_2_mbox_conf'] = x
flatten = Flatten(name='conv6_2_mbox_conf_flat')
net['conv6_2_mbox_conf_flat'] = flatten(net['conv6_2_mbox_conf'])
priorbox = PriorBox(img_size,
153.6,
max_size=230.4,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv6_2_mbox_priorbox')
net['conv6_2_mbox_priorbox'] = priorbox(net['conv6_2'])
# Prediction from conv7_2
num_priors = 6
x = Conv2D(num_priors * 4, (3, 3), padding='same',
name='conv7_2_mbox_loc')(net['conv7_2'])
net['conv7_2_mbox_loc'] = x
flatten = Flatten(name='conv7_2_mbox_loc_flat')
net['conv7_2_mbox_loc_flat'] = flatten(net['conv7_2_mbox_loc'])
name = 'conv7_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Conv2D(num_priors * num_classes, (3, 3), padding='same',
name=name)(net['conv7_2'])
net['conv7_2_mbox_conf'] = x
flatten = Flatten(name='conv7_2_mbox_conf_flat')
net['conv7_2_mbox_conf_flat'] = flatten(net['conv7_2_mbox_conf'])
priorbox = PriorBox(img_size,
230.4,
max_size=307.2,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv7_2_mbox_priorbox')
net['conv7_2_mbox_priorbox'] = priorbox(net['conv7_2'])
# Prediction from conv8_2
num_priors = 6
x = Conv2D(num_priors * 4, (3, 3), padding='same',
name='conv8_2_mbox_loc')(net['conv8_2'])
net['conv8_2_mbox_loc'] = x
flatten = Flatten(name='conv8_2_mbox_loc_flat')
net['conv8_2_mbox_loc_flat'] = flatten(net['conv8_2_mbox_loc'])
name = 'conv8_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Conv2D(num_priors * num_classes, (3, 3), padding='same',
name=name)(net['conv8_2'])
net['conv8_2_mbox_conf'] = x
flatten = Flatten(name='conv8_2_mbox_conf_flat')
net['conv8_2_mbox_conf_flat'] = flatten(net['conv8_2_mbox_conf'])
priorbox = PriorBox(img_size,
307.2,
max_size=384.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv8_2_mbox_priorbox')
net['conv8_2_mbox_priorbox'] = priorbox(net['conv8_2'])
# Prediction from conv9_2
num_priors = 4
x = Conv2D(num_priors * 4, (3, 3), padding='same',
name='conv9_2_mbox_loc')(net['conv9_2'])
net['conv9_2_mbox_loc'] = x
flatten = Flatten(name='conv9_2_mbox_loc_flat')
net['conv9_2_mbox_loc_flat'] = flatten(net['conv9_2_mbox_loc'])
name = 'conv9_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Conv2D(num_priors * num_classes, (3, 3), padding='same',
name=name)(net['conv9_2'])
net['conv9_2_mbox_conf'] = x
flatten = Flatten(name='conv9_2_mbox_conf_flat')
net['conv9_2_mbox_conf_flat'] = flatten(net['conv9_2_mbox_conf'])
priorbox = PriorBox(img_size,
384.0,
max_size=460.8,
aspect_ratios=[2],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv9_2_mbox_priorbox')
net['conv9_2_mbox_priorbox'] = priorbox(net['conv9_2'])
# Prediction from pool6
num_priors = 4
x = Dense(num_priors * 4, name='pool6_mbox_loc_flat')(net['pool6'])
net['pool6_mbox_loc_flat'] = x
name = 'pool6_mbox_conf_flat'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Dense(num_priors * num_classes, name=name)(net['pool6'])
net['pool6_mbox_conf_flat'] = x
priorbox = PriorBox(img_size,
460.8,
max_size=537.6,
aspect_ratios=[2],
variances=[0.1, 0.1, 0.2, 0.2],
name='pool6_mbox_priorbox')
if K.image_dim_ordering() == 'tf':
target_shape = (1, 1, 256)
else:
target_shape = (256, 1, 1)
net['pool6_reshaped'] = Reshape(target_shape,
name='pool6_reshaped')(net['pool6'])
net['pool6_mbox_priorbox'] = priorbox(net['pool6_reshaped'])
# Gather all predictions
net['mbox_loc'] = concatenate([
net['conv4_3_norm_mbox_loc_flat'], net['fc7_mbox_loc_flat'],
net['conv6_2_mbox_loc_flat'], net['conv7_2_mbox_loc_flat'],
net['conv8_2_mbox_loc_flat'], net['conv9_2_mbox_loc_flat'],
net['pool6_mbox_loc_flat']
],
axis=1,
name='mbox_loc')
net['mbox_conf'] = concatenate([
net['conv4_3_norm_mbox_conf_flat'], net['fc7_mbox_conf_flat'],
net['conv6_2_mbox_conf_flat'], net['conv7_2_mbox_conf_flat'],
net['conv8_2_mbox_conf_flat'], net['conv9_2_mbox_conf_flat'],
net['pool6_mbox_conf_flat']
],
axis=1,
name='mbox_conf')
net['mbox_priorbox'] = concatenate([
net['conv4_3_norm_mbox_priorbox'], net['fc7_mbox_priorbox'],
net['conv6_2_mbox_priorbox'], net['conv7_2_mbox_priorbox'],
net['conv8_2_mbox_priorbox'], net['conv9_2_mbox_priorbox'],
net['pool6_mbox_priorbox']
],
axis=1,
name='mbox_priorbox')
if hasattr(net['mbox_loc'], '_keras_shape'):
num_boxes = net['mbox_loc']._keras_shape[-1] // 4
elif hasattr(net['mbox_loc'], 'int_shape'):
num_boxes = K.int_shape(net['mbox_loc'])[-1] // 4
net['mbox_loc'] = Reshape((num_boxes, 4),
name='mbox_loc_final')(net['mbox_loc'])
net['mbox_conf'] = Reshape((num_boxes, num_classes),
name='mbox_conf_logits')(net['mbox_conf'])
net['mbox_conf'] = Activation('softmax',
name='mbox_conf_final')(net['mbox_conf'])
net['predictions'] = concatenate(
[net['mbox_loc'], net['mbox_conf'], net['mbox_priorbox']],
axis=2,
name='predictions')
model = Model(net['input'], net['predictions'])
return model
import sys
from os import path
sys.path.append(path.dirname(path.dirname(path.abspath("utility"))))
from utility.evaluation import evaluate_model
batch_size = 64
nb_classes10 = 10
nb_classes100 = 100
nb_epoch = 300
img_rows, img_cols = 32, 32
img_channels = 3
img_dim = (img_channels, img_rows,
img_cols) if K.image_dim_ordering() == "th" else (img_rows,
img_cols,
img_channels)
depth = 40
nb_dense_block = 3
growth_rate = 12
nb_filter = 12
dropout_rate = 0.0 # 0.0 for data augmentation
seed = 333
weight_decay = 0.0001
learning_rate = 0.1
weights_file_10 = "../../models/weights_densenet_40_c10.h5"
weights_file_100 = "../../models/weights_densenet_40_c100.h5"
class_mapping = {v: k for k, v in class_mapping.items()}
print(class_mapping)
class_to_color = {
class_mapping[v]: np.random.randint(0, 255, 3)
for v in class_mapping
}
C.num_rois = int(options.num_rois)
if C.network == 'resnet50':
num_features = 1024
else:
# may need to fix this up with your backbone..!
print("backbone is not resnet50. number of features chosen is 512")
num_features = 512
if K.image_dim_ordering() == 'th':
input_shape_img = (3, None, None)
input_shape_features = (num_features, None, None)
else:
input_shape_img = (None, None, 3)
input_shape_features = (None, None, num_features)
img_input = Input(shape=input_shape_img)
roi_input = Input(shape=(C.num_rois, 4))
feature_map_input = Input(shape=input_shape_features)
# define the base network (resnet here, can be VGG, Inception, etc)
shared_layers = nn.nn_base(img_input, trainable=True)
# define the RPN, built on the base layers
num_anchors = len(C.anchor_box_scales) * len(C.anchor_box_ratios)
def nn_base(input_tensor=None, trainable=False):
# Determine proper input shape
if K.image_dim_ordering() == 'th':
input_shape = (3, None, None)
else:
input_shape = (None, None, 3)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if K.image_dim_ordering() == 'tf':
bn_axis = 3
else:
bn_axis = 1
x = ZeroPadding2D((3, 3))(img_input)
x = Convolution2D(64, (7, 7),
strides=(2, 2),
name='conv1',
trainable=trainable)(x)
x = FixedBatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x = Activation('relu')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
x = conv_block(x,
3, [64, 64, 256],
stage=2,
block='a',
strides=(1, 1),
trainable=trainable)
x = identity_block(x,
3, [64, 64, 256],
stage=2,
block='b',
trainable=trainable)
x = identity_block(x,
3, [64, 64, 256],
stage=2,
block='c',
trainable=trainable)
x = conv_block(x,
3, [128, 128, 512],
stage=3,
block='a',
trainable=trainable)
x = identity_block(x,
3, [128, 128, 512],
stage=3,
block='b',
trainable=trainable)
x = identity_block(x,
3, [128, 128, 512],
stage=3,
block='c',
trainable=trainable)
x = identity_block(x,
3, [128, 128, 512],
stage=3,
block='d',
trainable=trainable)
x = conv_block(x,
3, [256, 256, 1024],
stage=4,
block='a',
trainable=trainable)
x = identity_block(x,
3, [256, 256, 1024],
stage=4,
block='b',
trainable=trainable)
x = identity_block(x,
3, [256, 256, 1024],
stage=4,
block='c',
trainable=trainable)
x = identity_block(x,
3, [256, 256, 1024],
stage=4,
block='d',
trainable=trainable)
x = identity_block(x,
3, [256, 256, 1024],
stage=4,
block='e',
trainable=trainable)
x = identity_block(x,
3, [256, 256, 1024],
stage=4,
block='f',
trainable=trainable)
return x
from keras.datasets import cifar10
from keras.utils import np_utils
from keras.preprocessing.image import ImageDataGenerator
from keras.optimizers import Adam
from keras.callbacks import ModelCheckpoint
from keras import backend as K
batch_size = 64
nb_classes = 10
nb_epoch = 250
img_rows, img_cols = 32, 32
img_channels = 3
img_dim = (img_channels, img_rows, img_cols) if K.image_dim_ordering() == "th" else (img_rows, img_cols, img_channels)
depth = 40
nb_dense_block = 3
growth_rate = 12
nb_filter = 16
dropout_rate = 0.0 # 0.0 for data augmentation
model = densenet.create_dense_net(nb_classes, img_dim, depth, nb_dense_block, growth_rate, nb_filter,
dropout_rate=dropout_rate)
print("Model created")
model.summary()
optimizer = Adam(lr=1e-4) # Using Adam instead of SGD to speed up training
model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=["accuracy"])
print("Finished compiling")
print("Building model...")
def train_model(data_dir, output_dir, model_file='', batch_size=32,
num_epochs=100, optimizer='adam', deconv_layers=5,
use_yale=False, use_jaffe=False,
kernels_per_layer=None, generate_intermediate=False,
verbose=False):
"""
Trains the model on the data, generating intermediate results every epoch.
Args:
data_dir (str): Directory where the data lives.
output_dir (str): Directory where outputs should be saved.
model_file (str): Model file to load. If none specified, a new model
will be created.
Args (optional):
batch_size (int): Size of the batch to use.
num_epochs (int): Number of epochs to train for.
optimizer (str): Keras optimizer to use.
deconv_layers (int): The number of deconvolution layers to use.
generate_intermediate (bool): Whether or not to generate intermediate results.
"""
data_dir = os.path.expanduser(data_dir)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
instances = (YaleInstances(data_dir) if use_yale
else JAFFEInstances(data_dir) if use_jaffe
else RaFDInstances(data_dir))
if verbose:
print("Found {} instances with {} identities".format(
instances.num_instances, instances.num_identities))
# Create FaceGen model to use
if model_file:
model = load_model(model_file)
if verbose:
print("Loaded model %d identities from {}".format(model.model_file))
else:
# TODO: Refactor this to a more elegant way to determine params by dataset
initial_shape = (5,4)
if use_yale:
initial_shape = (6,8)
if use_jaffe:
initial_shape = (4,4)
model = build_model(
identity_len = instances.num_identities,
deconv_layers = deconv_layers,
num_kernels = kernels_per_layer,
optimizer = optimizer,
initial_shape = initial_shape,
use_yale = use_yale,
use_jaffe = use_jaffe,
)
if verbose:
print("Built model with:")
print("\tDeconv layers: {}".format(deconv_layers))
print("\tOutput shape: {}".format(model.output_shape[1:]))
# Create training callbacks
callbacks = list()
if generate_intermediate:
intermediate_dir = os.path.join(output_dir, 'intermediate.d{}.{}'.format(deconv_layers, optimizer))
callbacks.append( GenerateIntermediate(intermediate_dir, instances.num_identities,
use_yale=use_yale, use_jaffe=use_jaffe) )
model_path = os.path.join(output_dir, 'FaceGen.{}.model.d{}.{}.h5'
.format('YaleFaces' if use_yale else 'JAFFE' if use_jaffe else 'RaFD', deconv_layers, optimizer))
callbacks.append(
ModelCheckpoint(
model_path,
monitor='loss', verbose=0, save_best_only=True,
)
)
callbacks.append(
EarlyStopping(monitor='loss', patience=8)
)
# Load data and begin training
if verbose:
print("Loading data...")
if K.image_dim_ordering() == 'th':
image_size = model.output_shape[2:4]
else:
image_size = model.output_shape[1:3]
inputs, outputs = instances.load_data(image_size, verbose=verbose)
if verbose:
print("Training...")
model.fit(inputs, outputs, batch_size=batch_size, nb_epoch=num_epochs,
callbacks=callbacks, shuffle=True, verbose=1)
if verbose:
print("Done!")
def SSD300(input_shape, num_classes=21):
"""SSD300 architecture.
# Arguments
input_shape: Shape of the input image,
expected to be either (300, 300, 3) or (3, 300, 300)(not tested).
num_classes: Number of classes including background.
# References
https://arxiv.org/abs/1512.02325
"""
net = {}
# Block 1
input_tensor = input_tensor = Input(shape=input_shape)
img_size = (input_shape[1], input_shape[0])
net['input'] = input_tensor
net['conv1_1'] = Convolution2D(64,
3,
3,
activation='relu',
border_mode='same',
name='conv1_1')(net['input'])
net['conv1_2'] = Convolution2D(64,
3,
3,
activation='relu',
border_mode='same',
name='conv1_2')(net['conv1_1'])
net['pool1'] = MaxPooling2D((2, 2),
strides=(2, 2),
border_mode='same',
name='pool1')(net['conv1_2'])
# Block 2
net['conv2_1'] = Convolution2D(128,
3,
3,
activation='relu',
border_mode='same',
name='conv2_1')(net['pool1'])
net['conv2_2'] = Convolution2D(128,
3,
3,
activation='relu',
border_mode='same',
name='conv2_2')(net['conv2_1'])
net['pool2'] = MaxPooling2D((2, 2),
strides=(2, 2),
border_mode='same',
name='pool2')(net['conv2_2'])
# Block 3
net['conv3_1'] = Convolution2D(256,
3,
3,
activation='relu',
border_mode='same',
name='conv3_1')(net['pool2'])
net['conv3_2'] = Convolution2D(256,
3,
3,
activation='relu',
border_mode='same',
name='conv3_2')(net['conv3_1'])
net['conv3_3'] = Convolution2D(256,
3,
3,
activation='relu',
border_mode='same',
name='conv3_3')(net['conv3_2'])
net['pool3'] = MaxPooling2D((2, 2),
strides=(2, 2),
border_mode='same',
name='pool3')(net['conv3_3'])
# Block 4
net['conv4_1'] = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='conv4_1')(net['pool3'])
net['conv4_2'] = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='conv4_2')(net['conv4_1'])
net['conv4_3'] = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='conv4_3')(net['conv4_2'])
net['pool4'] = MaxPooling2D((2, 2),
strides=(2, 2),
border_mode='same',
name='pool4')(net['conv4_3'])
# Block 5
net['conv5_1'] = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='conv5_1')(net['pool4'])
net['conv5_2'] = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='conv5_2')(net['conv5_1'])
net['conv5_3'] = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
name='conv5_3')(net['conv5_2'])
net['pool5'] = MaxPooling2D((3, 3),
strides=(1, 1),
border_mode='same',
name='pool5')(net['conv5_3'])
# FC6
net['fc6'] = AtrousConvolution2D(1024,
3,
3,
atrous_rate=(6, 6),
activation='relu',
border_mode='same',
name='fc6')(net['pool5'])
# x = Dropout(0.5, name='drop6')(x)
# FC7
net['fc7'] = Convolution2D(1024,
1,
1,
activation='relu',
border_mode='same',
name='fc7')(net['fc6'])
# x = Dropout(0.5, name='drop7')(x)
# Block 6
net['conv6_1'] = Convolution2D(256,
1,
1,
activation='relu',
border_mode='same',
name='conv6_1')(net['fc7'])
net['conv6_2'] = Convolution2D(512,
3,
3,
subsample=(2, 2),
activation='relu',
border_mode='same',
name='conv6_2')(net['conv6_1'])
# Block 7
net['conv7_1'] = Convolution2D(128,
1,
1,
activation='relu',
border_mode='same',
name='conv7_1')(net['conv6_2'])
net['conv7_2'] = ZeroPadding2D()(net['conv7_1'])
net['conv7_2'] = Convolution2D(256,
3,
3,
subsample=(2, 2),
activation='relu',
border_mode='valid',
name='conv7_2')(net['conv7_2'])
# Block 8
net['conv8_1'] = Convolution2D(128,
1,
1,
activation='relu',
border_mode='same',
name='conv8_1')(net['conv7_2'])
net['conv8_2'] = Convolution2D(256,
3,
3,
subsample=(2, 2),
activation='relu',
border_mode='same',
name='conv8_2')(net['conv8_1'])
# Last Pool
net['pool6'] = GlobalAveragePooling2D(name='pool6')(net['conv8_2'])
# Prediction from conv4_3
net['conv4_3_norm'] = Normalize(20, name='conv4_3_norm')(net['conv4_3'])
num_priors = 3
x = Convolution2D(num_priors * 4,
3,
3,
border_mode='same',
name='conv4_3_norm_mbox_loc')(net['conv4_3_norm'])
net['conv4_3_norm_mbox_loc'] = x
flatten = Flatten(name='conv4_3_norm_mbox_loc_flat')
net['conv4_3_norm_mbox_loc_flat'] = flatten(net['conv4_3_norm_mbox_loc'])
name = 'conv4_3_norm_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Convolution2D(num_priors * num_classes,
3,
3,
border_mode='same',
name=name)(net['conv4_3_norm'])
net['conv4_3_norm_mbox_conf'] = x
flatten = Flatten(name='conv4_3_norm_mbox_conf_flat')
net['conv4_3_norm_mbox_conf_flat'] = flatten(net['conv4_3_norm_mbox_conf'])
priorbox = PriorBox(img_size,
30.0,
aspect_ratios=[2],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv4_3_norm_mbox_priorbox')
net['conv4_3_norm_mbox_priorbox'] = priorbox(net['conv4_3_norm'])
# Prediction from fc7
num_priors = 6
net['fc7_mbox_loc'] = Convolution2D(num_priors * 4,
3,
3,
border_mode='same',
name='fc7_mbox_loc')(net['fc7'])
flatten = Flatten(name='fc7_mbox_loc_flat')
net['fc7_mbox_loc_flat'] = flatten(net['fc7_mbox_loc'])
name = 'fc7_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
net['fc7_mbox_conf'] = Convolution2D(num_priors * num_classes,
3,
3,
border_mode='same',
name=name)(net['fc7'])
flatten = Flatten(name='fc7_mbox_conf_flat')
net['fc7_mbox_conf_flat'] = flatten(net['fc7_mbox_conf'])
priorbox = PriorBox(img_size,
60.0,
max_size=114.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='fc7_mbox_priorbox')
net['fc7_mbox_priorbox'] = priorbox(net['fc7'])
# Prediction from conv6_2
num_priors = 6
x = Convolution2D(num_priors * 4,
3,
3,
border_mode='same',
name='conv6_2_mbox_loc')(net['conv6_2'])
net['conv6_2_mbox_loc'] = x
flatten = Flatten(name='conv6_2_mbox_loc_flat')
net['conv6_2_mbox_loc_flat'] = flatten(net['conv6_2_mbox_loc'])
name = 'conv6_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Convolution2D(num_priors * num_classes,
3,
3,
border_mode='same',
name=name)(net['conv6_2'])
net['conv6_2_mbox_conf'] = x
flatten = Flatten(name='conv6_2_mbox_conf_flat')
net['conv6_2_mbox_conf_flat'] = flatten(net['conv6_2_mbox_conf'])
priorbox = PriorBox(img_size,
114.0,
max_size=168.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv6_2_mbox_priorbox')
net['conv6_2_mbox_priorbox'] = priorbox(net['conv6_2'])
# Prediction from conv7_2
num_priors = 6
x = Convolution2D(num_priors * 4,
3,
3,
border_mode='same',
name='conv7_2_mbox_loc')(net['conv7_2'])
net['conv7_2_mbox_loc'] = x
flatten = Flatten(name='conv7_2_mbox_loc_flat')
net['conv7_2_mbox_loc_flat'] = flatten(net['conv7_2_mbox_loc'])
name = 'conv7_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Convolution2D(num_priors * num_classes,
3,
3,
border_mode='same',
name=name)(net['conv7_2'])
net['conv7_2_mbox_conf'] = x
flatten = Flatten(name='conv7_2_mbox_conf_flat')
net['conv7_2_mbox_conf_flat'] = flatten(net['conv7_2_mbox_conf'])
priorbox = PriorBox(img_size,
168.0,
max_size=222.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv7_2_mbox_priorbox')
net['conv7_2_mbox_priorbox'] = priorbox(net['conv7_2'])
# Prediction from conv8_2
num_priors = 6
x = Convolution2D(num_priors * 4,
3,
3,
border_mode='same',
name='conv8_2_mbox_loc')(net['conv8_2'])
net['conv8_2_mbox_loc'] = x
flatten = Flatten(name='conv8_2_mbox_loc_flat')
net['conv8_2_mbox_loc_flat'] = flatten(net['conv8_2_mbox_loc'])
name = 'conv8_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Convolution2D(num_priors * num_classes,
3,
3,
border_mode='same',
name=name)(net['conv8_2'])
net['conv8_2_mbox_conf'] = x
flatten = Flatten(name='conv8_2_mbox_conf_flat')
net['conv8_2_mbox_conf_flat'] = flatten(net['conv8_2_mbox_conf'])
priorbox = PriorBox(img_size,
222.0,
max_size=276.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv8_2_mbox_priorbox')
net['conv8_2_mbox_priorbox'] = priorbox(net['conv8_2'])
# Prediction from pool6
num_priors = 6
x = Dense(num_priors * 4, name='pool6_mbox_loc_flat')(net['pool6'])
net['pool6_mbox_loc_flat'] = x
name = 'pool6_mbox_conf_flat'
if num_classes != 21:
name += '_{}'.format(num_classes)
x = Dense(num_priors * num_classes, name=name)(net['pool6'])
net['pool6_mbox_conf_flat'] = x
priorbox = PriorBox(img_size,
276.0,
max_size=330.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='pool6_mbox_priorbox')
if K.image_dim_ordering() == 'tf':
target_shape = (1, 1, 256)
else:
target_shape = (256, 1, 1)
net['pool6_reshaped'] = Reshape(target_shape,
name='pool6_reshaped')(net['pool6'])
net['pool6_mbox_priorbox'] = priorbox(net['pool6_reshaped'])
# Gather all predictions
net['mbox_loc'] = Concatenate(axis=1, name='mbox_loc')([
net['conv4_3_norm_mbox_loc_flat'], net['fc7_mbox_loc_flat'],
net['conv6_2_mbox_loc_flat'], net['conv7_2_mbox_loc_flat'],
net['conv8_2_mbox_loc_flat'], net['pool6_mbox_loc_flat']
])
net['mbox_conf'] = Concatenate(axis=1, name='mbox_conf')([
net['conv4_3_norm_mbox_conf_flat'], net['fc7_mbox_conf_flat'],
net['conv6_2_mbox_conf_flat'], net['conv7_2_mbox_conf_flat'],
net['conv8_2_mbox_conf_flat'], net['pool6_mbox_conf_flat']
])
net['mbox_priorbox'] = Concatenate(axis=1, name='mbox_priorbox')([
net['conv4_3_norm_mbox_priorbox'], net['fc7_mbox_priorbox'],
net['conv6_2_mbox_priorbox'], net['conv7_2_mbox_priorbox'],
net['conv8_2_mbox_priorbox'], net['pool6_mbox_priorbox']
])
# NOT USE KERAS 2.0
# net['mbox_loc'] = merge([net['conv4_3_norm_mbox_loc_flat'],
# net['fc7_mbox_loc_flat'],
# net['conv6_2_mbox_loc_flat'],
# net['conv7_2_mbox_loc_flat'],
# net['conv8_2_mbox_loc_flat'],
# net['pool6_mbox_loc_flat']],
# mode='concat', concat_axis=1, name='mbox_loc')
# net['mbox_conf'] = merge([net['conv4_3_norm_mbox_conf_flat'],
# net['fc7_mbox_conf_flat'],
# net['conv6_2_mbox_conf_flat'],
# net['conv7_2_mbox_conf_flat'],
# net['conv8_2_mbox_conf_flat'],
# net['pool6_mbox_conf_flat']],
# mode='concat', concat_axis=1, name='mbox_conf')
# net['mbox_priorbox'] = merge([net['conv4_3_norm_mbox_priorbox'],
# net['fc7_mbox_priorbox'],
# net['conv6_2_mbox_priorbox'],
# net['conv7_2_mbox_priorbox'],
# net['conv8_2_mbox_priorbox'],
# net['pool6_mbox_priorbox']],
# mode='concat', concat_axis=1,
# name='mbox_priorbox')
if hasattr(net['mbox_loc'], '_keras_shape'):
num_boxes = net['mbox_loc']._keras_shape[-1] // 4
elif hasattr(net['mbox_loc'], 'int_shape'):
num_boxes = K.int_shape(net['mbox_loc'])[-1] // 4
net['mbox_loc'] = Reshape((num_boxes, 4),
name='mbox_loc_final')(net['mbox_loc'])
net['mbox_conf'] = Reshape((num_boxes, num_classes),
name='mbox_conf_logits')(net['mbox_conf'])
net['mbox_conf'] = Activation('softmax',
name='mbox_conf_final')(net['mbox_conf'])
net['predictions'] = Concatenate(axis=2, name='predictions')(
[net['mbox_loc'], net['mbox_conf'], net['mbox_priorbox']])
# NOT USE KERAS 2.0
# net['predictions'] = merge([net['mbox_loc'],
# net['mbox_conf'],
# net['mbox_priorbox']],
# mode='concat', concat_axis=2,
# name='predictions')
model = Model(net['input'], net['predictions'])
return model
def MusicTaggerCRNN(weights='msd', input_tensor=None, include_top=True):
'''Instantiate the MusicTaggerCRNN architecture,
optionally loading weights pre-trained
on Million Song Dataset. Note that when using TensorFlow,
for best performance you should set
`image_dim_ordering="tf"` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The dimension ordering
convention used by the model is the one
specified in your Keras config file.
For preparing mel-spectrogram input, see
`audio_conv_utils.py` in [applications](https://github.com/fchollet/keras/tree/master/keras/applications).
You will need to install [Librosa](http://librosa.github.io/librosa/)
to use it.
# Arguments
weights: one of `None` (random initialization)
or "msd" (pre-training on ImageNet).
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
include_top: whether to include the 1 fully-connected
layer (output layer) at the top of the network.
If False, the network outputs 32-dim features.
# Returns
A Keras model instance.
'''
if weights not in {'msd', None}:
raise ValueError('The `weights` argument should be either '
'`None` (random initialization) or `msd` '
'(pre-training on Million Song Dataset).')
# Determine proper input shape
if K.image_dim_ordering() == 'th':
input_shape = (1, 96, 1366)
else:
input_shape = (96, 1366, 1)
if input_tensor is None:
melgram_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
melgram_input = Input(tensor=input_tensor, shape=input_shape)
else:
melgram_input = input_tensor
# Determine input axis
if K.image_dim_ordering() == 'th':
channel_axis = 1
freq_axis = 2
time_axis = 3
else:
channel_axis = 3
freq_axis = 1
time_axis = 2
# Input block
x = ZeroPadding2D(padding=(0, 37))(melgram_input)
x = BatchNormalization(axis=freq_axis, name='bn_0_freq')(x)
# Conv block 1
x = Convolution2D(64, 3, 3, border_mode='same', name='conv1')(x)
x = BatchNormalization(axis=channel_axis, mode=0, name='bn1')(x)
x = ELU()(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool1')(x)
x = Dropout(0.1, name='dropout1')(x)
# Conv block 2
x = Convolution2D(128, 3, 3, border_mode='same', name='conv2')(x)
x = BatchNormalization(axis=channel_axis, mode=0, name='bn2')(x)
x = ELU()(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(3, 3), name='pool2')(x)
x = Dropout(0.1, name='dropout2')(x)
# Conv block 3
x = Convolution2D(128, 3, 3, border_mode='same', name='conv3')(x)
x = BatchNormalization(axis=channel_axis, mode=0, name='bn3')(x)
x = ELU()(x)
x = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool3')(x)
x = Dropout(0.1, name='dropout3')(x)
# Conv block 4
x = Convolution2D(128, 3, 3, border_mode='same', name='conv4')(x)
x = BatchNormalization(axis=channel_axis, mode=0, name='bn4')(x)
x = ELU()(x)
x = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool4')(x)
x = Dropout(0.1, name='dropout4')(x)
# reshaping
if K.image_dim_ordering() == 'th':
x = Permute((3, 1, 2))(x)
x = Reshape((15, 128))(x)
# GRU block 1, 2, output
x = GRU(32, return_sequences=True, name='gru1')(x)
x = GRU(32, return_sequences=False, name='gru2')(x)
x = Dropout(0.3)(x)
if include_top:
x = Dense(50, activation='sigmoid', name='output')(x)
# Create model
model = Model(melgram_input, x)
if weights is None:
return model
else:
# Load input
if K.image_dim_ordering() == 'tf':
raise RuntimeError("Please set image_dim_ordering == 'th'."
"You can set it at ~/.keras/keras.json")
model.load_weights('data/music_tagger_crnn_weights_%s.h5' % K._BACKEND,
by_name=True)
return model
random.shuffle(all_imgs)
num_imgs = len(all_imgs)
train_imgs = [s for s in all_imgs if s['imageset'] == 'trainval']
val_imgs = [s for s in all_imgs if s['imageset'] == 'test']
print('Num train samples {}'.format(len(train_imgs)))
print('Num val samples {}'.format(len(val_imgs)))
data_gen_train = data_generators.get_anchor_gt(train_imgs,
classes_count,
C,
nn.get_img_output_length,
K.image_dim_ordering(),
mode='train')
data_gen_val = data_generators.get_anchor_gt(val_imgs,
classes_count,
C,
nn.get_img_output_length,
K.image_dim_ordering(),
mode='val')
input_shape_img = (None, None, 3)
img_input = Input(shape=input_shape_img)
roi_input = Input(shape=(None, 4))
# define the base network (resnet here, can be VGG, Inception, etc)
shared_layers = nn.nn_base(img_input, trainable=True)
def getData(inp_path, img_size, dataAug, testing, num_channel=1):
if (inp_path == None):
PATH = os.getcwd()
data_path = PATH + '/data'
data_dir_list = os.listdir(data_path)
else:
data_dir_list = os.listdir(inp_path)
num_samples = len(data_dir_list)
print(num_samples)
img_data_list = []
for img in data_dir_list:
input_img = cv2.imread(data_path + '/' + img)
input_img = cv2.cvtColor(input_img, cv2.COLOR_BGR2GRAY)
input_img_resize = cv2.resize(input_img, img_size)
img_data_list.append(input_img_resize)
label_list = np.ones((num_samples, ), dtype=int)
label_list[0:59] = 0
label_list[59:122] = 1
label_list[122:194] = 2
label_list[194:267] = 3
label_list[267:323] = 4
label_list[323:385] = 5
label_list[385:437] = 6
label_list[437:496] = 7
label_list[496:551] = 8
label_list[551:616] = 9
label_list[616:666] = 10
label_list[666:729] = 11
label_list[729:781] = 12
label_list[781:846] = 13
label_list[846:906] = 14
label_list[906:962] = 15
label_list[962:1039] = 16
label_list[1039:1101] = 17
label_list[1101:1162] = 18
label_list[1162:1228] = 19
label_list[1228:1288] = 20
label_list[1288:1343] = 21
label_list[1343:1398] = 22
label_list[1398:1463] = 23
label_list[1463:1517] = 24
label_list[1517:1569] = 25
label_list[1569:1622] = 26
label_list[1622:1677] = 27
label_list[1677:1734] = 28
label_list[1734:1798] = 29
label_list[1798:1851] = 30
label_list[1851:1907] = 31
img_data = np.array(img_data_list)
img_data = img_data.astype('float32')
img_data /= 255
Y = np_utils.to_categorical(label_list, 32)
if dataAug:
datagen = ImageDataGenerator(featurewise_center=True,
featurewise_std_normalization=True,
rotation_range=20,
width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=True)
else:
datagen = None
if num_channel == 1:
if K.image_dim_ordering() == 'th':
img_data = np.expand_dims(img_data, axis=1)
print(img_data.shape)
else:
img_data = np.expand_dims(img_data, axis=4)
print(img_data.shape)
else:
if K.image_dim_ordering() == 'th':
img_data = np.rollaxis(img_data, 3, 1)
print(img_data.shape)
images, labels = shuffle(img_data, Y)
if testing:
X_test, y_test = images, labels
X_train, y_train = None, None
else:
X_train, X_test, y_train, y_test = train_test_split(images,
labels,
test_size=0.2)
return datagen, X_train, X_test, y_train, y_test, 32
from keras import metrics
from keras.models import Model, load_model
from keras.layers import (Input, Dense, BatchNormalization, Dropout, Lambda,
Activation, Concatenate, Conv2D, MaxPooling2D, Reshape,
TimeDistributed, Flatten, Bidirectional, LSTM, GRU)
import numpy as np
from data_generator import DataGenerator
import pandas as pd
from keras import metrics
#import tensorflow as tf
#config = tf.ConfigProto()
#config.gpu_options.allow_growth=True
#sess = tf.Session(config=config)
print (K.floatx())
print (K.epsilon())
print (K.image_dim_ordering())
print (K.image_data_format())
print (K.backend())
from train_utils import *
from sklearn.metrics import classification_report
import pandas as pd
import vggish_input
classes_num = 5
dropout_rate = 0.2
#batch_size = 32
#n_epoch = 22
dual_output = True
mode = 1
def to_plot(img):
if K.image_dim_ordering() == 'tf':
return np.rollaxis(img, 0, 1).astype(np.uint8)
else:
return np.rollaxis(img, 0, 3).astype(np.uint8)
nb_channels = 3
#############################################batch size manual change #######################
#batch_size = 16
#############################################batch size manual change #######################
# the data, shuffled and split between train and test sets
(X_train, y_train), (X_test, y_test) = cifar10.load_data()
y_test=y_test[:64]
X_test=X_test[:64]
trainingmean=np.mean(X_train,axis=0)
trainingstd=np.std(X_train,axis=0)
X_train=(X_train-trainingmean)/trainingstd
X_test=(X_test-trainingmean)/trainingstd
X_train = X_train.astype('float16')
X_test = X_test.astype('float16')
if K.image_dim_ordering() == 'th':
X_train = X_train.reshape(X_train.shape[0], nb_channels, img_rows, img_cols)
X_test = X_test.reshape(X_test.shape[0], nb_channels, img_rows, img_cols)
input_shape = (nb_channels, img_rows, img_cols)
else:
X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, nb_channels)
X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, nb_channels)
input_shape = (img_rows, img_cols, nb_channels)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')
# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
s = set(filepath)
for filename in sorted_nicely(s):
image = cv2.imread(filename)
input_img = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
input_img_resize = cv2.resize(input_img, (28, 28))
#img_data_list.append(input_img_resize)
im_list.append(input_img_resize)
img_data = np.array(im_list)
img_data = img_data.astype('float32')
img_data = (img_data) / 255.
print(img_data.shape)
if num_channel == 1:
if K.image_dim_ordering() == 'th':
img_data = np.expand_dims(img_data, axis=1)
print(img_data.shape)
else:
img_data = np.expand_dims(img_data, axis=4)
print(img_data.shape)
else:
if K.image_dim_ordering() == 'th':
img_data = np.rollaxis(img_data, 3, 1)
print(img_data.shape)
# Load trained model
model = load_model('cgi-bin/models/model.hdf5')
list_labels = []
for data in img_data:
def get_input_dim(self):
if K.image_dim_ordering() == 'tf':
return (self.ht, self.wd, 1)
else:
return (1, self.ht, self.wd)
def predict_single_image(img_path, model_rpn, model_classifier_only, cfg,
class_mapping):
st = time.time()
img = cv2.imread(img_path)
if img is None:
print('reading image failed.')
exit(0)
X, ratio = format_img(img, cfg)
if K.image_dim_ordering() == 'tf':
X = np.transpose(X, (0, 2, 3, 1))
# get the feature maps and output from the RPN
[Y1, Y2, F] = model_rpn.predict(X)
# this is result contains all boxes, which is [x1, y1, x2, y2]
result = roi_helpers.rpn_to_roi(Y1,
Y2,
cfg,
K.image_dim_ordering(),
overlap_thresh=0.7)
# convert from (x1,y1,x2,y2) to (x,y,w,h)
result[:, 2] -= result[:, 0]
result[:, 3] -= result[:, 1]
bbox_threshold = 0.8
# apply the spatial pyramid pooling to the proposed regions
boxes = dict()
for jk in range(result.shape[0] // cfg.num_rois + 1):
rois = np.expand_dims(result[cfg.num_rois * jk:cfg.num_rois *
(jk + 1), :],
axis=0)
if rois.shape[1] == 0:
break
if jk == result.shape[0] // cfg.num_rois:
# pad R
curr_shape = rois.shape
target_shape = (curr_shape[0], cfg.num_rois, curr_shape[2])
rois_padded = np.zeros(target_shape).astype(rois.dtype)
rois_padded[:, :curr_shape[1], :] = rois
rois_padded[0, curr_shape[1]:, :] = rois[0, 0, :]
rois = rois_padded
[p_cls, p_regr] = model_classifier_only.predict([F, rois])
for ii in range(p_cls.shape[1]):
if np.max(p_cls[0, ii, :]) < bbox_threshold or np.argmax(
p_cls[0, ii, :]) == (p_cls.shape[2] - 1):
continue
cls_num = np.argmax(p_cls[0, ii, :])
if cls_num not in boxes.keys():
boxes[cls_num] = []
(x, y, w, h) = rois[0, ii, :]
try:
(tx, ty, tw, th) = p_regr[0, ii, 4 * cls_num:4 * (cls_num + 1)]
tx /= cfg.classifier_regr_std[0]
ty /= cfg.classifier_regr_std[1]
tw /= cfg.classifier_regr_std[2]
th /= cfg.classifier_regr_std[3]
x, y, w, h = roi_helpers.apply_regr(x, y, w, h, tx, ty, tw, th)
except Exception as e:
print(e)
pass
boxes[cls_num].append([
cfg.rpn_stride * x, cfg.rpn_stride * y,
cfg.rpn_stride * (x + w), cfg.rpn_stride * (y + h),
np.max(p_cls[0, ii, :])
])
# add some nms to reduce many boxes
for cls_num, box in boxes.items():
boxes_nms = roi_helpers.non_max_suppression_fast(box,
overlap_thresh=0.5)
boxes[cls_num] = boxes_nms
print(class_mapping[cls_num] + ":")
for b in boxes_nms:
b[0], b[1], b[2], b[3] = get_real_coordinates(
ratio, b[0], b[1], b[2], b[3])
print('{} prob: {}'.format(b[0:4], b[-1]))
img = draw_boxes_and_label_on_image_cv2(img, class_mapping, boxes)
print('Elapsed time = {}'.format(time.time() - st))
# cv2.imshow('image', img)
result_path = './results_images/{}.png'.format(
os.path.basename(img_path).split('.')[0])
print('result saved into ', result_path)
cv2.imwrite(result_path, img)
cv2.waitKey(0)
def ResNet152(include_top=True, weights='imagenet',
input_tensor=None, input_shape=None, pooling=None, classes=1000):
""" Instantiates the ResNet152 architecture.
Optionally loads weights pre-trained
on ImageNet. Note that when using TensorFlow,
for best performance you should set
`image_data_format='channels_last'` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible only with
TensorFlow. The data format
convention used by the model is the one
specified in your Keras config file.
# Arguments
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(224, 224, 3)` (with `channels_last` data format)
or `(3, 224, 224)` (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 197.
E.g. `(200, 200, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
eps = 1.1e-5
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_size=224,
min_size=197,
data_format=K.image_data_format(),
require_flatten=include_top,
weights=weights)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape, name='data')
else:
img_input = input_tensor
# Handle dimension ordering for different backends
if K.image_dim_ordering() == 'tf':
bn_axis = 3
else:
bn_axis = 1
x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input)
x = Conv2D(64, (7, 7), strides=(2, 2), name='conv1', use_bias=False)(x)
x = BatchNormalization(epsilon=eps, axis=bn_axis, name='bn_conv1')(x)
x = Scale(axis=bn_axis, name='scale_conv1')(x)
x = Activation('relu', name='conv1_relu')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1', padding='same')(x)
x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
for i in range(1, 8):
x = identity_block(x, 3, [128, 128, 512], stage=3, block='b' + str(i))
x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
for i in range(1, 36):
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b' + str(i))
x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')
if include_top:
# Classification block
x = AveragePooling2D((7, 7), name='avg_pool')(x)
x = Flatten()(x)
x = Dense(classes, activation='softmax', name='fc1000')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model
model = Model(inputs, x, name='resnet152')
# Load weights
if weights == 'imagenet':
if include_top:
weights_path = WEIGHTS_PATH
else:
weights_path = get_file(
'resnet152_weights_tf_dim_ordering_tf_kernels_no_top.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
md5_hash='02cb9130cc51543cd703c79697baa592')
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
return model
def channel_idx(self):
dim_ordering = K.image_dim_ordering()
if dim_ordering == 'th':
return 1
else:
return 3
def densenet169_model(img_rows,
img_cols,
color_type=1,
nb_dense_block=4,
growth_rate=32,
nb_filter=64,
reduction=0.5,
dropout_rate=0.0,
weight_decay=1e-4,
num_classes=None):
'''
DenseNet 169 Model for Keras
Model Schema is based on
https://github.com/flyyufelix/DenseNet-Keras
ImageNet Pretrained Weights
Theano: https://drive.google.com/open?id=0Byy2AcGyEVxfN0d3T1F1MXg0NlU
TensorFlow: https://drive.google.com/open?id=0Byy2AcGyEVxfSEc5UC1ROUFJdmM
# Arguments
nb_dense_block: number of dense blocks to add to end
growth_rate: number of filters to add per dense block
nb_filter: initial number of filters
reduction: reduction factor of transition blocks.
dropout_rate: dropout rate
weight_decay: weight decay factor
classes: optional number of classes to classify images
weights_path: path to pre-trained weights
# Returns
A Keras model instance.
'''
layers = []
eps = 1.1e-5
# compute compression factor
compression = 1.0 - reduction
# Handle Dimension Ordering for different backends
global concat_axis
if K.image_dim_ordering() == 'tf':
concat_axis = 3
img_input = Input(shape=(img_rows, img_cols, 3), name='data')
else:
concat_axis = 1
img_input = Input(shape=(3, img_rows, img_cols), name='data')
# From architecture for ImageNet (Table 1 in the paper)
nb_filter = 64
nb_layers = [6, 12, 32, 32] # For DenseNet-169
# Initial convolution
x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input)
x = Convolution2D(nb_filter,
7,
7,
subsample=(2, 2),
name='conv1',
bias=False)(x)
x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv1_bn')(x)
x = Scale(axis=concat_axis, name='conv1_scale')(x)
x = Activation('relu', name='relu1')(x)
layers.append(x)
# x = ZeroPadding2D((1, 1), name='pool1_zeropadding')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='pool1')(x)
# Add dense blocks
for block_idx in range(nb_dense_block - 1):
stage = block_idx + 2
x, nb_filter = dense_block(x,
stage,
nb_layers[block_idx],
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
layers.append(x)
# Add transition_block
x = transition_block(x,
stage,
nb_filter,
compression=compression,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
nb_filter = int(nb_filter * compression)
final_stage = stage + 1
x, nb_filter = dense_block(x,
final_stage,
nb_layers[-1],
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
x = BatchNormalization(epsilon=eps,
axis=concat_axis,
name='conv' + str(final_stage) + '_blk_bn')(x)
x = Scale(axis=concat_axis,
name='conv' + str(final_stage) + '_blk_scale')(x)
x = Activation('relu', name='relu' + str(final_stage) + '_blk')(x)
# x_fc = GlobalAveragePooling2D(name='pool'+str(final_stage))(x)
# x_fc = Dense(1000, name='fc6')(x_fc)
# x_fc = Activation('softmax', name='prob')(x_fc)
# model = Model(img_input, x_fc, name='densenet')
model = Model(img_input, x, name='densenet')
if K.image_dim_ordering() == 'th':
# Use pre-trained weights for Theano backend
weights_path = '../imagenet_models/densenet169_weights_th.h5'
else:
# Use pre-trained weights for Tensorflow backend
weights_path = '../imagenet_models/densenet169_weights_tf.h5'
model.load_weights(weights_path, by_name=True)
return model, layers
def gray(img):
if K.image_dim_ordering() == 'tf':
return np.rollaxis(img, 0, 1).dot(to_bw)
else:
return np.rollaxis(img, 0, 3).dot(to_bw)
def predict(imm):
with grp.as_default():
print imm.shape
X = format_img(imm, C)
img_scaled = np.transpose(X.copy()[0, (2, 1, 0), :, :],
(1, 2, 0)).copy()
img_scaled[:, :, 0] += 123.68
img_scaled[:, :, 1] += 116.779
img_scaled[:, :, 2] += 103.939
img_scaled = img_scaled.astype(np.uint8)
if K.image_dim_ordering() == 'tf':
X = np.transpose(X, (0, 2, 3, 1))
# get the feature maps and output from the RPN
[Y1, Y2, F] = model_rpn.predict(X)
R = roi_helpers.rpn_to_roi(Y1,
Y2,
C,
K.image_dim_ordering(),
overlap_thresh=0.7)
# convert from (x1,y1,x2,y2) to (x,y,w,h)
R[:, 2] -= R[:, 0]
R[:, 3] -= R[:, 1]
# apply the spatial pyramid pooling to the proposed regions
bboxes = {}
probs = {}
for jk in range(R.shape[0] // C.num_rois + 1):
ROIs = np.expand_dims(R[C.num_rois * jk:C.num_rois * (jk + 1), :],
axis=0)
if ROIs.shape[1] == 0:
break
if jk == R.shape[0] // C.num_rois:
#pad R
curr_shape = ROIs.shape
target_shape = (curr_shape[0], C.num_rois, curr_shape[2])
ROIs_padded = np.zeros(target_shape).astype(ROIs.dtype)
ROIs_padded[:, :curr_shape[1], :] = ROIs
ROIs_padded[0, curr_shape[1]:, :] = ROIs[0, 0, :]
ROIs = ROIs_padded
[P_cls, P_regr] = model_classifier_only.predict([F, ROIs])
for ii in range(P_cls.shape[1]):
if np.max(P_cls[0, ii, :]) < bbox_threshold or np.argmax(
P_cls[0, ii, :]) == (P_cls.shape[2] - 1):
continue
cls_name = class_mapping[np.argmax(P_cls[0, ii, :])]
if cls_name not in bboxes:
bboxes[cls_name] = []
probs[cls_name] = []
(x, y, w, h) = ROIs[0, ii, :]
cls_num = np.argmax(P_cls[0, ii, :])
try:
(tx, ty, tw, th) = P_regr[0, ii,
4 * cls_num:4 * (cls_num + 1)]
tx /= C.classifier_regr_std[0]
ty /= C.classifier_regr_std[1]
tw /= C.classifier_regr_std[2]
th /= C.classifier_regr_std[3]
x, y, w, h = roi_helpers.apply_regr(
x, y, w, h, tx, ty, tw, th)
except:
pass
bboxes[cls_name].append(
[16 * x, 16 * y, 16 * (x + w), 16 * (y + h)])
probs[cls_name].append(np.max(P_cls[0, ii, :]))
res = []
for key in bboxes:
bbox = np.array(bboxes[key])
# print bbox.shape
cf = np.array(probs[key])
# print cf.shape
new_boxes, new_probs = roi_helpers.non_max_suppression_fast(
bbox, np.array(probs[key]), overlap_thresh=0.5)
for jk in range(new_boxes.shape[0]):
(x1, y1, x2, y2) = new_boxes[jk, :]
res.append([key, x1, y1, x2, y2, cf[jk]])
return res
def batch_generator_train(cnn_type, files, labels, augment=False):
import keras.backend as K
global FULL_IMAGE_ARRAY
dim_ordering = K.image_dim_ordering()
in_shape = get_input_shape(cnn_type)
batch_size = get_batch_size(cnn_type)
if len(FULL_IMAGE_ARRAY) == 0:
prepread_images()
while True:
index = random.sample(range(len(files)), batch_size)
batch_files = files[index]
batch_labels = labels[index]
image_list = []
mask_list = []
for i in range(len(batch_files)):
# image = cv2.imread(batch_files[i])
image = FULL_IMAGE_ARRAY[os.path.basename(batch_files[i])]
if cnn_type == 'INCEPTION_V3' or cnn_type == 'INCEPTION_V4' or cnn_type == 'XCEPTION':
random_border = 20
start0 = random.randint(0, random_border)
start1 = random.randint(0, random_border)
end0 = random.randint(0, random_border)
end1 = random.randint(0, random_border)
image = image[start0:image.shape[0] - end0,
start1:image.shape[1] - end1]
image = cv2.resize(image, (299, 299), cv2.INTER_LANCZOS4)
else:
box_size = random.randint(200, 256)
start0 = random.randint(0, image.shape[0] - box_size)
start1 = random.randint(0, image.shape[1] - box_size)
image = image[start0:start0 + box_size,
start1:start1 + box_size]
image = cv2.resize(image, in_shape, cv2.INTER_LANCZOS4)
if augment:
# all possible mirroring and flips
# (in total there are only 8 possible configurations)
mirror = random.randint(0, 1)
if mirror == 1:
# flipud
image = image[::-1, :, :]
angle = random.randint(0, 3)
if angle != 0:
image = np.rot90(image, k=angle)
# image = random_intensity_change(image, 10)
mask = batch_labels[i]
image_list.append(image.astype(np.float32))
mask_list.append(mask)
image_list = np.array(image_list)
image_list = image_list.transpose((0, 3, 1, 2))
image_list = preprocess_input_overall(cnn_type, image_list)
if dim_ordering == 'tf':
image_list = image_list.transpose((0, 2, 3, 1))
mask_list = np.array(mask_list)
yield image_list, mask_list
def resnet50_model(img_rows, img_cols, color_type=1, num_classes=None):
"""
Resnet 50 Model for Keras
Model Schema is based on
https://github.com/fchollet/deep-learning-models/blob/master/resnet50.py
ImageNet Pretrained Weights
https://github.com/fchollet/deep-learning-models/releases/download/v0.2/resnet50_weights_th_dim_ordering_th_kernels.h5
Parameters:
img_rows, img_cols - resolution of inputs
channel - 1 for grayscale, 3 for color
num_classes - number of class labels for our classification task
"""
# Handle Dimension Ordering for different backends
global bn_axis
if K.image_dim_ordering() == 'tf':
bn_axis = 3
img_input = Input(shape=(img_rows, img_cols, color_type))
else:
bn_axis = 1
img_input = Input(shape=(color_type, img_rows, img_cols))
x = ZeroPadding2D((3, 3))(img_input)
x = Convolution2D(64, 7, 7, subsample=(2, 2), name='conv1')(x)
x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x = Activation('relu')(x)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='b')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='d')
x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f')
x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')
# Fully Connected Softmax Layer
x_fc = AveragePooling2D((7, 7), name='avg_pool')(x)
x_fc = Flatten()(x_fc)
x_fc = Dense(1000, activation='softmax', name='fc1000')(x_fc)
# Create model
model = Model(img_input, x_fc)
# Load ImageNet pre-trained data
if K.image_dim_ordering() == 'th':
# Use pre-trained weights for Theano backend
weights_path = '../imagenet_models/resnet50_weights_th_dim_ordering_th_kernels.h5'
else:
# Use pre-trained weights for Tensorflow backend
weights_path = '../imagenet_models/resnet50_weights_tf_dim_ordering_tf_kernels.h5'
model.load_weights(weights_path)
# Truncate and replace softmax layer for transfer learning
# Cannot use model.layers.pop() since model is not of Sequential() type
# The method below works since pre-trained weights are stored in layers but not in the model
x_newfc = AveragePooling2D((7, 7), name='avg_pool')(x)
x_newfc = Flatten()(x_newfc)
x_newfc = Dense(num_classes, activation='softmax', name='fc10')(x_newfc)
# Create another model with our customized softmax
model = Model(img_input, x_newfc)
# Learning rate is changed to 0.001
sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd,
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
if pooltype == 1:
return AveragePooling2D((2, 2), strides=(2, 2))(x)
else:
return MaxPooling2D((2, 2), strides=(2, 2))(x)
# get tensor representations of our images
base_image = K.variable(
preprocess_image(base_image_path, True, read_mode=read_mode))
style_reference_images = []
for style_path in style_image_paths:
style_reference_images.append(K.variable(preprocess_image(style_path)))
# this will contain our generated image
if K.image_dim_ordering() == 'th':
combination_image = K.placeholder((1, 3, img_width, img_height))
else:
combination_image = K.placeholder((1, img_width, img_height, 3))
image_tensors = [base_image]
for style_image_tensor in style_reference_images:
image_tensors.append(style_image_tensor)
image_tensors.append(combination_image)
nb_tensors = len(image_tensors)
nb_style_images = nb_tensors - 2 # Content and Output image not considered
# combine the various images into a single Keras tensor
input_tensor = K.concatenate(image_tensors, axis=0)
def LearnKit19(include_top=True,
input_tensor=None,
input_shape=None,
classes=1000):
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_size=224,
min_size=139,
dim_ordering=K.image_dim_ordering(),
include_top=include_top)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
#1 224x224x32
x = conv2d_bn(img_input, 32, 3, 3, subsample=(1, 1))
# 112x112x32
x = MaxPooling2D((2, 2), strides=(2, 2))(x)
#2 112x112x64
x = conv2d_bn(x, 64, 3, 3)
# 56x56x64
x = MaxPooling2D((2, 2), strides=(2, 2))(x)
#3 56x56x128
x = conv2d_bn(x, 128, 3, 3)
#4 56x56x64
x = conv2d_bn(x, 64, 1, 1)
#5 56x56x128
x = conv2d_bn(x, 128, 3, 3)
# 28x28x128
x = MaxPooling2D((2, 2), strides=(2, 2))(x)
#6 28x28x256
x = conv2d_bn(x, 256, 3, 3)
#7 28x28x128
x = conv2d_bn(x, 128, 1, 1)
#8 28x28x256
x = conv2d_bn(x, 256, 3, 3)
# 14x14x256
x = MaxPooling2D((2, 2), strides=(2, 2))(x)
#9 14x14x512
x = conv2d_bn(x, 512, 3, 3)
#10 14x14x256
x = conv2d_bn(x, 256, 1, 1)
#11 14x14x512
x = conv2d_bn(x, 512, 3, 3)
#12 14x14x256
x = conv2d_bn(x, 256, 1, 1)
#13 14x14x512
x = conv2d_bn(x, 512, 3, 3)
# 7x7x512
x = MaxPooling2D((2, 2), strides=(2, 2))(x)
#14 7x7x1024
x = conv2d_bn(x, 1024, 3, 3)
#15 7x7x512
x = conv2d_bn(x, 512, 1, 1)
#16 7x7x1024
x = conv2d_bn(x, 1024, 3, 3)
#17 7x7x512
x = conv2d_bn(x, 512, 1, 1)
#18 7x7x1024
x = conv2d_bn(x, 1024, 3, 3)
#17 7x7x1024
x = conv2d_bn(x, 1024, 1, 1)
if include_top:
# Classification block
x = AveragePooling2D((7, 7), strides=None, name='avg_pool')(x)
x = Flatten(name='flatten')(x)
x = Dense(classes, activation='softmax', name='predictions')(x)
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='learnkit_19')
return model
parser = argparse.ArgumentParser()
parser.add_argument('--mode', choices=['train', 'test'], default='test')
parser.add_argument('--env-name', type=str, default='Breakout-v0')
parser.add_argument('--weights', type=str, default=None)
args = parser.parse_args()
# Get the environment and extract the number of actions.
env = gym.make(args.env_name)
np.random.seed(42)
env.seed(42)
nb_actions = env.action_space.n
# Next, we build our model. We use the same model that was described by Mnih et al. (2015).
input_shape = (WINDOW_LENGTH, ) + INPUT_SHAPE
model = Sequential()
if K.image_dim_ordering() == 'tf':
# (width, height, channels)
model.add(Permute((2, 3, 1), input_shape=input_shape))
elif K.image_dim_ordering() == 'th':
# (channels, width, height)
model.add(Permute((1, 2, 3), input_shape=input_shape))
else:
raise RuntimeError('Unknown image_dim_ordering.')
model.add(Conv2D(32, 8, strides=4))
model.add(Activation('relu'))
model.add(Conv2D(64, 4, strides=2))
model.add(Activation('relu'))
model.add(Conv2D(64, 3, strides=1))
model.add(Activation('relu'))
model.add(Flatten())
model.add(Dense(512))
def channel_axis():
if K.image_dim_ordering() == 'tf':
return 3
else:
return 1
parser.add_argument('--pooling_type',
default='avg',
type=str,
choices=['max', 'avg'],
help='VGG pooling type.')
parser.add_argument('--image_size',
default=256,
type=int,
help='Input image size.')
parser.add_argument('--max_iter',
default=600,
type=int,
help='Number of training iter.')
args = parser.parse_args()
dim_ordering = K.image_dim_ordering()
channels = 3
width = args.image_size
height = args.image_size
size = (height, width)
if dim_ordering == 'th':
input_shape = (channels, width, height)
else:
input_shape = (width, height, channels)
X_train = np.array([
load_image(args.content,
size=(height, width),
preprocess_type='vgg19',
verbose=True)
])
def get_weight_path():
if K.image_dim_ordering() == 'th':
return 'resnet50_weights_th_dim_ordering_th_kernels_notop.h5'
else:
return 'resnet50_weights_tf_dim_ordering_tf_kernels.h5'
