def google_vgg16_finetune(classes=3, size=256):
input_layer = Input(shape=(size, size, 3), name='image_input')
base_model = VGG16(weights='imagenet', include_top=False, input_tensor=input_layer)
x = Conv2D(name='squeeze', filters=256, kernel_size=(1, 1))(base_model.output) # squeeze channels
x = Flatten(name='avgpool')(x)
x = Dense(256, name='features', kernel_regularizer=regularizers.l2(0.01))(x)
x = Activation('relu')(x)
x = Dense(classes, activation='softmax', name='out')(x)
model = Model(inputs=base_model.input, outputs=x)
for layer in model.layers:
if layer.name in ['block5_conv1', 'block5_conv2', 'block5_conv3', 'features', 'out']:
layer.trainable = True
else:
layer.trainable = False
print(model.summary())
model.compile(
loss='categorical_crossentropy',
optimizer=tf.keras.optimizers.RMSprop(lr=1e-4),
metrics=['accuracy'])
return model
def SqueezeNet(include_top = True, weights="imagenet", model_input=None, non_top_pooling=None,
num_classes=1000, model_path = ""):
if(weights == "imagenet" and num_classes != 1000):
raise ValueError("You must parse in SqueezeNet model trained on the 1000 class ImageNet")
image_input = model_input
network = Conv2D(64, (3,3), strides=(2,2), padding="valid")(image_input)
network = Activation("relu")(network)
network = MaxPool2D( pool_size=(3,3) , strides=(2,2))(network)
network = squeezenet_fire_module(input=network, input_channel_small=16, input_channel_large=64)
network = squeezenet_fire_module(input=network, input_channel_small=16, input_channel_large=64)
network = MaxPool2D(pool_size=(3,3), strides=(2,2))(network)
network = squeezenet_fire_module(input=network, input_channel_small=32, input_channel_large=128)
network = squeezenet_fire_module(input=network, input_channel_small=32, input_channel_large=128)
network = MaxPool2D(pool_size=(3, 3), strides=(2, 2))(network)
network = squeezenet_fire_module(input=network, input_channel_small=48, input_channel_large=192)
network = squeezenet_fire_module(input=network, input_channel_small=48, input_channel_large=192)
network = squeezenet_fire_module(input=network, input_channel_small=64, input_channel_large=256)
network = squeezenet_fire_module(input=network, input_channel_small=64, input_channel_large=256)
if(include_top):
network = Dropout(0.5)(network)
network = Conv2D(num_classes, kernel_size=(1,1), padding="valid", name="last_conv")(network)
network = Activation("relu")(network)
network = GlobalAvgPool2D()(network)
network = Activation("softmax")(network)
else:
if(non_top_pooling == "Average"):
network = GlobalAvgPool2D()(network)
elif(non_top_pooling == "Maximum"):
network = GlobalMaxPool2D()(network)
elif(non_top_pooling == None):
pass
input_image = image_input
model = Model(inputs=input_image, outputs=network)
if(weights =="imagenet"):
weights_path = model_path
model.load_weights(weights_path)
elif(weights =="trained"):
weights_path = model_path
model.load_weights(weights_path)
return model
def ResNet50(include_top=True, non_top_pooling=None, model_input=None, num_classes=1000, weights='imagenet', model_path=""):
layers = [3,4,6,3]
channel_depths = [256, 512, 1024, 2048]
input_object = model_input
output = Conv2D(64, kernel_size=7, strides=2, padding="same")(input_object)
output = BatchNormalization()(output)
output = Activation("relu")(output)
output = MaxPool2D(pool_size=(3,3), strides=(2,2))(output)
output = resnet_first_block_first_module(output, channel_depths[0])
for i in range(4):
channel_depth = channel_depths[i]
num_layers = layers[i]
strided_pool_first = True
if(i == 0):
strided_pool_first = False
num_layers = num_layers - 1
output = resnet_block(output, channel_depth=channel_depth, num_layers=num_layers, strided_pool_first=strided_pool_first)
if(include_top):
output = GlobalAvgPool2D(name="global_avg_pooling")(output)
output = Dense(num_classes)(output)
output = Activation("softmax")(output)
else:
if (non_top_pooling == "Average"):
output = GlobalAvgPool2D()(output)
elif (non_top_pooling == "Maximum"):
output = GlobalMaxPool2D()(output)
elif (non_top_pooling == None):
pass
model = Model(inputs=input_object, outputs=output)
if(weights == "imagenet"):
weights_path = model_path
model.load_weights(weights_path)
elif (weights == "trained"):
weights_path = model_path
model.load_weights(weights_path)
return model
def FPNet(backbone,
input_shape,
inputs=None,
norm_method='whole_image',
use_imagenet=False,
pooling=None,
required_channels=3,
n_classes=3,
name='fpnet',
frames_per_batch=1,
**kwargs):
"""Creates a Feature Pyramid Network with a semantic segmentation head
Args:
backbone (str): A name of a supported backbone from [deepcell, resnet50]
input_shape (tuple): Shape of the input image.
inputs (keras.Layer): Optional preexisting layers.
norm_method (str): Normalization method, defaults to 'whole_image'
use_imagenet (bool): Whether to load imagenet-based pretrained weights.
pooling (str): Optional pooling mode for feature extraction
when include_top is False.
- None means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- 'avg' means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- 'max' means that global max pooling will
be applied.
required_channels (int): The required number of channels of the
backbone. 3 is the default for all current backbones.
n_classes (int): The number of classes to be predicted
name (str): Name to use for the model.
frames_per_batch (int): Size of z axis in generated batches.
If equal to 1, assumes 2D data.
Returns:
tensorflow.keras.models.Model: Feature pyramid network with a semantic
segmentation head as the output
"""
if inputs is None:
inputs = Input(shape=input_shape)
# force the channel size for backbone input to be required_channels
norm = ImageNormalization2D(norm_method=norm_method)(inputs)
fixed_inputs = TensorProduct(required_channels)(norm)
# force the input shape
fixed_input_shape = list(input_shape)
fixed_input_shape[-1] = required_channels
fixed_input_shape = tuple(fixed_input_shape)
model_kwargs = {
'include_top': False,
'weights': None,
'input_shape': fixed_input_shape,
'pooling': pooling
}
# Get backbone outputs
_, backbone_dict = get_backbone(backbone,
fixed_inputs,
use_imagenet=use_imagenet,
frames_per_batch=frames_per_batch,
return_dict=True,
**model_kwargs)
# Construct feature pyramid network
pyramid_dict = __create_pyramid_features(backbone_dict)
levels = [int(re.findall(r'\d+', k)[0]) for k in pyramid_dict]
target_level = min(levels)
x = __create_semantic_head(pyramid_dict,
n_classes=n_classes,
input_target=inputs,
target_level=target_level,
ndim=len(input_shape) - 1)
return Model(inputs=inputs, outputs=x, name=name)
class Kmodel:
def __init__(self, fl, hparams):
"""
Initialises new ANN model
:param fl: fl class
:param hparams: Dict containing hyperparameter information. Dict can be created using create_hparams() function
"""
self.features_dim = fl.features_c_dim
self.labels_dim = fl.labels_dim # Assuming that each task has only 1 dimensional output
self.hparams = hparams
self.normalise_labels = fl.normalise_labels
self.labels_scaler = fl.labels_scaler
features_in = Input(shape=(self.features_dim, ),
name='main_features_c_input')
# Build keras ANN model
x = Dense(units=hparams['nodes'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(l1=hparams['reg_l1'],
l2=hparams['reg_l2']),
name='Layer_' + str(0))(features_in)
x = Dense(units=hparams['nodes'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(l1=hparams['reg_l1'],
l2=hparams['reg_l2']),
name='Layer_' + str(1))(x)
x = Dense(units=hparams['nodes'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(l1=hparams['reg_l1'],
l2=hparams['reg_l2']),
name='Layer_' + str(2))(x)
# x = BatchNormalization()(x)
x = Dense(units=self.labels_dim,
activation='linear',
kernel_regularizer=regularizers.l1_l2(l1=hparams['reg_l1'],
l2=hparams['reg_l2']),
name='Final')(x)
self.model = Model(inputs=features_in, outputs=x)
optimizer = Adam(learning_rate=hparams['learning_rate'], clipnorm=1)
def mean_relative_error(y_true, y_pred):
diff = K.abs(
(y_true - y_pred) /
K.reshape(K.clip(K.abs(y_true[:, -1]), K.epsilon(), None),
(-1, 1)))
return 100. * K.mean(diff, axis=-1)
if hparams['loss'] == 'mape':
self.model.compile(optimizer=optimizer,
loss=MeanAbsolutePercentageError())
elif hparams['loss'] == 'mre':
self.model.compile(optimizer=optimizer, loss=mean_relative_error)
elif hparams['loss'] == 'mse':
self.model.compile(optimizer=optimizer, loss='mean_squared_error')
# self.model.summary()
def train_model(self,
fl,
i_fl,
save_name='mt.h5',
save_dir='./save/models/',
save_mode=False,
plot_name=None):
# Training model
training_features = fl.features_c_norm
val_features = i_fl.features_c_norm
if self.normalise_labels:
training_labels = fl.labels_norm
val_labels = i_fl.labels_norm
else:
training_labels = fl.labels
val_labels = i_fl.labels
# Plotting
if plot_name:
history = self.model.fit(training_features,
training_labels,
validation_data=(val_features,
val_labels),
epochs=self.hparams['epochs'],
batch_size=self.hparams['batch_size'],
verbose=self.hparams['verbose'])
# Debugging check to see features and prediction
# pprint.pprint(training_features)
# pprint.pprint(self.model.predict(training_features))
# pprint.pprint(training_labels)
# summarize history for accuracy
plt.semilogy(history.history['loss'], label=['train'])
plt.semilogy(history.history['val_loss'], label=['test'])
plt.plot([], [],
' ',
label='Final train: {:.3e}'.format(
history.history['loss'][-1]))
plt.plot([], [],
' ',
label='Final val: {:.3e}'.format(
history.history['val_loss'][-1]))
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(loc='upper right')
plt.savefig(plot_name, bbox_inches='tight')
plt.close()
else:
history = self.model.fit(training_features,
training_labels,
epochs=self.hparams['epochs'],
batch_size=self.hparams['batch_size'],
verbose=self.hparams['verbose'])
# Saving Model
if save_mode:
self.model.save(save_dir + save_name)
return self
def predict(self, eval_fl):
features = eval_fl.features_c_norm
predictions = self.model.predict(features)
return predictions # If labels is normalized, the prediction here is also normalized!
# Download the dataset of MNIST
(x_train, y_train), (x_test, y_test) = mnist.load_data()
# Preprocessing - exchange the scale of data
x_train = x_train.reshape(60000, 784)
x_train = x_train / 255
x_test = x_test.reshape(10000, 784)
x_test = x_test / 255
# Preprocessing - change class label to 1-hot vector
y_train = to_categorical(y_train, 10)
y_test = to_categorical(y_test, 10)
# Create the neural network
input = Input(shape=(784, ))
middle = Dense(units=64, activation='relu')(input)
output = Dense(units=10, activation='softmax')(middle)
model = Model(inputs=input, outputs=output)
# Learn the model
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
tsb = TensorBoard(log_dir='./logs')
history_adam = model.fit(x_train,
y_train,
batch_size=32,
epochs=20,
validation_split=0.2,
callbacks=[tsb])
def generate_vgg_model_large(classes_len: int):
"""
Function to create a VGG19 model pre-trained with custom FC Layers at the start of the network plus optional layers at
the end before the fully connected ones as well
If the "advanced" command line argument is selected, adds an extra convolutional layer with extra filters to support
larger images.
This model is a larger model that starts with two more sets of convolutional layers with less filters
:param classes_len: The number of classes (labels).
:return: The VGG19 model.
"""
model_base = Sequential()
# Reconfigure single channel input into a greyscale 3 channel input
img_input = Input(shape=(config.VGG_IMG_SIZE_LARGE['HEIGHT'],
config.VGG_IMG_SIZE_LARGE['WIDTH'], 1))
img_conc = Concatenate()([img_input, img_input, img_input])
input_model = Model(inputs=img_input, outputs=img_conc)
# Generate extra convolutional layers for model to put at the beginning
model_base.add(input_model)
model_base.add(Conv2D(16, (3, 3), activation='relu', padding='same'))
model_base.add(Conv2D(16, (3, 3), activation='relu', padding='same'))
model_base.add(MaxPooling2D((2, 2), strides=(2, 2)))
model_base.add(Conv2D(32, (3, 3), activation='relu', padding='same'))
model_base.add(Conv2D(32, (3, 3), activation='relu', padding='same'))
model_base.add(MaxPooling2D((2, 2), strides=(2, 2)))
# To ensure model fits with vgg model, we can remove the first layer from the vgg model to replace with this
model_base.add(Conv2D(64, (3, 3), activation='relu', padding='same'))
# Generate a VGG19 model with pre-trained ImageNet weights, input as given above, excluded fully connected layers.
vgg_model = VGG19(include_top=False,
weights='imagenet',
input_shape=[
config.VGG_IMG_SIZE['HEIGHT'],
config.VGG_IMG_SIZE['HEIGHT'], 3
])
# Crop vgg model to exlude input layer and first convolutional layer
vgg_model_cropped = Sequential()
for layer in vgg_model.layers[2:]: # go through until last layer
vgg_model_cropped.add(layer)
# Combine the models
combined_model = Sequential()
combined_model.add(model_base)
combined_model.add(vgg_model_cropped)
# Add fully connected layers
model = Sequential()
# Start with base model consisting of convolutional layers
model.add(combined_model)
# Generate additional convolutional layers
if config.model == "advanced":
model.add(Conv2D(1024, (3, 3), activation='relu', padding='same'))
model.add(Conv2D(1024, (3, 3), activation='relu', padding='same'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
# Flatten layer to convert each input into a 1D array (no parameters in this layer, just simple pre-processing).
model.add(Flatten())
# Add fully connected hidden layers.
model.add(Dense(units=512, activation='relu', name='Dense_Intermediate_1'))
model.add(Dense(units=32, activation='relu', name='Dense_Intermediate_2'))
# Possible dropout for regularisation can be added later and experimented with:
# model.add(Dropout(0.1, name='Dropout_Regularization'))
# Final output layer that uses softmax activation function (because the classes are exclusive).
if classes_len == 2:
model.add(Dense(1, activation='sigmoid', name='Output'))
else:
model.add(Dense(classes_len, activation='softmax', name='Output'))
# Print model details if running in debug mode.
if config.verbose_mode:
model.summary()
return model
def InceptionResNetV2(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000):
"""Instantiates the Inception-ResNet v2 architecture.
Optionally loads weights pre-trained on ImageNet.
Note that when using TensorFlow, for best performance you should
set `"image_data_format": "channels_last"` in your Keras config
at `~/.keras/keras.json`.
The model and the weights are compatible with TensorFlow, Theano and
CNTK backends. The data format convention used by the model is
the one specified in your Keras config file.
Note that the default input image size for this model is 299x299, instead
of 224x224 as in the VGG16 and ResNet models. Also, the input preprocessing
function is different (i.e., do not use `imagenet_utils.preprocess_input()`
with this model. Use `preprocess_input()` defined in this module instead).
Arguments:
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is `False` (otherwise the input shape
has to be `(299, 299, 3)` (with `'channels_last'` data format)
or `(3, 299, 299)` (with `'channels_first'` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 139.
E.g. `(150, 150, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the last convolutional layer.
- `'avg'` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `'max'` means that global max pooling will be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is `True`, and
if no `weights` argument is specified.
Returns:
A Keras `Model` instance.
Raises:
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
input_shape = _obtain_input_shape(
input_shape,
default_size=299,
min_size=139,
data_format=K.image_data_format(),
require_flatten=False,
weights=weights)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
# Stem block: 35 x 35 x 192
x = conv2d_bn(img_input, 32, 3, strides=2, padding='valid')
x = conv2d_bn(x, 32, 3, padding='valid')
x = conv2d_bn(x, 64, 3)
x = MaxPooling2D(3, strides=2)(x)
x = conv2d_bn(x, 80, 1, padding='valid')
x = conv2d_bn(x, 192, 3, padding='valid')
x = MaxPooling2D(3, strides=2)(x)
# Mixed 5b (Inception-A block): 35 x 35 x 320
branch_0 = conv2d_bn(x, 96, 1)
branch_1 = conv2d_bn(x, 48, 1)
branch_1 = conv2d_bn(branch_1, 64, 5)
branch_2 = conv2d_bn(x, 64, 1)
branch_2 = conv2d_bn(branch_2, 96, 3)
branch_2 = conv2d_bn(branch_2, 96, 3)
branch_pool = AveragePooling2D(3, strides=1, padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1)
branches = [branch_0, branch_1, branch_2, branch_pool]
channel_axis = 1 if K.image_data_format() == 'channels_first' else 3
x = Concatenate(axis=channel_axis, name='mixed_5b')(branches)
# 10x block35 (Inception-ResNet-A block): 35 x 35 x 320
for block_idx in range(1, 11):
x = inception_resnet_block(
x, scale=0.17, block_type='block35', block_idx=block_idx)
# Mixed 6a (Reduction-A block): 17 x 17 x 1088
branch_0 = conv2d_bn(x, 384, 3, strides=2, padding='valid')
branch_1 = conv2d_bn(x, 256, 1)
branch_1 = conv2d_bn(branch_1, 256, 3)
branch_1 = conv2d_bn(branch_1, 384, 3, strides=2, padding='valid')
branch_pool = MaxPooling2D(3, strides=2, padding='valid')(x)
branches = [branch_0, branch_1, branch_pool]
x = Concatenate(axis=channel_axis, name='mixed_6a')(branches)
# 20x block17 (Inception-ResNet-B block): 17 x 17 x 1088
for block_idx in range(1, 21):
x = inception_resnet_block(
x, scale=0.1, block_type='block17', block_idx=block_idx)
# Mixed 7a (Reduction-B block): 8 x 8 x 2080
branch_0 = conv2d_bn(x, 256, 1)
branch_0 = conv2d_bn(branch_0, 384, 3, strides=2, padding='valid')
branch_1 = conv2d_bn(x, 256, 1)
branch_1 = conv2d_bn(branch_1, 288, 3, strides=2, padding='valid')
branch_2 = conv2d_bn(x, 256, 1)
branch_2 = conv2d_bn(branch_2, 288, 3)
branch_2 = conv2d_bn(branch_2, 320, 3, strides=2, padding='valid')
branch_pool = MaxPooling2D(3, strides=2, padding='valid')(x)
branches = [branch_0, branch_1, branch_2, branch_pool]
x = Concatenate(axis=channel_axis, name='mixed_7a')(branches)
# 10x block8 (Inception-ResNet-C block): 8 x 8 x 2080
for block_idx in range(1, 10):
x = inception_resnet_block(
x, scale=0.2, block_type='block8', block_idx=block_idx)
x = inception_resnet_block(
x, scale=1., activation=None, block_type='block8', block_idx=10)
# Final convolution block: 8 x 8 x 1536
x = conv2d_bn(x, 1536, 1, name='conv_7b')
if include_top:
# Classification block
x = GlobalAveragePooling2D(name='avg_pool')(x)
x = Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`
if input_tensor is not None:
inputs = layer_utils.get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model
model = Model(inputs, x, name='inception_resnet_v2')
# Load weights
if weights == 'imagenet':
if include_top:
fname = 'inception_resnet_v2_weights_tf_dim_ordering_tf_kernels.h5'
weights_path = get_file(
fname,
BASE_WEIGHT_URL + fname,
cache_subdir='models',
file_hash='e693bd0210a403b3192acc6073ad2e96')
else:
fname = 'inception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.h5'
weights_path = get_file(
fname,
BASE_WEIGHT_URL + fname,
cache_subdir='models',
file_hash='d19885ff4a710c122648d3b5c3b684e4')
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
return model
def NASNet(input_shape=None,
penultimate_filters=4032,
num_blocks=6,
stem_block_filters=96,
skip_reduction=True,
filter_multiplier=2,
include_top=True,
weights=None,
input_tensor=None,
pooling=None,
classes=1000,
default_size=None):
"""Instantiates a NASNet model.
Note that only TensorFlow is supported for now,
therefore it only works with the data format
`image_data_format='channels_last'` in your Keras config
at `~/.keras/keras.json`.
Arguments:
input_shape: Optional shape tuple, the input shape
is by default `(331, 331, 3)` for NASNetLarge and
`(224, 224, 3)` for NASNetMobile.
It should have exactly 3 inputs channels,
and width and height should be no smaller than 32.
E.g. `(224, 224, 3)` would be one valid value.
penultimate_filters: Number of filters in the penultimate layer.
NASNet models use the notation `NASNet (N @ P)`, where:
- N is the number of blocks
- P is the number of penultimate filters
num_blocks: Number of repeated blocks of the NASNet model.
NASNet models use the notation `NASNet (N @ P)`, where:
- N is the number of blocks
- P is the number of penultimate filters
stem_block_filters: Number of filters in the initial stem block
skip_reduction: Whether to skip the reduction step at the tail
end of the network. Set to `False` for CIFAR models.
filter_multiplier: Controls the width of the network.
- If `filter_multiplier` < 1.0, proportionally decreases the number
of filters in each layer.
- If `filter_multiplier` > 1.0, proportionally increases the number
of filters in each layer.
- If `filter_multiplier` = 1, default number of filters from the
paper are used at each layer.
include_top: Whether to include the fully-connected
layer at the top of the network.
weights: `None` (random initialization) or
`imagenet` (ImageNet weights)
input_tensor: Optional Keras tensor (i.e. output of
`layers.Input()`)
to use as image input for the model.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model
will be the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a
2D tensor.
- `max` means that global max pooling will
be applied.
classes: Optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
default_size: Specifies the default image size of the model
Returns:
A Keras model instance.
Raises:
ValueError: In case of invalid argument for `weights`,
invalid input shape or invalid `penultimate_filters` value.
RuntimeError: If attempting to run this model with a
backend that does not support separable convolutions.
"""
if K.backend() != 'tensorflow':
raise RuntimeError('Only Tensorflow backend is currently supported, '
'as other backends do not support '
'separable convolution.')
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as ImageNet with `include_top` '
'as true, `classes` should be 1000')
if (isinstance(input_shape, tuple) and None in input_shape and
weights == 'imagenet'):
raise ValueError('When specifying the input shape of a NASNet'
' and loading `ImageNet` weights, '
'the input_shape argument must be static '
'(no None entries). Got: `input_shape=' +
str(input_shape) + '`.')
if default_size is None:
default_size = 331
# Determine proper input shape and default size.
input_shape = _obtain_input_shape(
input_shape,
default_size=default_size,
min_size=32,
data_format=K.image_data_format(),
require_flatten=False,
weights=weights)
if K.image_data_format() != 'channels_last':
logging.warning('The NASNet family of models is only available '
'for the input data format "channels_last" '
'(width, height, channels). '
'However your settings specify the default '
'data format "channels_first" (channels, width, height).'
' You should set `image_data_format="channels_last"` '
'in your Keras config located at ~/.keras/keras.json. '
'The model being returned right now will expect inputs '
'to follow the "channels_last" data format.')
K.set_image_data_format('channels_last')
old_data_format = 'channels_first'
else:
old_data_format = None
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if penultimate_filters % 24 != 0:
raise ValueError(
'For NASNet-A models, the value of `penultimate_filters` '
'needs to be divisible by 24. Current value: %d' % penultimate_filters)
channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
filters = penultimate_filters // 24
if not skip_reduction:
x = Conv2D(
stem_block_filters, (3, 3),
strides=(2, 2),
padding='valid',
use_bias=False,
name='stem_conv1',
kernel_initializer='he_normal')(
img_input)
else:
x = Conv2D(
stem_block_filters, (3, 3),
strides=(1, 1),
padding='same',
use_bias=False,
name='stem_conv1',
kernel_initializer='he_normal')(
img_input)
x = BatchNormalization(
axis=channel_dim, momentum=0.9997, epsilon=1e-3, name='stem_bn1')(
x)
p = None
if not skip_reduction: # imagenet / mobile mode
x, p = _reduction_a_cell(
x, p, filters // (filter_multiplier**2), block_id='stem_1')
x, p = _reduction_a_cell(
x, p, filters // filter_multiplier, block_id='stem_2')
for i in range(num_blocks):
x, p = _normal_a_cell(x, p, filters, block_id='%d' % (i))
x, p0 = _reduction_a_cell(
x, p, filters * filter_multiplier, block_id='reduce_%d' % (num_blocks))
p = p0 if not skip_reduction else p
for i in range(num_blocks):
x, p = _normal_a_cell(
x, p, filters * filter_multiplier, block_id='%d' % (num_blocks + i + 1))
x, p0 = _reduction_a_cell(
x,
p,
filters * filter_multiplier**2,
block_id='reduce_%d' % (2 * num_blocks))
p = p0 if not skip_reduction else p
for i in range(num_blocks):
x, p = _normal_a_cell(
x,
p,
filters * filter_multiplier**2,
block_id='%d' % (2 * num_blocks + i + 1))
x = Activation('relu')(x)
if include_top:
x = GlobalAveragePooling2D()(x)
x = Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = layer_utils.get_source_inputs(input_tensor)
else:
inputs = img_input
model = Model(inputs, x, name='NASNet')
# load weights
if weights == 'imagenet':
if default_size == 224: # mobile version
if include_top:
weight_path = NASNET_MOBILE_WEIGHT_PATH
model_name = 'nasnet_mobile.h5'
else:
weight_path = NASNET_MOBILE_WEIGHT_PATH_NO_TOP
model_name = 'nasnet_mobile_no_top.h5'
weights_file = get_file(model_name, weight_path, cache_subdir='models')
model.load_weights(weights_file)
elif default_size == 331: # large version
if include_top:
weight_path = NASNET_LARGE_WEIGHT_PATH
model_name = 'nasnet_large.h5'
else:
weight_path = NASNET_LARGE_WEIGHT_PATH_NO_TOP
model_name = 'nasnet_large_no_top.h5'
weights_file = get_file(model_name, weight_path, cache_subdir='models')
model.load_weights(weights_file)
else:
raise ValueError('ImageNet weights can only be loaded with NASNetLarge'
' or NASNetMobile')
elif weights is not None:
model.load_weights(weights)
if old_data_format:
K.set_image_data_format(old_data_format)
return model
plt.ylim(Y.min(), Y.max())
plt.title('Visualize last layer')
plt.show()
if __name__ == "__main__":
X_train, X_test, y_train, y_test, class_weights, list = load_dataset_beginner(
dataSet, className)
model = load_model(modelName)
for index, layer in enumerate(model.layers):
print(layer)
print(index)
feature = Model(inputs=model.input, outputs=model.layers[71].output)
a = feature.predict(X_train)
b = feature.predict(X_test)
pred = np.vstack((a, b))
print(pred.shape)
a = y_train.argmax(axis=1)
b = y_test.argmax(axis=1)
# b = [x + 3 for x in b]
lable = np.concatenate((a, b))
print(lable.shape)
tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)
low_dim_embs = tsne.fit_transform(pred)
train_data, test_data, classes = ImageWoof.load_data()
train_data = train_data.map(preprocess).batch(batch_size).prefetch(1)
test_data = test_data.map(preprocess).batch(batch_size).prefetch(1)
base_model = xception.Xception(weights="imagenet",
include_top=False,
input_shape=(300, 300, 3))
model = GlobalAveragePooling2D()(base_model.output)
model = Dropout(0.5)(model)
model = Dense(512, activation='relu')(model)
model = Dropout(0.5)(model)
model = Dense(256, activation='relu')(model)
model = Dropout(0.5)(model)
output = Dense(10, activation='softmax')(model)
model = Model(base_model.input, output)
# model.summary()
for layer in base_model.layers:
layer.trainable = False
model.compile(optimizer="Adam",
loss="sparse_categorical_crossentropy",
metrics=["accuracy"])
history = model.fit(train_data, epochs=5, validation_data=test_data)
helpers.plot_history("After 5 epochs:", history)
plt.show()
for layer in base_model.layers[:20]:
layer.trainable = True
data.append(i)
counts[label] += 1
#Creating the Model by concatenating it with the pre-trained model
frozen_model = MobileNetV2(include_top=False, weights='imagenet')
x = frozen_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(1024, activation='relu')(x)
x = Dropout(0.2)(x)
x = Dense(512, activation='relu')(x)
x = Dropout(0.2)(x)
x = Dense(128, activation='relu')(x)
x = Dense(16, activation='relu')(x)
predictions = Dense(5, activation='softmax')(x)
model = Model(inputs=frozen_model.input, outputs=predictions)
#Freezing the convolutional layers
for layer in frozen_model.layers:
layer.trainable = False
#compile the model
model.compile(optimizer='Adam',
loss='poisson',
metrics=['accuracy'])
#Training the model along with hyperparameter definitions
epochs,bs = 30,100
x_data,y_data = np.empty([bs,224,224,3]),np.empty([bs,1])
# build our classifier model based on pre-trained ResNet50:
# 1. we don't include the top (fully connected) layers of ResNet50
# 2. we add a DropOut layer followed by a Dense (fully connected)
# layer which generates softmax class score for each class
# 3. we compile the final model using an Adam optimizer, with a
# low learning rate (since we are 'fine-tuning')
net = ResNet50(include_top=False,
weights='imagenet',
input_tensor=None,
input_shape=(CROP_LENGTH, CROP_LENGTH, 3))
x = net.output
x = Flatten()(x)
x = Dropout(0.5)(x)
output_layer = Dense(NUM_CLASSES, activation='softmax', name='softmax')(x)
net_final = Model(inputs=net.input, outputs=output_layer)
for layer in net_final.layers[:FREEZE_LAYERS]:
layer.trainable = False
for layer in net_final.layers[FREEZE_LAYERS:]:
layer.trainable = True
net_final.compile(optimizer=Adam(lr=1e-5),
loss='categorical_crossentropy',
metrics=['accuracy'])
print(net_final.summary())
# train the model
net_final.fit_generator(train_crops,
steps_per_epoch=train_batches.samples // BATCH_SIZE,
validation_data=valid_crops,
validation_steps=valid_batches.samples // BATCH_SIZE,
epochs=NUM_EPOCHS)
def DenseNet(blocks,
include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000):
"""Instantiates the DenseNet architecture.
Optionally loads weights pre-trained
on ImageNet. Note that when using TensorFlow,
for best performance you should set
`image_data_format='channels_last'` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with
TensorFlow, Theano, and CNTK. The data format
convention used by the model is the one
specified in your Keras config file.
Arguments:
blocks: numbers of building blocks for the four dense layers.
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(224, 224, 3)` (with `channels_last` data format)
or `(3, 224, 224)` (with `channels_first` data format).
It should have exactly 3 inputs channels.
pooling: optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
Returns:
A Keras model instance.
Raises:
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
input_shape = _obtain_input_shape(
input_shape,
default_size=224,
min_size=221,
data_format=K.image_data_format(),
require_flatten=include_top,
weights=weights)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
bn_axis = 3 if K.image_data_format() == 'channels_last' else 1
x = ZeroPadding2D(padding=((3, 3), (3, 3)))(img_input)
x = Conv2D(64, 7, strides=2, use_bias=False, name='conv1/conv')(x)
x = BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name='conv1/bn')(x)
x = Activation('relu', name='conv1/relu')(x)
x = ZeroPadding2D(padding=((1, 1), (1, 1)))(x)
x = MaxPooling2D(3, strides=2, name='pool1')(x)
x = dense_block(x, blocks[0], name='conv2')
x = transition_block(x, 0.5, name='pool2')
x = dense_block(x, blocks[1], name='conv3')
x = transition_block(x, 0.5, name='pool3')
x = dense_block(x, blocks[2], name='conv4')
x = transition_block(x, 0.5, name='pool4')
x = dense_block(x, blocks[3], name='conv5')
x = BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name='bn')(x)
if include_top:
x = GlobalAveragePooling2D(name='avg_pool')(x)
x = Dense(classes, activation='softmax', name='fc1000')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D(name='avg_pool')(x)
elif pooling == 'max':
x = GlobalMaxPooling2D(name='max_pool')(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = layer_utils.get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
if blocks == [6, 12, 24, 16]:
model = Model(inputs, x, name='densenet121')
elif blocks == [6, 12, 32, 32]:
model = Model(inputs, x, name='densenet169')
elif blocks == [6, 12, 48, 32]:
model = Model(inputs, x, name='densenet201')
else:
model = Model(inputs, x, name='densenet')
# Load weights.
if weights == 'imagenet':
if include_top:
if blocks == [6, 12, 24, 16]:
weights_path = get_file(
'densenet121_weights_tf_dim_ordering_tf_kernels.h5',
DENSENET121_WEIGHT_PATH,
cache_subdir='models',
file_hash='0962ca643bae20f9b6771cb844dca3b0')
elif blocks == [6, 12, 32, 32]:
weights_path = get_file(
'densenet169_weights_tf_dim_ordering_tf_kernels.h5',
DENSENET169_WEIGHT_PATH,
cache_subdir='models',
file_hash='bcf9965cf5064a5f9eb6d7dc69386f43')
elif blocks == [6, 12, 48, 32]:
weights_path = get_file(
'densenet201_weights_tf_dim_ordering_tf_kernels.h5',
DENSENET201_WEIGHT_PATH,
cache_subdir='models',
file_hash='7bb75edd58cb43163be7e0005fbe95ef')
else:
if blocks == [6, 12, 24, 16]:
weights_path = get_file(
'densenet121_weights_tf_dim_ordering_tf_kernels_notop.h5',
DENSENET121_WEIGHT_PATH_NO_TOP,
cache_subdir='models',
file_hash='4912a53fbd2a69346e7f2c0b5ec8c6d3')
elif blocks == [6, 12, 32, 32]:
weights_path = get_file(
'densenet169_weights_tf_dim_ordering_tf_kernels_notop.h5',
DENSENET169_WEIGHT_PATH_NO_TOP,
cache_subdir='models',
file_hash='50662582284e4cf834ce40ab4dfa58c6')
elif blocks == [6, 12, 48, 32]:
weights_path = get_file(
'densenet201_weights_tf_dim_ordering_tf_kernels_notop.h5',
DENSENET201_WEIGHT_PATH_NO_TOP,
cache_subdir='models',
file_hash='1c2de60ee40562448dbac34a0737e798')
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
return model
def DenseNet(model_input, depth=40, nb_dense_block=3, growth_rate=12, nb_filter=-1, nb_layers_per_block=-1,
bottleneck=False, reduction=0.0, dropout_rate=0.0, weight_decay=1e-4, subsample_initial_block=False,
include_top=True, weights=None, input_tensor=None,
classes=10, activation='softmax', model_path = ''):
'''Instantiate the DenseNet architecture,
optionally loading weights pre-trained
on CIFAR-10. Note that when using TensorFlow,
for best performance you should set
`image_data_format='channels_last'` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The dimension ordering
convention used by the model is the one
specified in your Keras config file.
# Arguments
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(32, 32, 3)` (with `channels_last` dim ordering)
or `(3, 32, 32)` (with `channels_first` dim ordering).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 8.
E.g. `(200, 200, 3)` would be one valid value.
depth: number or layers in the DenseNet
nb_dense_block: number of dense blocks to add to end (generally = 3)
growth_rate: number of filters to add per dense block
nb_filter: initial number of filters. -1 indicates initial
number of filters is 2 * growth_rate
nb_layers_per_block: number of layers in each dense block.
Can be a -1, positive integer or a list.
If -1, calculates nb_layer_per_block from the network depth.
If positive integer, a set number of layers per dense block.
If list, nb_layer is used as provided. Note that list size must
be (nb_dense_block + 1)
bottleneck: flag to add bottleneck blocks in between dense blocks
reduction: reduction factor of transition blocks.
Note : reduction value is inverted to compute compression.
dropout_rate: dropout rate
weight_decay: weight decay rate
subsample_initial_block: Set to True to subsample the initial convolution and
add a MaxPool2D before the dense blocks are added.
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization) or
'imagenet' (pre-training on ImageNet)..
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
activation: Type of activation at the top layer. Can be one of 'softmax' or 'sigmoid'.
Note that if sigmoid is used, classes must be 1.
# Returns
A Keras model instance.
'''
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as ImageNet with `include_top`'
' as true, `classes` should be 1000')
if activation not in ['softmax', 'sigmoid']:
raise ValueError('activation must be one of "softmax" or "sigmoid"')
if activation == 'sigmoid' and classes != 1:
raise ValueError('sigmoid activation can only be used when classes = 1')
# Determine proper input shape
img_input = model_input
x = __create_dense_net(classes, img_input, include_top, depth, nb_dense_block,
growth_rate, nb_filter, nb_layers_per_block, bottleneck, reduction,
dropout_rate, weight_decay, subsample_initial_block, activation)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
inputs = img_input
# Create model.
model = Model(inputs, x, name='densenet')
# load weights
if weights == 'imagenet':
weights_loaded = False
if (depth == 121) and (nb_dense_block == 4) and (growth_rate == 32) and (nb_filter == 64) and \
(bottleneck is True) and (reduction == 0.5) and (dropout_rate == 0.0) and (subsample_initial_block):
weights_path = model_path
model.load_weights(weights_path)
weights_loaded = True
if weights_loaded:
if K.backend() == 'theano':
convert_all_kernels_in_model(model)
if K.image_data_format() == 'channels_first' and K.backend() == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image data format convention '
'(`image_data_format="channels_first"`). '
'For best performance, set '
'`image_data_format="channels_last"` in '
'your Keras config '
'at ~/.keras/keras.json.')
elif (weights == "trained"):
weights_path = model_path
model.load_weights(weights_path)
return model
def TrainLSTM(self):
"""
Train LSTM
"""
modelName = self.modelPath + 'my_model_weights.h5'
kerasModel = self.KerasModel()
kerasModel.load_weights(modelName)
kerasModelExtFea = Model(
inputs=kerasModel.input,
outputs=kerasModel.get_layer('fc_feature').output)
self.logger.info(kerasModelExtFea.summary())
modelLSTM = self.KerasModelLSTM()
if self.taskList == [0, 1]:
modelLSTM.compile(optimizer=tf.keras.optimizers.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0),
loss={
'out_task0': self.LossCCEPieceWise,
'out_task1': self.LossCCEPieceWise
},
loss_weights={
'out_task0': 1,
'out_task1': 1
},
metrics=['accuracy'])
elif self.taskList == [0]:
modelLSTM.compile(optimizer=tf.keras.optimizers.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0),
loss={'out_task0': self.LossCCEPieceWise},
metrics=['accuracy'])
feaTrain, labelTrain, feaTest, labelTest = self._ExtractFeature(
kerasModelExtFea)
if feaTrain is None:
self.logger.error('No image is found in {}'.format(
self.trainDataDir))
valAccList = list()
for trainIter in range(self.trainIter):
trainHistory = modelLSTM.fit(x=feaTrain,
y=labelTrain,
validation_data=(feaTest, labelTest),
batch_size=self.netLSTMBatchSize,
shuffle=1,
epochs=1)
trainAcc, valAcc = self.GetAcc(trainHistory)
valAccList.append(valAcc + trainAcc)
modelLSTM.save_weights(self.modelPath + 'my_model_weights_LSTM' +
np.str(trainIter) + '.h5')
logging.info(
'NetworkLSTM: Iter {}....{}: train_acc is {} and val_acc is {}'
.format(trainIter, self.trainIter, trainAcc, valAcc))
indexAccValMax = valAccList.index(np.max(valAccList))
logging.info(
'Use model from the {}th iteration'.format(indexAccValMax))
srcModelName = self.modelPath + 'my_model_weights_LSTM' + np.str(
indexAccValMax) + '.h5'
dstModelName = self.modelPath + 'my_model_weights_LSTM' + '.h5'
shutil.copyfile(srcModelName, dstModelName)
self.progressReport.SendTrainProgress(100)
kernel_size=kernel,
activation=activation)(model)
model = MaxPooling2D((2, 2))(model)
model = Conv2D(filters=n_filters * 2,
kernel_size=kernel,
activation=activation)(model)
model = MaxPooling2D((2, 2))(model)
model = Conv2D(filters=n_filters * 2,
kernel_size=kernel,
activation=activation,
padding='same')(model)
model = MaxPooling2D((2, 2))(model)
model = Flatten()(model)
model = Dense(dense_neurons, activation=activation)(model)
model = Dense(dense_neurons / 2, activation='tanh')(model)
output = Dense(n_classes, activation="softmax")(model)
cnn_model = Model(input_layer, output)
cnn_model.compile(loss="sparse_categorical_crossentropy",
optimizer=keras.optimizers.SGD(lr=learning_rate,
momentum=0.9),
metrics=["accuracy"])
cnn_model.summary()
history = cnn_model.fit(train_data, epochs=12, validation_data=test_data)
def layer_test(layer_cls, kwargs={}, input_shape=None, input_dtype=None,
input_data=None, expected_output=None,
expected_output_dtype=None, fixed_batch_size=False, supports_masking=False):
# generate input data
if input_data is None:
if not input_shape:
raise AssertionError()
if not input_dtype:
input_dtype = K.floatx()
input_data_shape = list(input_shape)
for i, e in enumerate(input_data_shape):
if e is None:
input_data_shape[i] = np.random.randint(1, 4)
input_mask = []
if all(isinstance(e, tuple) for e in input_data_shape):
input_data = []
for e in input_data_shape:
input_data.append(
(10 * np.random.random(e)).astype(input_dtype))
if supports_masking:
a = np.full(e[:2], False)
a[:, :e[1] // 2] = True
input_mask.append(a)
else:
input_data = (10 * np.random.random(input_data_shape))
input_data = input_data.astype(input_dtype)
if supports_masking:
a = np.full(input_data_shape[:2], False)
a[:, :input_data_shape[1] // 2] = True
print(a)
print(a.shape)
input_mask.append(a)
else:
if input_shape is None:
input_shape = input_data.shape
if input_dtype is None:
input_dtype = input_data.dtype
if expected_output_dtype is None:
expected_output_dtype = input_dtype
# instantiation
layer = layer_cls(**kwargs)
# test get_weights , set_weights at layer level
weights = layer.get_weights()
layer.set_weights(weights)
try:
expected_output_shape = layer.compute_output_shape(input_shape)
except Exception:
expected_output_shape = layer._compute_output_shape(input_shape)
# test in functional API
if isinstance(input_shape, list):
if fixed_batch_size:
x = [Input(batch_shape=e, dtype=input_dtype) for e in input_shape]
if supports_masking:
mask = [Input(batch_shape=e[0:2], dtype=bool)
for e in input_shape]
else:
x = [Input(shape=e[1:], dtype=input_dtype) for e in input_shape]
if supports_masking:
mask = [Input(shape=(e[1],), dtype=bool) for e in input_shape]
else:
if fixed_batch_size:
x = Input(batch_shape=input_shape, dtype=input_dtype)
if supports_masking:
mask = Input(batch_shape=input_shape[0:2], dtype=bool)
else:
x = Input(shape=input_shape[1:], dtype=input_dtype)
if supports_masking:
mask = Input(shape=(input_shape[1],), dtype=bool)
if supports_masking:
y = layer(Masking()(x), mask=mask)
else:
y = layer(x)
if not (K.dtype(y) == expected_output_dtype):
raise AssertionError()
# check with the functional API
if supports_masking:
model = Model([x, mask], y)
actual_output = model.predict([input_data, input_mask[0]])
else:
model = Model(x, y)
actual_output = model.predict(input_data)
actual_output_shape = actual_output.shape
for expected_dim, actual_dim in zip(expected_output_shape,
actual_output_shape):
if expected_dim is not None:
if not (expected_dim == actual_dim):
raise AssertionError("expected_shape", expected_output_shape, "actual_shape", actual_output_shape)
if expected_output is not None:
assert_allclose(actual_output, expected_output, rtol=1e-3)
# test serialization, weight setting at model level
model_config = model.get_config()
recovered_model = model.__class__.from_config(model_config)
if model.weights:
weights = model.get_weights()
recovered_model.set_weights(weights)
_output = recovered_model.predict(input_data)
assert_allclose(_output, actual_output, rtol=1e-3)
# test training mode (e.g. useful when the layer has a
# different behavior at training and testing time).
if has_arg(layer.call, 'training'):
model.compile('rmsprop', 'mse')
model.train_on_batch(input_data, actual_output)
# test instantiation from layer config
layer_config = layer.get_config()
layer_config['batch_input_shape'] = input_shape
layer = layer.__class__.from_config(layer_config)
# for further checks in the caller function
return actual_output
def InceptionV3(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000):
"""Instantiates the Inception v3 architecture.
Optionally loads weights pre-trained
on ImageNet. Note that when using TensorFlow,
for best performance you should set
`image_data_format='channels_last'` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The data format
convention used by the model is the one
specified in your Keras config file.
Note that the default input image size for this model is 299x299.
Arguments:
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(299, 299, 3)` (with `channels_last` data format)
or `(3, 299, 299)` (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 139.
E.g. `(150, 150, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
Returns:
A Keras model instance.
Raises:
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
input_shape = _obtain_input_shape(
input_shape,
default_size=299,
min_size=139,
data_format=K.image_data_format(),
require_flatten=False,
weights=weights)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if K.image_data_format() == 'channels_first':
channel_axis = 1
else:
channel_axis = 3
x = conv2d_bn(img_input, 32, 3, 3, strides=(2, 2), padding='valid')
x = conv2d_bn(x, 32, 3, 3, padding='valid')
x = conv2d_bn(x, 64, 3, 3)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv2d_bn(x, 80, 1, 1, padding='valid')
x = conv2d_bn(x, 192, 3, 3, padding='valid')
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
# mixed 0, 1, 2: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 32, 1, 1)
x = layers.concatenate(
[branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed0')
# mixed 1: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1, 1)
x = layers.concatenate(
[branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed1')
# mixed 2: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1, 1)
x = layers.concatenate(
[branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed2')
# mixed 3: 17 x 17 x 768
branch3x3 = conv2d_bn(x, 384, 3, 3, strides=(2, 2), padding='valid')
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(
branch3x3dbl, 96, 3, 3, strides=(2, 2), padding='valid')
branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = layers.concatenate(
[branch3x3, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed3')
# mixed 4: 17 x 17 x 768
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 128, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 128, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 128, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed4')
# mixed 5, 6: 17 x 17 x 768
for i in range(2):
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 160, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 160, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 160, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed' + str(5 + i))
# mixed 7: 17 x 17 x 768
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 192, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 192, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed7')
# mixed 8: 8 x 8 x 1280
branch3x3 = conv2d_bn(x, 192, 1, 1)
branch3x3 = conv2d_bn(branch3x3, 320, 3, 3, strides=(2, 2), padding='valid')
branch7x7x3 = conv2d_bn(x, 192, 1, 1)
branch7x7x3 = conv2d_bn(branch7x7x3, 192, 1, 7)
branch7x7x3 = conv2d_bn(branch7x7x3, 192, 7, 1)
branch7x7x3 = conv2d_bn(
branch7x7x3, 192, 3, 3, strides=(2, 2), padding='valid')
branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = layers.concatenate(
[branch3x3, branch7x7x3, branch_pool], axis=channel_axis, name='mixed8')
# mixed 9: 8 x 8 x 2048
for i in range(2):
branch1x1 = conv2d_bn(x, 320, 1, 1)
branch3x3 = conv2d_bn(x, 384, 1, 1)
branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3)
branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1)
branch3x3 = layers.concatenate(
[branch3x3_1, branch3x3_2], axis=channel_axis, name='mixed9_' + str(i))
branch3x3dbl = conv2d_bn(x, 448, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3)
branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3)
branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1)
branch3x3dbl = layers.concatenate(
[branch3x3dbl_1, branch3x3dbl_2], axis=channel_axis)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch3x3, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed' + str(9 + i))
if include_top:
# Classification block
x = GlobalAveragePooling2D(name='avg_pool')(x)
x = Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = layer_utils.get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='inception_v3')
# load weights
if weights == 'imagenet':
if include_top:
weights_path = get_file(
'inception_v3_weights_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH,
cache_subdir='models',
file_hash='9a0d58056eeedaa3f26cb7ebd46da564')
else:
weights_path = get_file(
'inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
file_hash='bcbd6486424b2319ff4ef7d526e38f63')
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
return model
class YOLO(object):
def __init__(self, backend, input_sizeW, input_sizeH, labels,
max_box_per_image, anchors):
self.input_sizeW = input_sizeW
self.input_sizeH = input_sizeH
# self.input_size = (input_sizeW,input_sizeH)
self.labels = list(labels)
self.nb_class = len(self.labels)
self.nb_box = len(anchors) // 2
self.class_wt = np.ones(self.nb_class, dtype='float32')
self.anchors = anchors
self.max_box_per_image = max_box_per_image
##########################
# Make the model
##########################
# make the feature extractor layers
input_image = Input(shape=(self.input_sizeW, self.input_sizeH, 3))
self.true_boxes = Input(shape=(1, 1, 1, max_box_per_image, 4))
if backend == 'Inception3':
self.feature_extractor = Inception3Feature(self.input_sizeW,
self.input_sizeH)
elif backend == 'SqueezeNet':
self.feature_extractor = SqueezeNetFeature(self.input_sizeW,
self.input_sizeH)
elif backend == 'MobileNet':
self.feature_extractor = MobileNetFeature(self.input_sizeW,
self.input_sizeH)
elif backend == 'Full Yolo':
self.feature_extractor = FullYoloFeature(input_sizeW, input_sizeH)
elif backend == 'Tiny Yolo':
self.feature_extractor = TinyYoloFeature(self.input_sizeW,
self.input_sizeH)
elif backend == 'VGG16':
self.feature_extractor = VGG16Feature(self.input_sizeW,
self.input_sizeH)
elif backend == 'ResNet50':
self.feature_extractor = ResNet50Feature(self.input_sizeW,
self.input_sizeH)
else:
raise Exception(
'Architecture not supported! Only support Full Yolo, Tiny Yolo, MobileNet, SqueezeNet, VGG16, ResNet50, and Inception3 at the moment!'
)
print(self.feature_extractor.get_output_shape())
self.grid_h, self.grid_w = self.feature_extractor.get_output_shape()
features = self.feature_extractor.extract(input_image)
# make the object detection layer
output = Conv2D(self.nb_box * (4 + 1 + self.nb_class), (1, 1),
strides=(1, 1),
padding='same',
name='DetectionLayer',
kernel_initializer='lecun_normal')(features)
output = Reshape((self.grid_h, self.grid_w, self.nb_box,
4 + 1 + self.nb_class))(output)
output = Lambda(lambda args: args[0])([output, self.true_boxes])
self.model = Model([input_image, self.true_boxes], output)
# initialize the weights of the detection layer
layer = self.model.layers[-4]
weights = layer.get_weights()
new_kernel = np.random.normal(size=weights[0].shape) / (self.grid_h *
self.grid_w)
new_bias = np.random.normal(size=weights[1].shape) / (self.grid_h *
self.grid_w)
layer.set_weights([new_kernel, new_bias])
# print a summary of the whole model
self.model.summary()
def custom_loss(self, y_true, y_pred):
mask_shape = tf.shape(y_true)[:4]
cell_x = tf.to_float(
tf.reshape(tf.tile(tf.range(self.grid_w), [self.grid_h]),
(1, self.grid_h, self.grid_w, 1, 1)))
cell_y = tf.transpose(cell_x, (0, 2, 1, 3, 4))
cell_grid = tf.tile(tf.concat([cell_x, cell_y], -1),
[self.batch_size, 1, 1, self.nb_box, 1])
coord_mask = tf.zeros(mask_shape)
conf_mask = tf.zeros(mask_shape)
class_mask = tf.zeros(mask_shape)
seen = tf.Variable(0.)
total_recall = tf.Variable(0.)
"""
Adjust prediction
"""
### adjust x and y
pred_box_xy = tf.sigmoid(y_pred[..., :2]) + cell_grid
### adjust w and h
pred_box_wh = tf.exp(y_pred[..., 2:4]) * np.reshape(
self.anchors, [1, 1, 1, self.nb_box, 2])
### adjust confidence
pred_box_conf = tf.sigmoid(y_pred[..., 4])
### adjust class probabilities
pred_box_class = y_pred[..., 5:]
"""
Adjust ground truth
"""
### adjust x and y
true_box_xy = y_true[...,
0:2] # relative position to the containing cell
### adjust w and h
true_box_wh = y_true[
..., 2:4] # number of cells accross, horizontally and vertically
### adjust confidence
true_wh_half = true_box_wh / 2.
true_mins = true_box_xy - true_wh_half
true_maxes = true_box_xy + true_wh_half
pred_wh_half = pred_box_wh / 2.
pred_mins = pred_box_xy - pred_wh_half
pred_maxes = pred_box_xy + pred_wh_half
intersect_mins = tf.maximum(pred_mins, true_mins)
intersect_maxes = tf.minimum(pred_maxes, true_maxes)
intersect_wh = tf.maximum(intersect_maxes - intersect_mins, 0.)
intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]
true_areas = true_box_wh[..., 0] * true_box_wh[..., 1]
pred_areas = pred_box_wh[..., 0] * pred_box_wh[..., 1]
union_areas = pred_areas + true_areas - intersect_areas
iou_scores = tf.truediv(intersect_areas, union_areas)
true_box_conf = iou_scores * y_true[..., 4]
### adjust class probabilities
true_box_class = tf.argmax(y_true[..., 5:], -1)
"""
Determine the masks
"""
### coordinate mask: simply the position of the ground truth boxes (the predictors)
coord_mask = tf.expand_dims(y_true[..., 4], axis=-1) * self.coord_scale
### confidence mask: penelize predictors + penalize boxes with low IOU
# penalize the confidence of the boxes, which have IOU with some ground truth box < 0.6
true_xy = self.true_boxes[..., 0:2]
true_wh = self.true_boxes[..., 2:4]
true_wh_half = true_wh / 2.
true_mins = true_xy - true_wh_half
true_maxes = true_xy + true_wh_half
pred_xy = tf.expand_dims(pred_box_xy, 4)
pred_wh = tf.expand_dims(pred_box_wh, 4)
pred_wh_half = pred_wh / 2.
pred_mins = pred_xy - pred_wh_half
pred_maxes = pred_xy + pred_wh_half
intersect_mins = tf.maximum(pred_mins, true_mins)
intersect_maxes = tf.minimum(pred_maxes, true_maxes)
intersect_wh = tf.maximum(intersect_maxes - intersect_mins, 0.)
intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]
true_areas = true_wh[..., 0] * true_wh[..., 1]
pred_areas = pred_wh[..., 0] * pred_wh[..., 1]
union_areas = pred_areas + true_areas - intersect_areas
iou_scores = tf.truediv(intersect_areas, union_areas)
best_ious = tf.reduce_max(iou_scores, axis=4)
conf_mask = conf_mask + tf.to_float(
best_ious < 0.6) * (1 - y_true[..., 4]) * self.no_object_scale
# penalize the confidence of the boxes, which are reponsible for corresponding ground truth box
conf_mask = conf_mask + y_true[..., 4] * self.object_scale
### class mask: simply the position of the ground truth boxes (the predictors)
class_mask = y_true[..., 4] * tf.gather(
self.class_wt, true_box_class) * self.class_scale
"""
Warm-up training
"""
no_boxes_mask = tf.to_float(coord_mask < self.coord_scale / 2.)
seen = tf.assign_add(seen, 1.)
true_box_xy, true_box_wh, coord_mask = tf.cond(tf.less(seen, self.warmup_batches+1),
lambda: [true_box_xy + (0.5 + cell_grid) * no_boxes_mask,
true_box_wh + tf.ones_like(true_box_wh) * \
np.reshape(self.anchors, [1,1,1,self.nb_box,2]) * \
no_boxes_mask,
tf.ones_like(coord_mask)],
lambda: [true_box_xy,
true_box_wh,
coord_mask])
"""
Finalize the loss
"""
nb_coord_box = tf.reduce_sum(tf.to_float(coord_mask > 0.0))
nb_conf_box = tf.reduce_sum(tf.to_float(conf_mask > 0.0))
nb_class_box = tf.reduce_sum(tf.to_float(class_mask > 0.0))
loss_xy = tf.reduce_sum(
tf.square(true_box_xy - pred_box_xy) *
coord_mask) / (nb_coord_box + 1e-6) / 2.
loss_wh = tf.reduce_sum(
tf.square(true_box_wh - pred_box_wh) *
coord_mask) / (nb_coord_box + 1e-6) / 2.
loss_conf = tf.reduce_sum(
tf.square(true_box_conf - pred_box_conf) *
conf_mask) / (nb_conf_box + 1e-6) / 2.
loss_class = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=true_box_class, logits=pred_box_class)
loss_class = tf.reduce_sum(
loss_class * class_mask) / (nb_class_box + 1e-6)
loss = tf.cond(tf.less(seen, self.warmup_batches + 1),
lambda: loss_xy + loss_wh + loss_conf + loss_class + 10,
lambda: loss_xy + loss_wh + loss_conf + loss_class)
if self.debug:
nb_true_box = tf.reduce_sum(y_true[..., 4])
nb_pred_box = tf.reduce_sum(
tf.to_float(true_box_conf > 0.5) *
tf.to_float(pred_box_conf > 0.3))
current_recall = nb_pred_box / (nb_true_box + 1e-6)
total_recall = tf.assign_add(total_recall, current_recall)
loss = tf.Print(loss, [loss_xy],
message='Loss XY \t',
summarize=1000)
loss = tf.Print(loss, [loss_wh],
message='Loss WH \t',
summarize=1000)
loss = tf.Print(loss, [loss_conf],
message='Loss Conf \t',
summarize=1000)
loss = tf.Print(loss, [loss_class],
message='Loss Class \t',
summarize=1000)
loss = tf.Print(loss, [loss],
message='Total Loss \t',
summarize=1000)
loss = tf.Print(loss, [current_recall],
message='Current Recall \t',
summarize=1000)
loss = tf.Print(loss, [total_recall / seen],
message='Average Recall \t',
summarize=1000)
return loss
def load_weights(self, weight_path):
self.model.load_weights(weight_path)
def train(
self,
train_imgs, # the list of images to train the model
valid_imgs, # the list of images used to validate the model
train_times, # the number of time to repeat the training set, often used for small datasets
valid_times, # the number of times to repeat the validation set, often used for small datasets
nb_epochs, # number of epoches
learning_rate, # the learning rate
batch_size, # the size of the batch
warmup_epochs, # number of initial batches to let the model familiarize with the new dataset
object_scale,
no_object_scale,
coord_scale,
class_scale,
saved_weights_name='best_weights.h5',
debug=False):
self.batch_size = batch_size
self.object_scale = object_scale
self.no_object_scale = no_object_scale
self.coord_scale = coord_scale
self.class_scale = class_scale
self.debug = debug
############################################
# Make train and validation generators
############################################
generator_config = {
'IMAGE_H': self.input_sizeH,
'IMAGE_W': self.input_sizeW,
'GRID_H': self.grid_h,
'GRID_W': self.grid_w,
'BOX': self.nb_box,
'LABELS': self.labels,
'CLASS': len(self.labels),
'ANCHORS': self.anchors,
'BATCH_SIZE': self.batch_size,
'TRUE_BOX_BUFFER': self.max_box_per_image,
}
train_generator = BatchGenerator(train_imgs,
generator_config,
norm=self.feature_extractor.normalize)
valid_generator = BatchGenerator(valid_imgs,
generator_config,
norm=self.feature_extractor.normalize,
jitter=False)
self.warmup_batches = warmup_epochs * (
train_times * len(train_generator) +
valid_times * len(valid_generator))
############################################
# Compile the model
############################################
optimizer = Adam(lr=learning_rate,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
self.model.compile(loss=self.custom_loss, optimizer=optimizer)
############################################
# Make a few callbacks
############################################
early_stop = EarlyStopping(monitor='val_loss',
min_delta=0.001,
patience=3,
mode='min',
verbose=1)
checkpoint = ModelCheckpoint(saved_weights_name,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min',
period=1)
tensorboard = TensorBoard(
log_dir=os.path.expanduser('~/logs/'),
histogram_freq=0,
#write_batch_performance=True,
write_graph=True,
write_images=False)
############################################
# Start the training process
############################################
self.model.fit_generator(
generator=train_generator,
steps_per_epoch=len(train_generator) * train_times,
epochs=warmup_epochs + nb_epochs,
verbose=1,
validation_data=valid_generator,
validation_steps=len(valid_generator) * valid_times,
callbacks=[early_stop, checkpoint, tensorboard],
workers=3,
max_queue_size=8)
############################################
# Compute mAP on the validation set
############################################
average_precisions = self.evaluate(valid_generator)
# print evaluation
for label, average_precision in average_precisions.items():
print(self.labels[label], '{:.4f}'.format(average_precision))
print('mAP: {:.4f}'.format(
sum(average_precisions.values()) / len(average_precisions)))
def evaluate(self,
generator,
iou_threshold=0.3,
score_threshold=0.3,
max_detections=100,
save_path=None):
""" Evaluate a given dataset using a given model.
code originally from https://github.com/fizyr/keras-retinanet
# Arguments
generator : The generator that represents the dataset to evaluate.
model : The model to evaluate.
iou_threshold : The threshold used to consider when a detection is positive or negative.
score_threshold : The score confidence threshold to use for detections.
max_detections : The maximum number of detections to use per image.
save_path : The path to save images with visualized detections to.
# Returns
A dict mapping class names to mAP scores.
"""
# gather all detections and annotations
all_detections = [[None for i in range(generator.num_classes())]
for j in range(generator.size())]
all_annotations = [[None for i in range(generator.num_classes())]
for j in range(generator.size())]
for i in range(generator.size()):
raw_image = generator.load_image(i)
raw_height, raw_width, raw_channels = raw_image.shape
# make the boxes and the labels
pred_boxes = self.predict(raw_image)
score = np.array([box.score for box in pred_boxes])
pred_labels = np.array([box.label for box in pred_boxes])
if len(pred_boxes) > 0:
pred_boxes = np.array([[
box.xmin * raw_width, box.ymin * raw_height,
box.xmax * raw_width, box.ymax * raw_height, box.score
] for box in pred_boxes])
else:
pred_boxes = np.array([[]])
# sort the boxes and the labels according to scores
score_sort = np.argsort(-score)
pred_labels = pred_labels[score_sort]
pred_boxes = pred_boxes[score_sort]
# copy detections to all_detections
for label in range(generator.num_classes()):
all_detections[i][label] = pred_boxes[pred_labels == label, :]
annotations = generator.load_annotation(i)
# copy detections to all_annotations
for label in range(generator.num_classes()):
all_annotations[i][label] = annotations[annotations[:, 4] ==
label, :4].copy()
# compute mAP by comparing all detections and all annotations
average_precisions = {}
for label in range(generator.num_classes()):
false_positives = np.zeros((0, ))
true_positives = np.zeros((0, ))
scores = np.zeros((0, ))
num_annotations = 0.0
for i in range(generator.size()):
detections = all_detections[i][label]
annotations = all_annotations[i][label]
num_annotations += annotations.shape[0]
detected_annotations = []
for d in detections:
scores = np.append(scores, d[4])
if annotations.shape[0] == 0:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
continue
overlaps = compute_overlap(np.expand_dims(d, axis=0),
annotations)
assigned_annotation = np.argmax(overlaps, axis=1)
max_overlap = overlaps[0, assigned_annotation]
if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:
false_positives = np.append(false_positives, 0)
true_positives = np.append(true_positives, 1)
detected_annotations.append(assigned_annotation)
else:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
# no annotations -> AP for this class is 0 (is this correct?)
if num_annotations == 0:
average_precisions[label] = 0
continue
# sort by score
indices = np.argsort(-scores)
false_positives = false_positives[indices]
true_positives = true_positives[indices]
# compute false positives and true positives
false_positives = np.cumsum(false_positives)
true_positives = np.cumsum(true_positives)
# compute recall and precision
recall = true_positives / num_annotations
precision = true_positives / np.maximum(
true_positives + false_positives,
np.finfo(np.float64).eps)
# compute average precision
average_precision = compute_ap(recall, precision)
average_precisions[label] = average_precision
return average_precisions
def predict(self, image):
image_h, image_w, _ = image.shape
image = cv2.resize(image, (self.input_sizeW, self.input_sizeH))
image = self.feature_extractor.normalize(image)
input_image = image[:, :, ::-1]
input_image = np.expand_dims(input_image, 0)
dummy_array = np.zeros((1, 1, 1, 1, self.max_box_per_image, 4))
netout = self.model.predict([input_image, dummy_array])[0]
boxes = decode_netout(netout, self.anchors, self.nb_class)
return boxes
def __init__(self, backend, input_sizeW, input_sizeH, labels,
max_box_per_image, anchors):
self.input_sizeW = input_sizeW
self.input_sizeH = input_sizeH
# self.input_size = (input_sizeW,input_sizeH)
self.labels = list(labels)
self.nb_class = len(self.labels)
self.nb_box = len(anchors) // 2
self.class_wt = np.ones(self.nb_class, dtype='float32')
self.anchors = anchors
self.max_box_per_image = max_box_per_image
##########################
# Make the model
##########################
# make the feature extractor layers
input_image = Input(shape=(self.input_sizeW, self.input_sizeH, 3))
self.true_boxes = Input(shape=(1, 1, 1, max_box_per_image, 4))
if backend == 'Inception3':
self.feature_extractor = Inception3Feature(self.input_sizeW,
self.input_sizeH)
elif backend == 'SqueezeNet':
self.feature_extractor = SqueezeNetFeature(self.input_sizeW,
self.input_sizeH)
elif backend == 'MobileNet':
self.feature_extractor = MobileNetFeature(self.input_sizeW,
self.input_sizeH)
elif backend == 'Full Yolo':
self.feature_extractor = FullYoloFeature(input_sizeW, input_sizeH)
elif backend == 'Tiny Yolo':
self.feature_extractor = TinyYoloFeature(self.input_sizeW,
self.input_sizeH)
elif backend == 'VGG16':
self.feature_extractor = VGG16Feature(self.input_sizeW,
self.input_sizeH)
elif backend == 'ResNet50':
self.feature_extractor = ResNet50Feature(self.input_sizeW,
self.input_sizeH)
else:
raise Exception(
'Architecture not supported! Only support Full Yolo, Tiny Yolo, MobileNet, SqueezeNet, VGG16, ResNet50, and Inception3 at the moment!'
)
print(self.feature_extractor.get_output_shape())
self.grid_h, self.grid_w = self.feature_extractor.get_output_shape()
features = self.feature_extractor.extract(input_image)
# make the object detection layer
output = Conv2D(self.nb_box * (4 + 1 + self.nb_class), (1, 1),
strides=(1, 1),
padding='same',
name='DetectionLayer',
kernel_initializer='lecun_normal')(features)
output = Reshape((self.grid_h, self.grid_w, self.nb_box,
4 + 1 + self.nb_class))(output)
output = Lambda(lambda args: args[0])([output, self.true_boxes])
self.model = Model([input_image, self.true_boxes], output)
# initialize the weights of the detection layer
layer = self.model.layers[-4]
weights = layer.get_weights()
new_kernel = np.random.normal(size=weights[0].shape) / (self.grid_h *
self.grid_w)
new_bias = np.random.normal(size=weights[1].shape) / (self.grid_h *
self.grid_w)
layer.set_weights([new_kernel, new_bias])
# print a summary of the whole model
self.model.summary()
class SemiAdversarial:
def __init__(self, autoencoder, gender_classifier, face_matcher):
self.autoencoder = autoencoder
self.models_dir = '../model'
input_shape=[224, 224]
# Inputs
img = Input(shape=(224, 224, 1), name="Original_img")
proto_sm = Input(shape=(224, 224, 3), name="SameGender_prototype")
proto_op = Input(shape=(224, 224, 3), name="OppositeGender_prototype")
label = Input(shape=(2,), name="Label")
label_reversed = Input(shape=(2,), name="Label_reversed")
vgg_ground = Input(shape=(2622,), name="vgg_ground")
inputs = [img, proto_sm, proto_op, label, label_reversed, vgg_ground]
#Here we obtain perturbed images from the autoencodet
img_perturbed = self.autoencoder ([inputs[0],inputs[1],inputs[2]])
#Here we obtain the prediction for gender according the same prototype
GP_sm = gender_classifier (img_perturbed[0])
#Gender prediction opposite gender
GP_op = gender_classifier (img_perturbed[1])
#Here we obtain the Face-Martcher feature vector
FM = face_matcher (img_perturbed[0])
#The total loss: in this case is the last layer of the SAN
y_model = TotalLoss(name="Total_loss_custom")([GP_sm, GP_op, FM, inputs[3], inputs[4], inputs[5]])
self.model = Model(inputs=inputs, outputs=y_model)
print(self.model.summary())
#plot_model(self.model, to_file='SemiAdversarialModelCustomLoss.png')
#da creare loss custom
self.model.compile(optimizer='adam', loss=None)
def savemodel(self, name):
filename = pathlib.Path(self.models_dir, name + '.h5')
try:
self.model.save(filename)
print("\nModel saved successfully on file %s\n" % filename)
except OSError:
print("\nUnable to save the model on file %s\n Please check the path and/or permissions" % filename)
def loadmodel(self, name):
filename = pathlib.Path(self.models_dir, name + '.h5')
try:
self.model = Model.load_model(filename)
print("\nModel loaded successclass_weightfully from file %s\n" % filename)
except OSError:
print("\nModel file %s not found!!!\n" % filename)
self.model = None
def DCN(
feature_dim_dict,
embedding_size='auto',
cross_num=2,
hidden_size=(
128,
128,
),
l2_reg_embedding=1e-5,
l2_reg_cross=1e-5,
l2_reg_deep=0,
init_std=0.0001,
seed=1024,
keep_prob=1,
use_bn=False,
activation='relu',
final_activation='sigmoid',
):
"""Instantiates the Deep&Cross Network architecture.
:param feature_dim_dict: dict,to indicate sparse field and dense field like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_4','field_5']}
:param embedding_size: positive int or str,sparse feature embedding_size.If set to "auto",it will be 6*pow(cardinality,025)
:param cross_num: positive integet,cross layer number
:param hidden_size: list,list of positive integer or empty list, the layer number and units in each layer of deep net
:param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
:param l2_reg_cross: float. L2 regularizer strength applied to cross net
:param l2_reg_deep: float. L2 regularizer strength applied to deep net
:param init_std: float,to use as the initialize std of embedding vector
:param seed: integer ,to use as random seed.
:param keep_prob: float in (0,1]. keep_prob used in deep net
:param use_bn: bool. Whether use BatchNormalization before activation or not.in deep net
:param activation: Activation function to use in deep net
:param final_activation: str,output activation,usually ``'sigmoid'`` or ``'linear'``
:return: A Keras model instance.
"""
if len(hidden_size) == 0 and cross_num == 0:
raise ValueError("Either hidden_layer or cross layer must > 0")
if not isinstance(
feature_dim_dict, dict
) or "sparse" not in feature_dim_dict or "dense" not in feature_dim_dict:
raise ValueError(
"feature_dim must be a dict like {'sparse':{'field_1':4,'field_2':3,'field_3':2},'dense':['field_5',]}"
)
sparse_input, dense_input = get_input(
feature_dim_dict,
None,
)
sparse_embedding = get_embeddings(feature_dim_dict, embedding_size,
init_std, seed, l2_reg_embedding)
embed_list = [
sparse_embedding[i](sparse_input[i]) for i in range(len(sparse_input))
]
deep_input = Flatten()(
Concatenate()(embed_list) if len(embed_list) > 1 else embed_list[0])
if len(dense_input) > 0:
if len(dense_input) == 1:
continuous_list = dense_input[0]
else:
continuous_list = Concatenate()(dense_input)
deep_input = Concatenate()([deep_input, continuous_list])
if len(hidden_size) > 0 and cross_num > 0: # Deep & Cross
deep_out = MLP(hidden_size, activation, l2_reg_deep, keep_prob, use_bn,
seed)(deep_input)
cross_out = CrossNet(cross_num, l2_reg=l2_reg_cross)(deep_input)
stack_out = Concatenate()([cross_out, deep_out])
final_logit = Dense(1, use_bias=False, activation=None)(stack_out)
elif len(hidden_size) > 0: # Only Deep
deep_out = MLP(hidden_size, activation, l2_reg_deep, keep_prob, use_bn,
seed)(deep_input)
final_logit = Dense(1, use_bias=False, activation=None)(deep_out)
elif cross_num > 0: # Only Cross
cross_out = CrossNet(cross_num, l2_reg=l2_reg_cross)(deep_input)
final_logit = Dense(1, use_bias=False, activation=None)(cross_out)
else: # Error
raise NotImplementedError
# Activation(self.final_activation)(final_logit)
output = PredictionLayer(final_activation)(final_logit)
model = Model(inputs=sparse_input + dense_input, outputs=output)
return model
def ResNet50(include_top=True, non_top_pooling=None, model_input=None, num_classes=1000, weights='imagenet', model_path="", initial_num_classes=None, transfer_with_full_training = True):
layers = [3,4,6,3]
channel_depths = [256, 512, 1024, 2048]
input_object = model_input
output = Conv2D(64, kernel_size=7, strides=2, padding="same")(input_object)
output = BatchNormalization()(output)
output = Activation("relu")(output)
output = MaxPool2D(pool_size=(3,3), strides=(2,2))(output)
output = resnet_first_block_first_module(output, channel_depths[0])
for i in range(4):
channel_depth = channel_depths[i]
num_layers = layers[i]
strided_pool_first = True
if(i == 0):
strided_pool_first = False
num_layers = num_layers - 1
output = resnet_block(output, channel_depth=channel_depth, num_layers=num_layers, strided_pool_first=strided_pool_first)
if(include_top):
output = GlobalAvgPool2D(name="global_avg_pooling")(output)
if(initial_num_classes != None):
output = Dense(initial_num_classes)(output)
else:
output = Dense(num_classes)(output)
output = Activation("softmax")(output)
else:
if (non_top_pooling == "Average"):
output = GlobalAvgPool2D()(output)
elif (non_top_pooling == "Maximum"):
output = GlobalMaxPool2D()(output)
elif (non_top_pooling == None):
pass
model = Model(inputs=input_object, outputs=output)
if(weights == "imagenet"):
weights_path = model_path
model.load_weights(weights_path)
return model
elif(weights == "trained"):
weights_path = model_path
model.load_weights(weights_path)
return model
elif(weights == "continued"):
weights_path = model_path
model.load_weights(weights_path)
return model
elif(weights == "transfer"):
weights_path = model_path
model.load_weights(weights_path)
if(transfer_with_full_training == False):
for eachlayer in model.layers:
eachlayer.trainable = False
print("Training with top layers of the Model")
else:
print("Training with all layers of the Model")
output2 = model.layers[-3].output
output2 = Dense(num_classes)(output2)
output2 = Activation("softmax")(output2)
new_model = Model(inputs=model.input, outputs = output2)
return new_model
elif(weights == "custom"):
return model
def dscn(starting_num_filters=64,
min_filters=24,
filters_decay_gamma=1.5,
nin_filters=64,
n_fe_layers=8,
scale=4):
nin_filters2 = nin_filters // 2
x_in = Input(shape=(None, None, 3))
normalised = Lambda(normalize)(x_in)
# Generate the 1st layer
# layer_1 = conv2d_weightnorm(num_filters, 3, padding='same', , use_bias=True, activation=prelu_activation())(x_in)
# layer_1 = conv2d_block(x_in, num_filters, 3, padding='same', initial_alpha=0.1)
# conv_layers = [layer_1]
############################ Build feature extraction layers ###########################
conv_layers = []
total_filter_layers = 0
previous_num_filters = starting_num_filters
for layer_no in range(n_fe_layers):
num_filters = int(
max(previous_num_filters // filters_decay_gamma, min_filters))
# Find the correct input layer
input_layer = conv_layers[-1] if len(conv_layers) > 0 else normalised
layer_n = conv2d_weightnorm(num_filters,
3,
padding='same',
use_bias=True,
activation=prelu_activation())(input_layer)
conv_layers.append(layer_n)
total_filter_layers += num_filters
previous_num_filters = num_filters
feature_extraction_layer = Concatenate(
name="Feature_Extraction_Layer")(conv_layers)
######################### Build Reconstruction layers ##################################
A1 = conv2d_weightnorm(
nin_filters,
1,
padding='same',
use_bias=True,
activation=prelu_activation())(feature_extraction_layer)
B1 = conv2d_weightnorm(
nin_filters2,
1,
padding='same',
use_bias=True,
activation=prelu_activation())(feature_extraction_layer)
B2 = conv2d_weightnorm(nin_filters2,
1,
padding='same',
use_bias=True,
activation=prelu_activation())(B1)
reconstruction_layer = Concatenate(name="Reconstruction_Layer")([A1, B2])
######################### Upsampling Layer ############################################
upsampled_conv = conv2d_weightnorm(3 * (scale**2),
5,
padding='same',
use_bias=True)(reconstruction_layer)
upsampled_layer = Lambda(pixel_shuffle(scale))(upsampled_conv)
######################## Reconstruction Layer ###########################################
denormalised = Lambda(denormalize)(upsampled_layer)
model = Model(x_in, denormalised, name="dscn")
return model
def getModels():
input_img = Input(shape=(imageSize, imageSize, 1))
x = Convolution2D(32, (3, 3), padding='same', input_shape=(imageSize, imageSize, 1))(input_img)
x = ELU()(x)
x = MaxPooling2D(padding='same')(x)
x = Convolution2D(64, (3, 3), padding='same')(x)
x = ELU()(x)
x = MaxPooling2D((2, 2), padding='same')(x)
# Latent space // bottleneck layer
x = Flatten()(x)
x = Dense(latent_dim)(x)
z = ELU()(x)
##### MODEL 1: ENCODER #####
encoder = Model(input_img, z)
# We instantiate these layers separately so as to reuse them for the decoder
# Dense from latent space to image dimension
x_decoded_dense1 = Dense(7 * 7 * 64)
# Reshape for image
x_decoded_reshape0 = Reshape((7, 7, 64))
x_decoded_upsample0 = UpSampling2D((2, 2))
x_decoded_conv0 = Convolution2D(32, (3, 3), padding='same')
x_decoded_upsample3 = UpSampling2D((2, 2))
x_decoded_conv3 = Convolution2D(1, (3, 3), activation='sigmoid', padding='same')
# Create second part of autoencoder
x_decoded = x_decoded_dense1(z)
x_decoded = ELU()(x_decoded)
x_decoded = x_decoded_reshape0(x_decoded)
x_decoded = x_decoded_upsample0(x_decoded)
x_decoded = x_decoded_conv0(x_decoded)
x_decoded = ELU()(x_decoded)
# Tanh layer
x_decoded = x_decoded_upsample3(x_decoded)
decoded_img = x_decoded_conv3(x_decoded)
##### MODEL 2: AUTO-ENCODER #####
autoencoder = Model(input_img, decoded_img)
# Create decoder
input_z = Input(shape=(latent_dim,))
x_decoded_decoder = x_decoded_dense1(input_z)
x_decoded_decoder = ELU()(x_decoded_decoder)
x_decoded_decoder = x_decoded_reshape0(x_decoded_decoder)
x_decoded_decoder = x_decoded_upsample0(x_decoded_decoder)
x_decoded_decoder = x_decoded_conv0(x_decoded_decoder)
x_decoded_decoder = ELU()(x_decoded_decoder)
# Tanh layer
x_decoded_decoder = x_decoded_upsample3(x_decoded_decoder)
decoded_decoder_img = x_decoded_conv3(x_decoded_decoder)
##### MODEL 3: DECODER #####
decoder = Model(input_z, decoded_decoder_img)
return autoencoder, encoder, decoder
inputs, pred = simple_detection_netowrk((128, 128, 3), n_anchors,
n_classes)
# Generate prior boxes
strides = [4, 8, 16]
scales = [10, 25, 40]
ratios = [(1, 1), (1.5, 0.5), (1.2, 0.8), (0.8, 1.2), (1.4, 1.4)]
prior = PriorBoxes(strides, scales, ratios)
prior_boxes = prior.generate(image_shape)
# Generate Dataset
trainset = DetectionDataset(data_type='train')
validset = DetectionDataset(data_type='validation')
traingen = DetectionGenerator(trainset.config, prior.config, batch_size=64)
validgen = DetectionGenerator(validset.config, prior.config, batch_size=64)
# Define Loss
ssd_loss = SSDLoss(1.0, 3.)
# Training
model = Model(inputs, pred)
model.compile(Adam(1e-3), loss=SSDLoss(1.0, 3.))
rlrop = ReduceLROnPlateau(factor=0.1, min_lr=1e-6, patience=5, cooldown=3)
callbacks = []
callbacks.append(rlrop)
model.fit_generator(traingen,
epochs=50,
validation_data=validgen,
use_multiprocessing=True,
workers=6,
callbacks=callbacks)
def FM(user_feature_columns,
item_feature_columns,
l2_reg_embedding=1e-6,
init_std=0.0001,
seed=1024,
metric='cos'):
"""Instantiates the FM architecture.
:param user_feature_columns: An iterable containing user's features used by the model.
:param item_feature_columns: An iterable containing item's features used by the model.
:param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
:param init_std: float,to use as the initialize std of embedding vector
:param seed: integer ,to use as random seed.
:param metric: str, ``"cos"`` for cosine or ``"ip"`` for inner product
:return: A Keras model instance.
"""
embedding_matrix_dict = create_embedding_matrix(user_feature_columns +
item_feature_columns,
l2_reg_embedding,
init_std,
seed,
seq_mask_zero=True)
user_features = build_input_features(user_feature_columns)
user_inputs_list = list(user_features.values())
user_sparse_embedding_list, user_dense_value_list = input_from_feature_columns(
user_features,
user_feature_columns,
l2_reg_embedding,
init_std,
seed,
support_dense=False,
embedding_matrix_dict=embedding_matrix_dict)
item_features = build_input_features(item_feature_columns)
item_inputs_list = list(item_features.values())
item_sparse_embedding_list, item_dense_value_list = input_from_feature_columns(
item_features,
item_feature_columns,
l2_reg_embedding,
init_std,
seed,
support_dense=False,
embedding_matrix_dict=embedding_matrix_dict)
user_dnn_input = concat_func(user_sparse_embedding_list, axis=1)
user_vector_sum = Lambda(lambda x: reduce_sum(x, axis=1, keep_dims=False))(
user_dnn_input)
item_dnn_input = concat_func(item_sparse_embedding_list, axis=1)
item_vector_sum = Lambda(lambda x: reduce_sum(x, axis=1, keep_dims=False))(
item_dnn_input)
score = Similarity(type=metric)([user_vector_sum, item_vector_sum])
output = PredictionLayer("binary", False)(score)
model = Model(inputs=user_inputs_list + item_inputs_list, outputs=output)
model.__setattr__("user_input", user_inputs_list)
model.__setattr__("user_embedding", user_vector_sum)
model.__setattr__("item_input", item_inputs_list)
model.__setattr__("item_embedding", item_vector_sum)
return model
def unet_model(output_channel_count):
# (256 x 256 x input_channel_count)
inputs = Input((INPUT_IMAGE_SIZE, INPUT_IMAGE_SIZE, input_channel_count))
# エンコーダーの作成
# (128 x 128 x N)
enc1 = ZeroPadding2D(CONV_PADDING)(inputs)
enc1 = Conv2D(first_layer_filter_count,
CONV_FILTER_SIZE,
strides=CONV_STRIDE)(enc1)
# (64 x 64 x 2N)
filter_count = first_layer_filter_count * 2
enc2 = _add_encoding_layer(filter_count, enc1)
# (32 x 32 x 4N)
filter_count = first_layer_filter_count * 4
enc3 = _add_encoding_layer(filter_count, enc2)
# (16 x 16 x 8N)
filter_count = first_layer_filter_count * 8
enc4 = _add_encoding_layer(filter_count, enc3)
# (8 x 8 x 8N)
enc5 = _add_encoding_layer(filter_count, enc4)
# (4 x 4 x 8N)
enc6 = _add_encoding_layer(filter_count, enc5)
# (2 x 2 x 8N)
enc7 = _add_encoding_layer(filter_count, enc6)
# (1 x 1 x 8N)
enc8 = _add_encoding_layer(filter_count, enc7)
# デコーダーの作成
# (2 x 2 x 8N)
dec1 = _add_decoding_layer(filter_count, True, enc8)
dec1 = concatenate([dec1, enc7], axis=CONCATENATE_AXIS)
# (4 x 4 x 8N)
dec2 = _add_decoding_layer(filter_count, True, dec1)
dec2 = concatenate([dec2, enc6], axis=CONCATENATE_AXIS)
# (8 x 8 x 8N)
dec3 = _add_decoding_layer(filter_count, True, dec2)
dec3 = concatenate([dec3, enc5], axis=CONCATENATE_AXIS)
# (16 x 16 x 8N)
dec4 = _add_decoding_layer(filter_count, False, dec3)
dec4 = concatenate([dec4, enc4], axis=CONCATENATE_AXIS)
# (32 x 32 x 4N)
filter_count = first_layer_filter_count * 4
dec5 = _add_decoding_layer(filter_count, False, dec4)
dec5 = concatenate([dec5, enc3], axis=CONCATENATE_AXIS)
# (64 x 64 x 2N)
filter_count = first_layer_filter_count * 2
dec6 = _add_decoding_layer(filter_count, False, dec5)
dec6 = concatenate([dec6, enc2], axis=CONCATENATE_AXIS)
# (128 x 128 x N)
filter_count = first_layer_filter_count
dec7 = _add_decoding_layer(filter_count, False, dec6)
dec7 = concatenate([dec7, enc1], axis=CONCATENATE_AXIS)
# (256 x 256 x output_channel_count)
dec8 = Activation(activation='relu')(dec7)
dec8 = Conv2DTranspose(output_channel_count,
DECONV_FILTER_SIZE,
strides=DECONV_STRIDE)(dec8)
dec8 = Activation(activation='sigmoid')(dec8)
return Model(inputs=inputs, outputs=dec8)
def KDD_DIN(dnn_feature_columns,
history_feature_list,
dnn_use_bn=False,
dnn_hidden_units=(200, 80),
dnn_activation='relu',
att_hidden_size=(80, 40),
att_activation="dice",
att_weight_normalization=False,
l2_reg_dnn=0,
l2_reg_embedding=1e-6,
dnn_dropout=0,
init_std=0.0001,
seed=1024,
task='binary'):
"""Instantiates the Deep Interest Network architecture.
:param dnn_feature_columns: An iterable containing all the features used by deep part of the model.
:param history_feature_list: list,to indicate sequence sparse field
:param dnn_use_bn: bool. Whether use BatchNormalization before activation or not in deep net
:param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of deep net
:param dnn_activation: Activation function to use in deep net
:param att_hidden_size: list,list of positive integer , the layer number and units in each layer of attention net
:param att_activation: Activation function to use in attention net
:param att_weight_normalization: bool.Whether normalize the attention score of local activation unit.
:param l2_reg_dnn: float. L2 regularizer strength applied to DNN
:param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
:param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate.
:param init_std: float,to use as the initialize std of embedding vector
:param seed: integer ,to use as random seed.
:param task: str, ``"binary"`` for binary logloss or ``"regression"`` for regression loss
:return: A Keras model instance.
"""
features = build_input_features(dnn_feature_columns)
sparse_feature_columns = list(
filter(lambda x: isinstance(x, SparseFeat),
dnn_feature_columns)) if dnn_feature_columns else []
dense_feature_columns = list(
filter(lambda x: isinstance(x, DenseFeat),
dnn_feature_columns)) if dnn_feature_columns else []
varlen_sparse_feature_columns = list(
filter(lambda x: isinstance(x, VarLenSparseFeat),
dnn_feature_columns)) if dnn_feature_columns else []
history_feature_columns = []
sparse_varlen_feature_columns = []
history_fc_names = list(map(lambda x: "hist_" + x, history_feature_list))
for fc in varlen_sparse_feature_columns:
feature_name = fc.name
if feature_name in history_fc_names:
history_feature_columns.append(fc)
else:
sparse_varlen_feature_columns.append(fc)
inputs_list = list(features.values())
embedding_dict = kdd_create_embedding_matrix(dnn_feature_columns,
l2_reg_embedding,
init_std,
seed,
prefix="")
query_emb_list = embedding_lookup(embedding_dict,
features,
sparse_feature_columns,
history_feature_list,
history_feature_list,
to_list=True)
keys_emb_list = embedding_lookup(embedding_dict,
features,
history_feature_columns,
history_fc_names,
history_fc_names,
to_list=True)
dnn_input_emb_list = embedding_lookup(embedding_dict,
features,
sparse_feature_columns,
mask_feat_list=history_feature_list,
to_list=True)
dense_value_list = get_dense_input(features, dense_feature_columns)
sequence_embed_dict = varlen_embedding_lookup(
embedding_dict, features, sparse_varlen_feature_columns)
sequence_embed_list = get_varlen_pooling_list(
sequence_embed_dict,
features,
sparse_varlen_feature_columns,
to_list=True)
dnn_input_emb_list += sequence_embed_list
keys_emb = concat_func(keys_emb_list, mask=True)
deep_input_emb = concat_func(dnn_input_emb_list)
query_emb = concat_func(query_emb_list, mask=True)
hist = AttentionSequencePoolingLayer(
att_hidden_size,
att_activation,
weight_normalization=att_weight_normalization,
supports_masking=True)([query_emb, keys_emb])
deep_input_emb = Concatenate()([NoMask()(deep_input_emb), hist])
deep_input_emb = Flatten()(deep_input_emb)
dnn_input = combined_dnn_input([deep_input_emb], dense_value_list)
output = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout,
dnn_use_bn, seed)(dnn_input)
final_logit = Dense(1, use_bias=False)(output)
output = PredictionLayer(task)(final_logit)
model = Model(inputs=inputs_list, outputs=output)
return model
def define_nmt(hidden_size, batch_size, en_timesteps, en_vsize, fr_timesteps, fr_vsize):
""" Defining a NMT model """
# Define an input sequence and process it.
if batch_size:
encoder_inputs = Input(batch_shape=(batch_size, en_timesteps, en_vsize), name='encoder_inputs')
decoder_inputs = Input(batch_shape=(batch_size, fr_timesteps - 1, fr_vsize), name='decoder_inputs')
else:
encoder_inputs = Input(shape=(en_timesteps, en_vsize), name='encoder_inputs')
decoder_inputs = Input(shape=(fr_timesteps - 1, fr_vsize), name='decoder_inputs')
# Encoder GRU
encoder_gru = Bidirectional(GRU(hidden_size, return_sequences=True, return_state=True, name='encoder_gru'), name='bidirectional_encoder')
encoder_out, encoder_fwd_state, encoder_back_state = encoder_gru(encoder_inputs)
# Set up the decoder GRU, using `encoder_states` as initial state.
decoder_gru = GRU(hidden_size*2, return_sequences=True, return_state=True, name='decoder_gru')
decoder_out, decoder_state = decoder_gru(
decoder_inputs, initial_state=Concatenate(axis=-1)([encoder_fwd_state, encoder_back_state])
)
# Attention layer
attn_layer = AttentionLayer(name='attention_layer')
attn_out, attn_states = attn_layer([encoder_out, decoder_out])
# Concat attention input and decoder GRU output
decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([decoder_out, attn_out])
# Dense layer
dense = Dense(fr_vsize, activation='softmax', name='softmax_layer')
dense_time = TimeDistributed(dense, name='time_distributed_layer')
decoder_pred = dense_time(decoder_concat_input)
# Full model
full_model = Model(inputs=[encoder_inputs, decoder_inputs], outputs=decoder_pred)
full_model.compile(optimizer='adam', loss='categorical_crossentropy')
full_model.summary()
""" Inference model """
batch_size = 1
""" Encoder (Inference) model """
encoder_inf_inputs = Input(batch_shape=(batch_size, en_timesteps, en_vsize), name='encoder_inf_inputs')
encoder_inf_out, encoder_inf_fwd_state, encoder_inf_back_state = encoder_gru(encoder_inf_inputs)
encoder_model = Model(inputs=encoder_inf_inputs, outputs=[encoder_inf_out, encoder_inf_fwd_state, encoder_inf_back_state])
""" Decoder (Inference) model """
decoder_inf_inputs = Input(batch_shape=(batch_size, 1, fr_vsize), name='decoder_word_inputs')
encoder_inf_states = Input(batch_shape=(batch_size, en_timesteps, 2*hidden_size), name='encoder_inf_states')
decoder_init_state = Input(batch_shape=(batch_size, 2*hidden_size), name='decoder_init')
decoder_inf_out, decoder_inf_state = decoder_gru(
decoder_inf_inputs, initial_state=decoder_init_state)
attn_inf_out, attn_inf_states = attn_layer([encoder_inf_states, decoder_inf_out])
decoder_inf_concat = Concatenate(axis=-1, name='concat')([decoder_inf_out, attn_inf_out])
decoder_inf_pred = TimeDistributed(dense)(decoder_inf_concat)
decoder_model = Model(inputs=[encoder_inf_states, decoder_init_state, decoder_inf_inputs],
outputs=[decoder_inf_pred, attn_inf_states, decoder_inf_state])
return full_model, encoder_model, decoder_model
train_label, # train_label,
batch_size=512,
epochs=20,
verbose=1,
validation_split=0.0,
)
# model.save_weights('SDM_weights.h5')
K.set_learning_phase(False)
# 4. Generate user features for testing and full item features for retrieval
test_user_model_input = test_model_input
all_item_model_input = {
"movie_id": item_profile['movie_id'].values,
}
user_embedding_model = Model(inputs=model.user_input,
outputs=model.user_embedding)
item_embedding_model = Model(inputs=model.item_input,
outputs=model.item_embedding)
user_embs = user_embedding_model.predict(test_user_model_input,
batch_size=2**12)
# user_embs = user_embs[:, i, :] # i in [0,k_max) if MIND
item_embs = item_embedding_model.predict(all_item_model_input,
batch_size=2**12)
print(user_embs.shape)
print(item_embs.shape)
test_true_label = {line[0]: [line[2]] for line in test_set}
import numpy as np
class Network(object):
"""
Network class for imitation learning
"""
def __init__(self, trainDataDir, testDataDir, trainClassDir, testClassDir,
cfgData):
self.logger = logging.getLogger('agent')
self.sampleBuf = {}
self.imageChannel = 3
self.featureDim = 200
self.isSmallNet = cfgData.get('isSmallNet') or False
self.isMax = cfgData.get('isMax') or False
self.randomRatio = cfgData.get('randomRatio') or 0
self.trainIter = cfgData.get('trainIter') or 20
self.timeStep = cfgData.get('timeStep') or 5
self.randomRatio = self.randomRatio * 1000
self.useClassBalance = cfgData.get('useClassBalance')
self.useResNet = cfgData.get('useResNet')
if self.isSmallNet:
self.imageSize = 50
else:
self.imageSize = 150
self.progressReport = ProgressReport()
self.progressReport.Init()
self.actionAheadNum = cfgData['actionAheadNum']
self.classImageTimes = cfgData['classImageTimes']
self.inputHeight = cfgData['inputHeight']
self.inputWidth = cfgData['inputWidth']
self.roiRegion = cfgData.get('roiRegion')
self.actionDefine = cfgData.get('actionDefine')
self.taskList, self.taskActionDict, self.actionNameDict = ObtainTaskDict(
self.actionDefine)
self.actionSpaceList = list()
for taskIndex in self.taskList:
actionName = self.actionNameDict[taskIndex]
self.actionSpaceList.append(len(actionName))
if len(self.taskList) > 2:
self.logger.error(
'Imitation learning only supports one or two tasks')
self.actionPriorWeights = [
cfgData['actionDefine'][n]['prior']
for n in range(len(cfgData['actionDefine']))
]
self.trainDataDir = trainDataDir
self.testDataDir = testDataDir
self.trainClassDir = trainClassDir
self.testClassDir = testClassDir
self.modelPath = os.path.join(DATA_ROOT_DIR + 'data/ImitationModel/')
if not os.path.exists(self.modelPath):
os.mkdir(self.modelPath)
self.netBatchSize = 32
self.kerasModel = None
self.netLSTMBatchSize = 64
self.kerasModelLSTM = None
def Init(self):
"""
Initialize function
"""
pass
def Finish(self):
"""
Finish fuction
"""
pass
def LossCCEPieceWise(self, y_true, y_pred):
"""
Define piece-wise class cross entropy loss
"""
state1 = 0.5 * y_pred + 10e-3
state2 = y_pred
y_pred = tf.where(y_pred < 2 * 10e-3, x=state1, y=state2)
loss = K.sum(-K.log(y_pred) * y_true)
return loss
def TrainGenerator(self):
"""
Train network using "fit_generator" function
"""
if self.useClassBalance:
self.trainFileName, self.trainLabel = ReadTxt(
os.path.join(self.trainClassDir, 'data.txt'))
else:
self.trainFileName, self.trainLabel = ReadTxt(
os.path.join(self.trainClassDir, 'dataOri.txt'))
self.trainFileNameOri, self.trainLabelOri = ReadTxt(
os.path.join(self.trainClassDir, 'dataOri.txt'))
self.testFileName, self.testLabel = ReadTxt(
os.path.join(self.testClassDir, 'data.txt'))
nb_train_samples = len(self.trainFileName)
nb_val_samples = len(self.testFileName)
kerasModel = self.KerasModel()
self.logger.info(kerasModel.summary())
if self.taskList == [0, 1]:
kerasModel.compile(optimizer=tf.keras.optimizers.Adam(
lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0),
loss={
'out_task0': self.LossCCEPieceWise,
'out_task1': self.LossCCEPieceWise
},
loss_weights={
'out_task0': 1,
'out_task1': 1
},
metrics=['accuracy'])
elif self.taskList == [0]:
kerasModel.compile(optimizer=tf.keras.optimizers.Adam(
lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0),
loss={'out_task0': self.LossCCEPieceWise},
metrics=['accuracy'])
valAccList = list()
for trainIter in range(self.trainIter):
# obtain input features for network
self.logger.info('trainIter is {}'.format(trainIter))
trainHistory = kerasModel.fit_generator(
DataGenerator(self.actionSpaceList,
imgFiles=self.trainFileName,
labels=self.trainLabel,
batchSize=self.netBatchSize,
dim=self.imageSize),
steps_per_epoch=nb_train_samples / self.netBatchSize,
epochs=1,
verbose=1,
validation_data=DataGenerator(self.actionSpaceList,
imgFiles=self.testFileName,
labels=self.testLabel,
batchSize=self.netBatchSize,
dim=self.imageSize),
validation_steps=nb_val_samples / self.netBatchSize)
trainAcc, valAcc = self.GetAcc(trainHistory)
valAccList.append(valAcc + trainAcc)
kerasModel.save_weights(self.modelPath + 'my_model_weights' +
np.str(trainIter) + '.h5')
logging.info(
'Network: Iteration {}....{}: train_acc is {} and val_acc is {}'
.format(trainIter, self.trainIter, trainAcc, valAcc))
if trainIter < (self.trainIter - 1):
self.progressReport.SendTrainProgress(
int((trainIter + 1) * 100 / self.trainIter))
indexAccValMax = valAccList.index(np.max(valAccList))
logging.info(
'Use model from the {}th iteration'.format(indexAccValMax))
srcModelName = self.modelPath + 'my_model_weights' + np.str(
indexAccValMax) + '.h5'
dstModelName = self.modelPath + 'my_model_weights' + '.h5'
shutil.copyfile(srcModelName, dstModelName)
def KerasModelSmallNet50(self, imgInput):
"""
Construct small net. The image size is 50*50, which is suitable for map.
"""
x = Conv2D(16, (3, 3), activation='tanh')(imgInput)
x = Conv2D(32, (3, 3), activation='relu')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Conv2D(32, (3, 3), activation='relu')(x)
x = Conv2D(32, (3, 3), activation='relu')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Conv2D(32, (3, 3), activation='relu')(x)
x = Conv2D(32, (3, 3), activation='relu')(x)
x = Flatten()(x)
x = Dense(self.featureDim,
kernel_regularizer=regularizers.l2(0.0002),
activity_regularizer=regularizers.l1(0.0002),
name='fc_feature')(x)
x = PReLU()(x)
return x
def KerasModelSmallNet150(self, imgInput):
"""
Construct small net. The image size is 150*150, which is suitable for image.
"""
x = Conv2D(32, (3, 3), activation='tanh')(imgInput)
x = Conv2D(32, (3, 3), activation='relu')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Conv2D(32, (3, 3), activation='relu')(x)
x = Conv2D(32, (3, 3), activation='relu')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Conv2D(32, (3, 3), activation='relu')(x)
x = Conv2D(32, (3, 3), activation='relu')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Conv2D(64, (3, 3), activation='relu')(x)
x = Conv2D(64, (3, 3), activation='relu')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Flatten()(x)
x = Dense(self.featureDim,
kernel_regularizer=regularizers.l2(0.0002),
activity_regularizer=regularizers.l1(0.0002),
name='fc_feature')(x)
x = PReLU()(x)
return x
def KerasModelResNet(self, imgInput):
"""
Construct resNet. The image size is 150*150, which is suitable for image.
"""
bn_axis = 3
x = ZeroPadding2D((3, 3))(imgInput)
x = Convolution2D(8, 7, strides=(2, 2), name='conv1')(x)
x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x = Activation('relu')(x)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv_block(x, 3, [8, 8, 16], stage=2, block='a', strides=(1, 1))
x = identity_block(x, 3, [8, 8, 16], stage=2, block='b')
x = identity_block(x, 3, [8, 8, 16], stage=2, block='c')
x = conv_block(x, 3, [16, 16, 32], stage=3, block='a')
x = identity_block(x, 3, [16, 16, 32], stage=3, block='b')
x = identity_block(x, 3, [16, 16, 32], stage=3, block='c')
x = identity_block(x, 3, [16, 16, 32], stage=3, block='d')
x = conv_block(x, 3, [32, 32, 64], stage=4, block='a')
x = identity_block(x, 3, [32, 32, 64], stage=4, block='b')
x = identity_block(x, 3, [32, 32, 64], stage=4, block='c')
x = identity_block(x, 3, [32, 32, 64], stage=4, block='d')
x = identity_block(x, 3, [32, 32, 64], stage=4, block='e')
x = identity_block(x, 3, [32, 32, 64], stage=4, block='f')
x = conv_block(x, 3, [64, 64, 128], stage=5, block='a')
x = identity_block(x, 3, [64, 64, 128], stage=5, block='b')
x = identity_block(x, 3, [64, 64, 128], stage=5, block='c')
x = conv_block(x, 3, [64, 64, 256], stage=6, block='a')
x = identity_block(x, 3, [64, 64, 256], stage=6, block='b')
x = identity_block(x, 3, [64, 64, 256], stage=6, block='c')
x = GlobalAveragePooling2D()(x)
# x = Flatten()(x)
x = Dense(self.featureDim,
kernel_regularizer=regularizers.l2(0.0002),
activity_regularizer=regularizers.l1(0.0002),
name='fc_feature')(x)
x = PReLU()(x)
return x
def KerasModel(self):
"""
Construct two structures of network
"""
input_shape = (self.imageSize, self.imageSize, self.imageChannel)
imgInput = Input(shape=input_shape)
if self.isSmallNet:
x = self.KerasModelSmallNet50(imgInput)
else:
if self.useResNet:
x = self.KerasModelResNet(imgInput)
else:
x = self.KerasModelSmallNet150(imgInput)
for taskIndex in self.taskList:
actionName = self.actionNameDict[taskIndex]
if taskIndex == 0:
out0 = Dense(len(actionName),
activation='softmax',
name='out_task0')(x)
elif taskIndex == 1:
out1 = Dense(len(actionName),
activation='softmax',
name='out_task1')(x)
else:
assert 'task for multi_task should be 0 or 1'
if self.taskList == [0, 1]:
model = Model(imgInput, [out0, out1])
elif self.taskList == [0]:
model = Model(imgInput, out0)
else:
assert 'taskList only sopports [0,1] and [0], ' \
'please check actionDefine in ImitationLearning.json'
return model
def KerasModelLSTM(self):
"""
Define structure of LSTM
"""
input_shape = (self.timeStep, self.featureDim)
featureInput = Input(shape=input_shape)
x = LSTM(100)(featureInput)
for taskIndex in self.taskList:
actionName = self.actionNameDict[taskIndex]
if taskIndex == 0:
out0 = Dense(len(actionName),
activation='softmax',
name='out_task0')(x)
elif taskIndex == 1:
out1 = Dense(len(actionName),
activation='softmax',
name='out_task1')(x)
else:
assert 'task for multi_task should be 0 or 1'
if self.taskList == [0, 1]:
model = Model(featureInput, [out0, out1])
elif self.taskList == [0]:
model = Model(featureInput, out0)
return model
def TrainLSTM(self):
"""
Train LSTM
"""
modelName = self.modelPath + 'my_model_weights.h5'
kerasModel = self.KerasModel()
kerasModel.load_weights(modelName)
kerasModelExtFea = Model(
inputs=kerasModel.input,
outputs=kerasModel.get_layer('fc_feature').output)
self.logger.info(kerasModelExtFea.summary())
modelLSTM = self.KerasModelLSTM()
if self.taskList == [0, 1]:
modelLSTM.compile(optimizer=tf.keras.optimizers.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0),
loss={
'out_task0': self.LossCCEPieceWise,
'out_task1': self.LossCCEPieceWise
},
loss_weights={
'out_task0': 1,
'out_task1': 1
},
metrics=['accuracy'])
elif self.taskList == [0]:
modelLSTM.compile(optimizer=tf.keras.optimizers.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0),
loss={'out_task0': self.LossCCEPieceWise},
metrics=['accuracy'])
feaTrain, labelTrain, feaTest, labelTest = self._ExtractFeature(
kerasModelExtFea)
if feaTrain is None:
self.logger.error('No image is found in {}'.format(
self.trainDataDir))
valAccList = list()
for trainIter in range(self.trainIter):
trainHistory = modelLSTM.fit(x=feaTrain,
y=labelTrain,
validation_data=(feaTest, labelTest),
batch_size=self.netLSTMBatchSize,
shuffle=1,
epochs=1)
trainAcc, valAcc = self.GetAcc(trainHistory)
valAccList.append(valAcc + trainAcc)
modelLSTM.save_weights(self.modelPath + 'my_model_weights_LSTM' +
np.str(trainIter) + '.h5')
logging.info(
'NetworkLSTM: Iter {}....{}: train_acc is {} and val_acc is {}'
.format(trainIter, self.trainIter, trainAcc, valAcc))
indexAccValMax = valAccList.index(np.max(valAccList))
logging.info(
'Use model from the {}th iteration'.format(indexAccValMax))
srcModelName = self.modelPath + 'my_model_weights_LSTM' + np.str(
indexAccValMax) + '.h5'
dstModelName = self.modelPath + 'my_model_weights_LSTM' + '.h5'
shutil.copyfile(srcModelName, dstModelName)
self.progressReport.SendTrainProgress(100)
def GetAcc(self, trainHistory):
"""
Calculate train and val accuracy
"""
if self.taskList == [0, 1]:
trainAcc = 0.5 * (trainHistory.history['out_task0_acc'][0] + \
trainHistory.history['out_task1_acc'][0])
valAcc = 0.5 * (trainHistory.history['val_out_task0_acc'][0] + \
trainHistory.history['val_out_task1_acc'][0])
else:
trainAcc = trainHistory.history['acc'][0]
valAcc = trainHistory.history['val_acc'][0]
return trainAcc, valAcc
def ExtractNetFeature(self, kerasModelExtFea, fileNameList, label):
"""
Extract feature for LSTM
"""
featureFromNet = dict()
labelFromNet = dict()
for n, fileName in enumerate(fileNameList):
img = cv2.imread(fileName).astype('float')
imgOut = PreprocessImage(img)
featureTmp = np.zeros(
[1, self.imageSize, self.imageSize, self.imageChannel])
featureTmp[0, ] = imgOut
featureFromNet[fileName] = kerasModelExtFea.predict(featureTmp)
labelFromNet[fileName] = label[n]
featureConcat, labelConcat = self._ConcatFeature(
featureFromNet, labelFromNet)
return featureConcat, labelConcat
def _ExtractFeature(self, kerasModelExtFea):
"""
Extract feature of training and test image set for LSTM
"""
featureTrainOri, labelTrainOri = self.ExtractNetFeature(
kerasModelExtFea, self.trainFileNameOri, self.trainLabelOri)
featureTrain, labelTrain = GetFeatureAndLabel(featureTrainOri,
labelTrainOri,
self.trainFileName,
self.actionSpaceList)
featureTestOri, labelTestOri = self.ExtractNetFeature(
kerasModelExtFea, self.testFileName, self.testLabel)
featureTest, labelTest = GetFeatureAndLabel(featureTestOri,
labelTestOri,
self.testFileName,
self.actionSpaceList)
return featureTrain, labelTrain, featureTest, labelTest
def _ConcatFeature(self, featureFromNet, label):
"""
Concat feature from network for lstm
"""
featureConcat = dict()
labelConCat = dict()
for fileName in featureFromNet.keys():
_, fileNameTmp = os.path.split(fileName)
fileNameNum = int(fileNameTmp.split('_')[-1][:-4])
noFile = 0
for n in range(self.timeStep):
fileNameNumTmp = fileNameNum - self.timeStep + n + 1
fileNameTmp = fileName[:-len(fileNameTmp.split('_')[-1])] +\
str(fileNameNumTmp) +\
fileName[-4:]
if fileNameTmp not in featureFromNet.keys():
noFile = 1
break
else:
noFile = 0
if noFile:
continue
featureTmp = np.zeros([self.timeStep, self.featureDim])
for n in range(self.timeStep):
fileNameNumTmp = fileNameNum - self.timeStep + n + 1
fileNameTmp = fileName[:-len(fileNameTmp.split('_')[-1])] + \
str(fileNameNumTmp) + fileName[-4:]
featureTmp[n, :] = featureFromNet[fileNameTmp]
featureConcat[fileName] = featureTmp
labelConCat[fileName] = label[fileName]
return featureConcat, labelConCat
def LoadWeights(self):
"""
Load weights of CNN and LSTM
"""
self.kerasModel = self.KerasModel()
self.kerasModel.load_weights(self.modelPath + 'my_model_weights.h5')
self.kerasModelExtFea = Model(
inputs=self.kerasModel.input,
outputs=self.kerasModel.get_layer('fc_feature').output)
self.kerasModelLSTM = self.KerasModelLSTM()
self.kerasModelLSTM.load_weights(self.modelPath +
'my_model_weights_LSTM.h5')
def Predict(self, image):
"""
Output action given a test image based on CNN
"""
inputData = self.PrepareData(image)
if len(self.taskList) == 2:
predAction = list()
for n in range(len(self.taskList)):
actionScore = self.kerasModel.predict(inputData)[n][0]
predAction.append(self.ChooseAction(actionScore))
else:
actionScore = self.kerasModel.predict(inputData)[0]
predAction = self.ChooseAction(actionScore)
return predAction
def PrepareData(self, image):
"""
Preprare data based on image
"""
if self.roiRegion is not None:
image = image[self.roiRegion[1]:self.roiRegion[1] +
self.roiRegion[3],
self.roiRegion[0]:self.roiRegion[0] +
self.roiRegion[2], :]
if image.shape[0] != self.imageSize or image.shape[1] != self.imageSize:
image = cv2.resize(image, (self.imageSize, self.imageSize))
image = PreprocessImage(image)
inputData = np.zeros(
[1, self.imageSize, self.imageSize, self.imageChannel])
inputData[0, :, :, :] = image
return inputData
def PredictLSTM(self, inputData):
"""
Output action given a test feature based on LSTM
"""
if len(self.taskList) == 2:
predAction = list()
for n in range(len(self.taskList)):
actionScore = self.kerasModelLSTM.predict(inputData)[n][0]
predAction.append(self.ChooseAction(actionScore))
else:
actionScore = self.kerasModelLSTM.predict(inputData)[0]
predAction = self.ChooseAction(actionScore)
return predAction
def ChooseAction(self, actionScore):
"""
choose action according to score of different actions
There are two modes: max or randomly choose action
"""
if self.isMax:
actionScore = [
max(np.floor(actionScore[n] * 1000), 10)
for n in range(len(actionScore))
]
actionScore = [
actionScore[n] * self.actionPriorWeights[n]
for n in range(len(actionScore))
]
self.logger.info('actionScore is {}'.format(actionScore))
predAction = np.argmax(actionScore)
else:
actionScore = [
np.floor(actionScore[n] * 1000) + self.randomRatio
for n in range(len(actionScore))
]
actionScore = [
actionScore[n] * self.actionPriorWeights[n]
for n in range(len(actionScore))
]
self.logger.info('actionScore is {}'.format(actionScore))
predAction = int(self.RandomIndex(actionScore))
return predAction
def RandomIndex(self, rate):
"""
Randomly select a index based on probability distribution
"""
start = 0
index = 0
randnum = random.randint(1, int(sum(rate)))
for index, scope in enumerate(rate):
start += scope
if randnum <= start:
break
return index
def ExtractFeature(self, image):
"""
Extract feature of the input image
"""
inputData = self.PrepareData(image)
feature = self.kerasModelExtFea.predict(inputData)
return feature
encoder_states = [state_h, state_c]
# Set up the decoder, using `encoder_states` as initial state.
decoder_inputs = Input(shape=(None, num_decoder_tokens))
# We set up our decoder to return full output sequences,
# and to return internal states as well. We don't use the
# return states in the training model, but we will use them in inference.
decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True)
decoder_outputs, _, _ = decoder_lstm(decoder_inputs,
initial_state=encoder_states)
decoder_dense = Dense(num_decoder_tokens, activation='softmax')
decoder_outputs = decoder_dense(decoder_outputs)
# Define the model that will turn
# `encoder_input_data` & `decoder_input_data` into `decoder_target_data`
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
# Run training
model.compile(optimizer='rmsprop', loss='categorical_crossentropy')
model.fit([encoder_input_data, decoder_input_data], decoder_target_data,
batch_size=batch_size,
epochs=epochs,
validation_split=0.2)
# Save model
model.save('s2s.h5')
# Next: inference mode (sampling).
# Here's the drill:
# 1) encode input and retrieve initial decoder state
# 2) run one step of decoder with this initial state
# and a "start of sequence" token as target.
def layer_test(layer_cls, kwargs={}, input_shape=None, input_dtype=None,
input_data=None, expected_output=None,
expected_output_dtype=None, fixed_batch_size=False):
# generate input data
if input_data is None:
if not input_shape:
raise AssertionError()
if not input_dtype:
input_dtype = K.floatx()
input_data_shape = list(input_shape)
for i, e in enumerate(input_data_shape):
if e is None:
input_data_shape[i] = np.random.randint(1, 4)
if all(isinstance(e, tuple) for e in input_data_shape):
input_data = []
for e in input_data_shape:
input_data.append(
(10 * np.random.random(e)).astype(input_dtype))
else:
input_data = (10 * np.random.random(input_data_shape))
input_data = input_data.astype(input_dtype)
else:
if input_shape is None:
input_shape = input_data.shape
if input_dtype is None:
input_dtype = input_data.dtype
if expected_output_dtype is None:
expected_output_dtype = input_dtype
# instantiation
layer = layer_cls(**kwargs)
# test get_weights , set_weights at layer level
weights = layer.get_weights()
layer.set_weights(weights)
try:
expected_output_shape = layer.compute_output_shape(input_shape)
except Exception:
expected_output_shape = layer._compute_output_shape(input_shape)
# test in functional API
if isinstance(input_shape, list):
if fixed_batch_size:
x = [Input(batch_shape=e, dtype=input_dtype) for e in input_shape]
else:
x = [Input(shape=e[1:], dtype=input_dtype) for e in input_shape]
else:
if fixed_batch_size:
x = Input(batch_shape=input_shape, dtype=input_dtype)
else:
x = Input(shape=input_shape[1:], dtype=input_dtype)
y = layer(x)
if not (K.dtype(y) == expected_output_dtype):
raise AssertionError()
# check with the functional API
model = Model(x, y)
actual_output = model.predict(input_data)
actual_output_shape = actual_output.shape
for expected_dim, actual_dim in zip(expected_output_shape,
actual_output_shape):
if expected_dim is not None:
if not (expected_dim == actual_dim):
raise AssertionError()
if expected_output is not None:
assert_allclose(actual_output, expected_output, rtol=1e-3)
# test serialization, weight setting at model level
model_config = model.get_config()
recovered_model = model.__class__.from_config(model_config)
if model.weights:
weights = model.get_weights()
recovered_model.set_weights(weights)
_output = recovered_model.predict(input_data)
assert_allclose(_output, actual_output, rtol=1e-3)
# test training mode (e.g. useful when the layer has a
# different behavior at training and testing time).
if has_arg(layer.call, 'training'):
model.compile('rmsprop', 'mse')
model.train_on_batch(input_data, actual_output)
# test instantiation from layer config
layer_config = layer.get_config()
layer_config['batch_input_shape'] = input_shape
layer = layer.__class__.from_config(layer_config)
# for further checks in the caller function
return actual_output
def run(self, batch_size: int, output_dir: str, image_type: CqDataType,
scale: int):
gpus = tf.config.experimental.list_physical_devices("GPU")
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
print(f"Number of GPUS: {len(gpus)}")
cq_dataset = CqData(image_type, scale_down=scale)
dataset = tf.data.Dataset.from_tensor_slices(cq_dataset.images) \
.shuffle(len(cq_dataset.images)) \
.repeat() \
.batch(batch_size, drop_remainder=True)
output_dir = f"{output_dir}_{cq_dataset.get_type_str()}"
output_num_x = 8
output_num_y = 8
num_output_image = output_num_x * output_num_y
img_width = cq_dataset.get_image_width()
img_height = cq_dataset.get_image_height()
num_channel = cq_dataset.get_channel_count()
lr = 0.0002
z_size = 256
num_cat = cq_dataset.get_count()
# num_cat = 0
output_interval = 100
opt_g = keras.optimizers.Adam(lr)
opt_d = keras.optimizers.Adam(lr)
opt_g = tf.keras.mixed_precision.LossScaleOptimizer(opt_g)
opt_d = tf.keras.mixed_precision.LossScaleOptimizer(opt_d)
if strategy_type == "mirror":
strategy = tf.distribute.MirroredStrategy()
elif strategy_type == "tpu":
resolver = tf.distribute.cluster_resolver.TPUClusterResolver(
tpu=f"grpc://{os.environ['COLAB_TPU_ADDR']}")
tf.config.experimental_connect_to_host(resolver.master())
tf.tpu.experimental.initialize_tpu_system(resolver)
strategy = tf.distribute.experimental.TPUStrategy(resolver)
else:
raise Exception(f"Wrong strategy type: {strategy_type}")
with strategy.scope():
gen = Generator(4, img_width, img_height, num_channel)
disc = Discriminator(4, num_cat)
gen.optimizer = opt_g
disc.optimizer = opt_d
iwgan = IWGanLoss(disc)
input_g = Input(batch_size=batch_size,
shape=z_size + num_cat,
name="z")
input_d = Input(batch_size=batch_size,
shape=(img_height, img_width, num_channel),
name="real_images")
input_eps = Input(batch_size=batch_size,
shape=(1, 1, 1),
name="eps")
disc_gen: tf.Tensor
disc_real: tf.Tensor
cat_output: tf.Tensor
gen_images: tf.Tensor = gen(input_g)
x_pn = input_eps * input_d + (1 - input_eps) * gen_images
iwgan_loss = iwgan(x_pn)
disc_gen, cat_output = disc(gen_images)
disc_real, _ = disc(input_d)
model_g = Model(inputs=[input_g], outputs=[disc_gen, cat_output])
model_d = Model(inputs=[input_g, input_d, input_eps],
outputs=[disc_gen, disc_real, iwgan_loss, cat_output])
dataset = strategy.experimental_distribute_dataset(dataset)
data_it = iter(dataset)
# Model
model_dir = os.path.join(output_dir, "model_save")
gen_model_dir = os.path.join(model_dir, "gen")
disc_model_dir = os.path.join(model_dir, "disc")
gen_ckpt = tf.train.Checkpoint(model=gen, optimizer=gen.optimizer)
disc_ckpt = tf.train.Checkpoint(model=disc, optimizer=disc.optimizer)
gen_ckpt_mgr = tf.train.CheckpointManager(gen_ckpt,
gen_model_dir,
max_to_keep=5)
disc_ckpt_mgr = tf.train.CheckpointManager(disc_ckpt,
disc_model_dir,
max_to_keep=5)
gen_latest = gen_ckpt_mgr.latest_checkpoint
disc_latest = disc_ckpt_mgr.latest_checkpoint
with strategy.scope():
if gen_latest:
gen_ckpt.restore(gen_latest)
if disc_latest:
disc_ckpt.restore(disc_latest)
if mode == "train":
# Image output
image_dir = os.path.join(output_dir, "images")
helper.clean_create_dir(image_dir)
test_input_images = cq_dataset.get_ordered_batch(
num_output_image, False)
test_input_images = tf.convert_to_tensor(test_input_images)
test_input_images = CqGAN.convert_to_save_format(
test_input_images, output_num_x, output_num_y, img_width,
img_height, num_channel)
tf.io.write_file(os.path.join(image_dir, "input.png"),
test_input_images)
# Summary
summary_dir = "cq_log"
summary_writer = tf.summary.create_file_writer(summary_dir)
metrics = {
"disc_gen": keras.metrics.Mean(),
"disc_real": keras.metrics.Mean(),
"loss_gen": keras.metrics.Mean(),
"loss_real": keras.metrics.Mean(),
"loss_cat": keras.metrics.Mean(),
"iwgan_loss": keras.metrics.Mean()
}
summary_writer.set_as_default()
graph(K.get_graph(), step=0)
# Test variables
z_fixed, _, cat_fixed = self.generate_fixed_z(
num_output_image, z_size, num_cat)
# Begin training
begin = datetime.datetime.now()
for step in range(num_iter):
train_step(strategy, data_it, disc, gen, model_g, model_d,
batch_size, z_size, num_cat, metrics)
# Output
if step % output_interval == 0 and step != 0:
now = datetime.datetime.now()
diff = now - begin
begin = now
output_count = step // output_interval
output_filename = f"output{output_count}.png"
output_images = gen(z_fixed)
output_images = CqGAN.convert_to_save_format(
output_images, output_num_x, output_num_y, img_width,
img_height, num_channel)
tf.io.write_file(os.path.join(image_dir, output_filename),
output_images)
tf.saved_model.save(disc, disc_model_dir)
tf.saved_model.save(gen, gen_model_dir)
gen_ckpt_mgr.save()
disc_ckpt_mgr.save()
print(
f"{output_count * output_interval} times done: {diff.total_seconds()}s passed, "
f"loss_gen: {metrics['loss_gen'].result()}, "
f"loss_real: {metrics['loss_real'].result()}, "
f"loss_cat: {metrics['loss_cat'].result()}")
# Summary
for metric_name, metric in metrics.items():
tf.summary.scalar(f"loss/{metric_name}", metric.result(),
step)
metric.reset_states()
elif mode == "predict":
# Image output
image_dir = os.path.join(output_dir, "predict_images")
helper.clean_create_dir(image_dir)
prr = plot_utils.Plot_Reproduce_Performance(
image_dir, output_num_x, output_num_y, img_width, img_height,
scale)
with strategy.scope():
fixed_z, real_z, _ = self.generate_nocat_z(
batch_size, z_size, num_cat)
fixed_cat_input = self.generate_fixed_cat(batch_size, num_cat)
fixed_z2 = tf.concat([real_z, fixed_cat_input], 1)
input_images1 = predict_image(strategy, gen, fixed_z).numpy()
input_images2 = predict_image(strategy, gen, fixed_z2).numpy()
prr.save_pngs(input_images1, num_channel, "input1.png")
prr.save_pngs(input_images2, num_channel, "input2.png")
for i in range(num_cat):
cat_input = self.generate_fixed_cat(
batch_size, num_cat, [i])
input_z = tf.concat([real_z, cat_input], 1)
gen_images = predict_image(strategy, gen, input_z)
gen_images = gen_images.numpy()
output_filename = f"output{i + 1}.png"
prr.save_pngs(gen_images, num_channel, output_filename)
def VGG19(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000):
"""Instantiates the VGG19 architecture.
Optionally loads weights pre-trained
on ImageNet. Note that when using TensorFlow,
for best performance you should set
`image_data_format='channels_last'` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The data format
convention used by the model is the one
specified in your Keras config file.
Arguments:
include_top: whether to include the 3 fully-connected
layers at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(224, 224, 3)` (with `channels_last` data format)
or `(3, 224, 224)` (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 48.
E.g. `(200, 200, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
Returns:
A Keras model instance.
Raises:
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
input_shape = _obtain_input_shape(
input_shape,
default_size=224,
min_size=48,
data_format=K.image_data_format(),
require_flatten=include_top,
weights=weights)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
# Block 1
x = Conv2D(
64, (3, 3), activation='relu', padding='same', name='block1_conv1')(
img_input)
x = Conv2D(
64, (3, 3), activation='relu', padding='same', name='block1_conv2')(
x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
# Block 2
x = Conv2D(
128, (3, 3), activation='relu', padding='same', name='block2_conv1')(
x)
x = Conv2D(
128, (3, 3), activation='relu', padding='same', name='block2_conv2')(
x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
# Block 3
x = Conv2D(
256, (3, 3), activation='relu', padding='same', name='block3_conv1')(
x)
x = Conv2D(
256, (3, 3), activation='relu', padding='same', name='block3_conv2')(
x)
x = Conv2D(
256, (3, 3), activation='relu', padding='same', name='block3_conv3')(
x)
x = Conv2D(
256, (3, 3), activation='relu', padding='same', name='block3_conv4')(
x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
# Block 4
x = Conv2D(
512, (3, 3), activation='relu', padding='same', name='block4_conv1')(
x)
x = Conv2D(
512, (3, 3), activation='relu', padding='same', name='block4_conv2')(
x)
x = Conv2D(
512, (3, 3), activation='relu', padding='same', name='block4_conv3')(
x)
x = Conv2D(
512, (3, 3), activation='relu', padding='same', name='block4_conv4')(
x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
# Block 5
x = Conv2D(
512, (3, 3), activation='relu', padding='same', name='block5_conv1')(
x)
x = Conv2D(
512, (3, 3), activation='relu', padding='same', name='block5_conv2')(
x)
x = Conv2D(
512, (3, 3), activation='relu', padding='same', name='block5_conv3')(
x)
x = Conv2D(
512, (3, 3), activation='relu', padding='same', name='block5_conv4')(
x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)
if include_top:
# Classification block
x = Flatten(name='flatten')(x)
x = Dense(4096, activation='relu', name='fc1')(x)
x = Dense(4096, activation='relu', name='fc2')(x)
x = Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='vgg19')
# load weights
if weights == 'imagenet':
if include_top:
weights_path = get_file(
'vgg19_weights_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH,
cache_subdir='models',
file_hash='cbe5617147190e668d6c5d5026f83318')
else:
weights_path = get_file(
'vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
file_hash='253f8cb515780f3b799900260a226db6')
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
return model
def model():
inputs = Input((28, 28, 3))
x = Flatten()(inputs)
x = Dense(100, activation='relu')(x)
x = Dense(1)(x)
return Model(inputs, x)
def MobileNet(input_shape=None,
alpha=1.0,
depth_multiplier=1,
dropout=1e-3,
include_top=True,
weights='imagenet',
input_tensor=None,
pooling=None,
classes=1000):
"""Instantiates the MobileNet architecture.
To load a MobileNet model via `load_model`, import the custom
objects `relu6` and pass them to the `custom_objects` parameter.
E.g.
model = load_model('mobilenet.h5', custom_objects={
'relu6': mobilenet.relu6})
Arguments:
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(224, 224, 3)` (with `channels_last` data format)
or (3, 224, 224) (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 32.
E.g. `(200, 200, 3)` would be one valid value.
alpha: controls the width of the network.
- If `alpha` < 1.0, proportionally decreases the number
of filters in each layer.
- If `alpha` > 1.0, proportionally increases the number
of filters in each layer.
- If `alpha` = 1, default number of filters from the paper
are used at each layer.
depth_multiplier: depth multiplier for depthwise convolution
(also called the resolution multiplier)
dropout: dropout rate
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of
`layers.Input()`)
to use as image input for the model.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model
will be the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a
2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
Returns:
A Keras model instance.
Raises:
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
RuntimeError: If attempting to run this model with a
backend that does not support separable convolutions.
"""
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as ImageNet with `include_top` '
'as true, `classes` should be 1000')
# Determine proper input shape and default size.
if input_shape is None:
default_size = 224
else:
if K.image_data_format() == 'channels_first':
rows = input_shape[1]
cols = input_shape[2]
else:
rows = input_shape[0]
cols = input_shape[1]
if rows == cols and rows in [128, 160, 192, 224]:
default_size = rows
else:
default_size = 224
input_shape = _obtain_input_shape(
input_shape,
default_size=default_size,
min_size=32,
data_format=K.image_data_format(),
require_flatten=include_top,
weights=weights)
if K.image_data_format() == 'channels_last':
row_axis, col_axis = (0, 1)
else:
row_axis, col_axis = (1, 2)
rows = input_shape[row_axis]
cols = input_shape[col_axis]
if weights == 'imagenet':
if depth_multiplier != 1:
raise ValueError('If imagenet weights are being loaded, '
'depth multiplier must be 1')
if alpha not in [0.25, 0.50, 0.75, 1.0]:
raise ValueError('If imagenet weights are being loaded, '
'alpha can be one of'
'`0.25`, `0.50`, `0.75` or `1.0` only.')
if rows != cols or rows not in [128, 160, 192, 224]:
if rows is None:
rows = 224
logging.warning('MobileNet shape is undefined.'
' Weights for input shape (224, 224) will be loaded.')
else:
raise ValueError('If imagenet weights are being loaded, '
'input must have a static square shape (one of '
'(128, 128), (160, 160), (192, 192), or (224, 224)).'
' Input shape provided = %s' % (input_shape,))
if K.image_data_format() != 'channels_last':
logging.warning('The MobileNet family of models is only available '
'for the input data format "channels_last" '
'(width, height, channels). '
'However your settings specify the default '
'data format "channels_first" (channels, width, height).'
' You should set `image_data_format="channels_last"` '
'in your Keras config located at ~/.keras/keras.json. '
'The model being returned right now will expect inputs '
'to follow the "channels_last" data format.')
K.set_image_data_format('channels_last')
old_data_format = 'channels_first'
else:
old_data_format = None
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
x = _conv_block(img_input, 32, alpha, strides=(2, 2))
x = _depthwise_conv_block(x, 64, alpha, depth_multiplier, block_id=1)
x = _depthwise_conv_block(
x, 128, alpha, depth_multiplier, strides=(2, 2), block_id=2)
x = _depthwise_conv_block(x, 128, alpha, depth_multiplier, block_id=3)
x = _depthwise_conv_block(
x, 256, alpha, depth_multiplier, strides=(2, 2), block_id=4)
x = _depthwise_conv_block(x, 256, alpha, depth_multiplier, block_id=5)
x = _depthwise_conv_block(
x, 512, alpha, depth_multiplier, strides=(2, 2), block_id=6)
x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=7)
x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=8)
x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=9)
x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=10)
x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=11)
x = _depthwise_conv_block(
x, 1024, alpha, depth_multiplier, strides=(2, 2), block_id=12)
x = _depthwise_conv_block(x, 1024, alpha, depth_multiplier, block_id=13)
if include_top:
if K.image_data_format() == 'channels_first':
shape = (int(1024 * alpha), 1, 1)
else:
shape = (1, 1, int(1024 * alpha))
x = GlobalAveragePooling2D()(x)
x = Reshape(shape, name='reshape_1')(x)
x = Dropout(dropout, name='dropout')(x)
x = Conv2D(classes, (1, 1), padding='same', name='conv_preds')(x)
x = Activation('softmax', name='act_softmax')(x)
x = Reshape((classes,), name='reshape_2')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='mobilenet_%0.2f_%s' % (alpha, rows))
# load weights
if weights == 'imagenet':
if K.image_data_format() == 'channels_first':
raise ValueError('Weights for "channels_first" format '
'are not available.')
if alpha == 1.0:
alpha_text = '1_0'
elif alpha == 0.75:
alpha_text = '7_5'
elif alpha == 0.50:
alpha_text = '5_0'
else:
alpha_text = '2_5'
if include_top:
model_name = 'mobilenet_%s_%d_tf.h5' % (alpha_text, rows)
weigh_path = BASE_WEIGHT_PATH + model_name
weights_path = get_file(model_name, weigh_path, cache_subdir='models')
else:
model_name = 'mobilenet_%s_%d_tf_no_top.h5' % (alpha_text, rows)
weigh_path = BASE_WEIGHT_PATH + model_name
weights_path = get_file(model_name, weigh_path, cache_subdir='models')
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
if old_data_format:
K.set_image_data_format(old_data_format)
return model
tower_3 = layers.LeakyReLU(alpha=0.01)(tower_3)
layer_x = layers.concatenate([tower_1, tower_2, tower_3], axis=-1)
# concatenate features of tower_1, tower_2, tower_3
layer_x = layers.Reshape((100, 96))(layer_x)
# 64 LSTM units
#CPU version
layer_x = layers.LSTM(64)(layer_x)
#GPU version, cannot run on CPU
#layer_x = layers.CuDNNLSTM(64)(layer_x)
# The last output layer uses a softmax activation function
output = layers.Dense(3, activation='softmax')(layer_x)
model = Model(input_tensor, output)
opt = Adam(
lr=0.01,
epsilon=1) # learning rate and epsilon are the same as paper DeepLOB
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
print(model.summary())
data_path = os.path.join(os.getcwd(), 'Train_Dst_NoAuction_ZScore_CF_9.txt')
data = read_data(data_path)
train_X, train_Y = get_model_data(data)
train_Y = train_Y.astype(int)
# #separate 5 target variables(next 1,2,3,5 and 10 events)
# train_y = to_categorical(train_Y[:,0])# y1 is the next event's mid price (k=1)
def Xception(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000):
"""Instantiates the Xception architecture.
Optionally loads weights pre-trained
on ImageNet. This model is available for TensorFlow only,
and can only be used with inputs following the TensorFlow
data format `(width, height, channels)`.
You should set `image_data_format='channels_last'` in your Keras config
located at ~/.keras/keras.json.
Note that the default input image size for this model is 299x299.
Arguments:
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(299, 299, 3)`.
It should have exactly 3 inputs channels,
and width and height should be no smaller than 71.
E.g. `(150, 150, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
Returns:
A Keras model instance.
Raises:
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
RuntimeError: If attempting to run this model with a
backend that does not support separable convolutions.
"""
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
if K.image_data_format() != 'channels_last':
logging.warning(
'The Xception model is only available for the '
'input data format "channels_last" '
'(width, height, channels). '
'However your settings specify the default '
'data format "channels_first" (channels, width, height). '
'You should set `image_data_format="channels_last"` in your Keras '
'config located at ~/.keras/keras.json. '
'The model being returned right now will expect inputs '
'to follow the "channels_last" data format.')
K.set_image_data_format('channels_last')
old_data_format = 'channels_first'
else:
old_data_format = None
# Determine proper input shape
input_shape = _obtain_input_shape(
input_shape,
default_size=299,
min_size=71,
data_format=K.image_data_format(),
require_flatten=False,
weights=weights)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
x = Conv2D(
32, (3, 3), strides=(2, 2), use_bias=False, name='block1_conv1')(
img_input)
x = BatchNormalization(name='block1_conv1_bn')(x)
x = Activation('relu', name='block1_conv1_act')(x)
x = Conv2D(64, (3, 3), use_bias=False, name='block1_conv2')(x)
x = BatchNormalization(name='block1_conv2_bn')(x)
x = Activation('relu', name='block1_conv2_act')(x)
residual = Conv2D(
128, (1, 1), strides=(2, 2), padding='same', use_bias=False)(
x)
residual = BatchNormalization()(residual)
x = SeparableConv2D(
128, (3, 3), padding='same', use_bias=False, name='block2_sepconv1')(
x)
x = BatchNormalization(name='block2_sepconv1_bn')(x)
x = Activation('relu', name='block2_sepconv2_act')(x)
x = SeparableConv2D(
128, (3, 3), padding='same', use_bias=False, name='block2_sepconv2')(
x)
x = BatchNormalization(name='block2_sepconv2_bn')(x)
x = MaxPooling2D(
(3, 3), strides=(2, 2), padding='same', name='block2_pool')(
x)
x = layers.add([x, residual])
residual = Conv2D(
256, (1, 1), strides=(2, 2), padding='same', use_bias=False)(
x)
residual = BatchNormalization()(residual)
x = Activation('relu', name='block3_sepconv1_act')(x)
x = SeparableConv2D(
256, (3, 3), padding='same', use_bias=False, name='block3_sepconv1')(
x)
x = BatchNormalization(name='block3_sepconv1_bn')(x)
x = Activation('relu', name='block3_sepconv2_act')(x)
x = SeparableConv2D(
256, (3, 3), padding='same', use_bias=False, name='block3_sepconv2')(
x)
x = BatchNormalization(name='block3_sepconv2_bn')(x)
x = MaxPooling2D(
(3, 3), strides=(2, 2), padding='same', name='block3_pool')(
x)
x = layers.add([x, residual])
residual = Conv2D(
728, (1, 1), strides=(2, 2), padding='same', use_bias=False)(
x)
residual = BatchNormalization()(residual)
x = Activation('relu', name='block4_sepconv1_act')(x)
x = SeparableConv2D(
728, (3, 3), padding='same', use_bias=False, name='block4_sepconv1')(
x)
x = BatchNormalization(name='block4_sepconv1_bn')(x)
x = Activation('relu', name='block4_sepconv2_act')(x)
x = SeparableConv2D(
728, (3, 3), padding='same', use_bias=False, name='block4_sepconv2')(
x)
x = BatchNormalization(name='block4_sepconv2_bn')(x)
x = MaxPooling2D(
(3, 3), strides=(2, 2), padding='same', name='block4_pool')(
x)
x = layers.add([x, residual])
for i in range(8):
residual = x
prefix = 'block' + str(i + 5)
x = Activation('relu', name=prefix + '_sepconv1_act')(x)
x = SeparableConv2D(
728, (3, 3), padding='same', use_bias=False, name=prefix + '_sepconv1')(
x)
x = BatchNormalization(name=prefix + '_sepconv1_bn')(x)
x = Activation('relu', name=prefix + '_sepconv2_act')(x)
x = SeparableConv2D(
728, (3, 3), padding='same', use_bias=False, name=prefix + '_sepconv2')(
x)
x = BatchNormalization(name=prefix + '_sepconv2_bn')(x)
x = Activation('relu', name=prefix + '_sepconv3_act')(x)
x = SeparableConv2D(
728, (3, 3), padding='same', use_bias=False, name=prefix + '_sepconv3')(
x)
x = BatchNormalization(name=prefix + '_sepconv3_bn')(x)
x = layers.add([x, residual])
residual = Conv2D(
1024, (1, 1), strides=(2, 2), padding='same', use_bias=False)(
x)
residual = BatchNormalization()(residual)
x = Activation('relu', name='block13_sepconv1_act')(x)
x = SeparableConv2D(
728, (3, 3), padding='same', use_bias=False, name='block13_sepconv1')(
x)
x = BatchNormalization(name='block13_sepconv1_bn')(x)
x = Activation('relu', name='block13_sepconv2_act')(x)
x = SeparableConv2D(
1024, (3, 3), padding='same', use_bias=False, name='block13_sepconv2')(
x)
x = BatchNormalization(name='block13_sepconv2_bn')(x)
x = MaxPooling2D(
(3, 3), strides=(2, 2), padding='same', name='block13_pool')(
x)
x = layers.add([x, residual])
x = SeparableConv2D(
1536, (3, 3), padding='same', use_bias=False, name='block14_sepconv1')(
x)
x = BatchNormalization(name='block14_sepconv1_bn')(x)
x = Activation('relu', name='block14_sepconv1_act')(x)
x = SeparableConv2D(
2048, (3, 3), padding='same', use_bias=False, name='block14_sepconv2')(
x)
x = BatchNormalization(name='block14_sepconv2_bn')(x)
x = Activation('relu', name='block14_sepconv2_act')(x)
if include_top:
x = GlobalAveragePooling2D(name='avg_pool')(x)
x = Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='xception')
# load weights
if weights == 'imagenet':
if include_top:
weights_path = get_file(
'xception_weights_tf_dim_ordering_tf_kernels.h5',
TF_WEIGHTS_PATH,
cache_subdir='models',
file_hash='0a58e3b7378bc2990ea3b43d5981f1f6')
else:
weights_path = get_file(
'xception_weights_tf_dim_ordering_tf_kernels_notop.h5',
TF_WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
file_hash='b0042744bf5b25fce3cb969f33bebb97')
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
if old_data_format:
K.set_image_data_format(old_data_format)
return model
class ImageModel(object):
def __init__(self,
base_route,
train_folder="train",
validation_folder="val",
epochs=10,
fine_tune: bool = False):
self.__model = None
self.__base_model = None
self.__width = self.__height = 64
self.__train_route = os.path.join(base_route, train_folder)
self.__validation_route = os.path.join(base_route, validation_folder)
self.__fine_tuning = fine_tune
self.__epochs = epochs
self.__batch_size = 256
self.__model_route = "model.h5"
self.__early_stop = EarlyStopping(monitor='val_acc',
min_delta=0,
patience=10,
verbose=1,
mode='auto')
self.__checkpoint = self._get_model_checkpoint()
self.train_size = 0
self.validation_size = 0
self.train_steps = 0
self.validation_steps = 0
def train(self):
train_directory_iterator = self._get_directory_iterator(
self.__train_route)
validation_directory_iterator = self._get_directory_iterator(
self.__validation_route)
self.train_size = train_directory_iterator.samples
self.validation_size = validation_directory_iterator.samples
self._build_model(train_directory_iterator.num_classes)
if self.__fine_tuning:
self._set_fine_tune()
else:
self._set_transfer_learning()
self.__model.fit_generator(
train_directory_iterator,
steps_per_epoch=self.train_steps,
epochs=self.__epochs,
validation_data=validation_directory_iterator,
validation_steps=self.validation_steps,
callbacks=[self.__checkpoint, self.__early_stop])
self.__model.save(self.__model_route)
metrics = self.__model.evaluate_generator(train_directory_iterator)
return metrics
def _build_model(self, num_classes: int):
self.__base_model = VGG19(weights='imagenet',
include_top=False,
input_shape=(self.__width, self.__height, 3))
x = self.__base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(512, activation='relu')(x)
predictions = Dense(num_classes, activation='softmax')(x)
self.__model = Model(inputs=self.__base_model.input,
outputs=predictions)
def _set_transfer_learning(self):
for layer in self.__base_model.layers:
layer.trainable = False
self.__model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
def _set_fine_tune(self):
layers_to_freeze = int(len(self.__model.layers) * 0.9)
for layer in self.__model.layers[:layers_to_freeze]:
layer.trainable = False
for layer in self.__model.layers[layers_to_freeze:]:
layer.trainable = True
self.__model.compile(optimizer=SGD(lr=0.0001, momentum=0.9),
loss='categorical_crossentropy',
metrics=['accuracy'])
def _get_model_checkpoint(self):
return ModelCheckpoint(self.__model_route,
monitor='val_acc',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='auto',
period=1)
def _get_directory_iterator(self, route):
image_generator = image.ImageDataGenerator(rescale=1.0 / 255)
return image_generator.flow_from_directory(directory=route,
target_size=(self.__width,
self.__height),
batch_size=self.batch_size,
class_mode="categorical")
@property
def train_size(self):
return self.__train_size
@train_size.setter
def train_size(self, train_size):
self.__train_size = train_size
self.train_steps = math.ceil(self.train_size / self.batch_size)
@property
def validation_size(self):
return self.__validation_size
@validation_size.setter
def validation_size(self, validation_size):
self.__validation_size = validation_size
self.validation_steps = math.ceil(self.validation_size /
self.batch_size)
@property
def batch_size(self):
return self.__batch_size
@batch_size.setter
def batch_size(self, batch_size):
self.__batch_size = batch_size
self.train_steps = math.ceil(self.train_size / self.batch_size)
self.validation_steps = math.ceil(self.validation_size /
self.batch_size)
def ResNet50(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000):
"""Instantiates the ResNet50 architecture.
Optionally loads weights pre-trained
on ImageNet. Note that when using TensorFlow,
for best performance you should set
`image_data_format='channels_last'` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The data format
convention used by the model is the one
specified in your Keras config file.
Arguments:
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(224, 224, 3)` (with `channels_last` data format)
or `(3, 224, 224)` (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 197.
E.g. `(200, 200, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
Returns:
A Keras model instance.
Raises:
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
input_shape = _obtain_input_shape(
input_shape,
default_size=224,
min_size=197,
data_format=K.image_data_format(),
require_flatten=include_top,
weights=weights)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if K.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
x = Conv2D(
64, (7, 7), strides=(2, 2), padding='same', name='conv1')(img_input)
x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x = Activation('relu')(x)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='b')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='d')
x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f')
x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')
x = AveragePooling2D((7, 7), name='avg_pool')(x)
if include_top:
x = Flatten()(x)
x = Dense(classes, activation='softmax', name='fc1000')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='resnet50')
# load weights
if weights == 'imagenet':
if include_top:
weights_path = get_file(
'resnet50_weights_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH,
cache_subdir='models',
md5_hash='a7b3fe01876f51b976af0dea6bc144eb')
else:
weights_path = get_file(
'resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
md5_hash='a268eb855778b3df3c7506639542a6af')
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
return model
def InceptionV3(include_top=True,
weights='imagenet',
input_tensor=None,
model_input=None,
pooling=None,
classes=1000, model_path=""):
"""Instantiates the Inception v3 architecture.
Optionally loads weights pre-trained
on ImageNet. Note that when using TensorFlow,
for best performance you should set
`image_data_format='channels_last'` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The data format
convention used by the model is the one
specified in your Keras config file.
Note that the default input image size for this model is 299x299.
# Arguments
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization)
or 'imagenet' (pre-training on ImageNet).
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(299, 299, 3)` (with `channels_last` data format)
or `(3, 299, 299)` (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 139.
E.g. `(150, 150, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
img_input = model_input
channel_axis = 3
x = conv2d_bn(img_input, 32, 3, 3, strides=(2, 2), padding='valid')
x = conv2d_bn(x, 32, 3, 3, padding='valid')
x = conv2d_bn(x, 64, 3, 3)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv2d_bn(x, 80, 1, 1, padding='valid')
x = conv2d_bn(x, 192, 3, 3, padding='valid')
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
# mixed 0, 1, 2: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 32, 1, 1)
x = layers.concatenate(
[branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed0')
# mixed 1: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1, 1)
x = layers.concatenate(
[branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed1')
# mixed 2: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1, 1)
x = layers.concatenate(
[branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed2')
# mixed 3: 17 x 17 x 768
branch3x3 = conv2d_bn(x, 384, 3, 3, strides=(2, 2), padding='valid')
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(
branch3x3dbl, 96, 3, 3, strides=(2, 2), padding='valid')
branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = layers.concatenate(
[branch3x3, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed3')
# mixed 4: 17 x 17 x 768
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 128, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 128, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 128, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed4')
# mixed 5, 6: 17 x 17 x 768
for i in range(2):
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 160, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 160, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 160, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D(
(3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed' + str(5 + i))
# mixed 7: 17 x 17 x 768
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 192, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 192, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed7')
# mixed 8: 8 x 8 x 1280
branch3x3 = conv2d_bn(x, 192, 1, 1)
branch3x3 = conv2d_bn(branch3x3, 320, 3, 3,
strides=(2, 2), padding='valid')
branch7x7x3 = conv2d_bn(x, 192, 1, 1)
branch7x7x3 = conv2d_bn(branch7x7x3, 192, 1, 7)
branch7x7x3 = conv2d_bn(branch7x7x3, 192, 7, 1)
branch7x7x3 = conv2d_bn(
branch7x7x3, 192, 3, 3, strides=(2, 2), padding='valid')
branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = layers.concatenate(
[branch3x3, branch7x7x3, branch_pool], axis=channel_axis, name='mixed8')
# mixed 9: 8 x 8 x 2048
for i in range(2):
branch1x1 = conv2d_bn(x, 320, 1, 1)
branch3x3 = conv2d_bn(x, 384, 1, 1)
branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3)
branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1)
branch3x3 = layers.concatenate(
[branch3x3_1, branch3x3_2], axis=channel_axis, name='mixed9_' + str(i))
branch3x3dbl = conv2d_bn(x, 448, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3)
branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3)
branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1)
branch3x3dbl = layers.concatenate(
[branch3x3dbl_1, branch3x3dbl_2], axis=channel_axis)
branch_pool = AveragePooling2D(
(3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch3x3, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed' + str(9 + i))
if include_top:
# Classification block
x = GlobalAveragePooling2D(name='avg_pool')(x)
x = Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
inputs = img_input
# Create model.
model = Model(inputs, x, name='inception_v3')
# load weights
if weights == 'imagenet':
if K.image_data_format() == 'channels_first':
if K.backend() == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image data format convention '
'(`image_data_format="channels_first"`). '
'For best performance, set '
'`image_data_format="channels_last"` in '
'your Keras config '
'at ~/.keras/keras.json.')
if include_top:
weights_path = model_path
model.load_weights(weights_path)
else:
weights_path = ""
elif (weights == "trained"):
weights_path = model_path
model.load_weights(weights_path)
return model
