def model_inception_pooled(input_shape=(None, None, 3), indexes=list(range(11)),
pool_size=(5, 5), name='', return_sizes=False):
"""
Returns the wide MLSP features, spatially pooled, from InceptionV3.
Similar to `model_inception_multigap`.
* input_shape: shape of the input images
* indexes: indices to use from the usual pools
* pool_size: spatial extend of the MLSP features
* name: name of the model
* return_sizes: return the sizes of each layer: (model, pool_sizes)
:return: model or (model, pool_sizes)
"""
print('Loading InceptionV3 multi-pooled with input_shape:', input_shape)
model_base = InceptionV3(weights = 'imagenet',
include_top = False,
input_shape = input_shape)
print('Creating multi-pooled model')
ImageResizer = Lambda(lambda x: tf.image.resize_area(x, pool_size),
name='feature_resizer')
feature_layers = [model_base.get_layer('mixed%d' % i) for i in indexes]
pools = [ImageResizer(l.output) for l in feature_layers]
conc_pools = Concatenate(name='conc_pools', axis=3)(pools)
model = Model(inputs = model_base.input,
outputs = conc_pools)
if name: model.name = name
if return_sizes:
pool_sizes = [[np.int32(x) for x in f.get_shape()[1:]] for f in pools]
return model, pool_sizes
else:
return model
def set_model(self, model_name="inception_v3"):
"""
# This is a function that composes a model, and proceeds to compile.
# [Reference] - https://www.tensorflow.org/api_docs/python/tf/keras/applications/inception_v3
"""
if model_name == "inception_v3":
self.model = InceptionV3(weights="imagenet", include_top=False)
elif model_name == "mobilenet_v2":
self.model = MobileNetV2(weights="imagenet", include_top=False)
self.model_name = model_name
x = self.model.output
x = GlobalAveragePooling2D()(x)
x = Dense(128, activation="relu")(x)
x = Dropout(0.2)(x)
pred = Dense(
len(self.class_list),
kernel_regularizer=regularizers.l2(0.005),
activation="softmax",
)(x)
self.model = Model(inputs=self.model.input, outputs=pred)
self.model.compile(
optimizer=SGD(lr=0.0001, momentum=0.9),
loss="categorical_crossentropy",
metrics=["accuracy"],
)
return 1
def test_anchor_images():
os.environ.clear()
alibi_model = os.path.join(
kfserving.Storage.download(IMAGENET_EXPLAINER_URI), EXPLAINER_FILENAME)
with open(alibi_model, "rb") as f:
model = InceptionV3(weights="imagenet")
predictor = lambda x: model.predict(x) # pylint:disable=unnecessary-lambda
alibi_model = dill.load(f)
anchor_images = AnchorImages(predictor,
alibi_model,
batch_size=25,
stop_on_first=True)
category = "Persian cat"
image_shape = (299, 299, 3)
data, _ = fetch_imagenet(category,
nb_images=10,
target_size=image_shape[:2],
seed=2,
return_X_y=True)
images = preprocess_input(data)
print(images.shape)
np.random.seed(0)
explanation = anchor_images.explain(images[0:1])
exp_json = json.loads(explanation.to_json())
assert exp_json["data"]["precision"] > 0.9
def getInceptionV3Architecture(classes, dropoutRate):
# create the base pre-trained model
base_model = InceptionV3(weights='imagenet',
include_top=False,
input_shape=(224, 224, 3))
# InceptionV3
x = base_model.output
x = GlobalAveragePooling2D()(x)
# let's add a fully-connected layer
x = Dense(1024,
activation='relu',
kernel_initializer='random_uniform',
bias_initializer='random_uniform',
bias_regularizer=regularizers.l2(0.01))(x)
# add Dropout regularizer
x = Dropout(dropoutRate)(x)
# and a logistic layer with all car classes
predictions = Dense(len(classes),
activation='softmax',
kernel_initializer='random_uniform',
bias_initializer='random_uniform',
bias_regularizer=regularizers.l2(0.01),
name='predictions')(x)
# this is the model we will train
model = Model(inputs=base_model.input, outputs=predictions)
for layer in enumerate(base_model.layers):
layer[1].trainable = False
return model
def __init__(self):
'''class constructor'''
self.reference_width = 299
self.reference_height = 299
self.base_model = InceptionV3(weights='imagenet',include_top=True, input_shape=(self.reference_width, self.reference_height, 3))
self.model = Model(inputs=self.base_model.input, outputs=self.base_model.output)
def main():
args = parser.parse_args()
print(args)
# prepare the inception v3 model
model = InceptionV3(include_top=False,
pooling='avg',
input_shape=(299, 299, 3))
pos = ['top_right', 'top_left', 'bottom_left', 'bottom_right']
# pos = ['center']
fid = np.array([0] * len(pos))
for i, p in enumerate(pos):
path1 = pathlib.Path(args.path[0])
files1 = list(path1.glob('*.png'))
images1 = np.array([imread(str(fn)) for fn in files1])
images1 = crop_images(images1, 299, 299, p)
images1 = preprocess_input(images1)
path2 = pathlib.Path(args.path[1])
files2 = list(path2.glob('*.png'))
images2 = np.array([imread(str(fn)) for fn in files2])
images2 = crop_images(images2, 299, 299, p)
images2 = preprocess_input(images2)
# fid between images1 and images2
fid[i] = calculate_fid(model, images1, images2)
fid = np.mean(fid)
print('FID: %.3f' % fid)
return 0
def __cnn3(self, time_size, freq_size):
from tensorflow.keras.applications.inception_v3 import InceptionV3
inception = InceptionV3(include_top=False,
weights=None,
pooling='max',
input_shape=(time_size, freq_size, 1))
dense = keras.layers.Dense(1024, name='dense1',
activation='relu')(inception.outputs[0])
dense = keras.layers.Dense(512, name='dense2',
activation='relu')(dense)
dense = keras.layers.Dense(1, name='dense_out')(dense)
new_model = keras.Model(inputs=inception.inputs[0],
outputs=dense,
name='Preciser_v2-3')
optimizer = keras.optimizers.SGD(lr=0.001,
decay=1e-6,
momentum=0.9,
nesterov=True)
# optimizer = keras.optimizers.Adam()
loss = keras.losses.mean_squared_error
metric0 = keras.metrics.mean_squared_error
metric1 = keras.metrics.mean_absolute_error
new_model.compile(loss=loss,
optimizer=optimizer,
metrics=[metric0, metric1])
metrics_names = ['loss', metric0.__name__, metric1.__name__]
return new_model, metrics_names
def make_inception(num_classes):
model = InceptionV3(input_tensor=image_inputA, include_top=True)
last_layer = model.get_layer('avg_pool').output
out = Dense(1, activation='sigmoid', name='output')(last_layer)
custommodel = Model(inputs=image_inputA, outputs=out)
# summarize the model
return custommodel
def build_inception_net(neurons_top_dense_layer=1024, learning_rate=0.0001):
urllib.request.urlretrieve(WEIGHTS_URL, WEIGHTS_FILE)
pre_trained_model = InceptionV3(input_shape=(256, 256, 3),
include_top=False,
weights=None)
pre_trained_model.load_weights(WEIGHTS_FILE)
for layer in pre_trained_model.layers:
layer.trainable = False
last_layer = pre_trained_model.get_layer('mixed7')
print('Last layer output shape: ', last_layer.output_shape)
last_output = last_layer.output
x = Flatten()(last_output)
x = Dense(neurons_top_dense_layer, activation='relu')(x)
x = Dense(1, activation='sigmoid')(x)
model = Model(pre_trained_model.input, x)
model_checkpoint = ModelCheckpoint(
filepath="./log/mars-rover-{epoch:02d}.hdf5",
save_best_only=False,
save_weights_only=True,
monitor='val_accuracy',
mode='max')
model.compile(optimizer=Adam(lr=learning_rate),
loss='binary_crossentropy',
metrics=['acc'])
return model, model_checkpoint
def feature_extraction_InV3(train_data_dir, num_image, epochs):
# 建立預訓練模型
base_model = InceptionV3(input_shape=(299, 299, 3),
weights='imagenet',
include_top=False)
x = GlobalAveragePooling2D()(base_model.output)
model = Model(inputs=base_model.input, outputs=x)
# 設定圖片生成器格式，載入圖片。
generator = ImageDataGenerator(rescale=1. / 255).\
flow_from_directory(train_data_dir,
target_size = (299, 299),
batch_size = 15,
class_mode = "categorical",
shuffle=False)
# 產生圖片檔資料集
# y_train是label，只需要手動設定類別數，並one-hot化。
# x_train是data，需要利用預訓練模型來事先處理一次。
train = generator.classes
y_train = np.zeros((num_image, 6))
y_train[np.arange(num_image), train] = 1
X_train = model.predict(generator, verbose=1)
return X_train, y_train
def get_base_model(model_name, weights_path, weight_decay=1e-4):
"""
Define base model used in transfer learning.
"""
if not weights_path:
weights_path = 'imagenet'
if model_name == 'VGG16':
base_model = VGG16(weights=weights_path, include_top=False)
elif model_name == 'VGG19':
base_model = VGG19(weights=weights_path, include_top=False)
elif model_name == 'ResNet50V1':
base_model = awsdet.models.backbones.ResNet50(weights=None, include_top=False, weight_decay=weight_decay)
elif model_name == 'ResNet50V2':
base_model = awsdet.models.backbones.ResNet50V2(weights=None, include_top=False, weight_decay=weight_decay)
elif model_name == 'Xception':
base_model = Xception(weights=weights_path, include_top=False)
elif model_name == 'InceptionV3':
base_model = InceptionV3(weights=weights_path, include_top=False)
elif model_name == 'InceptionResNetV2':
base_model = InceptionResNetV2(weights=weights_path,
include_top=False)
elif model_name == 'MobileNet':
base_model = MobileNet(weights=weights_path, include_top=False)
else:
raise ValueError(
'Valid base model values are: "VGG16","VGG19","ResNet50V1", "ResNet50V2", "Xception", \
"InceptionV3","InceptionResNetV2","MobileNet".'
)
return base_model
def transfer_learning():
import os
from tensorflow.keras import layers
from tensorflow.keras import Model
from tensorflow.keras.optimizers import RMSprop
#!wget - -no - check - certificate \
# https: // storage.googleapis.com / mledu - datasets / inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5 \
# - O / tmp / inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5
from tensorflow.keras.applications.inception_v3 import InceptionV3
local_weights_file = '/tmp/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5'
pre_trained_model = InceptionV3(input_shape=(150, 150, 3),
include_top=False,
weights=None)
pre_trained_model.load_weights(local_weights_file)
for layer in pre_trained_model.layers:
layer.trainable = False
pre_trained_model.summary()
last_layer = pre_trained_model.get_layer('mixed7')
print('last layer output shape: ', last_layer.output_shape)
last_output = last_layer.output
# Flatten the output layer to 1 dimension
x = layers.Flatten()(last_output)
# Add a fully connected layer with 1,024 hidden units and ReLU activation
x = layers.Dense(1024, activation='relu')(x)
# Add a dropout rate of 0.2
x = layers.Dropout(0.2)(x)
# Add a final sigmoid layer for classification
x = layers.Dense(1, activation='sigmoid')(x)
model = Model(pre_trained_model.input, x)
model.compile(optimizer=RMSprop(lr=0.0001),
loss='binary_crossentropy',
metrics=['accuracy'])
def extractImgFeature(self, filename, modelType):
if modelType == 'inceptionv3':
from tensorflow.keras.applications.inception_v3 import preprocess_input
target_size = (299, 299)
model = InceptionV3()
elif modelType == 'xception':
from tensorflow.keras.applications.xception import preprocess_input
target_size = (299, 299)
model = Xception()
elif modelType == 'vgg16':
from tensorflow.keras.applications.vgg16 import preprocess_input
target_size = (224, 224)
model = VGG16()
elif modelType == 'resnet50':
from tensorflow.keras.applications.resnet50 import preprocess_input
target_size = (224, 224)
model = ResNet50()
model.layers.pop()
model = Model(inputs=model.inputs, outputs=model.layers[-1].output)
image = load_img(filename,
target_size=target_size) # Loading and resizing image
image = img_to_array(
image) # Convert the image pixels to a numpy array
image = image.reshape((1, image.shape[0], image.shape[1],
image.shape[2])) # Reshape data for the model
image = preprocess_input(
image) # Prepare the image for the CNN Model model
features = model.predict(
image, verbose=0) # Pass image into model to get encoded features
return features
def merge_resnet_inceptionv3flat(NUM_CLASS):
inceptionv3 = InceptionV3(weights='imagenet', include_top=False)
resnet50 = ResNet50(weights='imagenet', include_top=False)
res_out = resnet50.output
res_out = GlobalAveragePooling2D()(res_out)
res_out = Dropout(0.5)(res_out)
res_out = Dense(2048)(res_out)
inc_out = inceptionv3.output
inc_out = GlobalAveragePooling2D()(inc_out)
inc_out = Dropout(0.5)(inc_out)
inc_out = Dense(2048)(inc_out)
i_flat = Flatten()(inc_out)
r_flat = Flatten()(res_out)
merge = Concatenate()([r_flat, i_flat])
x = Dense(2048)(merge)
x = Dropout(0.5)(x)
x = Dense(1024)(x)
x = Dense(512)(x)
x = Dropout(0.5)(x)
output = Dense(NUM_CLASS, activation='softmax', name='output_layer')(x)
model = Model(inputs=[resnet50.input, inceptionv3.input], outputs=output)
# plot graph
plot_model(model,
to_file='multiple_inputs.png',
show_shapes=True,
dpi=600,
expand_nested=False)
# model.summary()
return model
def calculate_inception_score(self, images, n_split=10, eps=1E-16):
# load inception v3 model
model = InceptionV3()
# enumerate splits of images/predictions
scores = list()
n_part = floor(images.shape[0] / n_split)
for i in range(n_split):
# retrieve images
ix_start, ix_end = i * n_part, (i + 1) * n_part
subset = images[ix_start:ix_end]
# convert from uint8 to float32
subset = subset.astype('float32')
# scale images to the required size
subset = self.scale_images(subset, (299, 299, 3))
# pre-process images, scale to [-1,1]
subset = preprocess_input(subset)
# predict p(y|x)
p_yx = model.predict(subset)
# calculate p(y)
p_y = expand_dims(p_yx.mean(axis=0), 0)
# calculate KL divergence using log probabilities
kl_d = p_yx * (log(p_yx + eps) - log(p_y + eps))
# sum over classes
sum_kl_d = kl_d.sum(axis=1)
# average over images
avg_kl_d = mean(sum_kl_d)
# undo the log
is_score = exp(avg_kl_d)
# store
scores.append(is_score)
# average across images
is_avg, is_std = mean(scores), std(scores)
return is_avg, is_std
def __init__(self):
goods_dataset = GoodsDataset("dataset-181018.list",
"dataset-181018.labels",
settings.IMAGE_SIZE, settings.train_batch,
settings.valid_batch, settings.multiply,
settings.valid_percentage)
train_set = goods_dataset.get_train_dataset()
valid_set = goods_dataset.get_valid_dataset()
input_tensor = keras.layers.Input(shape=(IMAGE_SIZE[0], IMAGE_SIZE[1],
3))
base_model = InceptionV3(weights='imagenet',
include_top=False,
pooling='avg',
input_tensor=input_tensor)
output_layer_number = 248
intermediate_layer_model = keras.Model(
inputs=base_model.input,
outputs=base_model.layers[output_layer_number].output)
def _intermediate_processing(images, labels):
images = intermediate_layer_model.predict(images, steps=77)
return images, labels
self.train_set = train_set.map(
_intermediate_processing) #, num_parallel_calls=8)
self.valid_set = valid_set.map(
_intermediate_processing) #, num_parallel_calls=8)
def create_model():
base_model = InceptionV3(weights='imagenet', include_top=False)
for i, layer in enumerate(base_model.layers):
print(i, layer.name)
# add a global spatial average pooling layer
x = base_model.output
x = GlobalAveragePooling2D()(x)
# let's add a fully-connected layer
x = Dense(512, activation='relu',
kernel_regularizer=regularizers.l1_l2(l1=1e-1, l2=1e-1))(x) #1e-4, 1e-4
x = Dropout(rate=0.5)(x)
# and a logistic layer
action = lambda x: tf.keras.activations.relu(x,max_value=4)
predictions = Dense(1, activation=action)(x)
#arg_predictions = Lambda(argmax_layer,name="argmax")(predictions)
# this is the model we will train
model = Model(inputs=base_model.input, outputs=predictions)
optimiser = tf.keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=0.1,amsgrad=False)
model.compile(optimizer=optimiser, loss='mse',
metrics=['accuracy'])
#loss_func ={
# 'argmax':"mse",
#}
#metrics={
# 'softmax':['sparse_categorical_crossentropy','accuracy'],
# 'argmax': ['mse']
#}
#model.compile(optimizer=optimiser, loss=loss_func,
# metrics=metrics)
return model,base_model
def calculate_fid(images1, images2, model=None, return_acts=True):
# Prepare the inception v3 model
if model is None:
model = InceptionV3(include_top=False,
pooling='avg',
input_shape=(256, 256, 3))
# Calculate activations
act1 = model.predict(images1, verbose=1)
act2 = model.predict(images2, verbose=1)
# Calculate mean and covariance statistics
mu1, sigma1 = act1.mean(axis=0), np.cov(act1, rowvar=False)
mu2, sigma2 = act2.mean(axis=0), np.cov(act2, rowvar=False)
# Calculate sum squared difference between means
ssdiff = np.sum((mu1 - mu2)**2.0)
# Calculate sqrt of product between cov
covmean = sqrtm(sigma1.dot(sigma2))
# Check and correct imaginary numbers from sqrt
if np.iscomplexobj(covmean):
covmean = covmean.real
# Calculate score
fid = ssdiff + np.trace(sigma1 + sigma2 - 2.0 * covmean)
if return_acts:
return fid, act1, act2
return fid
def __init__(self, img_size, batch_size):
self.train_data_gen = ImageDataGenerator(rescale=1.0 / 255.0)
self.test_data_gen = ImageDataGenerator(rescale=1.0 / 255.0)
self.train_dir = "./data/animal10/raw-img/train"
self.test_dir = "./data/animal10/raw-img/test"
self.image_size = img_size
self.batch_size = 32
self.InceptionV3_base_model = InceptionV3(weights='imagenet',
include_top=False)
# print(self.InceptionV3_base_model.summary())
self.label_dict = {
'0': "dog",
"1": "horse",
"2": "elephant",
"3": "butterfly",
"4": "chicken",
"5": "cat",
"6": "cow",
"7": "sheep",
'8': 'spider',
"9": "squirrel",
}
def create_model(self):
local_weights_file = 'dataset/model/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5'
pre_trained_model = InceptionV3(input_shape=(150, 150, 3),
include_top=False,
weights=None)
pre_trained_model.load_weights(local_weights_file)
for layer in pre_trained_model.layers:
layer.trainable = False
last_layer = pre_trained_model.get_layer('mixed7')
last_output = last_layer.output
print('last layer output shape:', last_layer.output_shape)
# pre_trained_model.summary()
# Flatten the output layer to 1 dimension
x = layers.Flatten()(last_output)
# Add a fully connected layer with 1,024 hidden units and ReLU activation
x = layers.Dense(1024, activation='relu')(x)
# Add a dropout rate of 0.2
x = layers.Dropout(0.2)(x)
# Add a final sigmoid layer for classification
x = layers.Dense(1, activation='sigmoid')(x)
# Configure and compile the model
model = Model(pre_trained_model.input, x)
model.compile(loss='binary_crossentropy',
optimizer=RMSprop(lr=0.0001),
metrics=['acc'])
return model
def create_model(hp):
base_model = InceptionV3(weights='imagenet')
y = base_model.get_layer('mixed5').output
y = GlobalAveragePooling2D()(y)
y = Dense(8, activation='softmax', name='predictions')(y)
model = Model(inputs=base_model.input, outputs=y)
optimizer_init = get_optimizer(
hp.Choice('optimizer',
['Adam', 'SGD', 'RMSprop', 'Adadelta', 'Adagrad']),
hp.Choice('learning_rate', [1e-1, 1e-2, 1e-3]),
hp.Choice('momentum', [0.6, 0.8, 0.9]))
activ_func = hp.Choice('dense_activation', ['relu', 'tanh', 'elu'])
for layer in model.layers:
if str(layer.__class__.__name__) == "Activation":
layer.activation = activ_func_dict[activ_func]
model.compile(loss='categorical_crossentropy',
optimizer=optimizer_init,
metrics=['accuracy'])
return model
def __init__(self):
self.inception_input_shape = (299, 299, 3)
self.inception_input_size = (299, 299)
self.inception_model = InceptionV3(
include_top=False,
pooling='avg',
input_shape=self.inception_input_shape)
def get_inception_model(trainable=False,
num_nodes=NUM_NODES,
dropout_rate=DROPOUT_RATE,
learning_rate=LEARNING_RATE,
print_summary=True):
"""
Builds and compiles InceptionV3 transfer learning model.
"""
transfer_model = InceptionV3(
input_shape=INPUT_SHAPE,
include_top=False, # leave out the last fully connected layer
weights='imagenet')
for layer in transfer_model.layers:
layer.trainable = trainable
hidden_layers = []
hidden_layers.append(layers.Flatten()(transfer_model.output))
hidden_layers.append(
layers.Dense(num_nodes, activation="relu")(hidden_layers[-1]))
hidden_layers.append(layers.Dropout(dropout_rate)(hidden_layers[-1]))
output_layer = layers.Dense(NUM_LABELS,
activation="softmax")(hidden_layers[-1])
model1 = Model(transfer_model.input, output_layer)
model1.compile(loss=losses.CategoricalCrossentropy(),
optimizer=optimizers.Adam(learning_rate=learning_rate),
metrics=METRICS)
if print_summary:
model1.summary()
return model1
def createModel(my_learning_rate):
# Load the inception V3 model
local_weights_file = '.\InceptionV3ModelWeights\inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5'
pretrained_inception_model = InceptionV3(input_shape=(150, 150, 3),
include_top=False,
weights=None)
pretrained_inception_model.load_weights(local_weights_file)
# By specifying the include_top=False argument, we load a network that doesn't include the classification layers at the top—ideal for feature extraction.
# Make the base model non-trainable
for layer in pretrained_inception_model.layers:
layer.trainable = False
# Get the mixed7 layer from the inception model and get its output
mixed7layer = pretrained_inception_model.get_layer('mixed7')
output = mixed7layer.output
# Add our custom layers on top of the pre-trained model
x = layers.Flatten()(output)
x = layers.Dense(1024, activation='relu')(x)
x = layers.Dropout(0.2)(x)
x = layers.Dense(61, activation='softmax')(
x) # We hve 61 distinct classes hence 61 logits
# Create the model
model = Model(pretrained_inception_model.input, x)
model.compile(
optimizer=tf.keras.optimizers.Adam(lr=my_learning_rate),
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy', 'sparse_categorical_crossentropy'])
return model
def get_InceptionV3_whole_model():
""" Combain together two parts of networks:
1-st part - 248 layers from InceptionV3
2-nd part - InceptionV3_top60_layers
"""
input_tensor = keras.layers.Input(shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3))
base_model = InceptionV3(weights='imagenet',
include_top=False,
pooling='avg',
input_tensor=input_tensor)
output_layer_number = 248
first_layers_model = keras.Model(
inputs=base_model.input,
outputs=base_model.layers[output_layer_number].output)
x = first_layers_model.output
x = InceptionV3_top60_layers(x,
classes=settings.num_classes,
pooling='avg')
model = keras.Model(inputs=base_model.input,
outputs=x,
name='inception_v3_whole_model')
#for layer in model.layers[:249]:
# layer.trainable = False
return model
def define_model(n_layers=45, BASE_MODEL='ResNet50V2'):
if BASE_MODEL == 'ResNet50V2':
# Pre-trained model with MobileNetV2
base_model = ResNet50V2(input_shape=(
IMG_WIDTH, IMG_HEIGHT, 3), include_top=False, weights='imagenet')
head_model = base_model
for layers in base_model.layers[:n_layers]:
layers.trainable = True
head_model = head_model.output
head_model = tf.keras.layers.GlobalMaxPooling2D()(head_model)
head_model = tf.keras.layers.Flatten(name="Flatten")(head_model)
head_model = tf.keras.layers.Dense(1600, activation='relu')(head_model)
head_model = tf.keras.layers.Dropout(0.2)(head_model)
prediction_layer = tf.keras.layers.Dense(
len(classes), activation='softmax')(head_model)
model = tf.keras.Model(inputs=base_model.input,
outputs=prediction_layer)
if BASE_MODEL == 'InceptionV3':
base_model = InceptionV3(input_shape=(
IMG_WIDTH, IMG_HEIGHT, 3), include_top=False, weights='imagenet')
head_model = base_model
for layers in base_model.layers[:n_layers]:
layers.trainable = False
head_model = head_model.output
head_model = tf.keras.layers.GlobalMaxPooling2D()(head_model)
head_model = tf.keras.layers.Flatten(name="Flatten")(head_model)
head_model = tf.keras.layers.Dense(1024, activation='relu')(head_model)
head_model = tf.keras.layers.Dropout(0.5)(head_model)
prediction_layer = tf.keras.layers.Dense(
len(classes), activation='softmax')(head_model)
model = tf.keras.Model(inputs=base_model.input,
outputs=prediction_layer)
return model
def create_cnn_model():
local_weights_file = './local_weights.h5'
pre_trained_model = InceptionV3(input_shape=(150, 150, 3),
include_top=False,
weights=None)
pre_trained_model.load_weights(local_weights_file)
for layer in pre_trained_model.layers:
layer.trainable = False
last_layer = pre_trained_model.get_layer('mixed7')
print('last layer output shape: ', last_layer.output_shape)
last_output = last_layer.output
# Flatten the output layer to 1 dimension
x = layers.Flatten()(last_output)
# Add a fully connected layer with 1,024 hidden units and ReLU activation
x = layers.Dense(512, activation='relu')(x)
# Add a dropout rate of 0.2
x = layers.Dropout(0.4)(x)
# Add a final sigmoid layer for classification
x = layers.Dense(5, activation='softmax')(x)
model = Model(pre_trained_model.input, x)
model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(lr=0.0001),
metrics=['accuracy'])
print(model.summary())
return model
def extract_features_inception_v3(frames_path, videos_selected):
"""
Extracting features from the Frames using InceptionV3 pretrained model. Output is of shape (n, 15, 2048): For n videos and 15 frames for each video
frames_path: path to the folder which contains the frames
videos_selected: list of final video names which meet the min frames threshold
"""
model = InceptionV3(weights='imagenet', include_top=True)
feature_extractor = Model(model.input, model.get_layer('avg_pool').output)
X = []
for video_name in videos_selected:
l = []
count = 0
for img_name in os.listdir(frames_path + video_name):
if count == 15:
break
img_path = 'dataset/msvd_videos/frames/' + video_name + "/" + img_name
img = image.load_img(img_path, target_size=(299, 299))
img_data = image.img_to_array(img)
img_data = np.expand_dims(img_data, axis=0)
img_data = preprocess_input_inception_v3(img_data)
features = feature_extractor.predict(img_data)
features = features.flatten()
l.append(features)
count += 1
print("Loading for", video_name)
X.append(l)
X = np.array(X)
return X
def get_InceptionV3_image_feature(image_path):
img = preprocess(image_path)
image_model = InceptionV3(include_top=False,
weights='imagenet',
pooling='avg')
feature = image_model.predict(img) # [1, 2048]
return feature
def calculate_inception_score(images, n_split=10, eps=1E-16):
# load inception v3 model
model = InceptionV3()
# convert from uint8 to float32
processed = images.astype('float32')
# pre-process raw images for inception v3 model
processed = preprocess_input(processed)
# predict class probabilities for images
yhat = model.predict(processed)
# enumerate splits of images/predictions
scores = list()
n_part = floor(images.shape[0] / n_split)
for i in range(n_split):
# retrieve p(y|x)
ix_start, ix_end = i * n_part, i * n_part + n_part
p_yx = yhat[ix_start:ix_end]
# calculate p(y)
p_y = expand_dims(p_yx.mean(axis=0), 0)
# calculate KL divergence using log probabilities
kl_d = p_yx * (log(p_yx + eps) - log(p_y + eps))
# sum over classes
sum_kl_d = kl_d.sum(axis=1)
# average over images
avg_kl_d = mean(sum_kl_d)
# undo the log
is_score = exp(avg_kl_d)
# store
scores.append(is_score)
# average across images
is_avg, is_std = mean(scores), std(scores)
return is_avg, is_std
