def predict_label_multi(model, labels, imagepath, preprocess=None):
img = tf.keras.preprocessing.image.load_img(os.path.join(imagepath), target_size=(224, 224))
img_array = tf.keras.preprocessing.image.img_to_array(img)
img_array = np.expand_dims(img_array, 0) # Create batch axis
if preprocess == 'vgg16':
img_array = preprocess_input_VGG16(img_array)
elif preprocess == 'vgg19':
img_array = preprocess_input_VGG19(img_array)
elif preprocess == 'MobileNetV2':
img_array = preprocess_input_MNV2(img_array)
elif preprocess == 'ResNet50':
img_array = Preprocess_RESNET50(img_array)
predictions = model.predict(img_array)[0]
index = np.argmax(predictions)
score = predictions[int(index)]
result = list(labels.values())[int(index)]
# imge = mpimg.imread(imagepath)
# plt.figure(figsize=(5, 5))
# plt.imshow(imge)
# plt.title(list(labels.keys())[int(score)])
# plt.xticks([])
# plt.yticks([])
# plt.show()
return result, score
def read_and_prep_image(image_path, model_name, img_width=IMAGE_WIDTH, img_height=IMAGE_HEIGHT):
image_raw = load_img(image_path, target_size=(img_height, img_width))
image_array = np.array([img_to_array(image_raw)])
if model_name == "ResNet50":
image = preprocess_input_ResNet50(image_array)
elif model_name == "VGG16":
image = preprocess_input_VGG16(image_array)
elif model_name == "VGG19":
image = preprocess_input_VGG19(image_array)
return image
def model_predictions(model,
features,
features_type,
seq_length,
categories_per_output,
img_width=224,
img_height=224,
cnnType='resnet',
batch_size=0):
'''
Used to make predictions, especially useful when input
is mixed with both preprocessed features and frames
Inputs:
model: a Keras model
features : [X_features, X_frames]
X_features: a numpy array [1, total_time_steps, features_number]
X_frames: a list of frame paths
features_type: 'features', 'frames' or 'both'
seq_length
categories_per_output: a list of number of categories for each output
batch_size: if 0, predictions are sequence per sequence
if >0, predictions are run by batches
Outputs:
predictions
'''
N_outputs = len(categories_per_output)
if features_type == 'frames':
total_length_round = (len(features[1]) // seq_length) * seq_length
elif features_type == 'features' or features_type == 'both':
total_length_round = (features[0].shape[1] // seq_length) * seq_length
feature_number = features[0].shape[2]
else:
sys.exit('Wrong features type')
if features_type == 'features' or features_type == 'both':
X_features = features[0][:, :total_length_round, :].reshape(
-1, seq_length, feature_number)
#batch_size_time = np.min([batch_size*seq_length, total_length_round])
if batch_size == 0:
if features_type == 'frames' or features_type == 'both':
X_frames = np.zeros(
(1, total_length_round, img_width, img_height, 3))
for iFrame in range(total_length_round):
if cnnType == 'resnet':
X_frames[0, iFrame, :, :, :] = preprocess_input_ResNet50(
img_to_array(
load_img(features[1][iFrame],
target_size=(img_width, img_height))))
elif cnnType == 'vgg':
X_frames[0, iFrame, :, :, :] = preprocess_input_VGG16(
img_to_array(
load_img(features[1][iFrame],
target_size=(img_width, img_height))))
elif cnnType == 'mobilenet':
X_frames[0, iFrame, :, :, :] = preprocess_input_MobileNet(
img_to_array(
load_img(features[1][iFrame],
target_size=(img_width, img_height))))
else:
sys.exit('Invalid CNN network model')
X_frames = X_frames.reshape(-1, seq_length, img_width, img_height,
3)
if features_type == 'features':
output = model.predict(X_features)
elif features_type == 'frames':
output = model.predict(X_frames)
else: #features_type == 'both':
output = model.predict([X_features, X_frames])
else:
if N_outputs > 1:
output = [
np.zeros((1, total_length_round, categories_per_output[i]))
for i in range(N_outputs)
]
else:
output = np.zeros(
(1, total_length_round, categories_per_output[0]))
N_full_batches = total_length_round // (batch_size * seq_length)
remainding_length = total_length_round - N_full_batches * batch_size * seq_length
# Full batches:
for i_batch in range(N_full_batches):
i_seq_start = i_batch * batch_size
i_seq_end = (i_batch + 1) * batch_size
i_frame_start = i_seq_start * seq_length
i_frame_end = i_seq_end * seq_length
if features_type == 'frames' or features_type == 'both':
X_frames_batch = np.zeros(
(1, batch_size * seq_length, img_width, img_height, 3))
for iFrame in range(i_frame_start, i_frame_end):
if cnnType == 'resnet':
X_frames_batch[
0, iFrame -
i_frame_start, :, :, :] = preprocess_input_ResNet50(
img_to_array(
load_img(features[1][iFrame],
target_size=(img_width,
img_height))))
elif cnnType == 'vgg':
X_frames_batch[
0, iFrame -
i_frame_start, :, :, :] = preprocess_input_VGG16(
img_to_array(
load_img(features[1][iFrame],
target_size=(img_width,
img_height))))
elif cnnType == 'mobilenet':
X_frames_batch[
0, iFrame -
i_frame_start, :, :, :] = preprocess_input_MobileNet(
img_to_array(
load_img(features[1][iFrame],
target_size=(img_width,
img_height))))
else:
sys.exit('Invalid CNN network model')
X_frames_batch = X_frames_batch.reshape(
-1, seq_length, img_width, img_height, 3)
if features_type == 'features':
pred = model.predict(X_features[i_seq_start:i_seq_end, :, :])
elif features_type == 'frames':
pred = model.predict(X_frames_batch)
else: #features_type == 'both':
pred = model.predict(
[X_features[i_seq_start:i_seq_end, :, :], X_frames_batch])
if N_outputs > 1:
for i_out in range(N_outputs):
output[i_out][
0, i_frame_start:i_frame_end, :] = pred[i_out].reshape(
1, -1, categories_per_output[i_out])
else:
output[0, i_frame_start:i_frame_end, :] = pred.reshape(
1, -1, categories_per_output[0])
# Last (incomplete) batch:
i_seq_start = N_full_batches * batch_size
i_seq_end = total_length_round // seq_length
i_frame_start = i_seq_start * seq_length
i_frame_end = i_seq_end * seq_length
if features_type == 'frames' or features_type == 'both':
X_frames_batch = np.zeros(
(1, remainding_length, img_width, img_height, 3))
for iFrame in range(i_frame_start, i_frame_end):
if cnnType == 'resnet':
X_frames_batch[
0, iFrame -
i_frame_start, :, :, :] = preprocess_input_ResNet50(
img_to_array(
load_img(features[1][iFrame],
target_size=(img_width, img_height))))
elif cnnType == 'vgg':
X_frames_batch[
0, iFrame -
i_frame_start, :, :, :] = preprocess_input_VGG16(
img_to_array(
load_img(features[1][iFrame],
target_size=(img_width, img_height))))
elif cnnType == 'mobilenet':
X_frames_batch[
0, iFrame -
i_frame_start, :, :, :] = preprocess_input_MobileNet(
img_to_array(
load_img(features[1][iFrame],
target_size=(img_width, img_height))))
else:
sys.exit('Invalid CNN network model')
X_frames_batch = X_frames_batch.reshape(-1, seq_length, img_width,
img_height, 3)
if features_type == 'features':
pred = model.predict(X_features[i_seq_start:, :, :])
elif features_type == 'frames':
pred = model.predict(X_frames_batch)
else: #features_type == 'both':
pred = model.predict(
[X_features[i_seq_start:, :, :], X_frames_batch])
if N_outputs > 1:
for i_out in range(N_outputs):
output[i_out][
0, i_frame_start:i_frame_end, :] = pred[i_out].reshape(
1, -1, categories_per_output[i_out])
else:
output[0, i_frame_start:i_frame_end, :] = pred.reshape(
1, -1, categories_per_output[0])
if N_outputs > 1:
for i_out in range(N_outputs):
output[i_out] = output[i_out].reshape(
-1, seq_length, categories_per_output[i_out])
else:
output = output.reshape(-1, seq_length, categories_per_output[0])
return output
def generator(features,
features_type,
annot,
batch_size,
seq_length,
output_form,
output_class_weights,
img_width,
img_height,
cnnType):
"""
Generator function for batch training models
features: [preprocessed features (numpy array (1, time_steps, nb_features)), images_path (list of strings)]
"""
if features_type == 'frames':
total_length_round = (len(features[1])//seq_length)*seq_length
elif features_type == 'features' or features_type == 'both':
total_length_round = (features[0].shape[1]//seq_length)*seq_length
feature_number = features[0].shape[2]
else:
sys.exit('Wrong features type')
batch_size_time = np.min([batch_size*seq_length, total_length_round])
if features_type == 'frames' or features_type == 'both':
batch_frames = np.zeros((1, batch_size_time, img_width, img_height, 3))
if features_type == 'features' or features_type == 'both':
batch_features = np.zeros((1, batch_size_time, feature_number))
if output_class_weights != []:
if output_form == 'mixed':
annot_labels_weight = []#np.ones((1, annot[0].shape[1]))
batch_labels_weight = []#np.zeros((1, batch_size_time))
labels_number = len(annot)
for i_label_cat in range(labels_number):
annot_labels_weight_tmp = np.zeros((1, annot[i_label_cat].shape[1]))
nClasses = annot[i_label_cat].shape[2]
for iClass in range(nClasses):
annot_labels_weight_tmp[0, np.argmax(annot[i_label_cat][0,:,:],axis=1)==iClass] = output_class_weights[i_label_cat][iClass]
annot_labels_weight.append(annot_labels_weight_tmp)# = annot_labels_weight*annot_labels_weight_tmp
batch_labels_weight.append(np.zeros((1, batch_size_time)))
elif output_form == 'sign_types':
nClasses = annot.shape[2]
annot_labels_weight=np.zeros((1, annot.shape[1]))
batch_labels_weight = np.zeros((1, batch_size_time))
for iClass in range(nClasses):
annot_labels_weight[0, np.argmax(annot[0,:,:],axis=1)==iClass] = output_class_weights[0][iClass]
if output_form == 'mixed':
batch_labels = []
labels_number = len(annot)
labels_shape = []
for i_label_cat in range(labels_number):
labels_shape.append(annot[i_label_cat].shape[2])
batch_labels.append(np.zeros((1, batch_size_time, labels_shape[i_label_cat])))
elif output_form == 'sign_types':
labels_shape = annot.shape[2]
batch_labels = np.zeros((1, batch_size_time, labels_shape))
else:
sys.exit('Wrong annotation format')
while True:
# Random start
random_ini = np.random.randint(0, total_length_round)
end = random_ini + batch_size_time
end_modulo = np.mod(end, total_length_round)
# Fill in batch features
if features_type == 'features' or features_type == 'both':
batch_features = batch_features.reshape(1, batch_size_time, feature_number)
if end <= total_length_round:
batch_features = np.copy(features[0][0, random_ini:end, :])
else:
batch_features[0, :(total_length_round - random_ini), :] = np.copy(features[0][0, random_ini:total_length_round, :])
batch_features[0, (total_length_round - random_ini):, :] = np.copy(features[0][0, 0:end_modulo, :])
batch_features = batch_features.reshape(-1, seq_length, feature_number)
if features_type == 'frames' or features_type == 'both':
batch_frames = batch_frames.reshape(1, batch_size_time, img_width, img_height, 3)
if end <= total_length_round:
for iFrame in range(random_ini, end):
if cnnType=='resnet':
batch_frames[0, iFrame-random_ini, :, :, :] = preprocess_input_ResNet50(img_to_array(load_img(features[1][iFrame], target_size=(img_width, img_height))))
elif cnnType=='vgg':
batch_frames[0, iFrame-random_ini, :, :, :] = preprocess_input_VGG16(img_to_array(load_img(features[1][iFrame], target_size=(img_width, img_height))))
elif cnnType=='mobilenet':
batch_frames[0, iFrame-random_ini, :, :, :] = preprocess_input_MobileNet(img_to_array(load_img(features[1][iFrame], target_size=(img_width, img_height))))
else:
sys.exit('Invalid CNN network model')
else:
for iFrame in range(random_ini,total_length_round):
if cnnType=='resnet':
batch_frames[0, iFrame-random_ini, :, :, :] = preprocess_input_ResNet50(img_to_array(load_img(features[1][iFrame], target_size=(img_width, img_height))))
elif cnnType=='vgg':
batch_frames[0, iFrame-random_ini, :, :, :] = preprocess_input_VGG16(img_to_array(load_img(features[1][iFrame], target_size=(img_width, img_height))))
elif cnnType=='mobilenet':
batch_frames[0, iFrame-random_ini, :, :, :] = preprocess_input_MobileNet(img_to_array(load_img(features[1][iFrame], target_size=(img_width, img_height))))
else:
sys.exit('Invalid CNN network model')
for iFrame in range(0, end_modulo):
if cnnType=='resnet':
batch_frames[0, iFrame+total_length_round-random_ini, :, :, :] = preprocess_input_ResNet50(img_to_array(load_img(features[1][iFrame], target_size=(img_width, img_height))))
elif cnnType=='vgg':
batch_frames[0, iFrame+total_length_round-random_ini, :, :, :] = preprocess_input_VGG16(img_to_array(load_img(features[1][iFrame], target_size=(img_width, img_height))))
elif cnnType=='mobilenet':
batch_frames[0, iFrame+total_length_round-random_ini, :, :, :] = preprocess_input_MobileNet(img_to_array(load_img(features[1][iFrame], target_size=(img_width, img_height))))
else:
sys.exit('Invalid CNN network model')
batch_frames = batch_frames.reshape(-1, seq_length, img_width, img_height, 3)
# Fill in batch weights
if output_class_weights != []:
if output_form == 'mixed':
for i_label_cat in range(labels_number):
batch_labels_weight[i_label_cat] = batch_labels_weight[i_label_cat].reshape(1, batch_size_time)
if end <= total_length_round:
batch_labels_weight[i_label_cat] = np.copy(annot_labels_weight[i_label_cat][0, random_ini:end])
else:
batch_labels_weight[i_label_cat][0, :(total_length_round - random_ini)] = np.copy(annot_labels_weight[i_label_cat][0, random_ini:total_length_round])
batch_labels_weight[i_label_cat][0, (total_length_round - random_ini):] = np.copy(annot_labels_weight[i_label_cat][0, 0:end_modulo])
batch_labels_weight[i_label_cat] = batch_labels_weight[i_label_cat].reshape(-1, seq_length)
elif output_form == 'sign_types':
batch_labels_weight = batch_labels_weight.reshape(1, batch_size_time)
if end <= total_length_round:
batch_labels_weight = np.copy(annot_labels_weight[0, random_ini:end])
else:
batch_labels_weight[0, :(total_length_round - random_ini)] = np.copy(annot_labels_weight[0, random_ini:total_length_round])
batch_labels_weight[0, (total_length_round - random_ini):] = np.copy(annot_labels_weight[0, 0:end_modulo])
batch_labels_weight = batch_labels_weight.reshape(-1, seq_length)
# Fill in batch annotations
if output_form == 'mixed':
for i_label_cat in range(labels_number):
batch_labels[i_label_cat] = batch_labels[i_label_cat].reshape(1, batch_size_time, labels_shape[i_label_cat])
if end <= total_length_round:
batch_labels[i_label_cat] = np.copy(annot[i_label_cat][0, random_ini:end, :])
else:
batch_labels[i_label_cat][0, :(total_length_round - random_ini), :] = np.copy(annot[i_label_cat][0, random_ini:total_length_round, :])
batch_labels[i_label_cat][0, (total_length_round - random_ini):, :] = np.copy(annot[i_label_cat][0, 0:end_modulo, :])
batch_labels[i_label_cat] = batch_labels[i_label_cat].reshape(-1, seq_length, labels_shape[i_label_cat])
elif output_form == 'sign_types':
batch_labels = batch_labels.reshape(1, batch_size_time, labels_shape)
if end <= total_length_round:
batch_labels = np.copy(annot[0, random_ini:end, :])
else:
batch_labels[0, :(total_length_round - random_ini), :] = np.copy(annot[0, random_ini:total_length_round, :])
batch_labels[0, (total_length_round - random_ini):, :] = np.copy(annot[0, 0:end_modulo, :])
batch_labels = batch_labels.reshape(-1, seq_length, labels_shape)
if output_class_weights != []:
if features_type == 'features':
yield batch_features, batch_labels, batch_labels_weight
elif features_type == 'frames':
yield batch_frames, batch_labels, batch_labels_weight
elif features_type == 'both':
yield [batch_features, batch_frames], batch_labels, batch_labels_weight
else:
if features_type == 'features':
yield batch_features, batch_labels
elif features_type == 'frames':
yield batch_frames, batch_labels
elif features_type == 'both':
yield [batch_features, batch_frames], batch_labels