def extract_litw_features(filename, model, my_preprocess):
"""
Given Logos in The Wild dataset, extract all logos from images and extract
features by applying truncated model (flattening W * H * n_filters features
from last layer).
Args:
filename: text file, where each line defines the logos in the image
specified. The format is:
path-to-file1.jpg xmin,ymin,xmax,ymax,class_id[,confidence] xmin,ymin,xmax,ymax,class_id[,confidence]
path-to-file2.jpg xmin,ymin,xmax,ymax,class_id[,confidence]
Returns:
features: (n_logos, n_features)-shaped np.array of features
all_logos: list of np.arrays for each logo
brand_map: brand id (in range 0,...,n_brands) for each extracted logo
"""
all_logos, brand_map = extract_litw_logos(filename)
# for i in all_logos:
# print(np.array(i).shape)
# logo.append(cv2.resize(i,(128,128)))
print("**************************************")
print(np.array(all_logos).shape)
features = utils.features_from_image(all_logos, model, my_preprocess)
return features, all_logos, brand_map
def load_brands_compute_cutoffs(img_input,
model_preproc,
features,
threshold=0.95,
timing=False):
"""
Given paths to input brand images, this is a wrapper to features_from_image()
and similarity_cutoff().
Args:
input_paths: list of paths to input images
model_preproc: (model, preprocess) tuple of model extractor and
image preprocessing function
features: (n_database, N) array of features for logo database
threshold: fractional threshold for setting the cutoff
Returns:
img_input: list of iamges (3D np.arrays)
feat_input: (n_input, F) array of 1D features extracted from input images
cutoff_list: list of cutoffs for each input
(bins, cdf_list): bins specifications and list of CDF distributions
for similarity of the logo database against each input.
"""
start = timer()
'''
img_input = []
for path in input_paths:
img = cv2.imread(path)
# apppend images in RGB color ordering
if img is not None:
img_input.append(img[:,:,::-1])
else:
print(path)
'''
t_read = timer() - start
model, my_preprocess = model_preproc
#img_input = np.array(img_input)
feat_input = features_from_image(img_input, model, my_preprocess)
t_feat = timer() - start
sim_cutoff, (bins, cdf_list) = similarity_cutoff(feat_input, features,
threshold, timing)
t_sim_cut = timer() - start
if timing:
print('Time spent in each section:')
print(
'-reading images: {:.2f}sec\n-features: {:.2f}sec\n-cosine similarity: {:.2f}sec'
.format(t_read, t_feat - t_read, t_sim_cut - t_feat))
print('Resulting 95% similarity threshold for targets:')
#for path, cutoff in zip(input_paths, sim_cutoff):
# print(' {} {:.2f}'.format(path, cutoff))
return img_input, feat_input, sim_cutoff, (bins, cdf_list)
def detect_and_match(model_preproc,
input_features_cdf_cutoff_labels,
image,
timestamp,
save_img=True,
save_img_path='./data/test/'):
'''
image = Image.open(img_path)
if image.mode != 'RGB':
image = image.convert("RGB")
'''
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
image = Image.fromarray(image)
image_array = np.array(image)
yolo, model, my_preprocess = model_preproc
prediction, new_image = yolo.detect_image(image)
#new_image.save(os.path.join(save_img_path, 'detect_' + os.path.basename(img_path)))
candidates, i_candidates_too_small = contents_of_bbox(
image_array, prediction)
prediction = [
pred for i, pred in enumerate(prediction)
if i not in i_candidates_too_small
]
features_cand = features_from_image(candidates, model, my_preprocess)
feat_input, sim_cutoff, bins, cdf_list, input_labels = input_features_cdf_cutoff_labels
matches, cos_sim = similar_matches(feat_input, features_cand, sim_cutoff,
bins, cdf_list)
match_pred = []
keys = list(matches.keys())
for idx in keys:
bb = prediction[idx]
if bb[-1] < 0.70:
matches.pop(idx)
continue
label = input_labels[matches[idx][0]]
pred = [bb[0], bb[1], bb[2], bb[3], label, bb[-1]] #, matches[idx][1]]
match_pred.append(pred)
print('Logo #{} - {} {} - classified as {} {:.2f}'.format(
idx, tuple(bb[:2]), tuple(bb[2:4]), label, matches[idx][1]))
with open('./data/test/pred_with_timestamp.pkl', 'wb') as f:
pickle.dump([match_pred, timestamp], f)
if save_img:
new_img = draw_matches(image_array, input_labels, prediction, matches)
saved = Image.fromarray(new_img).save(
os.path.join(save_img_path, 'output.jpg'))
return match_pred, timestamp
def match_logo(img_test, prediction, model_preproc,
input_features_cdf_cutoff_labels):
"""
Given an a path to an image and a list of predicted bounding boxes,
extract features and check each against input brand features. Declare
a match if the cosine similarity is smaller than an input-dependent
cutoff. Draw and annotate resulting boxes on image.
Args:
img_test: input image
prediction: bounding box candidates
model_preproc: (model, preprocess) tuple of the feature extractor model
and the preprocessing function to be applied to image before the model
input_features_cdf_cutoff_labels = (feat_input, sim_cutoff, bins, cdf_list, input_labels)
tuple of lists related to input brand, giving pre-computed features,
similarity cutoffs, cumulative similarity distribution and relative bins
specifications, and labels to be drawn when matches are found.
Returns:
outtxt: one line detailing input file path and resulting matched bounding
boxes, space-separated in format
(xmin,ymin,xmax,ymax,class_label,logo_confidence,similarity_percentile)
timing: timing for each step of the pipeline, namely image read, logog candidate
extraction, feature computation, matching to input brands
(optional, only if timing=True)
"""
model, my_preprocess = model_preproc
feat_input, sim_cutoff, bins, cdf_list, input_labels = input_features_cdf_cutoff_labels
candidates, i_candidates_too_small = contents_of_bbox(img_test, prediction)
# filter predicted bboxes to discard small logos
prediction = [
pred for i, pred in enumerate(prediction)
if i not in i_candidates_too_small
]
features_cand = features_from_image(candidates, model, my_preprocess)
matches, cos_sim = similar_matches(feat_input, features_cand, sim_cutoff,
bins, cdf_list)
outtxt = ''
for idx in matches:
bb = prediction[idx]
label = input_labels[matches[idx][0]]
print('Logo #{} - {} {} - classified as {} {:.2f}'.format(
idx, tuple(bb[:2]), tuple(bb[2:4]), label, matches[idx][1]))
outtxt += '{},{},{},{},{},{:.2f},{:.3f}\n'.format(
*bb[:4], label, bb[-1], matches[idx][1])
return outtxt
features, all_logos, brand_map = extract_litw_features('data_all_train.txt', model, my_preprocess)
print('Processed {} logos, transformed into feature vectors'.format(len(features)))
# save inception features at default size 299*299
utils.save_features('./model_poi/inception_logo_features.hdf5', features, brand_map, input_shape)
# save features for Inception with smaller input: 200 instead of 299 - last layer is 4*4 instead of 8*8
# Extract features at last layer as well as after last 3 inception blocks (mixed9,8,7)
input_shape = (200,200,3)
new_preprocess = lambda x: preprocess_input(utils.pad_image(x, input_shape))
trunc_layer = [-1, 279, 248, 228]
for i_layer in range(4):
model_out = Model(inputs=model.inputs, outputs=model.layers[trunc_layer[i_layer]].output)
features = utils.features_from_image(all_logos, model_out, new_preprocess)
extra = '_trunc{}'.format(i_layer) if i_layer > 0 else ''
utils.save_features('./model_poi/inception_logo_features_200{}.hdf5'.format(extra), features, brand_map, input_shape)
# save features for VGG16 at 3 different input scales
from keras.applications.vgg16 import VGG16
from keras.applications.vgg16 import preprocess_input
model = VGG16(weights='imagenet', include_top=False)
for n in [224,128,64]:
input_shape = (n,n,3)
new_preprocess = lambda x: preprocess_input(utils.pad_image(x, input_shape))
features = utils.features_from_image(all_logos, model, new_preprocess)
utils.save_features('./model_poi/vgg16_logo_features_{}.hdf5'.format(n), features, brand_map, input_shape)
def match_logo(img_test,
prediction,
model_preproc,
outtxt,
input_features_cdf_cutoff_labels,
save_img,
save_img_path='./',
timing=False):
"""
Given an a path to an image and a list of predicted bounding boxes,
extract features and check each against input brand features. Declare
a match if the cosine similarity is smaller than an input-dependent
cutoff. Draw and annotate resulting boxes on image.
Args:
img_test: input image
prediction: bounding box candidates
model_preproc: (model, preprocess) tuple of the feature extractor model
and the preprocessing function to be applied to image before the model
input_features_cdf_cutoff_labels = (feat_input, sim_cutoff, bins, cdf_list, input_labels)
tuple of lists related to input brand, giving pre-computed features,
similarity cutoffs, cumulative similarity distribution and relative bins
specifications, and labels to be drawn when matches are found.
save_img: bool flag to save annotated image
save_img_path: path to directory where to save image
timing: bool flag to output timing information for each step, make plot
Returns:
outtxt: one line detailing input file path and resulting matched bounding
boxes, space-separated in format
(xmin,ymin,xmax,ymax,class_label,logo_confidence,similarity_percentile)
timing: timing for each step of the pipeline, namely image read, logog candidate
extraction, feature computation, matching to input brands
(optional, only if timing=True)
"""
start = timer()
model, my_preprocess = model_preproc
feat_input, sim_cutoff, bins, cdf_list, input_labels = input_features_cdf_cutoff_labels
# from PIL image to np array
#img_test = np.array(image)
# img_test = cv2.imread(img_path) # could be removed by passing previous PIL image
t_read = timer() - start
candidates, i_candidates_too_small = contents_of_bbox(img_test, prediction)
# filter predicted bboxes to discard small logos
prediction = [
pred for i, pred in enumerate(prediction)
if i not in i_candidates_too_small
]
t_box = timer() - start
features_cand = features_from_image(candidates, model, my_preprocess)
t_feat = timer() - start
matches, cos_sim = similar_matches(feat_input, features_cand, sim_cutoff,
bins, cdf_list)
t_match = timer() - start
img_path = outtxt
for idx in matches:
bb = prediction[idx]
label = input_labels[matches[idx][0]]
print('Logo #{} - {} {} - classified as {} {:.2f}'.format(
idx, tuple(bb[:2]), tuple(bb[2:4]), label, matches[idx][1]))
outtxt += ' {},{},{},{},{},{:.2f},{:.3f}'.format(
*bb[:4], label, bb[-1], matches[idx][1])
outtxt += '\n'
new_img = draw_matches(img_test, input_labels, prediction, matches)
t_draw = timer() - start
if save_img == True:
save_img_path = os.path.abspath(save_img_path)
saved = Image.fromarray(new_img).save(
os.path.join(save_img_path, os.path.basename(img_path)))
# save with opencv, remember to flip RGB->BGR
# saved = cv2.imwrite(os.path.join(save_img_path, os.path.basename(img_path)), new_img[...,::-1])
t_save = timer() - start
if timing:
return outtxt, (t_read, t_box - t_read, t_feat - t_box,
t_match - t_feat, t_draw - t_match, t_save - t_draw)
return outtxt
