def extract_litw_features(filename, model, my_preprocess): """ Given Logos in The Wild dataset, extract all logos from images and extract features by applying truncated model (flattening W * H * n_filters features from last layer). Args: filename: text file, where each line defines the logos in the image specified. The format is: path-to-file1.jpg xmin,ymin,xmax,ymax,class_id[,confidence] xmin,ymin,xmax,ymax,class_id[,confidence] path-to-file2.jpg xmin,ymin,xmax,ymax,class_id[,confidence] Returns: features: (n_logos, n_features)-shaped np.array of features all_logos: list of np.arrays for each logo brand_map: brand id (in range 0,...,n_brands) for each extracted logo """ all_logos, brand_map = extract_litw_logos(filename) # for i in all_logos: # print(np.array(i).shape) # logo.append(cv2.resize(i,(128,128))) print("**************************************") print(np.array(all_logos).shape) features = utils.features_from_image(all_logos, model, my_preprocess) return features, all_logos, brand_map
def load_brands_compute_cutoffs(img_input, model_preproc, features, threshold=0.95, timing=False): """ Given paths to input brand images, this is a wrapper to features_from_image() and similarity_cutoff(). Args: input_paths: list of paths to input images model_preproc: (model, preprocess) tuple of model extractor and image preprocessing function features: (n_database, N) array of features for logo database threshold: fractional threshold for setting the cutoff Returns: img_input: list of iamges (3D np.arrays) feat_input: (n_input, F) array of 1D features extracted from input images cutoff_list: list of cutoffs for each input (bins, cdf_list): bins specifications and list of CDF distributions for similarity of the logo database against each input. """ start = timer() ''' img_input = [] for path in input_paths: img = cv2.imread(path) # apppend images in RGB color ordering if img is not None: img_input.append(img[:,:,::-1]) else: print(path) ''' t_read = timer() - start model, my_preprocess = model_preproc #img_input = np.array(img_input) feat_input = features_from_image(img_input, model, my_preprocess) t_feat = timer() - start sim_cutoff, (bins, cdf_list) = similarity_cutoff(feat_input, features, threshold, timing) t_sim_cut = timer() - start if timing: print('Time spent in each section:') print( '-reading images: {:.2f}sec\n-features: {:.2f}sec\n-cosine similarity: {:.2f}sec' .format(t_read, t_feat - t_read, t_sim_cut - t_feat)) print('Resulting 95% similarity threshold for targets:') #for path, cutoff in zip(input_paths, sim_cutoff): # print(' {} {:.2f}'.format(path, cutoff)) return img_input, feat_input, sim_cutoff, (bins, cdf_list)
def detect_and_match(model_preproc, input_features_cdf_cutoff_labels, image, timestamp, save_img=True, save_img_path='./data/test/'): ''' image = Image.open(img_path) if image.mode != 'RGB': image = image.convert("RGB") ''' image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) image = Image.fromarray(image) image_array = np.array(image) yolo, model, my_preprocess = model_preproc prediction, new_image = yolo.detect_image(image) #new_image.save(os.path.join(save_img_path, 'detect_' + os.path.basename(img_path))) candidates, i_candidates_too_small = contents_of_bbox( image_array, prediction) prediction = [ pred for i, pred in enumerate(prediction) if i not in i_candidates_too_small ] features_cand = features_from_image(candidates, model, my_preprocess) feat_input, sim_cutoff, bins, cdf_list, input_labels = input_features_cdf_cutoff_labels matches, cos_sim = similar_matches(feat_input, features_cand, sim_cutoff, bins, cdf_list) match_pred = [] keys = list(matches.keys()) for idx in keys: bb = prediction[idx] if bb[-1] < 0.70: matches.pop(idx) continue label = input_labels[matches[idx][0]] pred = [bb[0], bb[1], bb[2], bb[3], label, bb[-1]] #, matches[idx][1]] match_pred.append(pred) print('Logo #{} - {} {} - classified as {} {:.2f}'.format( idx, tuple(bb[:2]), tuple(bb[2:4]), label, matches[idx][1])) with open('./data/test/pred_with_timestamp.pkl', 'wb') as f: pickle.dump([match_pred, timestamp], f) if save_img: new_img = draw_matches(image_array, input_labels, prediction, matches) saved = Image.fromarray(new_img).save( os.path.join(save_img_path, 'output.jpg')) return match_pred, timestamp
def match_logo(img_test, prediction, model_preproc, input_features_cdf_cutoff_labels): """ Given an a path to an image and a list of predicted bounding boxes, extract features and check each against input brand features. Declare a match if the cosine similarity is smaller than an input-dependent cutoff. Draw and annotate resulting boxes on image. Args: img_test: input image prediction: bounding box candidates model_preproc: (model, preprocess) tuple of the feature extractor model and the preprocessing function to be applied to image before the model input_features_cdf_cutoff_labels = (feat_input, sim_cutoff, bins, cdf_list, input_labels) tuple of lists related to input brand, giving pre-computed features, similarity cutoffs, cumulative similarity distribution and relative bins specifications, and labels to be drawn when matches are found. Returns: outtxt: one line detailing input file path and resulting matched bounding boxes, space-separated in format (xmin,ymin,xmax,ymax,class_label,logo_confidence,similarity_percentile) timing: timing for each step of the pipeline, namely image read, logog candidate extraction, feature computation, matching to input brands (optional, only if timing=True) """ model, my_preprocess = model_preproc feat_input, sim_cutoff, bins, cdf_list, input_labels = input_features_cdf_cutoff_labels candidates, i_candidates_too_small = contents_of_bbox(img_test, prediction) # filter predicted bboxes to discard small logos prediction = [ pred for i, pred in enumerate(prediction) if i not in i_candidates_too_small ] features_cand = features_from_image(candidates, model, my_preprocess) matches, cos_sim = similar_matches(feat_input, features_cand, sim_cutoff, bins, cdf_list) outtxt = '' for idx in matches: bb = prediction[idx] label = input_labels[matches[idx][0]] print('Logo #{} - {} {} - classified as {} {:.2f}'.format( idx, tuple(bb[:2]), tuple(bb[2:4]), label, matches[idx][1])) outtxt += '{},{},{},{},{},{:.2f},{:.3f}\n'.format( *bb[:4], label, bb[-1], matches[idx][1]) return outtxt
features, all_logos, brand_map = extract_litw_features('data_all_train.txt', model, my_preprocess) print('Processed {} logos, transformed into feature vectors'.format(len(features))) # save inception features at default size 299*299 utils.save_features('./model_poi/inception_logo_features.hdf5', features, brand_map, input_shape) # save features for Inception with smaller input: 200 instead of 299 - last layer is 4*4 instead of 8*8 # Extract features at last layer as well as after last 3 inception blocks (mixed9,8,7) input_shape = (200,200,3) new_preprocess = lambda x: preprocess_input(utils.pad_image(x, input_shape)) trunc_layer = [-1, 279, 248, 228] for i_layer in range(4): model_out = Model(inputs=model.inputs, outputs=model.layers[trunc_layer[i_layer]].output) features = utils.features_from_image(all_logos, model_out, new_preprocess) extra = '_trunc{}'.format(i_layer) if i_layer > 0 else '' utils.save_features('./model_poi/inception_logo_features_200{}.hdf5'.format(extra), features, brand_map, input_shape) # save features for VGG16 at 3 different input scales from keras.applications.vgg16 import VGG16 from keras.applications.vgg16 import preprocess_input model = VGG16(weights='imagenet', include_top=False) for n in [224,128,64]: input_shape = (n,n,3) new_preprocess = lambda x: preprocess_input(utils.pad_image(x, input_shape)) features = utils.features_from_image(all_logos, model, new_preprocess) utils.save_features('./model_poi/vgg16_logo_features_{}.hdf5'.format(n), features, brand_map, input_shape)
def match_logo(img_test, prediction, model_preproc, outtxt, input_features_cdf_cutoff_labels, save_img, save_img_path='./', timing=False): """ Given an a path to an image and a list of predicted bounding boxes, extract features and check each against input brand features. Declare a match if the cosine similarity is smaller than an input-dependent cutoff. Draw and annotate resulting boxes on image. Args: img_test: input image prediction: bounding box candidates model_preproc: (model, preprocess) tuple of the feature extractor model and the preprocessing function to be applied to image before the model input_features_cdf_cutoff_labels = (feat_input, sim_cutoff, bins, cdf_list, input_labels) tuple of lists related to input brand, giving pre-computed features, similarity cutoffs, cumulative similarity distribution and relative bins specifications, and labels to be drawn when matches are found. save_img: bool flag to save annotated image save_img_path: path to directory where to save image timing: bool flag to output timing information for each step, make plot Returns: outtxt: one line detailing input file path and resulting matched bounding boxes, space-separated in format (xmin,ymin,xmax,ymax,class_label,logo_confidence,similarity_percentile) timing: timing for each step of the pipeline, namely image read, logog candidate extraction, feature computation, matching to input brands (optional, only if timing=True) """ start = timer() model, my_preprocess = model_preproc feat_input, sim_cutoff, bins, cdf_list, input_labels = input_features_cdf_cutoff_labels # from PIL image to np array #img_test = np.array(image) # img_test = cv2.imread(img_path) # could be removed by passing previous PIL image t_read = timer() - start candidates, i_candidates_too_small = contents_of_bbox(img_test, prediction) # filter predicted bboxes to discard small logos prediction = [ pred for i, pred in enumerate(prediction) if i not in i_candidates_too_small ] t_box = timer() - start features_cand = features_from_image(candidates, model, my_preprocess) t_feat = timer() - start matches, cos_sim = similar_matches(feat_input, features_cand, sim_cutoff, bins, cdf_list) t_match = timer() - start img_path = outtxt for idx in matches: bb = prediction[idx] label = input_labels[matches[idx][0]] print('Logo #{} - {} {} - classified as {} {:.2f}'.format( idx, tuple(bb[:2]), tuple(bb[2:4]), label, matches[idx][1])) outtxt += ' {},{},{},{},{},{:.2f},{:.3f}'.format( *bb[:4], label, bb[-1], matches[idx][1]) outtxt += '\n' new_img = draw_matches(img_test, input_labels, prediction, matches) t_draw = timer() - start if save_img == True: save_img_path = os.path.abspath(save_img_path) saved = Image.fromarray(new_img).save( os.path.join(save_img_path, os.path.basename(img_path))) # save with opencv, remember to flip RGB->BGR # saved = cv2.imwrite(os.path.join(save_img_path, os.path.basename(img_path)), new_img[...,::-1]) t_save = timer() - start if timing: return outtxt, (t_read, t_box - t_read, t_feat - t_box, t_match - t_feat, t_draw - t_match, t_save - t_draw) return outtxt