def show_image_debug(draw, boxes, scores, labels, masks, classes): from keras_retinanet.utils.visualization import draw_box, draw_caption from keras_maskrcnn.utils.visualization import draw_mask from keras_retinanet.utils.colors import label_color # visualize detections limit_conf = 0.2 for box, score, label, mask in zip(boxes, scores, labels, masks): # scores are sorted so we can break if score < limit_conf: break color = label_color(label) color_mask = (255, 0, 0) b = box.astype(int) draw_box(draw, b, color=color) mask = mask[:, :] draw_mask(draw, b, mask, color=color_mask) caption = "{} {:.3f}".format(classes[label], score) print(caption) draw_caption(draw, b, caption) draw = cv2.cvtColor(draw, cv2.COLOR_RGB2BGR) show_image(draw)
def get_detections(generator, model, out_dir, score_threshold=0.5): all_results = [None for _ in range(generator.num_images)] start = time() print("Processing %i images." % generator.num_images) for i, path, image, draw, scale in generator.load_images(): outputs = model.predict_on_batch(np.expand_dims(image, axis=0)) boxes = outputs[-4][0] scores = outputs[-3][0] labels = outputs[-2][0] masks = outputs[-1][0] boxes /= scale for box, score, label, mask in zip(boxes, scores, labels, masks): if score < score_threshold: break b = box.astype(int) mask = mask[:, :, label] draw_mask(draw, b, mask, color=label_color(label)) out_name = os.path.join(out_dir, path + '.png') cv2.imwrite(out_name, draw) rles = _masks_to_rles(masks, threshold=score_threshold) all_results[i] = (path, boxes, rles) end = time() print("Processed images in %is" % (end - start)) return all_results
def annotate_image(self, path_to_image, num_gpus=1, min_image_dimension=800, max_image_dimension=1024, steps_per_epoch=100, validation_steps=70): labels_to_names = self._load_labels() model = models.load_model(self._path_to_model, backbone_name='resnet50') # load image image = read_image_bgr(path_to_image) # copy to draw on draw = image.copy() draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB) # preprocess image for network image = preprocess_image(image) image, scale = resize_image(image) # process image start = time.time() outputs = model.predict_on_batch(np.expand_dims(image, axis=0)) print("processing time: ", time.time() - start) boxes = outputs[-4][0] scores = outputs[-3][0] labels = outputs[-2][0] masks = outputs[-1][0] # correct for image scale boxes /= scale # visualize detections for box, score, label, mask in zip(boxes, scores, labels, masks): if score < 0.5: break color = label_color(label) b = box.astype(int) draw_box(draw, b, color=color) mask = mask[:, :, label] draw_mask(draw, b, mask, color=label_color(label)) caption = "{} {:.3f}".format(labels_to_names[label], score) draw_caption(draw, b, caption) plt.figure(figsize=(15, 15)) plt.axis('off') plt.imshow(draw) plt.show() """config = ImageMonkeyConfig(len(labels), num_gpus, min_image_dimension, max_image_dimension, steps_per_epoch, validation_steps)
def retinamask(self): backbone = models.backbone('resnet50') batch_size = 1 train_generator, validation_generator = Retinamask.create_generators( batch_size, self.annotations, self.classes) freeze_backbone = 'store_true' weights = self.Input_weights_path print('Creating model, this may take a second...') model, training_model, prediction_model = Retinamask.create_models( backbone_retinanet=backbone.maskrcnn, num_classes=train_generator.num_classes(), weights=weights, freeze_backbone=freeze_backbone) #print(model.summary()) training_model.fit_generator(generator=train_generator, steps_per_epoch=1000, epochs=self.epoch, verbose=1, max_queue_size=1) training_model.save(self.trained_weights_path + 'retinamask.h5') #Testing model_path = self.trained_weights_path + 'retinamask.h5' model = models.load_model(model_path, backbone_name='resnet50') labels_to_names = {0: 'ship'} # load image image = read_image_bgr(test_image_path) # copy to draw on draw = image.copy() draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB) # preprocess image for network image = preprocess_image(image) image, scale = resize_image(image) # process image start = time.time() outputs = model.predict_on_batch(np.expand_dims(image, axis=0)) print("processing time: ", time.time() - start) boxes = outputs[-4][0] scores = outputs[-3][0] labels = outputs[-2][0] masks = outputs[-1][0] # correct for image scale boxes /= scale # visualize detections for box, score, label, mask in zip(boxes, scores, labels, masks): if score < 0.5: break color = label_color(label) b = box.astype(int) draw_box(draw, b, color=color) mask = mask[:, :, label] draw_mask(draw, b, mask, color=label_color(label)) caption = "{} {:.3f}".format(labels_to_names[label], score) draw_caption(draw, b, caption) plt.imsave(self.output_image_path + 'output.jpg', draw)
def apply_mask(model, image, draw=None, threshold_score=0.5, labels_to_names=None, image_segments=True): ''' Process image numpy matrix using model and return the annotations use draw if you want to draw over the image. the draw parameter is the image in RGB (image is in BGR instead) ''' from keras_retinanet.utils.image import preprocess_image, resize_image from keras_retinanet.utils.colors import label_color from keras_maskrcnn.utils.visualization import draw_mask from keras_retinanet.utils.visualization import draw_box, draw_caption, draw_annotations # import miscellaneous modules import numpy as np import time if not labels_to_names: # load label to names mapping for visualization purposes labels_to_names = { 0: 'bag', 1: 'belt', 2: 'boots', 3: 'footwear', 4: 'outer', 5: 'dress', 6: 'sunglasses', 7: 'pants', 8: 'top', 9: 'shorts', 10: 'skirt', 11: 'headwear', 12: 'scarf/tie' } if not draw.any(): # copy to draw on draw = image.copy() #switching to RGB from BGR draw[:, :, 2] = image[:, :, 0] draw[:, :, 0] = image[:, :, 2] draw_segment = draw.copy() # preprocess image for network image = preprocess_image(image) image, scale = resize_image(image) # process image start = time.time() outputs = model.predict_on_batch(np.expand_dims(image, axis=0)) print("processing time: ", time.time() - start) boxes = outputs[-4][0] scores = outputs[-3][0] labels = outputs[-2][0] masks = outputs[-1][0] # correct for image scale boxes /= scale annotations = [{ 'bbox': None, 'score': None, 'category': None, 'segment': None } for i in range( len([score for score in scores if score >= threshold_score]))] i = 0 # visualize detections for box, score, label, mask in zip(boxes, scores, labels, masks): if score < threshold_score: break color = label_color(label) drawclone = np.copy(draw_segment) b = box.astype(int) draw_box(draw, b, color=color) mask = mask[:, :, label] draw_mask_only(drawclone, b, mask, color=label_color(label)) draw_mask(draw, b, mask, color=label_color(label)) caption = "{} {:.3f}".format(labels_to_names[label], score) draw_caption(draw, b, caption) annotations[i]['bbox'] = [b[0], b[1], b[2] - b[0], b[3] - b[1]] annotations[i]['score'] = score annotations[i]['category'] = labels_to_names[label] if image_segments: annotations[i][ 'segment'] = drawclone # only the object inside the mask is shown, the rest is black i += 1 return annotations
def main(proc_img_path=None, proc_img_url=None, all_set=True, save_path=None, model_path=None, segments=False, annotations=False, threshold_score=0.5, limit=None, model=None, labels_to_names=None): # import keras import keras # import keras_retinanet from keras_maskrcnn import models from keras_maskrcnn.utils.visualization import draw_mask from keras_retinanet.utils.visualization import draw_box, draw_caption, draw_annotations from keras_retinanet.utils.image import read_image_bgr, preprocess_image, resize_image from keras_retinanet.utils.colors import label_color # import miscellaneous modules import matplotlib.pyplot as plt import cv2 import numpy as np import time # set tf backend to allow memory to grow, instead of claiming everything import tensorflow as tf # use this environment flag to change which GPU to use #os.environ["CUDA_VISIBLE_DEVICES"] = "1" with open( os.path.expanduser('~') + '/.maskrcnn-modanet/' + 'savedvars.json') as f: savedvars = json.load(f) path = savedvars['datapath'] img_path = path + "datasets/coco/images/" if not model: # set the modified tf session as backend in keras keras.backend.tensorflow_backend.set_session(get_session()) # adjust this to point to your trained model # get all models names in the results folder modelnames = [ f for f in os.listdir(snp_path) if os.path.isfile(os.path.join(snp_path, f)) ] import re def extract_number(f): s = re.findall("\d+$", f) return (int(s[0]) if s else -1, f) # get the model name with the highest epoch print(max(modelnames, key=extract_number)) model_path = os.path.join(snp_path, max(modelnames, key=extract_number)) # load retinanet model model = models.load_model(model_path, backbone_name='resnet50') if not labels_to_names: # load label to names mapping for visualization purposes labels_to_names = { 0: 'bag', 1: 'belt', 2: 'boots', 3: 'footwear', 4: 'outer', 5: 'dress', 6: 'sunglasses', 7: 'pants', 8: 'top', 9: 'shorts', 10: 'skirt', 11: 'headwear', 12: 'scarf/tie' } default_save_path = False if save_path == 'default': # set path to default save_path = path + 'results/processedimages/' #images/1.jpg' if not annotations: save_path += 'images/' elif annotations: save_path += 'annotations/' default_save_path = True SAVE_PATH = save_path # used for multiple images if annotations: # if save_path: save_path = path + 'results/processedimages/annotations/1.json' annotations = [{ 'bbox': None, 'score': None, 'category': None, 'part': None }] if all_set: # load images with open(ann_path + 'instances_val.json') as f: instances = json.load(f) images = instances['images'] for img in images: img['file_name'] = img_path + img['file_name'] elif proc_img_path: # just draw the image selected images = [{ 'file_name': img_path + proc_img_path if os.path.abspath(proc_img_path) != proc_img_path else proc_img_path }] elif proc_img_url: # just draw the image selected images = [{'file_name': proc_img_url}] try: #for each image in the dataset for i, img in enumerate(images): print(i, end=' ') if limit and i >= limit: break if all_set: image = read_image_bgr(img['file_name']) elif proc_img_path: image = read_image_bgr(img['file_name']) elif proc_img_url: import requests from io import BytesIO r = requests.get(img['file_name'], allow_redirects=True) image = read_image_bgr(BytesIO(r.content)) if save_path: if proc_img_path or all_set: img_file_name = img['file_name'].split("/")[-1] elif proc_img_url: img_file_name = 'urlimg.jpg' if not annotations: save_path += img_file_name elif annotations: save_path += img_file_name.split('.')[0] + '.json' if save_path and segments and not annotations: #remove the extension save_path = save_path.split('.')[0] # copy to draw on draw = image.copy() draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB) # preprocess image for network image = preprocess_image(image) image, scale = resize_image(image) # process image start = time.time() outputs = model.predict_on_batch(np.expand_dims(image, axis=0)) print("processing time: ", time.time() - start, "\t(Ctrl+c and close image to exit)") boxes = outputs[-4][0] scores = outputs[-3][0] labels = outputs[-2][0] masks = outputs[-1][0] # correct for image scale boxes /= scale if annotations: annotations = [{ 'bbox': None, 'score': None, 'category': None, 'part': None } for i in range( len([ score for score in scores if score >= threshold_score ]))] segment_id = 0 # visualize detections for box, score, label, mask in zip(boxes, scores, labels, masks): if score < threshold_score: break color = label_color(label) if not segments: b = box.astype(int) draw_box(draw, b, color=color) mask = mask[:, :, label] draw_mask(draw, b, mask, color=label_color(label)) caption = "{} {:.3f}".format(labels_to_names[label], score) draw_caption(draw, b, caption) elif segments: drawclone = np.copy(draw) b = box.astype(int) # draw_box(drawclone, b, color=color) mask = mask[:, :, label] draw_mask_only(drawclone, b, mask, color=label_color(label)) if not annotations: caption = "{} {:.3f}".format(labels_to_names[label], score) draw_caption(drawclone, b, caption) plt.figure(figsize=(15, 15)) plt.axis('off') plt.imshow(drawclone) if not save_path: plt.show() elif save_path: segment_path = '_segment_' + segment_id + '.jpg' save_path_segment = save_path + segment_path print(save_path_segment) plt.savefig(save_path_segment) plt.close() elif annotations: annotations[segment_id]['bbox'] = b annotations[segment_id]['score'] = score annotations[segment_id]['category'] = label annotations[segment_id][ 'part'] = drawclone # only the object inside the mask is shown, the rest is black segment_id += 1 if not segments: if not save_path: plt.figure(figsize=(15, 15)) plt.axis('off') plt.imshow(draw) if not proc_img_url: print(img['file_name']) plt.show() elif save_path: processed_image = Image.fromarray(draw, 'RGB') processed_image.save(save_path) del processed_image print(save_path) # plt.savefig(save_path) # plt.close() elif segments: if annotations: if save_path: print(save_path) with open(save_path, 'w') as outfile: json.dump(annotations, outfile) else: return annotations save_path = SAVE_PATH # restore path for next image except KeyboardInterrupt: pass
boxes = outputs[-4][0] scores = outputs[-3][0] labels = outputs[-2][0] masks = outputs[-1][0] # correct for image scale boxes /= scale # visualize detections for box, score, label, mask in zip(boxes, scores, labels, masks): if score < 0.5: break color = label_color(label) b = box.astype(int) draw_box(draw, b, color=color) mask = mask[:, :, label] draw_mask(draw, b, mask, color=label_color(label)) caption = "{} {:.3f}".format(labels_to_names[label], score) #draw_caption(draw, b, caption) plt.figure(figsize=(15, 15)) plt.axis('off') plt.savefig("output.png") plt.imshow(draw) plt.show()
def segmentation(imgpath, score_threshold=0.5, binarize_threshold=0.5): # adjust this to point to your downloaded/trained model model_path = "resnet50_weights/resnet50_coco_v0.2.0.h5" # load retinanet model model = models.load_model(model_path, backbone_name='resnet50') # load label to names mapping for visualization purposes labels_to_names = { 0: 'person', 1: 'bicycle', 2: 'car', 3: 'motorcycle', 4: 'airplane', 5: 'bus', 6: 'train', 7: 'truck', 8: 'boat', 9: 'traffic light', 10: 'fire hydrant', 11: 'stop sign', 12: 'parking meter', 13: 'bench', 14: 'bird', 15: 'cat', 16: 'dog', 17: 'horse', 18: 'sheep', 19: 'cow', 20: 'elephant', 21: 'bear', 22: 'zebra', 23: 'giraffe', 24: 'backpack', 25: 'umbrella', 26: 'handbag', 27: 'tie', 28: 'suitcase', 29: 'frisbee', 30: 'skis', 31: 'snowboard', 32: 'sports ball', 33: 'kite', 34: 'baseball bat', 35: 'baseball glove', 36: 'skateboard', 37: 'surfboard', 38: 'tennis racket', 39: 'bottle', 40: 'wine glass', 41: 'cup', 42: 'fork', 43: 'knife', 44: 'spoon', 45: 'bowl', 46: 'banana', 47: 'apple', 48: 'sandwich', 49: 'orange', 50: 'broccoli', 51: 'carrot', 52: 'hot dog', 53: 'pizza', 54: 'donut', 55: 'cake', 56: 'chair', 57: 'couch', 58: 'potted plant', 59: 'bed', 60: 'dining table', 61: 'toilet', 62: 'tv', 63: 'laptop', 64: 'mouse', 65: 'remote', 66: 'keyboard', 67: 'cell phone', 68: 'microwave', 69: 'oven', 70: 'toaster', 71: 'sink', 72: 'refrigerator', 73: 'book', 74: 'clock', 75: 'vase', 76: 'scissors', 77: 'teddy bear', 78: 'hair drier', 79: 'toothbrush' } # load image image = read_image_bgr(imgpath) # copy to draw on draw = image.copy() draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB) # preprocess image for network image = preprocess_image(image) image, scale = resize_image(image) # process image start = time.time() outputs = model.predict_on_batch(np.expand_dims(image, axis=0)) print("Segmentation took", round(time.time() - start, 2), "seconds.") boxes = outputs[-4][0] scores = outputs[-3][0] labels = outputs[-2][0] masks = outputs[-1][0] # correct for image scale boxes /= scale mask_list = [] box_list = [] for box, score, label, mask in zip(boxes, scores, labels, masks): if score < 0.5: break # save box coordinates in list box = box.astype(np.int16) box_list.append(box[:]) # resize to fit the box mask = mask.astype(np.float32) mask = cv2.resize(mask, (box[2] - box[0], box[3] - box[1])) # binarize the mask mask = (mask > binarize_threshold).astype(np.uint8) mask = cv2.normalize(src=mask, dst=None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8U) mask_list.append(mask[:, :, label]) # visualize detections for box, score, label, mask in zip(boxes, scores, labels, masks): if score < 0.5: break color = label_color(label) b = box.astype(int) draw_box(draw, b, color=color) mask = mask[:, :, label] draw_mask(draw, b, mask, color=label_color(label)) caption = "{} {:.3f}".format(labels_to_names[label], score) draw_caption(draw, b, caption) return mask_list, box_list, draw
def cloth_identifier_extract_text(input_imagefile): # load image image = read_image_bgr(input_imagefile) # copy to draw on draw = image.copy() draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB) # preprocess image for network image = preprocess_image(image) image, scale = resize_image(image) # process image outputs1 = model.predict_on_batch(np.expand_dims(image, axis=0)) boxes = outputs1[-4][0] scores = outputs1[-3][0] labels = outputs1[-2][0] masks = outputs1[-1][0] # correct for image scale boxes /= scale masks_dic = {} boxes_dic = {} labels_dic = {} counter = 0 # visualize detections for box, score, label, mask in zip(boxes, scores, labels, masks): if score < 0.5: break color = label_color(label) b = box.astype(int) draw_box(draw, b, color=color) mask = mask[:, :, label] draw_mask(draw, b, mask, color=label_color(label)) masks_dic[str(counter)] = mask boxes_dic[str(counter)] = box labels_dic[str(counter)] = label counter += 1 items_dic = {} counter = 0 masks = [] for box, mask, label in zip(boxes_dic.values(), masks_dic.values(), labels_dic.values()): b = box.astype(int) # resize to fit the box mask = mask.astype(np.float32) mask = cv2.resize(mask, (b[2] - b[0], b[3] - b[1])) # binarize the mask mask = (mask > 0.5).astype(np.uint8) # draw the mask in the image mask_image = np.zeros((draw.shape[0], draw.shape[1]), np.uint8) mask_image[b[1]:b[3], b[0]:b[2]] = mask mask = mask_image mask = (np.stack([mask] * 3, axis=2)) * draw items_dic[str(counter)] = mask counter += 1 return mask, label
def cloth(input_imagefile): # load image image = read_image_bgr(input_imagefile) # copy to draw on draw = image.copy() draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB) # preprocess image for network image = preprocess_image(image) image, scale = resize_image(image) # process image start = time.time() outputs = model.predict_on_batch(np.expand_dims(image, axis=0)) print("processing time: ", time.time() - start) boxes = outputs[-4][0] scores = outputs[-3][0] labels = outputs[-2][0] masks = outputs[-1][0] # correct for image scale boxes /= scale masks_dic = {} boxes_dic = {} labels_dic = {} counter = 0 # visualize detections for box, score, label, mask in zip(boxes, scores, labels, masks): if score < 0.5: break color = label_color(label) b = box.astype(int) draw_box(draw, b, color=color) mask = mask[:, :, label] draw_mask(draw, b, mask, color=label_color(label)) masks_dic[str(counter)] = mask boxes_dic[str(counter)] = box labels_dic[str(counter)] = label counter += 1 image = read_image_bgr(input_imagefile) # copy to draw on draw = image.copy() draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB) # visualize detections items_dic = {} counter = 0 for box, mask2, label2 in zip(boxes_dic.values(), masks_dic.values(), labels_dic.values()): b = box.astype(int) # resize to fit the box mask2 = mask2.astype(np.float32) mask2 = cv2.resize(mask2, (b[2] - b[0], b[3] - b[1])) # binarize the mask1 mask2 = (mask2 > 0.5).astype(np.uint8) # draw the mask2 in the image mask2_image = np.zeros((draw.shape[0], draw.shape[1]), np.uint8) mask2_image[b[1]:b[3], b[0]:b[2]] = mask2 mask2 = mask2_image mask2 = (np.stack([mask2] * 3, axis=2)) * draw items_dic[str(counter)] = mask2 counter += 1 # newfileneame=input_imagefile.split("/")[4].split('.')[0] # plt.ioff() # plt.figure(figsize=(15, 15)) # plt.axis('off') # plt.imshow(mask2) #plt.savefig('/home/ec2-user/SageMaker/'+str(newfileneame)+'-masked-'+str(label2)+'.jpg',bbox_inches='tight', pad_inches=0) #plt.show() plt.close('all') return mask2, label2