def do_detect_for_image(self,image): if not image is None: self.frame = image self.mask_frame = [image * self.mask] self.height, self.width, _ = image.shape self.data[self.frame_id]=[] detection = self.detect(self.mask_frame) res = detection[0] res_count = len(res['scores']) class_masks = np.zeros((self.num_classes, self.height, self.width), dtype = np.bool_) for track_id in range(res_count): box = res['rois'][i] i = 0 for k in range(len(res['class_ids'])): if Detector.check_boxes(res['rois'][k],box): i = k break class_id = self.track_per_class['classes'].get(track_id,None) score = res['scores'][i] if class_id is None: class_id = res['class_ids'][i] self.track_per_class['classes'][track_id]=class_id self.track_per_class['scores'][track_id]={} self.track_per_class['scores'][track_id][class_id] = self.track_per_class['scores'][track_id].get(class_id,0)+score s = sum(self.track_per_class['scores'][track_id].values()) mx = 0 mx_class_id = class_id for key in self.track_per_class['scores'][track_id]: self.track_per_class['scores'][track_id][key]/=s if mx<self.track_per_class['scores'][track_id][key]: mx=self.track_per_class['scores'][track_id][key] mx_class_id = key if mx_class_id!=class_id: self.track_per_class['classes'][track_id]=mx_class_id class_id = mx_class_id class_id = int(class_id) mask = res['masks'][:,:,i] class_masks[class_id] += mask self.data[self.frame_id].append(Detection(box=box,track_id = track_id, class_id = class_id, score = score)) cv2.putText(self.frame, str(track_id), (box[1] - 1, box[0] - 1), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 6) cv2.putText(self.frame, str(track_id), (box[1] - 3, box[0] - 3), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2) visualize.draw_box(self.frame, box, Detection.get_hash_color(track_id)) for class_id in range(self.num_classes): visualize.apply_mask(self.frame, class_masks[class_id], class_colors[class_id])
def highlight_with_diff_and_save_image(image_data, bboxes, nearest_lesion_indices, prefix): for index in nearest_lesion_indices: color = (200, 0, 0) # red x0, y0, x1, y1 = bboxes[index - 1] cv2.putText(image_data, str(index), (x0-2*LESION_BOX_MARGIN, y0-2*LESION_BOX_MARGIN), cv2.FONT_HERSHEY_SIMPLEX, 2, color, 3, cv2.LINE_AA) visualize.draw_box(image_data, (y0-LESION_BOX_MARGIN, x0-LESION_BOX_MARGIN, y1+LESION_BOX_MARGIN, x1+LESION_BOX_MARGIN), color, thickness=3) image_file_path = os.path.join(target_dir, prefix + "_img.jpg") Image.fromarray(image_data).save(image_file_path)
def render(result, rgb_image, target): N = result['rois'].shape[0] # 検出数 result_image = rgb_image.copy() mask = None colors = visualize.random_colors(N) for i in range(N): '''クラス関係なく1物体ごと処理を行う''' if class_names[result['class_ids'][i]] in target: # Color color = colors[i] rgb = (round(color[0] * 255), round(color[1] * 255), round(color[2] * 255)) font = cv2.FONT_HERSHEY_SIMPLEX # Bbox result_image = visualize.draw_box(result_image, result['rois'][i], rgb) # Class & Score text_top = f"ID{i:d} {class_names[result['class_ids'][i]]}: {result['scores'][i]:.3f}" result_image = cv2.putText( result_image, text_top, (result['rois'][i][1], result['rois'][i][0]), font, 0.7, rgb, 1, cv2.LINE_AA) # Mask mask = result['masks'][:, :, i] result_image = visualize.apply_mask(result_image, mask, color) # log print( f"ID: {i} | {class_names[result['class_ids'][i]]}: {result['scores'][i]}" ) return result_image, mask
def show_roi_and_mask(): g, num_random_rois, detection_targets = create_data_generator() [normalized_images, image_meta, rpn_match, rpn_bbox, gt_class_ids, gt_boxes, gt_masks, rpn_rois, rois], \ [mrcnn_class_ids, mrcnn_bbox, mrcnn_mask] = next(g) # Remove the last dim in mrcnn_class_ids. It's only added # to satisfy Keras restriction on target shape. # 原来的 mrcnn_class_idx 的 shape 是 (batch_size, num_rois, 1) mrcnn_class_ids = mrcnn_class_ids[:, :, 0] b = 0 # Restore original image (reverse normalization) sample_image = modellib.unmold_image(normalized_images[b], config) # Class aware bboxes # mrcnn_bbox 的 shape 是 (batch_size,num_rois,num_classes,4) # mrcnn_class_idx 的 shape 是 (batch_size, num_rois) # bbox_specific 的 shape 是 (num_train_rois, dataset.num_classes, 4) bbox_specific = mrcnn_bbox[b, np.arange(mrcnn_bbox.shape[1]), mrcnn_class_ids[b], :] # Refined ROIs refined_rois = utils.apply_box_deltas( rois[b].astype(np.float32), bbox_specific[:, :4] * config.BBOX_STD_DEV) # Class aware masks # mrcnn_mask 的 shape 是 (batch_size, num_rois, 28, 28, num_classes) # mask_specific 的 shape 是 (num_train_rois, 28, 28, dataset.num_classes) mask_specific = mrcnn_mask[b, np.arange(mrcnn_mask.shape[1]), :, :, mrcnn_class_ids[b]] # Dispalay ROIs and corresponding masks and bounding boxes ids = random.sample(range(rois.shape[1]), 8) images = [] titles = [] for i in ids: # 这里的 copy() 是为了在这个循环中的多个 id 互不影响 image = visualize.draw_box(sample_image.copy(), rois[b, i, :4].astype(np.int32), [255, 0, 0]) image = visualize.draw_box(image, refined_rois[i].astype(np.int64), [0, 255, 0]) images.append(image) titles.append("ROI {}".format(i)) images.append(mask_specific[i] * 255) titles.append(dataset.class_names[mrcnn_class_ids[b, i]][:20]) display_images(images, titles, cols=4, cmap="Blues", interpolation="none")
def detect(model): print("Running on {}".format(args.img)) # Read image image = skimage.io.imread(args.img) # Detect objects r = model.detect([image], verbose=1)[0] for i in range(len(r['rois'])): image = visualize.draw_box(image, r['rois'][i], (255, 0, 0)) image = visualize.apply_mask(image, r['masks'][i], (255, 0, 0)) # Save output file_name = "splash_{:%Y%m%dT%H%M%S}.png".format(datetime.now()) skimage.io.imsave(file_name, image)
def find_lesions(img_data, prefix): # img = image.load_img("c:/temp/70.jpg ") # img = image.load_img("c:/Users/eattulb/Downloads/change/img_020.JPG")#, target_size=(image_size, image_size)) img_data = img_data.copy() (image_height, image_width, _) = img_data.shape index = 0 lesion_masks = [] bboxes = [] for x in range(0, image_width, SUBIMAGE_SIZE): for y in range(0, image_height, SUBIMAGE_SIZE): # x = 1024 # y = 1024 print(x, y) subimage = get_subimage(img_data, x, y, x+SUBIMAGE_SIZE, y+SUBIMAGE_SIZE) results = model.detect([subimage], verbose=1) r = results[0] lesion_masks.append({"x": x, "y": y, "lesions": r}) rois = r['rois'] for roi in rois: print(str(roi)) index += 1 y0, x0, y1, x1 = roi bboxes.append([x + roi[1], y + roi[0], x + roi[3], y + roi[2]]) color = (0, 200, 0) # cv2.putText(subimage, str(index), (x0-2*LESION_BOX_MARGIN, y0-2*LESION_BOX_MARGIN), cv2.FONT_HERSHEY_SIMPLEX, 2, color, 3, cv2.LINE_AA) # subimage = visualize.draw_box(subimage, (y0-LESION_BOX_MARGIN, x0-LESION_BOX_MARGIN, y1+LESION_BOX_MARGIN, x1+LESION_BOX_MARGIN), color, thickness=3) cv2.putText(img_data, str(index), (x + x0-2*LESION_BOX_MARGIN, y + y0-2*LESION_BOX_MARGIN), cv2.FONT_HERSHEY_SIMPLEX, 2, color, 3, cv2.LINE_AA) img_data = visualize.draw_box(img_data, (y + y0-LESION_BOX_MARGIN, x + x0-LESION_BOX_MARGIN, y + y1+LESION_BOX_MARGIN, x + x1+LESION_BOX_MARGIN), color, thickness=3) img_data = img_data.astype(np.uint8) if prefix is not None: image_file_path = os.path.join(target_dir, prefix + "_img.jpg") Image.fromarray(img_data).save(image_file_path) return lesion_masks, bboxes, img_data
def mask_tile(image, detection, color, alpha=0.5, min_score=0.97): """ Return an RGBA image that is transparent except for the mask """ # alpha = np.zeros((256,256,4)) for n in range(3): alpha[:,:,n] = image[:,:,n] alpha = image scores = detection['scores'] masks = detection['masks'] rois = detection['rois'] # wipe out masks under a certain confidence level new_masks = np.copy(masks) above = 0 below = 0 for c in range(len(scores)): if scores[c] < min_score: below += 1 new_masks[:,:,c] = False else: above += 1 alpha = visualize.draw_box(alpha,rois[c],color) # print("masks above cf: "+str(above)) # print("masks below cf: " + str(below)) # print("total masks: " + str(len(masks))) # print("total rois: " + str(len(rois))) # print("total scores: " + str(len(rois))) mask = (np.sum(new_masks, -1, keepdims=True, dtype=int) >= 1) * 1 for c in range(3): # paint mask alpha[:, :, c] = np.where(mask[:, :, 0] == 1, alpha[:, :, c] * color[c], # (1 - alpha) + alpha * color[c] * 255, alpha[:, :, c]) # # remove other pixels # alpha[:, :, 3] = np.where(mask[:, :, 0] == 0, # alpha[:, :, 3] * 0, # alpha[:, :, 3]) return alpha
def put_frame(self, f, r, LABELS, fps=None): """ applies the detections on related frame; appends to self.frames_out (list) as [frame0, frame1, ...] """ frame = self.frames_inp[f] # pick the related frame # Visualize results for i in range(0, r["rois"].shape[0]): mask = r["masks"][:, :, i] frame = visualize.apply_mask(frame, mask, (1.0, 0.0, 0.0), alpha=0.5) frame = visualize.draw_box(frame, r["rois"][i], (1.0, 0.0, 0.0)) frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) for i in range(0, len(r["scores"])): (startY, startX, endY, end) = r["rois"][i] classID = r["class_ids"][i] label = LABELS[classID] score = r["scores"][i] text = "{}: {:.4f}".format(label, score) y = startY - 10 if startY - 10 > 10 else startY + 10 cv2.putText(frame, text, (startX, y), Video.FONT, 0.8, Video.GREEN, 2) # add fps and f if fps: cv2.putText(frame, f"FPS: {fps:.2f}", (10, 30), Video.FONT, 0.5, Video.RED, 1) cv2.putText(frame, f"{f}/{self.total_frame}", (10, 15), Video.FONT, 0.3, Video.RED, 1) self.frames_out.append(frame)
def process_raw_images(): """ process image by neural network on remote server """ # construct the argument parser and parse the arguments ap = argparse.ArgumentParser() ap.add_argument("-w", "--weights", help="optional path to pretrained weights") args = vars(ap.parse_args()) # initialize the inference configuration config = LettuceInferenceConfig() # initialize the Mask R-CNN model for inference model = modellib.MaskRCNN(mode="inference", config=config, model_dir=LOGS_AND_MODEL_DIR) # load our trained Mask R-CNN weights = args["weights"] if args["weights"] \ else model.find_last() model.load_weights(weights, by_name=True) #class_names = ["BG", "Lettuce"] #colors = visualize.random_colors(len(class_names)) raw_folderpath = 'vfarm' processed_folderpath = 'vfarm_processed' with open('harvest_log.csv', mode='a') as csv_file: fieldnames = ['Date', 'Time', 'Row&Column', 'Machine Validation'] writer = csv.DictWriter(csv_file, fieldnames=fieldnames) writer.writeheader() for infile in glob.glob(os.path.join(raw_folderpath, '*.*')): if infile.split(".")[-1].lower() in {"jpg"}: # load the input image img = cv2.imread(str(infile)) #convert it from BGR to RGB channel ordering img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) #resize the image img = imutils.resize(img, width=1024) # perform a forward pass of the network to obtain the results r = model.detect([img], verbose=1)[0] # loop over of the detected object's bounding boxes and # masks, drawing each as we go along for i in range(0, r["rois"].shape[0]): mask = r["masks"][:, :, i] img = visualize.apply_mask(img, mask, (1.0, 0.0, 0.0), alpha=0.5) img = visualize.draw_box(img, r["rois"][i], (1.0, 0.0, 0.0)) # convert the image back to BGR so we can use OpenCV's drawing function img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) # loop over the predicted scores and class labels for i in range(0, len(r["scores"])): # extract the bounding box information, class ID, label, # and predicted probability from the results (startY, startX, endY, end) = r["rois"][i] classID = r["class_ids"][i] label = CLASS_NAMES[classID] score = r["scores"][i] # draw the class label and score on the image text = "{}: {:.4f}".format(label, score) y = startY - 10 if startY - 10 > 10 else startY + 10 cv2.putText(img, text, (startX, y), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 255, 0), 2) #output = visualize.display_instances(img, r['rois'], r['masks'], r['class_ids'], # class_names, r['scores'], colors=colors) # resize the image so it more easily fits on our screen output = imutils.resize(img, width=512) #save the processed image to the vfarm folder cv2.imwrite( str(processed_folderpath) + "/" + str(infile), output) #if there are bounding boxes detected if len(r["rois"].shape[0]) > 0: rack_status.update({str(infile): "1"}) else: rack_status.update({str(infile): "0"}) writer.writerow({ 'Date': str(time.strftime("%H:%M:%S")), 'Time': str(time.strftime("%d/%m/%Y")), 'Row&Column': str(infile), 'Machine Validation': str(rack_status[str(infile)]) }) print(rack_status)
def inspect_data(dataset, config): print("Image Count: {}".format(len(dataset.image_ids))) print("Class Count: {}".format(dataset.num_classes)) for i, info in enumerate(dataset.class_info): print("{:3}. {:50}".format(i, info['name'])) # ## Display Samples # # Load and display images and masks. # In[4]: # Load and display random samples image_ids = np.random.choice(dataset.image_ids, 4) for image_id in image_ids: image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) visualize.display_top_masks(image, mask, class_ids, dataset.class_names) # ## Bounding Boxes # # Rather than using bounding box coordinates provided by the source datasets, we compute the bounding boxes from masks instead. This allows us to handle bounding boxes consistently regardless of the source dataset, and it also makes it easier to resize, rotate, or crop images because we simply generate the bounding boxes from the updates masks rather than computing bounding box transformation for each type of image transformation. # In[5]: # Load random image and mask. image_id = random.choice(dataset.image_ids) image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) # Compute Bounding box bbox = utils.extract_bboxes(mask) # Display image and additional stats print("image_id ", image_id, dataset.image_reference(image_id)) log("image", image) log("mask", mask) log("class_ids", class_ids) log("bbox", bbox) # Display image and instances visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names) # ## Resize Images # # To support multiple images per batch, images are resized to one size (1024x1024). Aspect ratio is preserved, though. If an image is not square, then zero padding is added at the top/bottom or right/left. # In[6]: # Load random image and mask. image_id = np.random.choice(dataset.image_ids, 1)[0] image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) original_shape = image.shape # Resize image, window, scale, padding, _ = utils.resize_image( image, min_dim=config.IMAGE_MIN_DIM, max_dim=config.IMAGE_MAX_DIM, mode=config.IMAGE_RESIZE_MODE) mask = utils.resize_mask(mask, scale, padding) # Compute Bounding box bbox = utils.extract_bboxes(mask) # Display image and additional stats print("image_id: ", image_id, dataset.image_reference(image_id)) print("Original shape: ", original_shape) log("image", image) log("mask", mask) log("class_ids", class_ids) log("bbox", bbox) # Display image and instances visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names) # ## Mini Masks # # Instance binary masks can get large when training with high resolution images. For example, if training with 1024x1024 image then the mask of a single instance requires 1MB of memory (Numpy uses bytes for boolean values). If an image has 100 instances then that's 100MB for the masks alone. # # To improve training speed, we optimize masks by: # * We store mask pixels that are inside the object bounding box, rather than a mask of the full image. Most objects are small compared to the image size, so we save space by not storing a lot of zeros around the object. # * We resize the mask to a smaller size (e.g. 56x56). For objects that are larger than the selected size we lose a bit of accuracy. But most object annotations are not very accuracy to begin with, so this loss is negligable for most practical purposes. Thie size of the mini_mask can be set in the config class. # # To visualize the effect of mask resizing, and to verify the code correctness, we visualize some examples. # In[7]: image_id = np.random.choice(dataset.image_ids, 1)[0] image, image_meta, class_ids, bbox, mask = modellib.load_image_gt( dataset, config, image_id, use_mini_mask=False) log("image", image) log("image_meta", image_meta) log("class_ids", class_ids) log("bbox", bbox) log("mask", mask) display_images([image] + [mask[:, :, i] for i in range(min(mask.shape[-1], 7))]) # In[8]: visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names) # In[9]: # Add augmentation and mask resizing. image, image_meta, class_ids, bbox, mask = modellib.load_image_gt( dataset, config, image_id, augment=True, use_mini_mask=True) log("mask", mask) display_images([image] + [mask[:, :, i] for i in range(min(mask.shape[-1], 7))]) # In[10]: mask = utils.expand_mask(bbox, mask, image.shape) visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names) # ## Anchors # # The order of anchors is important. Use the same order in training and prediction phases. And it must match the order of the convolution execution. # # For an FPN network, the anchors must be ordered in a way that makes it easy to match anchors to the output of the convolution layers that predict anchor scores and shifts. # * Sort by pyramid level first. All anchors of the first level, then all of the second and so on. This makes it easier to separate anchors by level. # * Within each level, sort anchors by feature map processing sequence. Typically, a convolution layer processes a feature map starting from top-left and moving right row by row. # * For each feature map cell, pick any sorting order for the anchors of different ratios. Here we match the order of ratios passed to the function. # # **Anchor Stride:** # In the FPN architecture, feature maps at the first few layers are high resolution. For example, if the input image is 1024x1024 then the feature meap of the first layer is 256x256, which generates about 200K anchors (256*256*3). These anchors are 32x32 pixels and their stride relative to image pixels is 4 pixels, so there is a lot of overlap. We can reduce the load significantly if we generate anchors for every other cell in the feature map. A stride of 2 will cut the number of anchors by 4, for example. # # In this implementation we use an anchor stride of 2, which is different from the paper. # In[11]: # Generate Anchors backbone_shapes = modellib.compute_backbone_shapes(config, config.IMAGE_SHAPE) anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES, config.RPN_ANCHOR_RATIOS, backbone_shapes, config.BACKBONE_STRIDES, config.RPN_ANCHOR_STRIDE) # Print summary of anchors num_levels = len(backbone_shapes) anchors_per_cell = len(config.RPN_ANCHOR_RATIOS) print("Count: ", anchors.shape[0]) print("Scales: ", config.RPN_ANCHOR_SCALES) print("ratios: ", config.RPN_ANCHOR_RATIOS) print("Anchors per Cell: ", anchors_per_cell) print("Levels: ", num_levels) anchors_per_level = [] for l in range(num_levels): num_cells = backbone_shapes[l][0] * backbone_shapes[l][1] anchors_per_level.append(anchors_per_cell * num_cells // config.RPN_ANCHOR_STRIDE**2) print("Anchors in Level {}: {}".format(l, anchors_per_level[l])) # Visualize anchors of one cell at the center of the feature map of a specific level. # In[12]: ## Visualize anchors of one cell at the center of the feature map of a specific level # Load and draw random image image_id = np.random.choice(dataset.image_ids, 1)[0] image, image_meta, _, _, _ = modellib.load_image_gt( dataset, config, image_id) fig, ax = plt.subplots(1, figsize=(10, 10)) ax.imshow(image) levels = len(backbone_shapes) for level in range(levels): colors = visualize.random_colors(levels) # Compute the index of the anchors at the center of the image level_start = sum( anchors_per_level[:level]) # sum of anchors of previous levels level_anchors = anchors[level_start:level_start + anchors_per_level[level]] print("Level {}. Anchors: {:6} Feature map Shape: {}".format( level, level_anchors.shape[0], backbone_shapes[level])) center_cell = backbone_shapes[level] // 2 center_cell_index = (center_cell[0] * backbone_shapes[level][1] + center_cell[1]) level_center = center_cell_index * anchors_per_cell center_anchor = anchors_per_cell * ( (center_cell[0] * backbone_shapes[level][1] / config.RPN_ANCHOR_STRIDE**2) \ + center_cell[1] / config.RPN_ANCHOR_STRIDE) level_center = int(center_anchor) # Draw anchors. Brightness show the order in the array, dark to bright. for i, rect in enumerate(level_anchors[level_center:level_center + anchors_per_cell]): y1, x1, y2, x2 = rect p = patches.Rectangle( (x1, y1), x2 - x1, y2 - y1, linewidth=2, facecolor='none', edgecolor=(i + 1) * np.array(colors[level]) / anchors_per_cell) ax.add_patch(p) # ## Data Generator # # In[13]: # Create data generator random_rois = 2000 g = modellib.data_generator(dataset, config, shuffle=True, random_rois=random_rois, batch_size=4, detection_targets=True) # In[14]: # Uncomment to run the generator through a lot of images # to catch rare errors # for i in range(1000): # print(i) # _, _ = next(g) # In[15]: # Get Next Image if random_rois: [ normalized_images, image_meta, rpn_match, rpn_bbox, gt_class_ids, gt_boxes, gt_masks, rpn_rois, rois ], [mrcnn_class_ids, mrcnn_bbox, mrcnn_mask] = next(g) log("rois", rois) log("mrcnn_class_ids", mrcnn_class_ids) log("mrcnn_bbox", mrcnn_bbox) log("mrcnn_mask", mrcnn_mask) else: [ normalized_images, image_meta, rpn_match, rpn_bbox, gt_boxes, gt_masks ], _ = next(g) log("gt_class_ids", gt_class_ids) log("gt_boxes", gt_boxes) log("gt_masks", gt_masks) log( "rpn_match", rpn_match, ) log("rpn_bbox", rpn_bbox) image_id = modellib.parse_image_meta(image_meta)["image_id"][0] print("image_id: ", image_id, dataset.image_reference(image_id)) # Remove the last dim in mrcnn_class_ids. It's only added # to satisfy Keras restriction on target shape. mrcnn_class_ids = mrcnn_class_ids[:, :, 0] # In[16]: b = 0 # Restore original image (reverse normalization) sample_image = modellib.unmold_image(normalized_images[b], config) # Compute anchor shifts. indices = np.where(rpn_match[b] == 1)[0] refined_anchors = utils.apply_box_deltas( anchors[indices], rpn_bbox[b, :len(indices)] * config.RPN_BBOX_STD_DEV) log("anchors", anchors) log("refined_anchors", refined_anchors) # Get list of positive anchors positive_anchor_ids = np.where(rpn_match[b] == 1)[0] print("Positive anchors: {}".format(len(positive_anchor_ids))) negative_anchor_ids = np.where(rpn_match[b] == -1)[0] print("Negative anchors: {}".format(len(negative_anchor_ids))) neutral_anchor_ids = np.where(rpn_match[b] == 0)[0] print("Neutral anchors: {}".format(len(neutral_anchor_ids))) # ROI breakdown by class for c, n in zip(dataset.class_names, np.bincount(mrcnn_class_ids[b].flatten())): if n: print("{:23}: {}".format(c[:20], n)) # Show positive anchors fig, ax = plt.subplots(1, figsize=(16, 16)) visualize.draw_boxes(sample_image, boxes=anchors[positive_anchor_ids], refined_boxes=refined_anchors, ax=ax) # In[17]: # Show negative anchors visualize.draw_boxes(sample_image, boxes=anchors[negative_anchor_ids]) # In[18]: # Show neutral anchors. They don't contribute to training. visualize.draw_boxes(sample_image, boxes=anchors[np.random.choice( neutral_anchor_ids, 100)]) # ## ROIs # In[19]: if random_rois: # Class aware bboxes bbox_specific = mrcnn_bbox[b, np.arange(mrcnn_bbox.shape[1]), mrcnn_class_ids[b], :] # Refined ROIs refined_rois = utils.apply_box_deltas( rois[b].astype(np.float32), bbox_specific[:, :4] * config.BBOX_STD_DEV) # Class aware masks mask_specific = mrcnn_mask[b, np.arange(mrcnn_mask.shape[1]), :, :, mrcnn_class_ids[b]] visualize.draw_rois(sample_image, rois[b], refined_rois, mask_specific, mrcnn_class_ids[b], dataset.class_names) # Any repeated ROIs? rows = np.ascontiguousarray(rois[b]).view( np.dtype((np.void, rois.dtype.itemsize * rois.shape[-1]))) _, idx = np.unique(rows, return_index=True) print("Unique ROIs: {} out of {}".format(len(idx), rois.shape[1])) # In[20]: if random_rois: # Dispalay ROIs and corresponding masks and bounding boxes ids = random.sample(range(rois.shape[1]), 8) images = [] titles = [] for i in ids: image = visualize.draw_box(sample_image.copy(), rois[b, i, :4].astype(np.int32), [255, 0, 0]) image = visualize.draw_box(image, refined_rois[i].astype(np.int64), [0, 255, 0]) images.append(image) titles.append("ROI {}".format(i)) images.append(mask_specific[i] * 255) titles.append(dataset.class_names[mrcnn_class_ids[b, i]][:20]) display_images(images, titles, cols=4, cmap="Blues", interpolation="none") # In[21]: # Check ratio of positive ROIs in a set of images. if random_rois: limit = 10 temp_g = modellib.data_generator(dataset, config, shuffle=True, random_rois=10000, batch_size=1, detection_targets=True) total = 0 for i in range(limit): _, [ids, _, _] = next(temp_g) positive_rois = np.sum(ids[0] > 0) total += positive_rois print("{:5} {:5.2f}".format(positive_rois, positive_rois / ids.shape[1])) print("Average percent: {:.2f}".format(total / (limit * ids.shape[1])))
def do_detect(self,image = None, tracking = True): if self.writer is None or (self.video_stream is None and image is None) or self.mask is None: raise ValueError('input stream or output stream not defined!') if not image is None: self.frame = image self.mask_frame = [image * self.mask] self.height, self.width, _ = image.shape self.data[self.frame_id]=[] detection = self.detect(self.mask_frame) res = detection[0] for_tracker = np.concatenate([res['rois'], [[x] for x in res['scores']]], axis=1) good_rois = [] for j in range(len(res['rois'])): if res['class_ids'][j] == 14: continue max_oui = 0 for k in good_rois: max_oui = np.maximum(max_oui, iou(res['rois'][j], res['rois'][k])) if max_oui < 0.95: good_rois.append(j) if not tracking: det = np.concatenate([res['rois'], [[i] for i in range(len(res['scores']))]], axis=1) else: det = self.update(for_tracker[good_rois]) class_masks = np.zeros((self.num_classes, self.height, self.width), dtype = np.bool_) for predict in det: box = np.array(predict[:4],dtype=np.int32) i = 0 for k in range(len(res['class_ids'])): if Detector.check_boxes(res['rois'][k],box): i = k break track_id = int(predict[4]) if track_id != 20: print (track_id) continue class_id = self.track_per_class['classes'].get(track_id,None) score = res['scores'][i] if class_id is None: class_id = res['class_ids'][i] self.track_per_class['classes'][track_id]=class_id self.track_per_class['scores'][track_id]={} self.track_per_class['scores'][track_id][class_id] = self.track_per_class['scores'][track_id].get(class_id,0)+score s = sum(self.track_per_class['scores'][track_id].values()) mx = 0 mx_class_id = class_id for key in self.track_per_class['scores'][track_id]: self.track_per_class['scores'][track_id][key]/=s if mx<self.track_per_class['scores'][track_id][key]: mx=self.track_per_class['scores'][track_id][key] mx_class_id = key if mx_class_id!=class_id: self.track_per_class['classes'][track_id]=mx_class_id class_id = mx_class_id class_id = int(class_id) mask = res['masks'][:,:,i] class_masks[class_id] += mask self.data[self.frame_id].append(Detection(box=box,track_id = track_id, class_id = class_id, score = score)) cv2.putText(self.frame, str(track_id), (box[1] - 1, box[0] - 1), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 6) cv2.putText(self.frame, str(track_id), (box[1] - 3, box[0] - 3), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2) visualize.draw_box(self.frame, box, Detection.get_hash_color(track_id)) for class_id in range(self.num_classes): visualize.apply_mask(self.frame, class_masks[class_id], class_colors[class_id]) self.writer.write(self.frame) self.frame_id +=1 if not self.video_stream is None: if self.video_stream.isOpened(): ret, self.frame = self.video_stream.read() if ret: self.mask_frame = [self.frame * self.mask] self.stream_is_open = self.video_stream.isOpened() and ret
# Visualize results r = results[0] visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], class_names, r['scores']) image_c = image.copy() upper_lx = 9999 upper_ly = 9999 lower_rx = 0 lower_ry = 0 car_size = 0 for i in range(len(r['class_ids'])): if r['class_ids'][i] == 3: visualize.draw_box(image_c, r['rois'][i], (255, 0, 0)) r_size = (r['rois'][i][2] - r['rois'][i][0]) * (r['rois'][i][3] - r['rois'][i][1]) if r_size > car_size: car_size = r_size upper_lx = r['rois'][i][0] upper_ly = r['rois'][i][1] lower_rx = r['rois'][i][2] lower_ry = r['rois'][i][3] cv2.rectangle(image_c, (upper_ly, upper_lx), (lower_ry, lower_rx), (0, 0, 255), 3) print(upper_lx, upper_ly, lower_rx, lower_ry) skimage.io.imsave('car_detect.jpg', image_c)
def get_labels(self, labels): dims = labels.shape unlabeled_labels = np.zeros((dims[0], dims[1], 1)) building_labels = np.zeros((dims[0], dims[1], 1)) fence_labels = np.zeros((dims[0], dims[1], 1)) other_labels = np.zeros((dims[0], dims[1], 1)) pedestrian_labels = np.zeros((dims[0], dims[1], 1)) pole_labels = np.zeros((dims[0], dims[1], 1)) road_line_labels = np.zeros((dims[0], dims[1], 1)) road_labels = np.zeros((dims[0], dims[1], 1)) sidewalk_labels = np.zeros((dims[0], dims[1], 1)) vegetation_labels = np.zeros((dims[0], dims[1], 1)) car_labels = np.zeros((dims[0], dims[1], 1)) wall_labels = np.zeros((dims[0], dims[1], 1)) traffic_sign_labels = np.zeros((dims[0], dims[1], 1)) unlabeled_index = np.all(labels == (0, 0, 0), axis=-1) building_index = np.all(labels == (70, 70, 70), axis=-1) fence_index = np.all(labels == (190, 153, 153), axis=-1) other_index = np.all(labels == (250, 170, 160), axis=-1) pedestrian_index = np.all(labels == (220, 20, 60), axis=-1) pole_index = np.all(labels == (153, 153, 153), axis=-1) road_line_index = np.all(labels == (157, 234, 50), axis=-1) road_index = np.all(labels == (128, 64, 128), axis=-1) sidewalk_index = np.all(labels == (244, 35, 232), axis=-1) vegetation_index = np.all(labels == (107, 142, 35), axis=-1) car_index = np.all(labels == (0, 0, 142), axis=-1) wall_index = np.all(labels == (102, 102, 156), axis=-1) traffic_sign_index = np.all(labels == (220, 220, 70), axis=-1) unlabeled_labels[unlabeled_index] = 1 building_labels[building_index] = 10 fence_labels[fence_index] = 10 other_labels[other_index] = 10 pedestrian_labels[pedestrian_index] = 10 pole_labels[pole_index] = 10 road_line_labels[road_line_index] = 10 road_labels[road_index] = 10 sidewalk_labels[sidewalk_index] = 10 vegetation_labels[vegetation_index] = 1 car_labels[car_index] = 10 wall_labels[wall_index] = 10 traffic_sign_labels[traffic_sign_index] = 10 return np.dstack([unlabeled_labels, building_labels, fence_labels, return np.dstack([unlabeled_labels, building_labels, fence_labels, other_labels, pedestrian_labels, pole_labels, road_line_labels, road_labels, sidewalk_labels, vegetation_labels, car_labels, wall_labels, traffic_sign_labels]) def image_reference(self, image_id): """Return the carla data of the image.""" info = self.image_info[image_id] if info["source"] == "carla": return info["id"] else: super(self.__class__).image_reference(self, image_id) config = CarlaConfig() config.STEPS_PER_EPOCH = NUMBER_OF_TRAIN_DATA//config.BATCH_SIZE config.VALIDATION_STEPS = NUMBER_OF_VAL_DATA//config.BATCH_SIZE config.display() dataset = carlaDataset() dataset.load_images(dir=RGB_TRAIN_DIR, type='train') # mask, a = train.load_mask(50) # print(a) dataset.prepare() print("Image Count: {}".format(len(dataset.image_ids))) print("Class Count: {}".format(dataset.num_classes)) for i, info in enumerate(dataset.class_info): print("{:3}. {:50}".format(i, info['name'])) image_ids = np.random.choice(dataset.image_ids, 4) for image_id in image_ids: image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) visualize.display_top_masks(image, mask, class_ids, dataset.class_names) # Load random image and mask. image_id = random.choice(dataset.image_ids) image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) # Compute Bounding box bbox = utils.extract_bboxes(mask) # Display image and additional stats print("image_id ", image_id) log("image", image) log("mask", mask) log("class_ids", class_ids) log("bbox", bbox) # Display image and instances visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names) # Load random image and mask. image_id = np.random.choice(dataset.image_ids, 1)[0] image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) original_shape = image.shape # Resize image, window, scale, padding, _ = utils.resize_image( image, min_dim=config.IMAGE_MIN_DIM, max_dim=config.IMAGE_MAX_DIM, mode=config.IMAGE_RESIZE_MODE) mask = utils.resize_mask(mask, scale, padding) # Compute Bounding box bbox = utils.extract_bboxes(mask) # Display image and additional stats print("image_id: ", image_id) print("Original shape: ", original_shape) log("image", image) log("mask", mask) log("class_ids", class_ids) log("bbox", bbox) # Display image and instances visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names) image_id = np.random.choice(dataset.image_ids, 1)[0] image, image_meta, class_ids, bbox, mask = modellib.load_image_gt( dataset, config, image_id, use_mini_mask=False) log("image", image) log("image_meta", image_meta) log("class_ids", class_ids) log("bbox", bbox) log("mask", mask) display_images([image]+[mask[:,:,i] for i in range(min(mask.shape[-1], 7))]) visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names) # Generate Anchors backbone_shapes = modellib.compute_backbone_shapes(config, config.IMAGE_SHAPE) anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES, config.RPN_ANCHOR_RATIOS, backbone_shapes, config.BACKBONE_STRIDES, config.RPN_ANCHOR_STRIDE) # Print summary of anchors num_levels = len(backbone_shapes) anchors_per_cell = len(config.RPN_ANCHOR_RATIOS) print("Count: ", anchors.shape[0]) print("Scales: ", config.RPN_ANCHOR_SCALES) print("ratios: ", config.RPN_ANCHOR_RATIOS) print("Anchors per Cell: ", anchors_per_cell) print("Levels: ", num_levels) anchors_per_level = [] for l in range(num_levels): num_cells = backbone_shapes[l][0] * backbone_shapes[l][1] anchors_per_level.append(anchors_per_cell * num_cells // config.RPN_ANCHOR_STRIDE**2) print("Anchors in Level {}: {}".format(l, anchors_per_level[l])) ## Visualize anchors of one cell at the center of the feature map of a specific level # Load and draw random image image_id = np.random.choice(dataset.image_ids, 1)[0] image, image_meta, _, _, _ = modellib.load_image_gt(dataset, config, image_id) fig, ax = plt.subplots(1, figsize=(10, 10)) ax.imshow(image) levels = len(backbone_shapes) for level in range(levels): colors = visualize.random_colors(levels) # Compute the index of the anchors at the center of the image level_start = sum(anchors_per_level[:level]) # sum of anchors of previous levels level_anchors = anchors[level_start:level_start+anchors_per_level[level]] print("Level {}. Anchors: {:6} Feature map Shape: {}".format(level, level_anchors.shape[0], backbone_shapes[level])) center_cell = backbone_shapes[level] // 2 center_cell_index = (center_cell[0] * backbone_shapes[level][1] + center_cell[1]) level_center = center_cell_index * anchors_per_cell center_anchor = anchors_per_cell * ( (center_cell[0] * backbone_shapes[level][1] / config.RPN_ANCHOR_STRIDE**2) \ + center_cell[1] / config.RPN_ANCHOR_STRIDE) level_center = int(center_anchor) # Draw anchors. Brightness show the order in the array, dark to bright. for i, rect in enumerate(level_anchors[level_center:level_center+anchors_per_cell]): y1, x1, y2, x2 = rect p = patches.Rectangle((x1, y1), x2-x1, y2-y1, linewidth=2, facecolor='none', edgecolor=(i+1)*np.array(colors[level]) / anchors_per_cell) ax.add_patch(p) # Create data generator random_rois = 4000 g = modellib.data_generator( dataset, config, shuffle=True, random_rois=random_rois, batch_size=4, detection_targets=True) # Get Next Image if random_rois: [normalized_images, image_meta, rpn_match, rpn_bbox, gt_class_ids, gt_boxes, gt_masks, rpn_rois, rois], \ [mrcnn_class_ids, mrcnn_bbox, mrcnn_mask] = next(g) log("rois", rois) log("mrcnn_class_ids", mrcnn_class_ids) log("mrcnn_bbox", mrcnn_bbox) log("mrcnn_mask", mrcnn_mask) else: [normalized_images, image_meta, rpn_match, rpn_bbox, gt_boxes, gt_masks], _ = next(g) log("gt_class_ids", gt_class_ids) log("gt_boxes", gt_boxes) log("gt_masks", gt_masks) log("rpn_match", rpn_match, ) log("rpn_bbox", rpn_bbox) image_id = modellib.parse_image_meta(image_meta)["image_id"][0] print("image_id: ", image_id, dataset.image_reference(image_id)) # Remove the last dim in mrcnn_class_ids. It's only added # to satisfy Keras restriction on target shape. mrcnn_class_ids = mrcnn_class_ids[:, :, 0] b = 0 # Restore original image (reverse normalization) sample_image = modellib.unmold_image(normalized_images[b], config) # Compute anchor shifts. indices = np.where(rpn_match[b] == 1)[0] refined_anchors = utils.apply_box_deltas(anchors[indices], rpn_bbox[b, :len(indices)] * config.RPN_BBOX_STD_DEV) log("anchors", anchors) log("refined_anchors", refined_anchors) # Get list of positive anchors positive_anchor_ids = np.where(rpn_match[b] == 1)[0] print("Positive anchors: {}".format(len(positive_anchor_ids))) negative_anchor_ids = np.where(rpn_match[b] == -1)[0] print("Negative anchors: {}".format(len(negative_anchor_ids))) neutral_anchor_ids = np.where(rpn_match[b] == 0)[0] print("Neutral anchors: {}".format(len(neutral_anchor_ids))) # ROI breakdown by class for c, n in zip(dataset.class_names, np.bincount(mrcnn_class_ids[b].flatten())): if n: print("{:23}: {}".format(c[:20], n)) # Show positive anchors visualize.draw_boxes(sample_image, boxes=anchors[positive_anchor_ids], refined_boxes=refined_anchors) # Show negative anchors visualize.draw_boxes(sample_image, boxes=anchors[negative_anchor_ids]) # Show neutral anchors. They don't contribute to training. visualize.draw_boxes(sample_image, boxes=anchors[np.random.choice(neutral_anchor_ids, 100)]) if random_rois: # Class aware bboxes bbox_specific = mrcnn_bbox[b, np.arange(mrcnn_bbox.shape[1]), mrcnn_class_ids[b], :] # Refined ROIs refined_rois = utils.apply_box_deltas(rois[b].astype(np.float32), bbox_specific[:, :4] * config.BBOX_STD_DEV) # Class aware masks mask_specific = mrcnn_mask[b, np.arange(mrcnn_mask.shape[1]), :, :, mrcnn_class_ids[b]] visualize.draw_rois(sample_image, rois[b], refined_rois, mask_specific, mrcnn_class_ids[b], dataset.class_names) # Any repeated ROIs? rows = np.ascontiguousarray(rois[b]).view(np.dtype((np.void, rois.dtype.itemsize * rois.shape[-1]))) _, idx = np.unique(rows, return_index=True) print("Unique ROIs: {} out of {}".format(len(idx), rois.shape[1])) if random_rois: # Dispalay ROIs and corresponding masks and bounding boxes ids = random.sample(range(rois.shape[1]), 8) images = [] titles = [] for i in ids: image = visualize.draw_box(sample_image.copy(), rois[b,i,:4].astype(np.int32), [255, 0, 0]) image = visualize.draw_box(image, refined_rois[i].astype(np.int64), [0, 255, 0]) images.append(image) titles.append("ROI {}".format(i)) images.append(mask_specific[i] * 255) titles.append(dataset.class_names[mrcnn_class_ids[b,i]][:20]) display_images(images, titles, cols=4, cmap="Blues", interpolation="none") # Check ratio of positive ROIs in a set of images. if random_rois: limit = 10 temp_g = modellib.data_generator( dataset, config, shuffle=True, random_rois=10000, batch_size=1, detection_targets=True) total = 0 for i in range(limit): _, [ids, _, _] = next(temp_g) positive_rois = np.sum(ids[0] > 0) total += positive_rois print("{:5} {:5.2f}".format(positive_rois, positive_rois/ids.shape[1])) print("Average percent: {:.2f}".format(total/(limit*ids.shape[1]))) exit()
def layer_instances(image, result, boxes, masks, class_ids, class_names, scores, colors, show_mask=True, show_bbox=True, captions=None): """ Create images by the classes. {class1: {rois: [[x,y,x1,y1],...,[x,y,x1,y1]], scores: [s1, ..., sn], name: class name }} boxes: [num_instance, (y1, x1, y2, x2, class_id)] in image coordinates. masks: [height, width, num_instances] class_ids: [num_instances] class_names: list of class names of the dataset scores: (optional) confidence scores for each box show_mask, show_bbox: To show masks and bounding boxes or not colors: (optional) An array or colors to use with each object captions: (optional) A list of strings to use as captions for each object """ layer_map = create_layer_data(result) # Number of instances if not len(layer_map.keys()): print("\n*** No instances to display *** \n") return ([]) # Show area outside image boundaries. height, width, d = image.shape layer_images = [] for k, v in layer_map.items(): # add alpha channle and BG is transparent. tmp_image = np.zeros((height, width, d + 1)) tmp_image.astype(np.uint32) class_name = class_names[k] logger.info('Processing... %s', class_name) color = colors.get(class_name) if isinstance(color, list): color.append(255) else: c = [] c.append(color[0]) c.append(color[1]) c.append(color[2]) c.append(255) color = c # Bounding box boxes = v.get('rois') masks = v.get('masks') for i in range(len(boxes)): if show_bbox: logger.debug('BBox %s in color: %s', boxes[i], color) tmp_image = visualize.draw_box(tmp_image, boxes[i], color) # Mask mask = masks[i] if show_mask: tmp_image = visualize.apply_mask(tmp_image, mask, color, 150) # y1, x1, y2, x2 = boxes[i] # print(tmp_image) # # font = ImageFont.truetype("/usr/share/fonts/dejavu/DejaVuSans.ttf",25) # text_img = Image.new("RGBA", (200, 200), (120, 20, 20)) # draw = ImageDraw2.Draw(text_img) # draw.text((x1, y1+5), class_name, color) # draw = ImageDraw.Draw(text_img) # img.save("a_test.png") # # Label # if not captions: # class_id = class_ids[i] # score = scores[i] if scores is not None else None # label = class_names[class_id] # caption = "{} {:.3f}".format(label, score) if score else label # else: # caption = captions[i] # ax.text(x1, y1 + 8, caption, # color='w', size=11, backgroundcolor="none") tmp_image = tmp_image.astype(np.uint8) # print('shape ', tmp_image.shape) # for i in tmp_image: # for j in i: # print('-->', j) # new_data = [] # for item in tmp_image: # if item[0] == 255 and item[1] == 200 and item[2] == 255: # newData.append((255, 255, 255, 0)) # else: # newData.append(item) layer_images.append(tmp_image) return (layer_images)
# load the input image, convert it from BGR to RGB channel # ordering, and resize the image image = cv2.imread(args["image"]) image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) image = imutils.resize(image, width=640) # perform a forward pass of the network to obtain the results r = model.detect([image], verbose=1)[0] # loop over of the detected object's bounding boxes and # masks, drawing each as we go along for i in range(0, r["rois"].shape[0]): mask = r["masks"][:, :, i] image = visualize.apply_mask(image, mask, (1.0, 0.0, 0.0), alpha=0.5) image = visualize.draw_box(image, r["rois"][i], (1.0, 0.0, 0.0)) # convert the image back to BGR so we can use OpenCV's # drawing functions image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) # loop over the predicted scores and class labels for i in range(0, len(r["scores"])): # extract the bounding box information, class ID, label, # and predicted probability from the results (startY, startX, endY, end) = r["rois"][i] classID = r["class_ids"][i] label = CLASS_NAMES[classID] score = r["scores"][i] # draw the class label and score on the image
def upload_image(): if 'image' not in request.files: return render_template( 'ImageML.html', prediction='No posted image. Should be attribute named image') file = request.files['image'] if file.filename == '': return render_template('ImageML.html', prediction='You did not select an image') if file and allowed_file(file.filename): items = [] use_tf_keras = False filename = file.filename originalFile = splitext( os.path.basename(filename))[0] + "_original_image" + splitext( os.path.basename(filename))[1] input_path = UPLOAD_FOLDER print("File Name --> " + filename) split_filename = filename.split(".")[0] split_filename = split_filename.split("_")[0] print("Image prefix value --> " + split_filename) if os.path.exists(input_path) and os.path.isdir(input_path): shutil.rmtree(input_path) print('input directory removed') ImageFile.LOAD_TRUNCATED_IMAGES = False img = Image.open(BytesIO(file.read())) K.clear_session() config = AcneInferenceConfig() model = modellib.MaskRCNN(mode="inference", config=config, model_dir='./') model.load_weights(COCO_TRAINED_MODEL, by_name=True) image = cv2.cvtColor(np.asarray(img), cv2.COLOR_BGR2RGB) actual_image = image.copy() actual_image = cv2.cvtColor(np.asarray(img), cv2.COLOR_BGR2RGB) actual_image = cv2.cvtColor(np.asarray(actual_image), cv2.COLOR_RGB2BGR) actual_image = imutils.resize(actual_image, width=600) image = imutils.resize(image, width=600) r = model.detect([image], verbose=1)[0] for i in range(0, r["rois"].shape[0]): mask = r["masks"][:, :, i] image = visualize.apply_mask(image, mask, (1.0, 0.0, 0.0), alpha=0.5) image = visualize.draw_box(image, r["rois"][i], (1.0, 0.0, 0.0)) image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) for i in range(0, len(r["scores"])): (startY, startX, endY, end) = r["rois"][i] classID = r["class_ids"][i] label = CLASS_NAMES[classID] score = r["scores"][i] text = "{}: {:.4f}".format(label, score) y = startY - 10 if startY - 10 > 10 else startY + 10 cv2.putText(image, text, (startX, y), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (0, 255, 0), 1) if not os.path.exists(input_path): os.makedirs(input_path) imageio.imwrite(input_path + filename, image) imageio.imwrite(input_path + originalFile, actual_image) #time.sleep(5) if len(r["scores"]) > 1: items.append('Acne Detected') else: items.append('No Acne Detected') response = {'Prediction': items} basepath = 'static/uploads' imagespath = os.path.join(basepath, '*g') images_list = {} for imag in glob.glob(imagespath): if "_original_image" in splitext(os.path.basename(imag))[0]: images_list[0] = originalFile else: images_list[1] = filename flash( 'Image successfully uploaded and displayed : {}'.format(response)) return render_template('ImageML.html', filename=images_list[1], filename1=images_list[0]) else: return render_template('ImageML.html', prediction='Invalid File extension')
visualize.draw_rois(sample_image, rois[b], refined_rois, mask_specific, mrcnn_class_ids[b], dataset.class_names) # Any repeated ROIs? rows = np.ascontiguousarray(rois[b]).view(np.dtype((np.void, rois.dtype.itemsize * rois.shape[-1]))) _, idx = np.unique(rows, return_index=True) print("Unique ROIs: {} out of {}".format(len(idx), rois.shape[1])) if random_rois: # Dispalay ROIs and corresponding masks and bounding boxes ids = random.sample(range(rois.shape[1]), 8) images = [] titles = [] for i in ids: image = visualize.draw_box(sample_image.copy(), rois[b,i,:4].astype(np.int32), [255, 0, 0]) image = visualize.draw_box(image, refined_rois[i].astype(np.int64), [0, 255, 0]) images.append(image) titles.append("ROI {}".format(i)) images.append(mask_specific[i] * 255) titles.append(dataset.class_names[mrcnn_class_ids[b,i]][:20]) display_images(images, titles, cols=4, cmap="Blues", interpolation="none") # Check ratio of positive ROIs in a set of images. if random_rois: limit = 10 temp_g = modellib.data_generator( dataset, config, shuffle=True, random_rois=10000, batch_size=1, detection_targets=True)
result = results[0] N = result['rois'].shape[0] # 検出数 result_image = padding_image.copy() colors = visualize.random_colors(N) for i in range(N): '''クラス関係なく1物体ごと処理を行う''' if class_names[result['class_ids'][i]] in filtered_classNames: # Color color = colors[i] rgb = (round(color[0] * 255), round(color[1] * 255), round(color[2] * 255)) font = cv2.FONT_HERSHEY_SIMPLEX # Bbox result_image = visualize.draw_box(result_image, result['rois'][i], rgb) # Class & Score print(result['rois'][i]) text_top = class_names[result['class_ids'][i]] + ':' + str(result['scores'][i]) result_image = cv2.putText(result_image, text_top, (result['rois'][i][1], result['rois'][i][2] + 15), font, 0.7, rgb, 1, cv2.LINE_AA) # Mask mask = result['masks'][:, :, i] result_image = visualize.apply_mask(result_image, mask, color) mask = mask[240:720, 320:960] # Distance mask_binary = mask.astype('uint8') center_pos = calc_center(mask_binary)
def output_image(img_list, mask_path, model_path): log_path = os.path.join(os.getcwd(), "logs") if not os.path.exists(log_path): os.mkdir(log_path) #Read configurations config = TargetConfig() DEVICE = "/gpu:0" # /cpu:0 or /gpu:0 start = time.time() with tf.device(DEVICE): model = modellib.MaskRCNN(mode="inference", model_dir=log_path, config=config) modellib_time = time.time() print("modellib_time:", modellib_time - start) start = time.time() #Read model model.load_weights(model_path, by_name=True) model_time = time.time() print("Time to read model:", model_time - start) class_names = ['BG', 'target', 'target_2'] for img_path in img_list: print("*" * 5 + "Info" + "*" * 5) print("file:", img_path) start = time.time() #Start to predict a nail image = cv2.imread(img_path) image_bb = image.copy() image_RGB = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) results = model.detect([image_RGB], verbose=0) r = results[0] predict_time = time.time() #Create Mask N = r['rois'].shape[0] print("N:", N) print("r[`class_ids`]:", r['class_ids']) print("r['rois'][i]", r['rois']) colors = visualize.random_colors(N) start = time.time() font = cv2.FONT_HERSHEY_SIMPLEX print("r['masks'].shape", r['masks'].shape) for i in range(N): mask = r['masks'][:, :, i] if r['class_ids'][i] == 0: print("class[0]_colors", colors[i]) image = visualize.apply_mask(image, mask, [0, 1, 0]) # if r['class_ids'][i] == 1: print("class[1]_colors", colors[i]) text = class_names[r['class_ids'][i]] + ':' + str( r['scores'][i]) result_image = cv2.putText(image, text, (r['rois'][i][1], r['rois'][i][0]), font, 0.5, [0, 255, 0], 1, cv2.LINE_AA) image = visualize.draw_box(image, r['rois'][i], [1, 0, 0]) image = visualize.apply_mask( image, mask, [0, 1, 0]) #fingers="B"GR + image has to BGR not RGB if r['class_ids'][i] == 2: print("class[2]_colors:", colors[i]) text = class_names[r['class_ids'][i]] + ':' + str( r['scores'][i]) result_image = cv2.putText(image, text, (r['rois'][i][1], r['rois'][i][0]), font, 0.5, [0, 0, 255], 1, cv2.LINE_AA) print("mask:", mask) image = visualize.apply_mask(image, mask, [0, 0, 1]) #nail=BG"R" fileName = os.path.join( mask_path, os.path.basename(img_path).split(".")[0] + "_mask.png") cv2.imwrite(fileName, image) drawing_time = time.time() print("drawing_time:", drawing_time - start) print("*" * 10) # fileName_bb = os.path.join(mask_path,os.path.basename(img_path).split(".")[0] + "_mask_bb.png") # cv2.imwrite(fileName_bb, image_bb) return
def all_steps(dataset, datacfg, dnncfg): ''' ## Single entry point for all the steps for inspecting dataset ''' ## Uncomment for debugging # inspectdata.load_and_display_dataset(dataset, datacfg) # In[7]: log.info("[7]. ---------------") log.info("Load and display random images and masks---------------") log.info("Bounding Boxes---------------") load_and_display_random_sample(dataset, datacfg) # In[9]: log.info("[9]. ---------------") log.info("Resize Images---------------") load_and_resize_images(dataset, datacfg, dnncfg) # In[10]: log.info("[10]. ---------------") log.info("Mini Masks---------------") image_id = load_mini_masks(dataset, datacfg, dnncfg) log.info("image_id: {}".format(image_id)) # In[11]: log.info("[11]. ---------------") log.info("Add augmentation and mask resizing---------------") add_augmentation(dataset, datacfg, dnncfg, image_id) info = dataset.image_info[image_id] log.debug("info: {}".format(info)) # In[12]: log.info("[12]. ---------------") log.info("Anchors---------------") backbone_shapes, anchors, anchors_per_level, anchors_per_cell = generate_anchors(dnncfg) # In[13]: log.info("[13]. ---------------") log.info("Visualize anchors of one cell at the center of the feature map of a specific level---------------") visualize_anchors_at_center(dataset, datacfg, dnncfg, backbone_shapes, anchors, anchors_per_level, anchors_per_cell) # In[14]: log.info("[14]. ---------------") log.info("info---------------") image_ids = dataset.image_ids log.info(image_ids) image_index = -1 image_index = (image_index + 1) % len(image_ids) log.info("image_index:{}".format(image_index)) # In[15]: log.info("[15]. ---------------") log.info("data_generator---------------") ## Data Generator # Create data generator random_rois = 2000 g = modellib.data_generator( dataset, datacfg, dnncfg, shuffle=True, random_rois=random_rois, batch_size=4, detection_targets=True) # Uncomment to run the generator through a lot of images # to catch rare errors # for i in range(1000): # log.debug(i) # _, _ = next(g) # Get Next Image if random_rois: [normalized_images, image_meta, rpn_match, rpn_bbox, gt_class_ids, gt_boxes, gt_masks, rpn_rois, rois], [mrcnn_class_ids, mrcnn_bbox, mrcnn_mask] = next(g) customlog("rois", rois) customlog("mrcnn_class_ids", mrcnn_class_ids) customlog("mrcnn_bbox", mrcnn_bbox) customlog("mrcnn_mask", mrcnn_mask) else: [normalized_images, image_meta, rpn_match, rpn_bbox, gt_boxes, gt_masks], _ = next(g) customlog("gt_class_ids", gt_class_ids) customlog("gt_boxes", gt_boxes) customlog("gt_masks", gt_masks) customlog("rpn_match", rpn_match, ) customlog("rpn_bbox", rpn_bbox) image_id = modellib.parse_image_meta(image_meta)["image_id"][0] # Remove the last dim in mrcnn_class_ids. It's only added # to satisfy Keras restriction on target shape. mrcnn_class_ids = mrcnn_class_ids[:,:,0] # In[16]: log.info("[16]. ---------------") b = 0 # Restore original image (reverse normalization) sample_image = modellib.unmold_image(normalized_images[b], dnncfg) # Compute anchor shifts. indices = np.where(rpn_match[b] == 1)[0] refined_anchors = utils.apply_box_deltas(anchors[indices], rpn_bbox[b, :len(indices)] * dnncfg.RPN_BBOX_STD_DEV) customlog("anchors", anchors) customlog("refined_anchors", refined_anchors) # Get list of positive anchors positive_anchor_ids = np.where(rpn_match[b] == 1)[0] log.info("Positive anchors: {}".format(len(positive_anchor_ids))) negative_anchor_ids = np.where(rpn_match[b] == -1)[0] log.info("Negative anchors: {}".format(len(negative_anchor_ids))) neutral_anchor_ids = np.where(rpn_match[b] == 0)[0] log.info("Neutral anchors: {}".format(len(neutral_anchor_ids))) log.info("ROI breakdown by class---------------") # ROI breakdown by class for c, n in zip(dataset.class_names, np.bincount(mrcnn_class_ids[b].flatten())): if n: log.info("{:23}: {}".format(c[:20], n)) log.info("Show positive anchors---------------") # Show positive anchors fig, ax = plt.subplots(1, figsize=(16, 16)) visualize.draw_boxes(sample_image, boxes=anchors[positive_anchor_ids], refined_boxes=refined_anchors, ax=ax) # In[17]: log.info("[17]. ---------------") log.info("Show negative anchors---------------") # Show negative anchors visualize.draw_boxes(sample_image, boxes=anchors[negative_anchor_ids]) # In[18]: log.info("[18]. ---------------") log.info("Show neutral anchors. They don't contribute to training---------------") # Show neutral anchors. They don't contribute to training. visualize.draw_boxes(sample_image, boxes=anchors[np.random.choice(neutral_anchor_ids, 100)]) # In[19]: log.info("[19]. ---------------") log.info("ROIs---------------") ## ROIs if random_rois: # Class aware bboxes bbox_specific = mrcnn_bbox[b, np.arange(mrcnn_bbox.shape[1]), mrcnn_class_ids[b], :] # Refined ROIs refined_rois = utils.apply_box_deltas(rois[b].astype(np.float32), bbox_specific[:,:4] * dnncfg.BBOX_STD_DEV) # Class aware masks mask_specific = mrcnn_mask[b, np.arange(mrcnn_mask.shape[1]), :, :, mrcnn_class_ids[b]] visualize.draw_rois(sample_image, rois[b], refined_rois, mask_specific, mrcnn_class_ids[b], dataset.class_names) # Any repeated ROIs? rows = np.ascontiguousarray(rois[b]).view(np.dtype((np.void, rois.dtype.itemsize * rois.shape[-1]))) _, idx = np.unique(rows, return_index=True) log.info("Unique ROIs: {} out of {}".format(len(idx), rois.shape[1])) # In[20]: log.info("[20]. ---------------") log.info("Dispalay ROIs and corresponding masks and bounding boxes---------------") if random_rois: # Dispalay ROIs and corresponding masks and bounding boxes ids = random.sample(range(rois.shape[1]), 8) images = [] titles = [] for i in ids: image = visualize.draw_box(sample_image.copy(), rois[b,i,:4].astype(np.int32), [255, 0, 0]) image = visualize.draw_box(image, refined_rois[i].astype(np.int64), [0, 255, 0]) images.append(image) titles.append("ROI {}".format(i)) images.append(mask_specific[i] * 255) titles.append(dataset.class_names[mrcnn_class_ids[b,i]][:20]) visualize.display_images(images, titles, cols=4, cmap="Blues", interpolation="none") # In[21]: log.info("[21]. ---------------") log.info("Check ratio of positive ROIs in a set of images.---------------") # Check ratio of positive ROIs in a set of images. if random_rois: limit = 10 temp_g = modellib.data_generator( dataset, datacfg, dnncfg, shuffle=True, random_rois=10000, batch_size=1, detection_targets=True) total = 0 for i in range(limit): _, [ids, _, _] = next(temp_g) positive_rois = np.sum(ids[0] > 0) total += positive_rois log.info("{:5} {:5.2f}".format(positive_rois, positive_rois/ids.shape[1])) log.info("Average percent: {:.2f}".format(total/(limit*ids.shape[1])))