def visualize(model, model_path, nr_visualize=100, output_dir='output'): """ Visualize some intermediate results (proposals, raw predictions) inside the pipeline. """ df = get_train_dataflow() # we don't visualize mask stuff df.reset_state() pred = OfflinePredictor(PredictConfig( model=model, session_init=get_model_loader(model_path), input_names=['image', 'gt_boxes', 'gt_labels'], output_names=[ 'generate_{}_proposals/boxes'.format('fpn' if cfg.MODE_FPN else 'rpn'), 'generate_{}_proposals/scores'.format('fpn' if cfg.MODE_FPN else 'rpn'), 'fastrcnn_all_scores', 'output/boxes', 'output/scores', 'output/labels', ])) if os.path.isdir(output_dir): shutil.rmtree(output_dir) utils.fs.mkdir_p(output_dir) with tqdm.tqdm(total=nr_visualize) as pbar: for idx, dp in itertools.islice(enumerate(df), nr_visualize): img = dp[0] if cfg.MODE_MASK: gt_boxes, gt_labels, gt_masks = dp[-3:] else: gt_boxes, gt_labels = dp[-2:] rpn_boxes, rpn_scores, all_scores, \ final_boxes, final_scores, final_labels = pred(img, gt_boxes, gt_labels) # draw groundtruth boxes gt_viz = draw_annotation(img, gt_boxes, gt_labels) # draw best proposals for each groundtruth, to show recall proposal_viz, good_proposals_ind = draw_proposal_recall(img, rpn_boxes, rpn_scores, gt_boxes) # draw the scores for the above proposals score_viz = draw_predictions(img, rpn_boxes[good_proposals_ind], all_scores[good_proposals_ind]) results = [DetectionResult(*args) for args in zip(final_boxes, final_scores, final_labels, [None] * len(final_labels))] final_viz = draw_final_outputs(img, results) viz = tpviz.stack_patches([ gt_viz, proposal_viz, score_viz, final_viz], 2, 2) if os.environ.get('DISPLAY', None): tpviz.interactive_imshow(viz) cv2.imwrite("{}/{:03d}.png".format(output_dir, idx), viz) pbar.update()
def visualize(model_path, nr_visualize=50, output_dir='output'): df = get_train_dataflow() # we don't visualize mask stuff df.reset_state() pred = OfflinePredictor( PredictConfig(model=Model(), session_init=get_model_loader(model_path), input_names=['image', 'gt_boxes', 'gt_labels'], output_names=[ 'generate_rpn_proposals/boxes', 'generate_rpn_proposals/probs', 'fastrcnn_all_probs', 'final_boxes', 'final_probs', 'final_labels', ])) if os.path.isdir(output_dir): shutil.rmtree(output_dir) utils.fs.mkdir_p(output_dir) with tqdm.tqdm(total=nr_visualize) as pbar: for idx, dp in itertools.islice(enumerate(df.get_data()), nr_visualize): img, _, _, gt_boxes, gt_labels = dp rpn_boxes, rpn_scores, all_probs, \ final_boxes, final_probs, final_labels = pred(img, gt_boxes, gt_labels) # draw groundtruth boxes gt_viz = draw_annotation(img, gt_boxes, gt_labels) # draw best proposals for each groundtruth, to show recall proposal_viz, good_proposals_ind = draw_proposal_recall( img, rpn_boxes, rpn_scores, gt_boxes) # draw the scores for the above proposals score_viz = draw_predictions(img, rpn_boxes[good_proposals_ind], all_probs[good_proposals_ind]) results = [ DetectionResult(*args) for args in zip(final_boxes, final_probs, final_labels, [None] * len(final_labels)) ] final_viz = draw_final_outputs(img, results) viz = tpviz.stack_patches( [gt_viz, proposal_viz, score_viz, final_viz], 2, 2) if os.environ.get('DISPLAY', None): tpviz.interactive_imshow(viz) cv2.imwrite("{}/{:03d}.png".format(output_dir, idx), viz) pbar.update()
def do_visualize(model, model_path, nr_visualize=100, output_dir='output'): """ Visualize some intermediate results (proposals, raw predictions) inside the pipeline. """ df = get_train_dataflow() df.reset_state() pred = OfflinePredictor(PredictConfig( model=model, session_init=SmartInit(model_path), input_names=['image', 'gt_boxes', 'gt_labels'], output_names=[ 'generate_{}_proposals/boxes'.format('fpn' if cfg.MODE_FPN else 'rpn'), 'generate_{}_proposals/scores'.format('fpn' if cfg.MODE_FPN else 'rpn'), 'fastrcnn_all_scores', 'output/boxes', 'output/scores', 'output/labels', ])) if os.path.isdir(output_dir): shutil.rmtree(output_dir) fs.mkdir_p(output_dir) with tqdm.tqdm(total=nr_visualize) as pbar: for idx, dp in itertools.islice(enumerate(df), nr_visualize): img, gt_boxes, gt_labels = dp['image'], dp['gt_boxes'], dp['gt_labels'] rpn_boxes, rpn_scores, all_scores, \ final_boxes, final_scores, final_labels = pred(img, gt_boxes, gt_labels) # draw groundtruth boxes gt_viz = draw_annotation(img, gt_boxes, gt_labels) # draw best proposals for each groundtruth, to show recall proposal_viz, good_proposals_ind = draw_proposal_recall(img, rpn_boxes, rpn_scores, gt_boxes) # draw the scores for the above proposals score_viz = draw_predictions(img, rpn_boxes[good_proposals_ind], all_scores[good_proposals_ind]) results = [DetectionResult(*args) for args in zip(final_boxes, final_scores, final_labels, [None] * len(final_labels))] final_viz = draw_final_outputs(img, results) viz = tpviz.stack_patches([ gt_viz, proposal_viz, score_viz, final_viz], 2, 2) # if os.environ.get('DISPLAY', None): # tpviz.interactive_imshow(viz) cv2.imwrite("{}/{:03d}.png".format(output_dir, idx), viz) pbar.update()
def visualize(model_path, nr_visualize=50, output_dir='output'): pred = OfflinePredictor( PredictConfig(model=Model(), session_init=get_model_loader(model_path), input_names=['image', 'gt_boxes', 'gt_labels'], output_names=[ 'generate_rpn_proposals/boxes', 'generate_rpn_proposals/probs', 'fastrcnn_all_probs', 'fastrcnn_fg_probs', 'fastrcnn_fg_boxes', ])) df = get_train_dataflow() df.reset_state() if os.path.isdir(output_dir): shutil.rmtree(output_dir) utils.fs.mkdir_p(output_dir) with tqdm.tqdm(total=nr_visualize) as pbar: for idx, dp in itertools.islice(enumerate(df.get_data()), nr_visualize): img, _, _, gt_boxes, gt_labels = dp rpn_boxes, rpn_scores, all_probs, fg_probs, fg_boxes = pred( img, gt_boxes, gt_labels) gt_viz = draw_annotation(img, gt_boxes, gt_labels) proposal_viz, good_proposals_ind = draw_proposal_recall( img, rpn_boxes, rpn_scores, gt_boxes) score_viz = draw_predictions(img, rpn_boxes[good_proposals_ind], all_probs[good_proposals_ind]) fg_boxes = clip_boxes(fg_boxes, img.shape[:2]) fg_viz = draw_predictions(img, fg_boxes, fg_probs) results = nms_fastrcnn_results(fg_boxes, fg_probs) final_viz = draw_final_outputs(img, results) viz = tpviz.stack_patches( [gt_viz, proposal_viz, score_viz, fg_viz, final_viz], 2, 3) if os.environ.get('DISPLAY', None): tpviz.interactive_imshow(viz) cv2.imwrite("{}/{:03d}.png".format(output_dir, idx), viz) pbar.update()
def preprocess(fname): print("start preproc mapillary") start = time.time() label_fname = fname.replace(train_img_path, train_label_path).replace(".jpg", ".png") pil_label = Image.open(label_fname) label = np.array(pil_label) instances = np.unique(label) instance_classes = [x // 256 for x in instances] # filter by categories we use instances_valid = [ cls in config.MAPILLARY_CAT_IDS_TO_USE for cls in instance_classes ] instances = [ inst for inst, valid in zip(instances, instances_valid) if valid ] instance_classes = [ cls for cls, valid in zip(instance_classes, instances_valid) if valid ] if len(instances) == 0: print("no instances") pil_label.close() return None if map_to_coco: instance_classes = [ config.MAPILLARY_TO_COCO_MAP[cls] for cls in instance_classes ] instance_classes = [ config.VOID_LABEL if cls == config.VOID_LABEL else COCOMeta.category_id_to_class_id[cls] for cls in instance_classes ] else: # remap to contiguous numbers starting with 1 instance_classes = [ config.MAPILLARY_CAT_IDS_TO_USE.index(cls) + 1 for cls in instance_classes ] masks = np.array([label == inst for inst in instances], dtype=np.uint8) #import cProfile #start1 = time.time() boxes1 = np.array( [get_bbox_from_segmentation_mask(mask) for mask in masks], dtype=np.float32) #boxes1_time = time.time() - start1 #pr = cProfile.Profile() #pr.enable() #start1 = time.time() #boxes2 = get_bboxes_from_segmentation_masks(masks) #print("boxes1", boxes1_time, "boxes2", time.time() - start1) #pr.disable() #pr.print_stats(sort="cumulative") #assert (boxes1 == boxes2).all(), (boxes1, boxes2) boxes = boxes1 second_klass = np.array(instance_classes, dtype=np.int) klass = np.ones_like(second_klass) is_crowd = np.zeros_like(second_klass) res = preproc_img(fname, boxes, klass, second_klass, is_crowd, aug) if res is None: print("mapillary: preproc_img returned None on", fname) pil_label.close() return None ret, params = res if add_mask: do_flip, h, w = params[1] assert do_flip in (True, False), do_flip # augment label label = np.array(pil_label.resize((w, h), Image.NEAREST)) if do_flip: label = label[:, ::-1] # create augmented masks masks = np.array([label == inst for inst in instances], dtype=np.uint8) ret.append(masks) end = time.time() elapsed = end - start print("mapillary example done, elapsed:", elapsed) VISUALIZE = False if VISUALIZE: from viz import draw_annotation, draw_mask config.CLASS_NAMES = [str(idx) for idx in range(81)] im = ret[0] boxes = ret[3] draw_klass = ret[-2] viz = draw_annotation(im, boxes, draw_klass) for mask in masks: viz = draw_mask(viz, mask) tpviz.interactive_imshow(viz) pil_label.close() return ret
def preprocess(fname): # print("start preproc mapillary") start = time.time() label_fname = fname.replace(train_img_path, train_label_path).replace(".jpg", ".png") pil_label = Image.open(label_fname) label = np.array(pil_label) instances = np.unique(label) instance_classes = [x // 256 for x in instances] if len(instances) == 0: print("no instances") pil_label.close() return None masks = np.array([label == inst for inst in instances], dtype=np.uint8) boxes1 = np.array( [get_bbox_from_segmentation_mask(mask) for mask in masks], dtype=np.float32) boxes = boxes1 # second_klass = np.array(instance_classes, dtype=np.int) second_klass = np.zeros_like(instance_classes, dtype=np.int) klass = np.ones_like(second_klass) is_crowd = np.zeros_like(second_klass) res = preproc_img(fname, boxes, klass, second_klass, is_crowd, aug) if res is None: print("davis: preproc_img returned None on", fname) pil_label.close() return None ret, params = res if add_mask: do_flip, h, w = params[1] assert do_flip in (True, False), do_flip # augment label label = np.array(pil_label.resize((w, h), Image.NEAREST)) if do_flip: label = label[:, ::-1] # create augmented masks masks = np.array([label == inst for inst in instances], dtype=np.uint8) ret.append(masks) end = time.time() elapsed = end - start # print("davis example done, elapsed:", elapsed) VISUALIZE = False if VISUALIZE: from viz import draw_annotation, draw_mask config.CLASS_NAMES = [str(idx) for idx in range(81)] im = ret[0] boxes = ret[3] draw_klass = ret[-2] viz = draw_annotation(im, boxes, draw_klass) for mask in masks: viz = draw_mask(viz, mask) tpviz.interactive_imshow(viz) pil_label.close() return ret
break orig_shape = batch_image.shape[:2] feed_dict = {image_P: batch_image} final_boxes_, final_labels_, final_probs_ = sess.run( [final_boxes, final_labels, final_probs], feed_dict) final_boxes_ = clip_boxes(final_boxes_, orig_shape) final_boxes_ = sess.run(final_boxes_) final_boxes_ = final_boxes_.astype('int32') if np.any(final_boxes_): tags = [ "{},{:.2f}".format(cfg.DATA.CLASS_NAMES[lb], score) for lb, score in zip(final_labels_, final_probs_) ] final_viz = viz.draw_boxes(batch_image, final_boxes_, tags) gt_viz = draw_annotation(batch_image, batch_gt_boxes, batch_gt_labels) img_out = np.hstack((final_viz, gt_viz)) imageio.imwrite(os.path.join(save_path, str(iter) + ".jpg"), img_out) Detection = [] for ik in range(final_boxes_.shape[0]): Detection.append([ cfg.DATA.CLASS_NAMES[final_labels_[ik]], float(final_probs_[ik]), final_boxes_[ik, 0], final_boxes_[ik, 1], final_boxes_[ik, 2] - final_boxes_[ik, 0], final_boxes_[ik, 3] - final_boxes_[ik, 1] ]) Detection = np.array(Detection) #np.savetxt(os.path.join(save_folder, 'detections/',str(iter) + '.txt'), Detection, fmt='%s %1.2f %1.0f %1.0f %1.0f %1.0f') np.savetxt(os.path.join(save_folder, 'detections/',
from dataset.display_virtual import DisplayDemo, register_display from viz import draw_annotation from dataset.tensorpack.utils.viz import interactive_imshow as imshow import cv2 import os from config import finalize_configs if __name__ == '__main__': base_dir = './data' roibds = DisplayDemo(base_dir, "train").training_roidbs() print("#images:", len(roibds)) register_display(base_dir) finalize_configs(True) i = 0 for r in roibds: i = i + 1 if i == 100: break im = cv2.imread(r["file_name"]) im_out = os.path.join('./output', r["file_name"].split('/')[-1]) print(im_out) vis = draw_annotation(im, r["boxes"], r["class"]) cv2.imwrite(im_out, vis)
def do_visualize(model, model_path, nr_visualize=100, output_dir='output'): """ Visualize some intermediate results (proposals, raw predictions) inside the pipeline. """ df = get_train_dataflow() # we don't visualize mask stuff df.reset_state() pred = OfflinePredictor( PredictConfig( model=model, session_init=get_model_loader(model_path), input_names=['images', 'orig_image_dims', 'gt_boxes', 'gt_labels'], output_names=[ 'generate_{}_proposals_topk_per_image/boxes'.format( 'fpn' if cfg.MODE_FPN else 'rpn'), 'generate_{}_proposals_topk_per_image/scores'.format( 'fpn' if cfg.MODE_FPN else 'rpn'), 'fastrcnn_all_scores', 'output/boxes', 'output/scores', 'output/labels', ])) if os.path.isdir(output_dir): shutil.rmtree(output_dir) utils.fs.mkdir_p(output_dir) with tqdm.tqdm(total=nr_visualize) as pbar: for idx, dp in itertools.islice(enumerate(df), nr_visualize): img, gt_boxes, gt_labels = dp['images'], dp['gt_boxes'], dp[ 'gt_labels'] orig_shape = img.shape[:2] rpn_boxes, rpn_scores, all_scores, \ final_boxes, final_scores, final_labels = pred(np.expand_dims(img, axis=0), np.expand_dims(np.array(img.shape), axis=0), np.expand_dims(gt_boxes, axis=0), np.expand_dims(gt_labels, axis=0)) # draw groundtruth boxes gt_viz = draw_annotation(img, gt_boxes, gt_labels) # draw best proposals for each groundtruth, to show recall # custom op creates different shape for boxes, convert back to original rpn_boxes = np.array([i[1:] for i in rpn_boxes]) proposal_viz, good_proposals_ind = draw_proposal_recall( img, rpn_boxes, rpn_scores, gt_boxes) # draw the scores for the above proposals score_viz = draw_predictions(img, rpn_boxes[good_proposals_ind], all_scores[good_proposals_ind]) results = [ DetectionResult(*args) for args in zip(final_boxes, final_scores, final_labels, [None] * len(final_labels)) ] final_viz = draw_final_outputs(img, results) viz = tpviz.stack_patches( [gt_viz, proposal_viz, score_viz, final_viz], 2, 2) if os.environ.get('DISPLAY', None): tpviz.interactive_imshow(viz) cv2.imwrite("{}/{:03d}.png".format(output_dir, idx), viz) pbar.update()