def represent(self, batch, pred): images, labels = batch['image'], batch['label'] mask = torch.ones(pred.shape[0], dtype=torch.int).to(pred.device) for i in range(pred.shape[1]): mask = (1 - (pred[:, i] == self.charset.blank).type(torch.int)) * mask pred[:, i] = pred[:, i] * mask + self.charset.blank * (1 - mask) output = [] for i in range(labels.shape[0]): label_str = self.label_to_string(labels[i]) pred_str = self.label_to_string(pred[i]) if False and label_str != pred_str: print('label: %s , pred: %s' % (label_str, pred_str)) img = (np.clip( images[i].cpu().data.numpy().transpose(1, 2, 0) + 0.5, 0, 1) * 255).astype('uint8') webcv.imshow( '【 pred: <%s> , label: <%s> 】' % (pred_str, label_str), np.array(img, dtype=np.uint8)) if webcv.waitKey() == ord('q'): continue output.append({'label_string': label_str, 'pred_string': pred_str}) return output
def visualize_batch(self, batch, output): visualization = dict() for index, output_dict in enumerate(output): image = batch['image'][index] image = NormalizeImage.restore(image) mask = output_dict['mask'] mask = cv2.resize(Visualize.visualize_weights(mask), image.shape[:2][::-1]) classify = output_dict['classify'] classify = cv2.resize( Visualize.visualize_heatmap(classify, format='CHW'), image.shape[:2][::-1]) canvas = np.concatenate([image, mask, classify], axis=0) key = "【%s-%s】" % (output_dict['label_string'], output_dict['pred_string']) vis_dict = {key: canvas} if self.eager_show: for k, v in vis_dict.items(): # if output_dict['label_string'] != output_dict['pred_string']: webcv2.imshow(k, v) visualization.update(mask=mask, classify=classify, image=image) if self.eager_show: webcv2.waitKey() return visualization
def output(vis, fname): if args.show: print(fname) webcv2.imshow(fname, vis.get_image()[:, :, ::-1]) webcv2.waitKey() else: filepath = os.path.join(dirname, fname) print("Saving to {} ...".format(filepath)) vis.save(filepath)
def boxes_from_bitmap(self, pred, _bitmap, dest_width, dest_height): ''' _bitmap: single map with shape (1, H, W), whose values are binarized as {0, 1} ''' assert _bitmap.size(0) == 1 bitmap = _bitmap.data.cpu().numpy()[0] # The first channel pred = pred.cpu().detach().numpy()[0] height, width = bitmap.shape boxes = [] _, contours, _ = cv2.findContours( (bitmap*255).astype(np.uint8), cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE) if self.debug: bitmap = cv2.cvtColor(pred * 255, cv2.COLOR_GRAY2BGR) for contour in contours[:self.max_candidates]: points, sside = self.get_mini_boxes(contour) if sside < self.min_size: continue points = np.array(points) score = self.box_score_fast(pred, points.reshape(-1, 2)) if self.debug: points = points.astype(np.int32) bitmap = cv2.polylines( bitmap, [points.reshape(-1, 2)], True, (255, 0, 0), 3) bitmap = cv2.putText( bitmap, str(round(score, 3)), (points[:, 0].min(), points[:, 1].min()), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0)) if self.box_thresh > score: continue box = self.unclip(points).reshape(-1, 1, 2) box, sside = self.get_mini_boxes(box) if sside < self.min_size + 2: continue box = np.array(box) if not self.resize: dest_width = width dest_height = height box[:, 0] = np.clip( np.round(box[:, 0] / width * dest_width), 0, dest_width) box[:, 1] = np.clip( np.round(box[:, 1] / height * dest_height), 0, dest_height) boxes.append(box.tolist()) if self.debug: webcv2.imshow('mask', bitmap) return boxes, bitmap
def single_visualize(self, batch, index, boxes, pred): image = batch['image'][index] polygons = batch['polygons'][index] if isinstance(polygons, torch.Tensor): polygons = polygons.cpu().data.numpy() ignore_tags = batch['ignore_tags'][index] original_shape = batch['shape'][index] filename = batch['filename'][index] std = np.array([0.229, 0.224, 0.225]).reshape(3, 1, 1) mean = np.array([0.485, 0.456, 0.406]).reshape(3, 1, 1) image = (image.cpu().numpy() * std + mean).transpose(1, 2, 0) * 255 pred_canvas = image.copy().astype(np.uint8) pred_canvas = cv2.resize(pred_canvas, (original_shape[1], original_shape[0])) if isinstance(pred, dict) and 'binary' in pred: binary = self._visualize_heatmap(pred['binary'][index]) if isinstance(pred, dict) and 'thresh' in pred: thresh = self._visualize_heatmap(pred['thresh'][index]) if isinstance(pred, dict) and 'thresh_binary' in pred: thresh_binary = self._visualize_heatmap( pred['thresh_binary'][index]) MakeICDARData.polylines(self, thresh_binary, polygons, ignore_tags) for box in boxes: box = np.array(box).astype(np.int32).reshape(-1, 2) cv2.polylines(pred_canvas, [box], True, (0, 255, 0), 2) if isinstance(pred, dict) and 'thresh_binary' in pred: cv2.polylines(thresh_binary, [box], True, (0, 255, 0), 1) if self.eager_show: webcv2.imshow(filename + ' output', cv2.resize(pred_canvas, (1024, 1024))) if isinstance(pred, dict) and 'thresh' in pred: webcv2.imshow(filename + ' thresh', cv2.resize(thresh, (1024, 1024))) webcv2.imshow(filename + ' pred', cv2.resize(pred_canvas, (1024, 1024))) if isinstance(pred, dict) and 'thresh_binary' in pred: webcv2.imshow(filename + ' thresh_binary', cv2.resize(thresh_binary, (1024, 1024))) return {} else: if isinstance(pred, dict) and 'thresh' in pred: return { filename + '_output': pred_canvas, filename + '_thresh': thresh, filename + '_pred': thresh_binary, filename + '_binary': binary } else: return { filename + '_output': pred_canvas, filename + '_pred': thresh_binary, filename + '_binary': binary }
def visualize_batch(self, batch, output): images, labels, lengths = batch['image'], batch['label'], batch[ 'length'] for i in range(images.shape[0]): image = NormalizeImage.restore(images[i]) gt = self.charset.label_to_string(labels[i]) webcv2.imshow(output[i]['pred_string'] + '_' + str(i) + '_' + gt, image) # folder = 'images/dropout/lexicon/' # np.save(folder + output[i]['pred_string'] + '_' + gt + '_' + batch['data_ids'][i], image) webcv2.waitKey() return { 'image': (np.clip( batch['image'][0].cpu().data.numpy().transpose(1, 2, 0) + 0.5, 0, 1) * 255).astype('uint8') }
def main(input_path, output_path, show=False): coco = COCO(input_path) if show: data_dir = "datasets/coco/train2017/" dicts = DatasetCatalog.get("coco_2017_train") metadata = MetadataCatalog.get("coco_2017_train") for dic in dicts: for ann in dic["annotations"]: coco.compute_rbox(ann) ann["bbox"] = ann["rbox"] ann["bbox_mode"] = ann["rbox_mode"] print(ann["bbox"]) image_path = dic["file_name"] img = utils.convert_PIL_to_numpy( Image.open(image_path), "RGB") visualizer = Visualizer(img, metadata=metadata, scale=1) vis = visualizer.draw_dataset_dict(dic) webcv2.imshow(image_path+"bbox", vis.get_image()[:, :, ::-1]) webcv2.waitKey() skip = False if "val" in input_path: skip = True anns = [] for ann_id, ann in tqdm.tqdm(coco.anns.items()): if skip and ann["iscrowd"]: continue ann = coco.compute_rbox(ann) ann["bbox"] = ann["rbox"] anns.append(ann) info = dict(date_created=str(datetime.datetime.now()), description="Rbox version of {}.".format( input_path)) coco_dict = dict( info=info, categories=coco.dataset["categories"], annotations=anns, images=coco.dataset["images"], license=None) with open(output_path, "wt") as writer: json.dump(coco_dict, writer)
def vis_boxes(box_1, box_2, steps=100, distance=l2): diff = (box_2 - box_1) / 100 xs = [] ious = [] losses = [] fig, iou_scores = plt.subplots() max_val = max(box_1.max(), box_2.max()) * 4 iou_scores.set_xlim(-0.5, 1.5) loss_scores = iou_scores.twinx() iou_scores.set_ylabel("IoU with %s" % str(distance)) loss_scores.set_ylabel("L1 Loss") for i in range(steps): iou_i = iou(box_1, box_2, distance) xs.append(i / steps) ious.append(iou_i) losses.append(np.abs(box_1 - box_2).sum()) if i % (steps // 3) == 0: patch = plt.Rectangle((i / steps, 0), width=(box_1[2] - box_1[0]) / max_val, height=(box_1[3] - box_1[1]) / max_val, fill=False) iou_scores.add_patch(patch) patch = plt.Rectangle((i / steps, 0), width=(box_2[2] - box_2[0]) / max_val, height=(box_2[3] - box_2[1]) / max_val, fill=False) iou_scores.add_patch(patch) box_1 += diff # examples.add_patch(patch) losses = np.array(losses) iou_scores.plot(xs, ious, "r", label="IoU") loss_scores.plot(xs, losses, "b", label="L1 loss") iou_scores.legend(loc="lower right") loss_scores.legend() image = fig2image(fig) webcv2.imshow("image", image) webcv2.waitKey()
def process_single(metadata, dic, args): masks = BorderMasks([ x['segmentation'] for x in dic['annotations'] if not isinstance(x['segmentation'], dict) ]) img = utils.read_image(dic["file_name"], "RGB") borders, centers, sizes = masks.masks(mask_size=img.shape[:2]) if args.show: visualizer = Visualizer(img, metadata=metadata) vis = visualizer.draw_dataset_dict(dic) webcv2.imshow(dic["file_name"], vis.get_image()[:, :, ::-1]) webcv2.imshow(dic["file_name"] + "-border", borders * 255) webcv2.imshow(dic["file_name"] + "-centers", centers * 255) webcv2.imshow(dic["file_name"] + "-sizes", (sizes / sizes.max()).sum(-1) * 255) webcv2.waitKey() else: file_name = os.path.basename(dic["file_name"]) save(borders, cfg.MASK_DIRECTORY, split_name, "borders", file_name) save(centers, cfg.MASK_DIRECTORY, split_name, "centers", file_name) save(sizes, cfg.MASK_DIRECTORY, split_name, "sizes", file_name)
def dataset_id_map(ds_id): return metadata.thing_dataset_id_to_contiguous_id[ds_id] elif "lvis" in args.dataset: # LVIS results are in the same format as COCO results, but have a different # mapping from dataset category id to contiguous category id in [0, #categories - 1] def dataset_id_map(ds_id): return ds_id - 1 else: raise ValueError("Unsupported dataset: {}".format(args.dataset)) os.makedirs(args.output, exist_ok=True) for dic in tqdm.tqdm(dicts): img = cv2.imread(dic["file_name"], cv2.IMREAD_COLOR)[:, :, ::-1] basename = os.path.basename(dic["file_name"]) predictions = create_instances(pred_by_image[dic["image_id"]], img.shape[:2]) vis = Visualizer(img, metadata) vis_pred = vis.draw_instance_predictions(predictions).get_image() vis = Visualizer(img, metadata) vis_gt = vis.draw_dataset_dict(dic).get_image() concat = np.concatenate((vis_pred, vis_gt), axis=1) webcv2.imshow(basename, concat[:, :, ::-1]) webcv2.waitKey() cv2.imwrite(os.path.join(args.output, basename), concat[:, :, ::-1])
def output(vis, fname): if args.show: print(fname) webcv2.imshow("window", vis.get_image()[:, :, ::-1]) webcv2.waitKey()
img = cv2.imread(dic["file_name"], cv2.IMREAD_COLOR)[:, :, ::-1] basename = os.path.basename(dic["file_name"]) vis = Visualizer(img, metadata) vis_pred = vis.draw_proposals_separately( proposal_by_image[dic["image_id"]], img.shape[:2], args.conf_threshold) predictions = create_instances(pred_by_image[dic["image_id"]], img.shape[:2]) vis = Visualizer(img, metadata) pred = vis.draw_instance_predictions(predictions).get_image() vis_pred.append(pred) vis = Visualizer(img, metadata) gt = vis.draw_dataset_dict(dic).get_image() concat = vis.smart_concatenate(vis_pred, min_side=1960) vis = np.concatenate([pred, gt], axis=1) if args.show: webcv2.imshow(basename + ' - Press D for details', vis[:, :, ::-1]) key = webcv2.waitKey() if key == 100: # 'd' webcv2.imshow(basename, concat[:, :, ::-1]) webcv2.waitKey() else: cv2.imwrite(os.path.join(args.output, basename), vis[:, :, ::-1]) cv2.imwrite( os.path.join(args.output, basename) + '-proposals.jpg', concat[:, :, ::-1])
def main( result_path="train_log/retinanet/R_50_FPN/baseline/inference/instances_predictions_coco.pkl", result_path_2="../train_log/retinanet/R_50_FPN/baseline/inference/coco_objects365_val_with_masks/instances_predictions.pkl"): result_path = smart_path(result_path) results = pickle.loads(result_path.read_bytes()) result_path_2 = smart_path(result_path_2) results_2 = pickle.loads(result_path_2.read_bytes()) # shape (T, K, R, A), where T, K, R, A are number of thresholds, # classes, ratios, areas, respectively. ar = results["ar"] stats = results["ar"]["ar-stats"]["recalls"] * 100 ar_2 = results_2["ar"] stats_2 = results_2["ar"]["ar-stats"]["recalls"] * 100 plt.style.use(['ieee', "no-latex"]) fig, axs = plt.subplots(3, 3, sharey=True, figsize=(9, 9)) markers = ['*', '_', '+', 'x'] # plot overall AR xs = np.arange(4) + 1 ys = np.array([ar["AR@100"], ar["AR- 0 - 1/5@100"], ar["AR-1/5 - 1/3@100"], ar["AR-1/3 - 3/1@100"]]) scatter_with_markers(axs[0, 0], xs, ys, markers, c="r") axs[0, 0].plot(xs, ys, linestyle="dotted") axs[0, 0].set_xticks(xs) title(axs[0, 0], "all objects") axs[0, 0].set_ylabel("mAR-COCO") axs[0, 0].set_xlabel("slenderness") axs[0, 0].set_xticklabels(["all", "XS", "S", "R"]) # merge [ax.remove() for ax in axs[0, 1:]] ax = fig.add_subplot(axs[0, 2].get_gridspec()[0, 1:]) # plot thresholds T = stats.shape[0] xs = np.array([1]) x_labels = [] thresh = 0.5 stride = 0.05 xticks = [] for i in range(T): ys = stats[i, :-1, 0:4, 0].mean(0) xs = np.arange(4) + xs.max() + 1 scatter_with_markers(ax, xs, ys, markers, c='r') ax.plot(xs, ys, c='black', linestyle="dotted") x_labels += ["", str(int(thresh * 100) / 100), "", ""] thresh += stride xticks.append(xs) ax.set_xticks(np.concatenate(xticks)) ax.set_xticklabels(x_labels) title(ax, "all objects") ax.set_xlabel("threshold") ax.legend( [Line2D([0], [0], color="b", linewidth=1, linestyle="none", marker=m, c="r") for m in markers], ["all", "XS", "S", "R"], loc="upper right") # plot overall AR xs = np.arange(4) + 1 ys = np.array([ar_2["AR@100"], ar_2["AR- 0 - 1/5@100"], ar_2["AR-1/5 - 1/3@100"], ar_2["AR-1/3 - 3/1@100"]]) ax = axs[1, 0] scatter_with_markers(ax, xs, ys, markers, c="g") ax.plot(xs, ys, linestyle="dotted") ax.set_xticks(xs) title(ax, "all objects") ax.set_ylabel("mAR-COCO+") ax.set_xlabel("slenderness") ax.set_xticklabels(["all", "XS", "S", "R"]) # merge [ax.remove() for ax in axs[1, 1:]] ax = fig.add_subplot(axs[1, 2].get_gridspec()[1, 1:]) # plot thresholds T = stats.shape[0] xs = np.array([1]) x_labels = [] thresh = 0.5 stride = 0.05 xticks = [] for i in range(T): ys = stats_2[i, :-1, 0:4, 0].mean(0) xs = np.arange(4) + xs.max() + 1 scatter_with_markers(ax, xs, ys, markers, c='g') ax.plot(xs, ys, c='black', linestyle="dotted") x_labels += ["", str(int(thresh * 100) / 100), "", ""] thresh += stride xticks.append(xs) ax.set_xticks(np.concatenate(xticks)) ax.set_xticklabels(x_labels) title(ax, "all objects") ax.set_xlabel("threshold") ax.legend( [Line2D([0], [0], color="b", linewidth=1, linestyle="none", marker=m, c="g") for m in markers], ["all", "XS", "S", "R"], loc="upper right") # plot small objects ax = axs[2, 0] xs = np.arange(4) ys = np.array([stats[:, :-1, 0, 1].mean(), stats[:, :-1, 1, 1].mean(), stats[:, :-1, 2, 1].mean(), stats[:, :-1, 3, 1].mean()]) ys_2 = np.array([stats_2[:, :-1, 0, 1].mean(), stats_2[:, :-1, 1, 1].mean(), stats_2[:, :-1, 2, 1].mean(), stats_2[:, :-1, 3, 1].mean()]) # scatter_with_markers(ax, xs, ys, markers, c='g') scatter_with_markers(ax, xs, ys_2, markers, c='g') # ax.plot(xs, ys, c="g", linestyle="dotted") ax.plot(xs, ys_2, c="black", linestyle="dotted") ax.set_xticks(xs) title(ax, "small objects") ax.set_ylabel("mAR-COCO+") ax.set_xlabel("slenderness") ax.set_xticklabels(["all", "XS", "S", "R"]) # plot medium objects ax = axs[2, 1] xs = np.arange(4) ys = np.array([stats[:, :-1, 0, 2].mean(), stats[:, :-1, 1, 2].mean(), stats[:, :-1, 2, 2].mean(), stats[:, :-1, 3, 2].mean()]) ys_2 = np.array([stats_2[:, :-1, 0, 2].mean(), stats_2[:, :-1, 1, 2].mean(), stats_2[:, :-1, 2, 2].mean(), stats_2[:, :-1, 3, 2].mean()]) # scatter_with_markers(ax, xs, ys, markers, c='g') scatter_with_markers(ax, xs, ys_2, markers, c='g') # ax.plot(xs, ys, c="g", linestyle="dotted") ax.plot(xs, ys_2, c="black", linestyle="dotted") ax.set_xticks(xs) title(ax, "medium objects") ax.set_xlabel("slenderness") ax.set_xticklabels(["all", "XS", "S", "R"]) # plot large objects ax = axs[2, 2] xs = np.arange(4) ys = np.array([stats[:, :-1, 0, 3].mean(), stats[:, :-1, 1, 3].mean(), stats[:, :-1, 2, 3].mean(), stats[:, :-1, 3, 3].mean()]) ys_2 = np.array([stats_2[:, :-1, 0, 3].mean(), stats_2[:, :-1, 1, 3].mean(), stats_2[:, :-1, 2, 3].mean(), stats_2[:, :-1, 3, 3].mean()]) # scatter_with_markers(ax, xs, ys, markers, c='g') scatter_with_markers(ax, xs, ys_2, markers, c='g') # ax.plot(xs, ys, c="g", linestyle="dotted") ax.plot(xs, ys_2, c="black", linestyle="dotted") ax.set_xticks(xs) title(ax, "large objects") ax.set_xlabel("slenderness") ax.set_xticklabels(["all", "XS", "S", "R"]) group = fig2image(fig) webcv2.imshow("group", group) webcv2.waitKey()
def PlotAll(dataset): dicts = list(DatasetCatalog.get(dataset)) metadata = MetadataCatalog.get(dataset) labels = metadata.thing_classes ratios = { "0-1/5": [0, 1 / 5], "1/5-1/3": [1 / 5, 1 / 3], "1/3-1": [1 / 3, 1] } bars = dict() for key in ratios.keys(): bars[key] = [0 for _ in range(len(labels))] all_ratios = [] ratio_counts = {k: 0 for k in ratios.keys()} for dic in tqdm(dicts): for obj in dic["annotations"]: ratio = COCO.compute_ratio(obj, oriented=True)["ratio"] for key, ratio_range in ratios.items(): if between(ratio, ratio_range): bars[key][obj["category_id"]] += 1 ratio_counts[key] += 1 all_ratios.append(ratio) print("images", len(dicts)) print("counts", ratio_counts) fig, ax = plt.subplots() ax.set_yscale("symlog") prev = np.zeros((len(labels), )) for key, bar in bars.items(): bar = np.array(bar) ax.bar(labels, bar, bottom=prev, label=key) prev = prev + bar ax.legend() fig.set_size_inches(18.5, 10.5) ax.set_xticklabels(labels, rotation="vertical") group = fig2image(fig) # cv2.imwrite('./group.png', group) webcv2.imshow("group", group) fig, ax = plt.subplots() all_ratios = sorted(all_ratios) numbers = [0] tick = 0.01 seperator = tick count = 0 for r in all_ratios: count += 1 while seperator < r: seperator += tick numbers.append(count) ax.plot(np.arange(0, 1, tick), np.array(numbers) / count) ax.set_xlabel("slenderness") ax.set_title("cumulative distribution function") number = fig2image(fig) # cv2.imwrite('./number.png', number) webcv2.imshow("number", number) webcv2.waitKey()
vis = Visualizer(img, metadata, scale=scale) topk_boxes, topk_indices = vis.topk_iou_boxes( predictions.pred_boxes, Boxes([ BoxMode.convert(x["bbox"], BoxMode.XYWH_ABS, BoxMode.XYXY_ABS) for x in grouped_gt[range_name] ])) topk_indices = topk_indices.reshape((-1, )) # Transform indices to list since shape 1 tensors will be regarded as scalars. vis.draw_dataset_dict({"annotations": grouped_gt[range_name]}) vis_boxes = vis.draw_instance_predictions( predictions[topk_indices.tolist()]) if args.show: webcv2.imshow(basename + "-boxes@" + range_name, vis_boxes.get_image()[..., ::-1]) else: save(vis_boxes.get_image()[..., ::-1], args.output, "boxes", basename + "@%s.jpg" % range_name) vis_anchor = Visualizer(img, metadata) anchors = predictions.anchors.tensor[topk_indices] vis_anchor = vis_anchor.overlay_instances( boxes=anchors.reshape(-1, 4), labels=predictions.scores[topk_indices.reshape(-1).tolist()]) if args.show: webcv2.imshow(basename + "-anchors@" + range_name, vis_anchor.get_image()[..., ::-1]) else: save(vis_anchor.get_image()[..., ::-1], args.output, "anchors",