def visualize(model, model_path, nr_visualize=100, output_dir='output'):
"""
Visualize some intermediate results (proposals, raw predictions) inside the pipeline.
"""
df = get_train_dataflow() # we don't visualize mask stuff
df.reset_state()
pred = OfflinePredictor(PredictConfig(
model=model,
session_init=get_model_loader(model_path),
input_names=['image', 'gt_boxes', 'gt_labels'],
output_names=[
'generate_{}_proposals/boxes'.format('fpn' if cfg.MODE_FPN else 'rpn'),
'generate_{}_proposals/scores'.format('fpn' if cfg.MODE_FPN else 'rpn'),
'fastrcnn_all_scores',
'output/boxes',
'output/scores',
'output/labels',
]))
if os.path.isdir(output_dir):
shutil.rmtree(output_dir)
utils.fs.mkdir_p(output_dir)
with tqdm.tqdm(total=nr_visualize) as pbar:
for idx, dp in itertools.islice(enumerate(df), nr_visualize):
img = dp[0]
if cfg.MODE_MASK:
gt_boxes, gt_labels, gt_masks = dp[-3:]
else:
gt_boxes, gt_labels = dp[-2:]
rpn_boxes, rpn_scores, all_scores, \
final_boxes, final_scores, final_labels = pred(img, gt_boxes, gt_labels)
# draw groundtruth boxes
gt_viz = draw_annotation(img, gt_boxes, gt_labels)
# draw best proposals for each groundtruth, to show recall
proposal_viz, good_proposals_ind = draw_proposal_recall(img, rpn_boxes, rpn_scores, gt_boxes)
# draw the scores for the above proposals
score_viz = draw_predictions(img, rpn_boxes[good_proposals_ind], all_scores[good_proposals_ind])
results = [DetectionResult(*args) for args in
zip(final_boxes, final_scores, final_labels,
[None] * len(final_labels))]
final_viz = draw_final_outputs(img, results)
viz = tpviz.stack_patches([
gt_viz, proposal_viz,
score_viz, final_viz], 2, 2)
if os.environ.get('DISPLAY', None):
tpviz.interactive_imshow(viz)
cv2.imwrite("{}/{:03d}.png".format(output_dir, idx), viz)
pbar.update()
def visualize(model_path, nr_visualize=50, output_dir='output'):
df = get_train_dataflow() # we don't visualize mask stuff
df.reset_state()
pred = OfflinePredictor(
PredictConfig(model=Model(),
session_init=get_model_loader(model_path),
input_names=['image', 'gt_boxes', 'gt_labels'],
output_names=[
'generate_rpn_proposals/boxes',
'generate_rpn_proposals/probs',
'fastrcnn_all_probs',
'final_boxes',
'final_probs',
'final_labels',
]))
if os.path.isdir(output_dir):
shutil.rmtree(output_dir)
utils.fs.mkdir_p(output_dir)
with tqdm.tqdm(total=nr_visualize) as pbar:
for idx, dp in itertools.islice(enumerate(df.get_data()),
nr_visualize):
img, _, _, gt_boxes, gt_labels = dp
rpn_boxes, rpn_scores, all_probs, \
final_boxes, final_probs, final_labels = pred(img, gt_boxes, gt_labels)
# draw groundtruth boxes
gt_viz = draw_annotation(img, gt_boxes, gt_labels)
# draw best proposals for each groundtruth, to show recall
proposal_viz, good_proposals_ind = draw_proposal_recall(
img, rpn_boxes, rpn_scores, gt_boxes)
# draw the scores for the above proposals
score_viz = draw_predictions(img, rpn_boxes[good_proposals_ind],
all_probs[good_proposals_ind])
results = [
DetectionResult(*args)
for args in zip(final_boxes, final_probs, final_labels,
[None] * len(final_labels))
]
final_viz = draw_final_outputs(img, results)
viz = tpviz.stack_patches(
[gt_viz, proposal_viz, score_viz, final_viz], 2, 2)
if os.environ.get('DISPLAY', None):
tpviz.interactive_imshow(viz)
cv2.imwrite("{}/{:03d}.png".format(output_dir, idx), viz)
pbar.update()
def do_visualize(model, model_path, nr_visualize=100, output_dir='output'):
"""
Visualize some intermediate results (proposals, raw predictions) inside the pipeline.
"""
df = get_train_dataflow()
df.reset_state()
pred = OfflinePredictor(PredictConfig(
model=model,
session_init=SmartInit(model_path),
input_names=['image', 'gt_boxes', 'gt_labels'],
output_names=[
'generate_{}_proposals/boxes'.format('fpn' if cfg.MODE_FPN else 'rpn'),
'generate_{}_proposals/scores'.format('fpn' if cfg.MODE_FPN else 'rpn'),
'fastrcnn_all_scores',
'output/boxes',
'output/scores',
'output/labels',
]))
if os.path.isdir(output_dir):
shutil.rmtree(output_dir)
fs.mkdir_p(output_dir)
with tqdm.tqdm(total=nr_visualize) as pbar:
for idx, dp in itertools.islice(enumerate(df), nr_visualize):
img, gt_boxes, gt_labels = dp['image'], dp['gt_boxes'], dp['gt_labels']
rpn_boxes, rpn_scores, all_scores, \
final_boxes, final_scores, final_labels = pred(img, gt_boxes, gt_labels)
# draw groundtruth boxes
gt_viz = draw_annotation(img, gt_boxes, gt_labels)
# draw best proposals for each groundtruth, to show recall
proposal_viz, good_proposals_ind = draw_proposal_recall(img, rpn_boxes, rpn_scores, gt_boxes)
# draw the scores for the above proposals
score_viz = draw_predictions(img, rpn_boxes[good_proposals_ind], all_scores[good_proposals_ind])
results = [DetectionResult(*args) for args in
zip(final_boxes, final_scores, final_labels,
[None] * len(final_labels))]
final_viz = draw_final_outputs(img, results)
viz = tpviz.stack_patches([
gt_viz, proposal_viz,
score_viz, final_viz], 2, 2)
# if os.environ.get('DISPLAY', None):
# tpviz.interactive_imshow(viz)
cv2.imwrite("{}/{:03d}.png".format(output_dir, idx), viz)
pbar.update()
def visualize(model_path, nr_visualize=50, output_dir='output'):
pred = OfflinePredictor(
PredictConfig(model=Model(),
session_init=get_model_loader(model_path),
input_names=['image', 'gt_boxes', 'gt_labels'],
output_names=[
'generate_rpn_proposals/boxes',
'generate_rpn_proposals/probs',
'fastrcnn_all_probs',
'fastrcnn_fg_probs',
'fastrcnn_fg_boxes',
]))
df = get_train_dataflow()
df.reset_state()
if os.path.isdir(output_dir):
shutil.rmtree(output_dir)
utils.fs.mkdir_p(output_dir)
with tqdm.tqdm(total=nr_visualize) as pbar:
for idx, dp in itertools.islice(enumerate(df.get_data()),
nr_visualize):
img, _, _, gt_boxes, gt_labels = dp
rpn_boxes, rpn_scores, all_probs, fg_probs, fg_boxes = pred(
img, gt_boxes, gt_labels)
gt_viz = draw_annotation(img, gt_boxes, gt_labels)
proposal_viz, good_proposals_ind = draw_proposal_recall(
img, rpn_boxes, rpn_scores, gt_boxes)
score_viz = draw_predictions(img, rpn_boxes[good_proposals_ind],
all_probs[good_proposals_ind])
fg_boxes = clip_boxes(fg_boxes, img.shape[:2])
fg_viz = draw_predictions(img, fg_boxes, fg_probs)
results = nms_fastrcnn_results(fg_boxes, fg_probs)
final_viz = draw_final_outputs(img, results)
viz = tpviz.stack_patches(
[gt_viz, proposal_viz, score_viz, fg_viz, final_viz], 2, 3)
if os.environ.get('DISPLAY', None):
tpviz.interactive_imshow(viz)
cv2.imwrite("{}/{:03d}.png".format(output_dir, idx), viz)
pbar.update()
def preprocess(fname):
print("start preproc mapillary")
start = time.time()
label_fname = fname.replace(train_img_path,
train_label_path).replace(".jpg", ".png")
pil_label = Image.open(label_fname)
label = np.array(pil_label)
instances = np.unique(label)
instance_classes = [x // 256 for x in instances]
# filter by categories we use
instances_valid = [
cls in config.MAPILLARY_CAT_IDS_TO_USE for cls in instance_classes
]
instances = [
inst for inst, valid in zip(instances, instances_valid) if valid
]
instance_classes = [
cls for cls, valid in zip(instance_classes, instances_valid)
if valid
]
if len(instances) == 0:
print("no instances")
pil_label.close()
return None
if map_to_coco:
instance_classes = [
config.MAPILLARY_TO_COCO_MAP[cls] for cls in instance_classes
]
instance_classes = [
config.VOID_LABEL if cls == config.VOID_LABEL else
COCOMeta.category_id_to_class_id[cls]
for cls in instance_classes
]
else:
# remap to contiguous numbers starting with 1
instance_classes = [
config.MAPILLARY_CAT_IDS_TO_USE.index(cls) + 1
for cls in instance_classes
]
masks = np.array([label == inst for inst in instances], dtype=np.uint8)
#import cProfile
#start1 = time.time()
boxes1 = np.array(
[get_bbox_from_segmentation_mask(mask) for mask in masks],
dtype=np.float32)
#boxes1_time = time.time() - start1
#pr = cProfile.Profile()
#pr.enable()
#start1 = time.time()
#boxes2 = get_bboxes_from_segmentation_masks(masks)
#print("boxes1", boxes1_time, "boxes2", time.time() - start1)
#pr.disable()
#pr.print_stats(sort="cumulative")
#assert (boxes1 == boxes2).all(), (boxes1, boxes2)
boxes = boxes1
second_klass = np.array(instance_classes, dtype=np.int)
klass = np.ones_like(second_klass)
is_crowd = np.zeros_like(second_klass)
res = preproc_img(fname, boxes, klass, second_klass, is_crowd, aug)
if res is None:
print("mapillary: preproc_img returned None on", fname)
pil_label.close()
return None
ret, params = res
if add_mask:
do_flip, h, w = params[1]
assert do_flip in (True, False), do_flip
# augment label
label = np.array(pil_label.resize((w, h), Image.NEAREST))
if do_flip:
label = label[:, ::-1]
# create augmented masks
masks = np.array([label == inst for inst in instances],
dtype=np.uint8)
ret.append(masks)
end = time.time()
elapsed = end - start
print("mapillary example done, elapsed:", elapsed)
VISUALIZE = False
if VISUALIZE:
from viz import draw_annotation, draw_mask
config.CLASS_NAMES = [str(idx) for idx in range(81)]
im = ret[0]
boxes = ret[3]
draw_klass = ret[-2]
viz = draw_annotation(im, boxes, draw_klass)
for mask in masks:
viz = draw_mask(viz, mask)
tpviz.interactive_imshow(viz)
pil_label.close()
return ret
def preprocess(fname):
# print("start preproc mapillary")
start = time.time()
label_fname = fname.replace(train_img_path,
train_label_path).replace(".jpg", ".png")
pil_label = Image.open(label_fname)
label = np.array(pil_label)
instances = np.unique(label)
instance_classes = [x // 256 for x in instances]
if len(instances) == 0:
print("no instances")
pil_label.close()
return None
masks = np.array([label == inst for inst in instances], dtype=np.uint8)
boxes1 = np.array(
[get_bbox_from_segmentation_mask(mask) for mask in masks],
dtype=np.float32)
boxes = boxes1
# second_klass = np.array(instance_classes, dtype=np.int)
second_klass = np.zeros_like(instance_classes, dtype=np.int)
klass = np.ones_like(second_klass)
is_crowd = np.zeros_like(second_klass)
res = preproc_img(fname, boxes, klass, second_klass, is_crowd, aug)
if res is None:
print("davis: preproc_img returned None on", fname)
pil_label.close()
return None
ret, params = res
if add_mask:
do_flip, h, w = params[1]
assert do_flip in (True, False), do_flip
# augment label
label = np.array(pil_label.resize((w, h), Image.NEAREST))
if do_flip:
label = label[:, ::-1]
# create augmented masks
masks = np.array([label == inst for inst in instances],
dtype=np.uint8)
ret.append(masks)
end = time.time()
elapsed = end - start
# print("davis example done, elapsed:", elapsed)
VISUALIZE = False
if VISUALIZE:
from viz import draw_annotation, draw_mask
config.CLASS_NAMES = [str(idx) for idx in range(81)]
im = ret[0]
boxes = ret[3]
draw_klass = ret[-2]
viz = draw_annotation(im, boxes, draw_klass)
for mask in masks:
viz = draw_mask(viz, mask)
tpviz.interactive_imshow(viz)
pil_label.close()
return ret
break
orig_shape = batch_image.shape[:2]
feed_dict = {image_P: batch_image}
final_boxes_, final_labels_, final_probs_ = sess.run(
[final_boxes, final_labels, final_probs], feed_dict)
final_boxes_ = clip_boxes(final_boxes_, orig_shape)
final_boxes_ = sess.run(final_boxes_)
final_boxes_ = final_boxes_.astype('int32')
if np.any(final_boxes_):
tags = [
"{},{:.2f}".format(cfg.DATA.CLASS_NAMES[lb], score)
for lb, score in zip(final_labels_, final_probs_)
]
final_viz = viz.draw_boxes(batch_image, final_boxes_, tags)
gt_viz = draw_annotation(batch_image, batch_gt_boxes, batch_gt_labels)
img_out = np.hstack((final_viz, gt_viz))
imageio.imwrite(os.path.join(save_path, str(iter) + ".jpg"), img_out)
Detection = []
for ik in range(final_boxes_.shape[0]):
Detection.append([
cfg.DATA.CLASS_NAMES[final_labels_[ik]],
float(final_probs_[ik]), final_boxes_[ik, 0], final_boxes_[ik, 1],
final_boxes_[ik, 2] - final_boxes_[ik, 0],
final_boxes_[ik, 3] - final_boxes_[ik, 1]
])
Detection = np.array(Detection)
#np.savetxt(os.path.join(save_folder, 'detections/',str(iter) + '.txt'), Detection, fmt='%s %1.2f %1.0f %1.0f %1.0f %1.0f')
np.savetxt(os.path.join(save_folder, 'detections/',
from dataset.display_virtual import DisplayDemo, register_display
from viz import draw_annotation
from dataset.tensorpack.utils.viz import interactive_imshow as imshow
import cv2
import os
from config import finalize_configs
if __name__ == '__main__':
base_dir = './data'
roibds = DisplayDemo(base_dir, "train").training_roidbs()
print("#images:", len(roibds))
register_display(base_dir)
finalize_configs(True)
i = 0
for r in roibds:
i = i + 1
if i == 100:
break
im = cv2.imread(r["file_name"])
im_out = os.path.join('./output', r["file_name"].split('/')[-1])
print(im_out)
vis = draw_annotation(im, r["boxes"], r["class"])
cv2.imwrite(im_out, vis)
def do_visualize(model, model_path, nr_visualize=100, output_dir='output'):
"""
Visualize some intermediate results (proposals, raw predictions) inside the pipeline.
"""
df = get_train_dataflow() # we don't visualize mask stuff
df.reset_state()
pred = OfflinePredictor(
PredictConfig(
model=model,
session_init=get_model_loader(model_path),
input_names=['images', 'orig_image_dims', 'gt_boxes', 'gt_labels'],
output_names=[
'generate_{}_proposals_topk_per_image/boxes'.format(
'fpn' if cfg.MODE_FPN else 'rpn'),
'generate_{}_proposals_topk_per_image/scores'.format(
'fpn' if cfg.MODE_FPN else 'rpn'),
'fastrcnn_all_scores',
'output/boxes',
'output/scores',
'output/labels',
]))
if os.path.isdir(output_dir):
shutil.rmtree(output_dir)
utils.fs.mkdir_p(output_dir)
with tqdm.tqdm(total=nr_visualize) as pbar:
for idx, dp in itertools.islice(enumerate(df), nr_visualize):
img, gt_boxes, gt_labels = dp['images'], dp['gt_boxes'], dp[
'gt_labels']
orig_shape = img.shape[:2]
rpn_boxes, rpn_scores, all_scores, \
final_boxes, final_scores, final_labels = pred(np.expand_dims(img, axis=0),
np.expand_dims(np.array(img.shape), axis=0),
np.expand_dims(gt_boxes, axis=0),
np.expand_dims(gt_labels, axis=0))
# draw groundtruth boxes
gt_viz = draw_annotation(img, gt_boxes, gt_labels)
# draw best proposals for each groundtruth, to show recall
# custom op creates different shape for boxes, convert back to original
rpn_boxes = np.array([i[1:] for i in rpn_boxes])
proposal_viz, good_proposals_ind = draw_proposal_recall(
img, rpn_boxes, rpn_scores, gt_boxes)
# draw the scores for the above proposals
score_viz = draw_predictions(img, rpn_boxes[good_proposals_ind],
all_scores[good_proposals_ind])
results = [
DetectionResult(*args)
for args in zip(final_boxes, final_scores, final_labels,
[None] * len(final_labels))
]
final_viz = draw_final_outputs(img, results)
viz = tpviz.stack_patches(
[gt_viz, proposal_viz, score_viz, final_viz], 2, 2)
if os.environ.get('DISPLAY', None):
tpviz.interactive_imshow(viz)
cv2.imwrite("{}/{:03d}.png".format(output_dir, idx), viz)
pbar.update()
