def par_generate_gt(config, pair_rec, flow_depth_rendered=None):
from lib.pair_matching.flow import calc_flow
target_size, max_size = config.SCALES[0][0], config.SCALES[0][1]
if flow_depth_rendered is None:
flow_depth_rendered = cv2.imread(pair_rec['depth_rendered'],
cv2.IMREAD_UNCHANGED).astype(np.float32)
flow_depth_rendered /= config.dataset.DEPTH_FACTOR
flow_depth_rendered, _ = resize(flow_depth_rendered, target_size, max_size)
if 'depth_render_real' in pair_rec:
flow_depth_real = cv2.imread(pair_rec['depth_render_real'],
cv2.IMREAD_UNCHANGED).astype(np.float32)
flow_depth_real /= config.dataset.DEPTH_FACTOR
else:
print('not using render_real depth in par_generate_gt')
flow_depth_real = cv2.imread(pair_rec['depth_real'],
cv2.IMREAD_UNCHANGED).astype(np.float32)
flow_depth_real /= config.dataset.DEPTH_FACTOR
flow_depth_real, _ = resize(flow_depth_real, target_size, max_size)
if 'mask_real_gt' or 'mask_real_est' in pair_rec:
mask_real_path = pair_rec['mask_real_gt']
assert os.path.exists(mask_real_path), '%s does not exist'.format(pair_rec['mask_real_gt'])
mask_real = cv2.imread(mask_real_path, cv2.IMREAD_UNCHANGED)
mask_real, _ = resize(mask_real, target_size, max_size)
flow_depth_real[mask_real != pair_rec['mask_idx']] = 0
if config.network.FLOW_I2R:
if config.FLOW_CLASS_AGNOSTIC:
flow_i2r, visible, _ = calc_flow(flow_depth_rendered, pair_rec['pose_est'], pair_rec['pose_real'],
config.dataset.INTRINSIC_MATRIX, flow_depth_real,
standard_rep=config.network.STANDARD_FLOW_REP)
flow_i2r_list = [flow_i2r, visible, np.logical_and(visible == 0, flow_depth_rendered == 0)]
else:
raise Exception('NOT_IMPLEMENTED')
else:
flow_i2r_list = None
if config.network.FLOW_R2I:
if config.FLOW_CLASS_AGNOSTIC:
flow_r2i, visible, _ = calc_flow(flow_depth_real, pair_rec['pose_real'], pair_rec['pose_est'],
config.dataset.INTRINSIC_MATRIX, flow_depth_rendered,
standard_rep=config.network.STANDARD_FLOW_REP)
flow_r2i_list = [flow_r2i, visible, np.logical_and(visible == 0, flow_depth_real == 0)]
else:
raise Exception('NOT_IMPLEMENTED')
else:
flow_r2i_list = None
return {'flow_i2r': flow_i2r_list, 'flow_r2i': flow_r2i_list}
def par_generate_gt(config, pair_rec, flow_depth_rendered=None):
from lib.pair_matching.flow import calc_flow
target_size, max_size = config.SCALES[0][0], config.SCALES[0][1]
if flow_depth_rendered is None:
flow_depth_rendered = cv2.imread(pair_rec["depth_rendered"],
cv2.IMREAD_UNCHANGED).astype(
np.float32)
flow_depth_rendered /= config.dataset.DEPTH_FACTOR
flow_depth_rendered, _ = resize(flow_depth_rendered, target_size,
max_size)
if "depth_gt_observed" in pair_rec:
flow_depth_observed = cv2.imread(pair_rec["depth_gt_observed"],
cv2.IMREAD_UNCHANGED).astype(
np.float32)
flow_depth_observed /= config.dataset.DEPTH_FACTOR
else:
logger.info("not using gt_observed depth in par_generate_gt")
flow_depth_observed = cv2.imread(pair_rec["depth_observed"],
cv2.IMREAD_UNCHANGED).astype(
np.float32)
flow_depth_observed /= config.dataset.DEPTH_FACTOR
flow_depth_observed, _ = resize(flow_depth_observed, target_size, max_size)
if "mask_gt_observed" or "mask_observed" in pair_rec:
mask_observed_path = pair_rec["mask_gt_observed"]
assert os.path.exists(mask_observed_path), "%s does not exist".format(
pair_rec["mask_gt_observed"])
mask_observed = cv2.imread(mask_observed_path, cv2.IMREAD_UNCHANGED)
mask_observed, _ = resize(mask_observed, target_size, max_size)
flow_depth_observed[mask_observed != pair_rec["mask_idx"]] = 0
if config.network.PRED_FLOW:
flow_i2r, visible, _ = calc_flow(
flow_depth_rendered,
pair_rec["pose_rendered"],
pair_rec["pose_observed"],
config.dataset.INTRINSIC_MATRIX,
flow_depth_observed,
standard_rep=config.network.STANDARD_FLOW_REP,
)
flow_i2r_list = [
flow_i2r, visible,
np.logical_and(visible == 0, flow_depth_rendered == 0)
]
return {"flow": flow_i2r_list}
def generate_batch(im):
"""
preprocess image, return batch
:param im: cv2.imread returns [height, width, channel] in BGR
:return:
data_batch: MXNet input batch
data_names: names in data_batch
im_scale: float number
"""
SHORT_SIDE = config.SCALES[0][0]
LONG_SIDE = config.SCALES[0][1]
PIXEL_MEANS = config.network.PIXEL_MEANS
DATA_NAMES = ['data', 'im_info']
im_array, im_scale = resize(im, SHORT_SIDE, LONG_SIDE)
im_array = transform(im_array, PIXEL_MEANS)
im_info = np.array([[im_array.shape[2], im_array.shape[3], im_scale]],
dtype=np.float32)
data = [[mx.nd.array(im_array), mx.nd.array(im_info)]]
data_shapes = [[('data', im_array.shape), ('im_info', im_info.shape)]]
data_batch = mx.io.DataBatch(data=data,
label=[None],
provide_data=data_shapes,
provide_label=[None])
return data_batch, DATA_NAMES, [im_scale]
def generate_batch(im):
"""
preprocess image, return batch
:param im: cv2.imread returns [height, width, channel] in BGR
:return:
data_batch: MXNet input batch
im_scale: float number
"""
SHORT_SIDE = config.SCALES[0][0]
LONG_SIDE = config.SCALES[0][1]
PIXEL_MEANS = config.network.PIXEL_MEANS
im_array, im_scale = resize(im, SHORT_SIDE, LONG_SIDE)
im_array = transform(im_array, PIXEL_MEANS)
im_array = im_array.astype(np.float32)
im_info = np.array([[im_array.shape[2], im_array.shape[3], im_scale]],
dtype=np.float32)
data_shapes = [[('data', im_array.shape), ('im_info', im_info.shape)]]
return {
'data_shapes': data_shapes,
'im_scale': [im_scale],
'im_array': im_array,
'im_info': im_info
}
def update_data_batch(config, data_batch, update_package):
import mxnet.ndarray as nd
for ctx_idx, data in enumerate(data_batch.data):
package = update_package[ctx_idx]
for blob_idx, blob in enumerate(data):
blob_name = data_batch.provide_data[ctx_idx][blob_idx][0]
blob_shape = data_batch.provide_data[ctx_idx][blob_idx][1]
if blob_name not in package:
continue
update_data = package[blob_name]
if blob_name.startswith("image"):
target_size = min(blob_shape[2:])
max_size = max(blob_shape[2:])
image_update, _ = image.resize(update_data, target_size,
max_size)
image_update = image.transform(image_update,
config.network.PIXEL_MEANS)
image_update = nd.array(image_update)
data_batch.data[ctx_idx][blob_idx] = image_update
elif blob_name.startswith("src_pose"):
src_pose_update = nd.array(update_data[np.newaxis, :, :])
data_batch.data[ctx_idx][blob_idx] = src_pose_update
elif blob_name.startswith("depth"):
target_size = min(blob_shape[2:])
max_size = max(blob_shape[2:])
depth_update, _ = image.resize(update_data, target_size,
max_size)
depth_update = nd.array(depth_update[np.newaxis,
np.newaxis, :, :])
data_batch.data[ctx_idx][blob_idx] = depth_update
elif blob_name.startswith("mask_observed"):
if config.TEST.UPDATE_MASK == "box_rendered":
update_data = np.copy(package["mask_rendered"])
mask_observed = np.zeros(update_data.shape)
x_max = np.max(update_data, 0)
y_max = np.max(update_data, 1)
nz_x = np.nonzero(x_max)[0]
nz_y = np.nonzero(y_max)[0]
x_start = np.min(nz_x)
x_end = np.max(nz_x)
y_start = np.min(nz_y)
y_end = np.max(nz_y)
mask_observed[y_start:y_end,
x_start:x_end] = 1.0 # rectangle
elif config.TEST.UPDATE_MASK == "box_observed":
mask_observed = np.zeros(update_data.shape)
x_max = np.max(update_data, 0)
y_max = np.max(update_data, 1)
nz_x = np.nonzero(x_max)[0]
nz_y = np.nonzero(y_max)[0]
if len(nz_x) == 0 or len(nz_y) == 0:
raise Exception("no point valid in mask_observed")
x_start = np.min(nz_x)
x_end = np.max(nz_x)
y_start = np.min(nz_y)
y_end = np.max(nz_y)
mask_observed[y_start:y_end,
x_start:x_end] = 1.0 # rectangle
else:
mask_observed = update_data
mask_update = nd.array(mask_observed[np.newaxis,
np.newaxis, :, :])
data_batch.data[ctx_idx][blob_idx] = mask_update
elif blob_name.startswith("mask_rendered"):
mask_update = nd.array(update_data[np.newaxis,
np.newaxis, :, :])
data_batch.data[ctx_idx][blob_idx] = mask_update
else:
raise Exception("NOT_IMPLEMENTED")
return data_batch
def main():
# get symbol
ctx_id = [int(i) for i in config.gpus.split(',')]
# ctx_id = [mx.cpu()]
print(ctx_id)
pprint.pprint(config)
sym_instance = eval(config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
# set up class names
num_classes = 2
classes = ['nuclei']
# load demo data
# image_root = '/home/daiab/machine_disk/work/kaggle_nuclei/data/stage1_test_images'
image_root = '/home/daiab/machine_disk/work/kaggle_nuclei/data/LikeVOC/img'
image_names = glob.glob(os.path.join(image_root, '*.png'))
data = []
for im_name in image_names:
im = cv2.imread(im_name,
cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
# get predictor
data_names = ['data', 'im_info']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names]
for i in range(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])]
for i in range(len(data))]
provide_label = [None for i in range(len(data))]
arg_params, aux_params = load_param(
'/home/daiab/machine_disk/projects/FCIS/'
'output/fcis/nuclei/nuclei_end2end/SDS_train/e2e',
2,
process=True)
data = data
max_data_shape = max_data_shape
provide_data = provide_data
predictor = Predictor(sym,
data_names,
label_names,
context=[mx.gpu(ctx_id[0])],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
# warm up
for i in range(2):
data_batch = mx.io.DataBatch(data=[data[0]],
label=[],
pad=0,
index=0,
provide_data=[[
(k, v.shape)
for k, v in zip(data_names, data[0])
]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in range(len(data_batch.data))
]
# print('-----------', data_batch)
_, _, _, _ = im_detect(predictor, data_batch, data_names, scales,
config)
# test
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(
data=[data[idx]],
label=[],
pad=0,
index=idx,
provide_data=[[(k, v.shape)
for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in range(len(data_batch.data))
]
tic()
scores, boxes, masks, data_dict = im_detect(predictor, data_batch,
data_names, scales, config)
im_shapes = [
data_batch.data[i][0].shape[2:4]
for i in range(len(data_batch.data))
]
if not config.TEST.USE_MASK_MERGE:
all_boxes = [[] for _ in range(num_classes)]
all_masks = [[] for _ in range(num_classes)]
nms = py_nms_wrapper(config.TEST.NMS)
for j in range(1, num_classes):
indexes = np.where(scores[0][:, j] > 0.7)[0]
cls_scores = scores[0][indexes, j, np.newaxis]
cls_masks = masks[0][indexes, 1, :, :]
try:
if config.CLASS_AGNOSTIC:
cls_boxes = boxes[0][indexes, :]
else:
raise Exception()
except:
cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
all_boxes[j] = cls_dets[keep, :]
all_masks[j] = cls_masks[keep, :]
dets = [all_boxes[j] for j in range(1, num_classes)]
masks = [all_masks[j] for j in range(1, num_classes)]
else:
masks = masks[0][:, 1:, :, :]
im_height = np.round(im_shapes[0][0] / scales[0]).astype('int')
im_width = np.round(im_shapes[0][1] / scales[0]).astype('int')
print(im_height, im_width)
boxes = clip_boxes(boxes[0], (im_height, im_width))
result_masks, result_dets = gpu_mask_voting(
masks, boxes, scores[0], num_classes, 100, im_width, im_height,
config.TEST.NMS, config.TEST.MASK_MERGE_THRESH,
config.BINARY_THRESH, ctx_id[0])
dets = [result_dets[j] for j in range(1, num_classes)]
masks = [
result_masks[j][:, 0, :, :] for j in range(1, num_classes)
]
# print(dets)
# print(masks)
print('testing {} {:.4f}s'.format(im_name, toc()))
# visualize
for i in range(len(dets)):
keep = np.where(dets[i][:, -1] > 0.7)
dets[i] = dets[i][keep]
masks[i] = masks[i][keep]
im = cv2.imread(im_name)
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
show_masks(im, dets, masks, classes, config)
print('done')
def main():
# get symbol
ctx_id = [int(i) for i in config.gpus.split(',')]
# ctx_id = [mx.cpu()]
print(ctx_id)
pprint.pprint(config)
sym_instance = eval(config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
# set up class names
num_classes = 81
classes = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat',
'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse',
'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie',
'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove',
'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon',
'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut',
'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse',
'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book',
'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush']
# load demo data
image_names = ['COCO_test2015_000000000275.jpg', 'COCO_test2015_000000001412.jpg', 'COCO_test2015_000000073428.jpg',
'COCO_test2015_000000393281.jpg']
data = []
for im_name in image_names:
assert os.path.exists(cur_path + '/../demo/' + im_name), ('%s does not exist'.format('../demo/' + im_name))
im = cv2.imread(cur_path + '/../demo/' + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
# get predictor
data_names = ['data', 'im_info']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names] for i in range(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in range(len(data))]
provide_label = [None for i in range(len(data))]
arg_params, aux_params = load_param(cur_path + '/../model/fcis_coco', 0, process=True)
data = data
max_data_shape = max_data_shape
provide_data = provide_data
predictor = Predictor(sym, data_names, label_names,
context=[mx.gpu(ctx_id[0])],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
# warm up
for i in range(2):
data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0,
provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]],
provide_label=[None])
scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in range(len(data_batch.data))]
# print('-----------', data_batch)
_, _, _, _ = im_detect(predictor, data_batch, data_names, scales, config)
# test
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx,
provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in range(len(data_batch.data))]
tic()
scores, boxes, masks, data_dict = im_detect(predictor, data_batch, data_names, scales, config)
im_shapes = [data_batch.data[i][0].shape[2:4] for i in range(len(data_batch.data))]
if not config.TEST.USE_MASK_MERGE:
all_boxes = [[] for _ in range(num_classes)]
all_masks = [[] for _ in range(num_classes)]
nms = py_nms_wrapper(config.TEST.NMS)
for j in range(1, num_classes):
indexes = np.where(scores[0][:, j] > 0.7)[0]
cls_scores = scores[0][indexes, j, np.newaxis]
cls_masks = masks[0][indexes, 1, :, :]
try:
if config.CLASS_AGNOSTIC:
cls_boxes = boxes[0][indexes, :]
else:
raise Exception()
except:
cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
all_boxes[j] = cls_dets[keep, :]
all_masks[j] = cls_masks[keep, :]
dets = [all_boxes[j] for j in range(1, num_classes)]
masks = [all_masks[j] for j in range(1, num_classes)]
else:
print('=============')
masks = masks[0][:, 1:, :, :]
im_height = np.round(im_shapes[0][0] / scales[0]).astype('int')
im_width = np.round(im_shapes[0][1] / scales[0]).astype('int')
print(im_height, im_width)
boxes = clip_boxes(boxes[0], (im_height, im_width))
result_masks, result_dets = gpu_mask_voting(masks, boxes, scores[0], num_classes,
100, im_width, im_height,
config.TEST.NMS, config.TEST.MASK_MERGE_THRESH,
config.BINARY_THRESH, ctx_id[0])
dets = [result_dets[j] for j in range(1, num_classes)]
masks = [result_masks[j][:, 0, :, :] for j in range(1, num_classes)]
# print(dets)
# print(masks)
print('testing {} {:.4f}s'.format(im_name, toc()))
# visualize
for i in range(len(dets)):
keep = np.where(dets[i][:,-1]>0.7)
dets[i] = dets[i][keep]
masks[i] = masks[i][keep]
im = cv2.imread(cur_path + '/../demo/' + im_name)
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
show_masks(im, dets, masks, classes, config)
print('done')
def par_generate_gt(config, pair_rec, flow_depth_rendered=None):
from lib.pair_matching.flow import calc_flow
target_size, max_size = config.SCALES[0][0], config.SCALES[0][1]
if flow_depth_rendered is None:
flow_depth_rendered = cv2.imread(
pair_rec["depth_rendered"], cv2.IMREAD_UNCHANGED
).astype(np.float32)
flow_depth_rendered /= config.dataset.DEPTH_FACTOR
flow_depth_rendered, _ = resize(flow_depth_rendered, target_size, max_size)
if "depth_render_real" in pair_rec:
flow_depth_real = cv2.imread(
pair_rec["depth_render_real"], cv2.IMREAD_UNCHANGED
).astype(np.float32)
flow_depth_real /= config.dataset.DEPTH_FACTOR
else:
print("not using render_real depth in par_generate_gt")
flow_depth_real = cv2.imread(
pair_rec["depth_real"], cv2.IMREAD_UNCHANGED
).astype(np.float32)
flow_depth_real /= config.dataset.DEPTH_FACTOR
flow_depth_real, _ = resize(flow_depth_real, target_size, max_size)
if "mask_gt_observed" or "mask_observed" in pair_rec:
mask_observed_path = pair_rec["mask_gt_observed"]
assert os.path.exists(mask_observed_path), "%s does not exist".format(
pair_rec["mask_gt_observed"]
)
mask_observed = cv2.imread(mask_observed_path, cv2.IMREAD_UNCHANGED)
mask_observed, _ = resize(mask_observed, target_size, max_size)
flow_depth_real[mask_observed != pair_rec["mask_idx"]] = 0
if config.network.FLOW_I2R:
if config.FLOW_CLASS_AGNOSTIC:
flow_i2r, visible, _ = calc_flow(
flow_depth_rendered,
pair_rec["pose_rendered"],
pair_rec["pose_observed"],
config.dataset.INTRINSIC_MATRIX,
flow_depth_real,
standard_rep=config.network.STANDARD_FLOW_REP,
)
flow_i2r_list = [
flow_i2r,
visible,
np.logical_and(visible == 0, flow_depth_rendered == 0),
]
else:
raise Exception("NOT_IMPLEMENTED")
else:
flow_i2r_list = None
if config.network.FLOW_R2I:
if config.FLOW_CLASS_AGNOSTIC:
flow_r2i, visible, _ = calc_flow(
flow_depth_real,
pair_rec["pose_observed"],
pair_rec["pose_rendered"],
config.dataset.INTRINSIC_MATRIX,
flow_depth_rendered,
standard_rep=config.network.STANDARD_FLOW_REP,
)
flow_r2i_list = [
flow_r2i,
visible,
np.logical_and(visible == 0, flow_depth_real == 0),
]
else:
raise Exception("NOT_IMPLEMENTED")
else:
flow_r2i_list = None
return {"flow_i2r": flow_i2r_list, "flow_r2i": flow_r2i_list}