def image_segmentation(gray):
"""
:param gray: grayscale image
:return: components, processed grayscale image
"""
gray = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY_INV,
ADAPTIVE_THRESHOLD_KERNEL_SIZE, ADAPTIVE_THRESHOLD_C)
gray = cv2.medianBlur(gray, 3)
gray = erode_dilate(gray)
_, labeled_img = cv2.connectedComponentsWithAlgorithm(gray, 8, cv2.CV_32S, cv2.CCL_GRANA)
labels = np.unique(labeled_img)
labels = labels[labels != 0]
intermediate_global_mask = np.zeros_like(labeled_img, dtype=np.uint8)
for label in labels:
mask = np.zeros_like(labeled_img, dtype=np.uint8)
mask[labeled_img == label] = 255
# Compute the convex hull
if get_opencv_major_version(cv2.__version__) in ['2', '3']:
mask, contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
else:
contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
hull = []
for cnt in contours:
hull.append(cv2.convexHull(cnt, False))
hull_mask = np.zeros((mask.shape[0], mask.shape[1]), dtype=np.uint8)
for i in range(len(contours)):
hull_mask = cv2.drawContours(hull_mask, hull, i, 255, -1, 8)
intermediate_global_mask = np.clip(intermediate_global_mask + hull_mask, 0, 255)
return connected_components_segmentation(intermediate_global_mask), gray
def connected_components_segmentation(intermediate_global_mask):
"""
:param intermediate_global_mask: black and white image
:return: components
"""
_, labeled_img = cv2.connectedComponentsWithAlgorithm(
intermediate_global_mask, 8, cv2.CV_32S, cv2.CCL_GRANA)
labels = np.unique(labeled_img)
labels = labels[labels != 0]
components = []
for label in labels:
mask = np.zeros_like(labeled_img, dtype=np.uint8)
mask[labeled_img == label] = 255
# Compute the convex hull
if get_opencv_major_version(cv2.__version__) in ['2', '3']:
mask, contours, _ = cv2.findContours(mask, cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
else:
contours, _ = cv2.findContours(mask, cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
hull = []
for cnt in contours:
hull.append(cv2.convexHull(cnt, False))
hull_mask = np.zeros((mask.shape[0], mask.shape[1]), dtype=np.uint8)
for i in range(len(contours)):
hull_mask = cv2.drawContours(hull_mask, hull, i, 255, -1, 8)
single_component, flag = draw_border_for_picture_parts(hull_mask)
_, connected_component, stats, _ = cv2.connectedComponentsWithStatsWithAlgorithm(
single_component, 8, cv2.CV_32S, cv2.CCL_GRANA)
valid_labels = np.argwhere(
stats[:, cv2.CC_STAT_AREA] >= LABEL_AREA_THRESHOLD)
if valid_labels[0] == 0:
valid_labels = valid_labels[1:]
for valid_label in valid_labels:
component = Component(valid_label, connected_component,
stats[valid_label], flag)
components.append(component)
components.sort(key=lambda x: x.area, reverse=True)
return components
def image_segmentation(img):
kernel = np.ones((5, 5), np.uint8)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
cv2.THRESH_BINARY, 13,
3) # 11,2 at the beginning
# cv2.imshow("thresh", thresh)
medianBlur = cv2.medianBlur(thresh, 3)
# cv2.imshow("medianBlur", medianBlur)
post_proc = erosion_dilation(medianBlur)
# cv2.imshow("post_proc", post_proc)
num_labels, labels_im = cv2.connectedComponentsWithAlgorithm(
post_proc, connectivity=8, ltype=cv2.CV_32S, ccltype=cv2.CCL_GRANA)
labeled_image = draw_components(labels_im)
# cv2.imshow("CCL", labeled_image)
return labeled_image
def eval_f(fp):
import warnings
warnings.filterwarnings("ignore")
# gt
if '2015' in args.dataset:
mask_gt = cv2.imread(maskp[fp], 0)
flow_gt = read_flow(maskp[fp].replace('obj_map',
'flow_occ')).astype(np.float32)
validmask = flow_gt[:, :, -1] == 1
bgmask_gt = np.logical_and(validmask, mask_gt == 0).astype(float)
else:
bgmask_gt = cv2.imread(maskp[fp], 0) > 0
shape = bgmask_gt.shape[:2]
validmask = np.ones(shape).astype(bool)
shape = bgmask_gt.shape[:2]
# pred
flowpath = flow_paths[fp]
# KITTI
if '2015' in args.dataset:
idnum = int(flowpath.split('/')[-1].split('_')[-2])
if args.method == 'ours':
maskp_pred = '%s/%s/pm-%06d_10.pfm' % (args.path, args.dataset,
idnum)
mask_pred = readPFM(maskp_pred)[0].astype(int)
bgmask_pred = mask_pred == 0
elif args.method == 'detectron2':
maskp_pred = '/data/gengshay/code/moseg/moseg-detectron2/%06d.png' % idnum
mask_pred = cv2.imread(maskp_pred, 0)
bgmask_pred = mask_pred == 0
elif args.method == 'mrflowss':
bgmask_pred = np.load(
'/home/gengshay/code/SF/mrflow/semantic_rigidity_cvpr2017/output_kitti_smodel/%06d_10.npy'
% idnum)
bgmask_pred = cv2.resize(bgmask_pred.astype(float),
shape[::-1]).astype(bool)
elif args.method == 'mrflowsk':
bgmask_pred = cv2.imread(
'/home/gengshay/code/SF/mrflow/semantic_rigidity_cvpr2017/output_kitti_kmodel/%06d_10.png'
% idnum, 0).astype(bool)
elif args.method == 'u2net':
bgmask_pred = cv2.imread(
'/data/gengshay/code/U-2-Net/test_data/u2net_results/%06d_10.png'
% idnum, 0) < 128
elif args.method == 'rtn':
bgmask_pred = ~cv2.imread(
'/data/gengshay/code/learningrigidity_mod/output_kitti/%06d_10.png'
% idnum, 0).astype(bool)
elif args.method == 'fusionseg':
bgmask_pred = ~np.load(
'/data/gengshay/code/fusionseg/python/kitti/results_liu/joint_davis_val/%06d_10_mask.npy'
% idnum).astype(bool)
elif args.method == 'fusionsegm':
bgmask_pred = ~np.load(
'/data/gengshay/code/fusionseg/python/kitti/results_liu/motion/%06d_10_mask.npy'
% idnum).astype(bool)
elif args.method == 'cosnet':
bgmask_pred = cv2.resize(
cv2.imread(
'/data/gengshay/code/COSNet/result/test/davis_iteration_conf/Results/kitti/%06d_10.png'
% idnum, 0), shape[::-1]) < 128
elif args.method == 'matnet':
bgmask_pred = cv2.imread(
'/data/gengshay/code/MATNet/output/kitti_liu/%06d_10.png' %
idnum, 0) < 128
elif args.method == 'cc':
bgmask_pred = cv2.resize(
np.load('/data/gengshay/code/cc/output/mask-joint/%03d.npy' %
idnum)[0], shape[::-1]) == 1
elif args.method == 'ccm':
bgmask_pred = cv2.resize(
np.load('/data/gengshay/code/cc/output/mask-motion/%03d.npy' %
idnum)[0], shape[::-1]) == 1
elif args.method == 'angle':
maskp_pred = '%s/%s/pm-%06d_10.pfm' % (args.path, args.dataset,
idnum)
mask_pred = readPFM(maskp_pred)[0].astype(int)
bgmask_pred = mask_pred == 0
else:
exit()
# Sintel
else:
passname = flowpath.split('/')[-1].split('_')[-4]
seqname1 = flowpath.split('/')[-1].split('_')[-3]
seqname2 = flowpath.split('/')[-1].split('_')[-2]
framename = int(flowpath.split('/')[-1].split('_')[-1].split('.')[0])
if args.method == 'ours':
maskp_pred = '%s/%s/pm-Sintel_%s_%s_%s_%02d.pfm' % (
args.path, args.dataset, passname, seqname1, seqname2,
framename)
mask_pred = readPFM(maskp_pred)[
0] # label map in {1,...N} given N rigid body predictions
bgmask_pred = mask_pred == 0
elif args.method == 'detectron2':
maskp_pred = '/data/gengshay/code/moseg/smoseg-detectron2/Sintel_%s_%s_%s_%02d.png' % (
passname, seqname1, seqname2, framename)
mask_pred = cv2.imread(maskp_pred, 0)
bgmask_pred = mask_pred == 0
elif args.method == 'mrflowss':
bgmask_pred = np.load(
'/home/gengshay/code/SF/mrflow/semantic_rigidity_cvpr2017/output_sintel_smodel/Sintel_%s_%s_%s_%02d.npy'
% (passname, seqname1, seqname2, framename)).astype(bool)
elif args.method == 'mrflowsk':
bgmask_pred = cv2.imread(
'/home/gengshay/code/SF/mrflow/semantic_rigidity_cvpr2017/output_sintel_kmodel/Sintel_%s_%s_%s_%02d.png'
% (passname, seqname1, seqname2, framename), 0).astype(bool)
elif args.method == 'u2net':
bgmask_pred = cv2.imread(
'/data/gengshay/code/U-2-Net/test_data/u2net_results_sintel/Sintel_%s_%s_%s_%02d.png'
% (passname, seqname1, seqname2, framename), 0) < 128
elif args.method == 'rtn':
bgmask_pred = ~cv2.imread(
'/data/gengshay/code/learningrigidity_mod/output_sintel/Sintel_%s_%s_%s_%02d.png'
% (passname, seqname1, seqname2, framename), 0).astype(bool)
elif args.method == 'fusionseg':
bgmask_pred = ~np.load(
'/data/gengshay/code/fusionseg/python/sintel/results/joint_davis_val/Sintel_%s_%s_%s_%02d_mask.npy'
% (passname, seqname1, seqname2, framename)).astype(bool)
elif args.method == 'fusionsegm':
bgmask_pred = ~np.load(
'/data/gengshay/code/fusionseg/python/sintel/results/motion/Sintel_%s_%s_%s_%02d_mask.npy'
% (passname, seqname1, seqname2, framename)).astype(bool)
elif args.method == 'cosnet':
bgmask_pred = cv2.resize(
cv2.imread(
'/data/gengshay/code/COSNet/result/test/davis_iteration_conf/Results/sintel/Sintel_%s_%s_%s_%02d.png'
% (passname, seqname1, seqname2, framename), 0),
shape[::-1]) < 128
elif args.method == 'matnet':
bgmask_pred = cv2.imread(
'/data/gengshay/code/MATNet/output/sintel/Sintel_%s_%s_%s_%02d.png'
% (passname, seqname1, seqname2, framename), 0) < 128
elif args.method == 'angle':
maskp_pred = '%s/%s/pm-Sintel_%s_%s_%s_%02d.pfm' % (
args.path, args.dataset, passname, seqname1, seqname2,
framename)
mask_pred = readPFM(maskp_pred)[0]
bgmask_pred = mask_pred == 0
else:
exit()
if args.method != 'ours' and args.method != 'detectron2':
_, mask_pred = cv2.connectedComponentsWithAlgorithm(
(1 - bgmask_pred.astype(np.uint8)),
connectivity=8,
ltype=cv2.CV_16U,
ccltype=cv2.CCL_WU)
# bg-iou
try:
bgiou = np.logical_and(
bgmask_gt.astype(bool)[validmask],
bgmask_pred[validmask]).sum() / np.logical_or(
bgmask_gt.astype(bool)[validmask],
bgmask_pred[validmask]).sum()
except:
pdb.set_trace()
# obj-F (penalizes false positive)
# defined in https://arxiv.org/abs/1902.03715 Sec.4
# F=2PR/P+R => 2/(1/P+1/R)
# since P = sum(|c and g|) / sum(|c|) and R = sum(|c and g|) / sum(|g|),
# where c is prediction and g is GT
# 1/P+1/R = [sum(|c|) + sum(|g|)] / sum(|c and g|)
# therefore F = 2*sum(|c and g|) / [sum(|c|) + sum(|g|)]
if '2015' in args.dataset:
gtlist = list(set(mask_gt.flatten()))
M = len(gtlist) - 1
imatrix = np.zeros((M, mask_pred.max()))
fmatrix = np.zeros((M, mask_pred.max()))
for i in range(M): # for all bg instances
objx_mask_gt = mask_gt == gtlist[i + 1]
for j in range(mask_pred.max()):
objx_mask_pred = mask_pred == j + 1
imatrix[i, j] = float(
np.logical_and(objx_mask_gt,
objx_mask_pred)[validmask].sum())
fmatrix[i,
j] = imatrix[i,
j] / (objx_mask_gt[validmask].sum() +
objx_mask_pred[validmask].sum()) * 2
ridx, cidx = scipy.optimize.linear_sum_assignment(1 - fmatrix)
objf = imatrix[ridx, cidx].sum() / ((mask_pred > 0)[validmask].sum() +
(mask_gt > 0)[validmask].sum()) * 2
else:
objf = 0.
return bgiou, objf
A = cv.imread("%s/%s" % (pwd, filename), cv.IMREAD_GRAYSCALE)
B = cv.threshold(A, thresh, 255, cv.THRESH_BINARY_INV)[1]
X = np.ones((ysize, np.shape(B)[1]))
C = cv.morphologyEx(B, cv.MORPH_CLOSE, X)
N, D = cv.connectedComponents(C)
k = 0
for i in range(1, N):
(H, W) = np.where(D == i)
F = B[min(H):max(H) + 1, :]
ARef = A[min(H):max(H) + 1, :]
base = imgProc_util.getBase(ARef)
X2 = np.ones((ysize, xsize))
C2 = cv.morphologyEx(F, cv.MORPH_DILATE, X2)
C2 = cv.rotate(C2, cv.ROTATE_90_CLOCKWISE)
N2, D2 = cv.connectedComponentsWithAlgorithm(C2,
8,
cv.CV_16U,
ccltype=cv.CCL_WU)
D2 = cv.rotate(D2, cv.ROTATE_90_COUNTERCLOCKWISE)
for j in range(1, N2):
(H2, W2) = np.where(D2 == j)
F2 = F[:, min(W2):max(W2) + 1]
ARef2 = ARef[:, min(W2):max(W2) + 1]
(H2, W2) = np.where(F2 != 0)
final = ARef2[:, min(W2):max(W2) + 1]
filepath = "%s/%s/w%03d_%03d.png" % (pwd, dirname, k, base)
k = k + 1
cv.imwrite(filepath, final)
print(
"Segmentation done. Check the WordSegmentResult folder for proper result, then excecute \"paragraph.py\""
)
return self._fields[name]
frames=[]
for i,path in enumerate(sorted(glob.glob('%s/pm*'%args.datapath))):
print(path)
pred = readPFM(path)[0]
center_img = cv2.imread(path.replace('pm', 'mvis').replace('.pfm', '.jpg'))
img = cv2.imread('%s/%s.png'%(args.imgpath,path.split('/')[-1].split('pm-')[1].split('.pfm')[0]))
if img is None:
img = cv2.imread('%s/%s.jpg'%(args.imgpath,path.split('/')[-1].split('pm-')[1].split('.pfm')[0]))
shape = pred.shape[:2]
num_instances = int(pred.max())
# if no object detected
if num_instances==0:
_, pred =cv2.connectedComponentsWithAlgorithm((1-(pred==0).astype(np.uint8)),connectivity=8,ltype=cv2.CV_16U,ccltype=cv2.CCL_WU)
num_instances = pred.max()
if num_instances>0:
pred_masks = torch.zeros((num_instances,)+shape).bool()
for k in range(num_instances):
pred_masks[k] = torch.Tensor(pred==(k+1))
obj = Object()
obj.image_height = shape[0]
obj.image_width = shape[1]
obj._fields = {}
obj._fields["pred_masks"] = pred_masks
v = Visualizer(img, coco_metadata, scale=0.5, instance_mode=ColorMode.IMAGE_BW)
try: