def crf_inference(img,
crf_config,
category_num,
feat=None,
pred=None,
gt_prob=0.7,
use_log=False):
'''
feat: the feature map of cnn, shape [h,w,c] or pred, shape [h,w], float32
img: the origin img, shape [h,w,3], uint8
crf_config: {"g_sxy":3,"g_compat":3,"bi_sxy":5,"bi_srgb":5,"bi_compat":10,"iterations":5}
'''
img = img.astype(np.uint8)
h, w = img.shape[0:2]
crf = dcrf.DenseCRF2D(w, h, category_num)
if feat is not None:
feat = feat.astype(np.float32)
if use_log is True:
feat = np.exp(feat - np.max(feat, axis=2, keepdims=True)) + 1e-5
feat /= np.sum(feat, axis=2, keepdims=True)
unary = -np.log(feat)
else:
unary = -feat
unary = np.reshape(unary, (-1, category_num))
unary = np.swapaxes(unary, 0, 1)
unary = np.copy(unary, order="C")
crf.setUnaryEnergy(unary)
else:
pred = pred.astype(np.float32)
# unary energy
unary = unary_from_labels(pred,
category_num,
gt_prob,
zero_unsure=False)
crf.setUnaryEnergy(unary)
# pairwise energy
crf.addPairwiseGaussian(sxy=crf_config["g_sxy"],
compat=crf_config["g_compat"])
crf.addPairwiseBilateral(sxy=crf_config["bi_sxy"],
srgb=crf_config["bi_srgb"],
rgbim=img,
compat=crf_config["bi_compat"])
Q = crf.inference(crf_config["iterations"])
Q = np.array(Q)
Q = np.reshape(Q, [category_num, h, w])
Q = np.transpose(Q, axes=[1, 2, 0]) # new shape: [h,w,c]
return Q
def run_single(self, sm_mask, img,
depth): # the input array are detached numpy already
batch_size = sm_mask.shape[0]
result_big = np.zeros((sm_mask.shape[0], sm_mask.shape[1],
self.input_size[0], self.input_size[1]))
result_small = np.zeros(sm_mask.shape)
for i in range(batch_size):
sm_u = np.transpose(
resize(np.transpose(sm_mask[i], [1, 2, 0]),
self.input_size,
mode='constant'), [2, 0, 1])
U = unary_from_softmax(sm_u)
d = dcrf.DenseCRF2D(self.input_size[1], self.input_size[0],
self.num_class)
d.setUnaryEnergy(U)
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# d.addPairwiseBilateral(sxy=(30,30), srgb=(13,13,13), rgbim=img[i].astype(np.uint8), compat=20, kernel=dcrf.DIAG_KERNEL, normalization=dcrf.NORMALIZE_SYMMETRIC)
d.addPairwiseBilateral(sxy=(80, 80),
srgb=(13, 13, 13),
rgbim=img[i].astype(np.uint8),
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
if not (depth is None):
pairwise_depth_energy = create_pairwise_bilateral(
sdims=(10, 10),
schan=(0.01, ),
img=depth[i].reshape(
(depth[i].shape[0], depth[i].shape[1], 1)),
chdim=2)
d.addPairwiseEnergy(pairwise_depth_energy, compat=15)
Q = d.inference(self.num_iter)
result_big[i] = np.array(Q).reshape(
(self.num_class, self.input_size[0], self.input_size[1]))
result_small[i] = np.transpose(
resize(np.transpose(result_big[i], [1, 2, 0]),
self.mask_size,
mode='constant'), [2, 0, 1])
return result_big, result_small
def crf(img, scores):
'''
scores: prob numpy array after softmax with shape (_, C, H, W)
img: image of shape (H, W, C)
CRF parameters: bi_w = 4, bi_xy_std = 121, bi_rgb_std = 5, pos_w = 3, pos_xy_std = 3
'''
img = np.asarray(img, dtype=np.uint8) # image.shape = [366,500,3]
scores = np.asarray(scores.cpu().detach(), dtype=np.float32)
scores = np.ascontiguousarray(scores)
img = np.ascontiguousarray(img)
n_classes, h, w = scores.shape
d = dcrf.DenseCRF2D(w, h, n_classes)
U = -np.log(scores)
U = U.reshape((n_classes, -1))
d.setUnaryEnergy(U)
feats = create_pairwise_gaussian(sdims=(3, 3), shape=img.shape[:2])
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.FULL_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This creates the color-dependent features --
# because the segmentation that we get from CNN are too coarse
# and we can use local color features to refine them
feats = create_pairwise_bilateral(sdims=(40, 40),
schan=(7, 7, 7),
img=img,
chdim=2)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.FULL_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
Q = np.array(Q).reshape((2, h, w))
for ii in range(Q.shape[0]):
Q[ii] = cv2.medianBlur(Q[ii], 5)
# Q[ii] = cv2.GaussianBlur(Q[ii], (7, 7), 1)
Q_score = copy.deepcopy(Q)
Q = np.argmax(Q, axis=0)
return Q, Q_score
def test_call_dcrf2d():
d = dcrf.DenseCRF2D(10, 10, 2)
unary = test_utils._get_simple_unary()
img = test_utils._get_simple_img()
d.setUnaryEnergy(-np.log(unary))
# d.setUnaryEnergy(PyConstUnary(-np.log(Up)))
d.addPairwiseBilateral(sxy=2, srgb=2, rgbim=img, compat=3)
# d.addPairwiseBilateral(2, 2, img, 3)
res = np.argmax(d.inference(10), axis=0).reshape(10, 10)
np.all(res == img[:, :, 0] / 255)
def _apply_crf(im, un, sxy=80, srgb=30, compat=1):
W, H, n_classes = un.shape
d = dcrf.DenseCRF2D(H, W, n_classes)
U = un.transpose(2, 0, 1).reshape((n_classes, -1))
U = U.copy(order='C')
im = im.copy(order='C')
d.setUnaryEnergy(U)
d.addPairwiseBilateral(sxy, srgb, im, compat)
Q = d.inference(5)
#print("KL-divergance: {}".format(d.klDivergence(Q)))
_map = np.argmax(Q, axis=0)
proba = np.array(Q)
_map = _map.reshape((W, H))
return _map
def crf_inference_label(img, labels, t=10, n_labels=21, gt_prob=0.7):
h, w = img.shape[:2]
d = dcrf.DenseCRF2D(w, h, n_labels)
unary = unary_from_labels(labels, n_labels, gt_prob=gt_prob, zero_unsure=False)
d.setUnaryEnergy(unary)
d.addPairwiseGaussian(sxy=3, compat=3)
d.addPairwiseBilateral(sxy=50, srgb=5, rgbim=np.ascontiguousarray(np.copy(img)), compat=10)
q = d.inference(t)
return np.argmax(np.array(q).reshape((n_labels, h, w)), axis=0)
def denseCRF(img, pred):
import pydensecrf.densecrf as dcrf
from pydensecrf.utils import unary_from_softmax
N,H,W = pred.shape
d = dcrf.DenseCRF2D(W, H, N) # width, height, nlabels
U = unary_from_softmax(pred)
d.setUnaryEnergy(U)
d.addPairwiseGaussian(sxy=3, compat=5)
Q = d.inference(5)
Q = np.array(Q).reshape((N,H,W)).transpose(1,2,0)
return Q
def densecrf(img, mask, dim=512):
if dim != None:
img = resize_short(img, dim)
w, h = img.size
mask = mask.resize((w, h), resample=Image.BILINEAR)
img_arr = np.array(img, dtype=np.uint8)
mask_arr = np.array(mask, dtype=np.uint8)
# start = time.time()
bg_thresh, fg_thresh = color_hist_adaptive(img_arr, mask_arr)
label_map = -np.ones(mask_arr.shape, dtype=np.int32)
label_map[mask_arr < bg_thresh] = 0
label_map[mask_arr > fg_thresh] = 1
rst = np.zeros(mask_arr.shape, dtype=np.uint8) + 128
rst[mask_arr < bg_thresh] = 0
rst[mask_arr > fg_thresh] = 255
if 128 not in rst:
return Image.fromarray(rst)
unary = unary_from_labels(label_map, 3)
d = dcrf.DenseCRF2D(w, h, 3)
d.setUnaryEnergy(unary)
d.addPairwiseGaussian(sxy=3,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# change compat to 5 to make is sharper
d.addPairwiseBilateral(sxy=max(w, h) // 4 + 20,
srgb=13,
rgbim=img_arr,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
out = np.argmax(Q, axis=0).reshape(mask_arr.shape).astype(np.uint8)
for i in range(h):
for j in range(w):
if rst[i][j] == 128:
if out[i][j] == 0:
rst[i][j] = 0
else:
rst[i][j] = 255
# print(time.time() - start)
rst = remove_iland(rst, 3)
return Image.fromarray(rst)
def getCRF_justcol(img,
Lc,
theta,
n_iter,
label_lines,
compat_col=40,
scale=5,
prob=0.5):
if np.ndim(img) == 2:
img = np.dstack((img, img, img))
H = img.shape[0]
W = img.shape[1]
d = dcrf.DenseCRF2D(H, W, len(label_lines) + 1)
U = unary_from_labels(Lc.astype('int'), len(label_lines) + 1, gt_prob=prob)
d.setUnaryEnergy(U)
del U
# sdims = The scaling factors per dimension.
# schan = The scaling factors per channel in the image.
# This creates the color-dependent features and then add them to the CRF
feats = create_pairwise_bilateral(
sdims=(theta, theta),
schan=(scale, scale, scale), #11,11,11
img=img,
chdim=2)
del img
d.addPairwiseEnergy(feats,
compat=compat_col,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
del feats
Q = d.inference(n_iter)
preds = np.array(Q, dtype=np.float32).reshape(
(len(label_lines) + 1, H, W)).transpose(1, 2, 0)
preds = np.expand_dims(preds, 0)
preds = np.squeeze(preds)
return np.argmax(Q, axis=0).reshape(
(H, W)), preds #, p, R, np.abs(d.klDivergence(Q)/ (H*W))
def crf_PIL(original_image, annotated_image, output_image, use_2d=True):
annotated_labels = annotated_image.flatten()
colors, labels = np.unique(annotated_labels, return_inverse=True)
print(len(labels))
print(labels)
print(len(colors))
print(colors)
#Setting up the CRF model
if use_2d:
d = dcrf.DenseCRF2D(original_image.shape[1], original_image.shape[0],
len(colors))
# get unary potentials (neg log probability)
U = unary_from_labels(labels,
len(colors),
gt_prob=0.7,
zero_unsure=False)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
d.addPairwiseBilateral(sxy=(80, 80),
srgb=(13, 13, 13),
rgbim=original_image,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
#Run Inference for 5 steps
Q = d.inference(5)
# Find out the most probable class for each pixel.
MAP = np.argmax(Q, axis=0)
print(MAP.shape)
# C将地图(标签)转换回相应的颜色并保存图像。
# 注意，这里不再有“未知”，不管我们一开始拥有什么。
MAP = MAP.reshape(original_image.shape[1], original_image.shape[0], 1)
cv2.imwrite(output_image, MAP)
#MAP = array_to_img(MAP)
#MAP.save(output_image)
return MAP
def crf_layer(fc8_sec_softmax, downscaled, iternum):
"""
:param fc8_sec_softmax: (batch_size, num_class(including background), height // 8, width // 8)
:param downscaled: (batch_size, height, width, 3 (RGB))
:param iternum: times that calculation CRF inference repeatedly
:return: crf inference results
"""
unary = np.asarray(fc8_sec_softmax.cpu().data)
imgs = downscaled
N = unary.shape[0] # batch_size
result = np.zeros(unary.shape) # (batch_size, num_class, height, width)
for i in range(N):
d = dcrf.DenseCRF2D(
imgs[i].shape[1], imgs[i].shape[0],
unary[i].shape[0]) # DenseCRF(width, height, num_class)
# get unary
U = unary_from_softmax(unary[i])
# set unary potentials
d.setUnaryEnergy(U)
# This creates the color-independent features and then add them to the CRF
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
img = imgs[i].cpu().numpy()
d.addPairwiseBilateral(sxy=(80, 80),
srgb=(13, 13, 13),
rgbim=img,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(iternum)
proba = np.asarray(Q)
result[i] = np.reshape(proba,
(unary[i].shape[0], img.shape[0], img.shape[1]))
result[result < min_prob] = min_prob
result = result / np.sum(result, axis=1, keepdims=True)
crf_result = np.log(result)
return crf_result
def crf(sm_mask_one, img_one, num_class, input_size, mask_size, num_iter, adaptive_crf_setting):
map = np.zeros([adaptive_crf_setting['num_maps'], self.num_maps, self.H, self.W])
sm_u = np.transpose(resize(np.transpose(sm_mask_one, [1,2,0]), input_size, mode='constant'),[2,0,1])
U = unary_from_softmax(sm_u)
d = dcrf.DenseCRF2D(input_size[0], input_size[1], num_class)
d.setUnaryEnergy(U)
d.addPairwiseGaussian(sxy=(3,3), compat=3, kernel=dcrf.DIAG_KERNEL, normalization=dcrf.NORMALIZE_SYMMETRIC)
# d.addPairwiseBilateral(sxy=(30,30), srgb=(13,13,13), rgbim=img_one.astype(np.uint8), compat=20, kernel=dcrf.DIAG_KERNEL, normalization=dcrf.NORMALIZE_SYMMETRIC)
d.addPairwiseBilateral(sxy=(80,80), srgb=(13,13,13), rgbim=img_one.astype(np.uint8), compat=10, kernel=dcrf.DIAG_KERNEL, normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(num_iter)
return np.array(Q).reshape((num_class, input_size[0], input_size[1]))
def test_compact_wrong():
# Tests whether expection is indeed raised
##########################################
# Via e-mail: crash when non-float32 compat
d = dcrf.DenseCRF2D(10, 10, 2)
d.setUnaryEnergy(np.ones((2, 10 * 10), dtype=np.float32))
compat = np.array([1.0, 2.0])
with pytest.raises(ValueError):
d.addPairwiseBilateral(sxy=(3, 3),
srgb=(3, 3, 3),
rgbim=np.zeros((10, 10, 3), np.uint8),
compat=compat)
d.inference(2)
def dense_crf(probs, resized_img):
unary = probs
assert(len(unary.shape) == 3)
unary = -np.log(unary)
unary = unary.transpose(2, 1, 0)
w, h, c = unary.shape
unary = unary.transpose(2, 0, 1).reshape(2, -1)
unary = np.ascontiguousarray(unary)
resized_img = resized_img.transpose(2, 1, 0)
resized_img = np.ascontiguousarray(resized_img)
d = dcrf.DenseCRF2D(w, h, 2)
d.setUnaryEnergy(unary)
d.addPairwiseBilateral(sxy=5, srgb=3, rgbim=resized_img, compat=1)
q = d.inference(50)
mask = np.argmax(q, axis=0).reshape(w, h).transpose(1, 0)
return mask
def post_processing(img, output_probs):
h = output_probs.shape[0]
w = output_probs.shape[1]
output_probs = np.expand_dims(output_probs, 0)
output_probs = np.append(1 - output_probs, output_probs, axis=0)
d = dcrf.DenseCRF2D(w, h, 2)
U = -np.log(output_probs + 1e-8)
U = U.reshape((2, -1))
U = np.ascontiguousarray(U)
img = np.ascontiguousarray(img) * 256.0
d.setUnaryEnergy(U.astype(np.float32))
d.addPairwiseGaussian(sxy=1e8, compat=np.array([1e-8,1e-8]).astype(np.float32))
d.addPairwiseBilateral(sxy=150, srgb=1e8, rgbim=img.astype(np.uint8), compat=np.array([1e-8,1e-8]).astype(np.float32))
Q = d.inference(10)
Q = np.argmax(np.array(Q), axis=0).reshape((h, w))
return Q
def crf_inference(sigm_val, H, W, proc_im):
sigm_val = np.squeeze(sigm_val)
d = densecrf.DenseCRF2D(W, H, 2)
U = np.expand_dims(-np.log(sigm_val + 1e-8), axis=0)
U_ = np.expand_dims(-np.log(1 - sigm_val + 1e-8), axis=0)
unary = np.concatenate((U_, U), axis=0)
unary = unary.reshape((2, -1))
d.setUnaryEnergy(unary)
d.addPairwiseGaussian(sxy=3, compat=3)
d.addPairwiseBilateral(sxy=20, srgb=3, rgbim=proc_im, compat=10)
Q = d.inference(5)
pred_raw_dcrf = np.argmax(Q, axis=0).reshape((H, W)).astype(np.float32)
# predicts_dcrf = im_processing.resize_and_crop(pred_raw_dcrf, mask.shape[0], mask.shape[1])
return pred_raw_dcrf
def crf_inference(img, probs, t=3, scale_factor=1, labels=21):
import pydensecrf.densecrf as dcrf
from pydensecrf.utils import unary_from_softmax
h, w = img.shape[:2]
n_labels = labels
d = dcrf.DenseCRF2D(w, h, n_labels)
unary = unary_from_softmax(probs)
unary = np.ascontiguousarray(unary)
d.setUnaryEnergy(unary)
d.addPairwiseGaussian(sxy=3 / scale_factor, compat=3)
d.addPairwiseBilateral(sxy=80 / scale_factor,
srgb=13,
rgbim=np.copy(img),
compat=10)
Q = d.inference(t)
return np.array(Q).reshape((n_labels, h, w))
def crf_inference_psa(img, probs, CRF_parameter, scale_factor=1, labels=21):
"""
this setting is different from PSA
IoU=62.44
"""
import pydensecrf.densecrf as dcrf
from pydensecrf.utils import unary_from_softmax
h, w = img.shape[:2]
n_labels = labels
d = dcrf.DenseCRF2D(w, h, n_labels)
pred_softmax = torch.nn.Softmax(dim=0)
probs = pred_softmax(torch.tensor(probs)).numpy()
# probs = pred_softmax(torch.from_numpy(probs).float()).numpy()
unary = unary_from_softmax(probs)
unary = np.ascontiguousarray(unary)
d.setUnaryEnergy(unary)
# =======================================================
# === setting in PSA for la_CRF,ha_CRF
# d.addPairwiseGaussian(sxy=3 / scale_factor, compat=3)
# d.addPairwiseBilateral(sxy=80 / scale_factor,
# srgb=13,
# rgbim=np.copy(img),
# compat=10)
# =======================================================
# setting in deeplab
# CRF_parameter = args.CRF
# CRF_parameter = args.CRF_psa
d.addPairwiseGaussian(sxy=CRF_parameter["pos_xy_std"] / scale_factor,
compat=CRF_parameter["pos_w"])
d.addPairwiseBilateral(sxy=CRF_parameter["bi_xy_std"] / scale_factor,
srgb=CRF_parameter["bi_rgb_std"],
rgbim=np.copy(img),
compat=CRF_parameter["bi_w"])
t = CRF_parameter["iter_max"]
# =======================================================
# d.addPairwiseGaussian(sxy=4 / scale_factor, compat=3)
# d.addPairwiseBilateral(sxy=83 / scale_factor,
# srgb=5,
# rgbim=np.copy(img),
# compat=3)
Q = d.inference(t)
return np.array(Q).reshape((n_labels, h, w))
def dense_crf_tgs2(original_image, mask_img):
# Converting annotated image to RGB if it is Gray scale
if (len(mask_img.shape) < 3):
mask_img = gray2rgb(mask_img).astype(np.uint8)
# #Converting the annotations RGB color to single 32 bit integer
annotated_label = mask_img[:, :, 0] + (mask_img[:, :, 1] << 8) + (
mask_img[:, :, 2] << 16)
# # Convert the 32bit integer color to 0,1, 2, ... labels.
colors, labels = np.unique(annotated_label, return_inverse=True)
n_labels = 2
# Setting up the CRF model
d = dcrf.DenseCRF2D(original_image.shape[1], original_image.shape[0],
n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=False)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
# d.addPairwiseGaussian(sxy=(3, 3), compat=3, kernel=dcrf.DIAG_KERNEL,
# normalization=dcrf.NORMALIZE_SYMMETRIC)
# MAX_ITER = 10
# POS_W = 3
# POS_XY_STD = 1
# Bi_W = 4
# Bi_XY_STD = 3
# Bi_RGB_STD = 10
img = np.ascontiguousarray(mask_img)
d.addPairwiseGaussian(sxy=POS_XY_STD, compat=POS_W)
d.addPairwiseBilateral(sxy=Bi_XY_STD,
srgb=Bi_RGB_STD,
rgbim=img,
compat=Bi_W)
# Run Inference for 10 steps
Q = d.inference(10)
# Find out the most probable class for each pixel.
MAP = np.argmax(Q, axis=0)
return MAP.reshape((original_image.shape[0], original_image.shape[1]))
def lineSeg_predict(img, predict_func):
"""use lineSeg model to predict the mask from an image
# Arguments
img: given image
predict_func: predict functions
# Returns
mask: the predict mask of given image
"""
# slice the image
slices = utils.slice_even(img, (200,200))
h_cnt, w_cnt, h, w = slices.shape
# get the predictions, with the shape of (h, w, 2)
# expand 2 dims, axis 0 is the batch dim, axis 3 is the channel dim
predictions = [predict_func([np.reshape(j, (1, j.shape[0], j.shape[1], 1))]) for i in slices for j in i ]
predictions = np.reshape(predictions, (h_cnt, w_cnt, h, w, 2))
# merge predictions
predictions = utils.merge(predictions,0,0,200,200)
predictions = predictions[:img.shape[0],:img.shape[1]]
# CRF
w, h, c = predictions.shape
d = dcrf.DenseCRF2D(w, h, c)
# set unary potential
predictions = np.transpose(predictions, (2, 0, 1))
U = unary_from_softmax(predictions)
d.setUnaryEnergy(U)
# set pairwise potential
# This creates the color-independent features and then add them to the CRF
d.addPairwiseGaussian(sxy=(3, 3), compat=3, kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
iter_num = 5
result = np.argmax(d.inference(iter_num), axis=0)
result = np.reshape(result, (img.shape[0],img.shape[1]))
# return the mask
result = (1 - result) * 255
result = np.uint8(result)
misc.imsave('result.jpg', result)
return result
def CRFs(image,
prediction,
num_iter=10,
sdims=(10, 10),
schan=(0.01, ),
compat=10):
"""
:param image:
:param prediction:
:return:
"""
W, H = image.shape
NLABELS = 2 # 1 class plus 1 background
probs = np.stack([1 - prediction, prediction], axis=0)
U = unary_from_softmax(probs) # note: num classes is first dim
# Create the pairwise bilateral term from the above image.
# The two `s{dims,chan}` parameters are model hyper-parameters defining
# the strength of the location and image content bilaterals, respectively.
pairwise_energy = create_pairwise_bilateral(sdims=sdims,
schan=schan,
img=image.reshape(W, H, 1),
chdim=2)
d = dcrf.DenseCRF2D(W, H, NLABELS)
d.setUnaryEnergy(U)
d.addPairwiseEnergy(
pairwise_energy,
compat=compat) # `compat` is the "strength" of this potential.
# This time, let's do inference in steps ourselves
# so that we can look at intermediate solutions
# as well as monitor KL-divergence, which indicates
# how well we have converged.
# PyDenseCRF also requires us to keep track of two
# temporary buffers it needs for computations.
Q, tmp1, tmp2 = d.startInference()
for _ in range(num_iter):
d.stepInference(Q, tmp1, tmp2)
# kl divergence, but is negative, dont know why
# kl1 = d.klDivergence(Q) / (H * W)
map_soln = np.argmax(Q, axis=0).reshape((H, W))
# And let's have a look.
return map_soln
def dense_crf(probs, img=None, n_iters=10,
sxy_gaussian=(1, 1), compat_gaussian=4,
kernel_gaussian=dcrf.DIAG_KERNEL,
normalisation_gaussian=dcrf.NORMALIZE_SYMMETRIC,
sxy_bilateral=(49, 49), compat_bilateral=5,
srgb_bilateral=(13, 13, 13),
kernel_bilateral=dcrf.DIAG_KERNEL,
normalisation_bilateral=dcrf.NORMALIZE_SYMMETRIC):
"""DenseCRF over unnormalised predictions.
More details on the arguments at https://github.com/lucasb-eyer/pydensecrf.
Args:
probs: class probabilities per pixel.
img: if given, the pairwise bilateral potential on raw RGB values will be computed.
n_iters: number of iterations of MAP inference.
sxy_gaussian: standard deviations for the location component of the colour-independent term.
compat_gaussian: label compatibilities for the colour-independent term (can be a number, a 1D array, or a 2D array).
kernel_gaussian: kernel precision matrix for the colour-independent term (can take values CONST_KERNEL, DIAG_KERNEL, or FULL_KERNEL).
normalisation_gaussian: normalisation for the colour-independent term (possible values are NO_NORMALIZATION, NORMALIZE_BEFORE, NORMALIZE_AFTER, NORMALIZE_SYMMETRIC).
sxy_bilateral: standard deviations for the location component of the colour-dependent term.
compat_bilateral: label compatibilities for the colour-dependent term (can be a number, a 1D array, or a 2D array).
srgb_bilateral: standard deviations for the colour component of the colour-dependent term.
kernel_bilateral: kernel precision matrix for the colour-dependent term (can take values CONST_KERNEL, DIAG_KERNEL, or FULL_KERNEL).
normalisation_bilateral: normalisation for the colour-dependent term (possible values are NO_NORMALIZATION, NORMALIZE_BEFORE, NORMALIZE_AFTER, NORMALIZE_SYMMETRIC).
Returns:
Refined predictions after MAP inference.
"""
# _, h, w, _ = probs.shape
_, h, w, n_classes = probs.shape
probs = probs[0].transpose(2, 0, 1).copy(order='C') # Need a contiguous array.
d = dcrf.DenseCRF2D(w, h, n_classes) # Define DenseCRF model.
U = -np.log(probs) # Unary potential.x
U = U.reshape((n_classes, -1)) # Needs to be flat.
d.setUnaryEnergy(U)
d.addPairwiseGaussian(sxy=sxy_gaussian, compat=compat_gaussian,
kernel=kernel_gaussian, normalization=normalisation_gaussian)
if img is not None:
assert (img.shape[1:3] == (h, w)), "The image height and width must coincide with dimensions of the logits."
d.addPairwiseBilateral(sxy=sxy_bilateral, compat=compat_bilateral,
kernel=kernel_bilateral, normalization=normalisation_bilateral,
srgb=srgb_bilateral, rgbim=img[0])
Q = d.inference(n_iters)
preds = np.array(Q, dtype=np.float32).reshape((n_classes, h, w)).transpose(1, 2, 0)
return np.expand_dims(preds, 0)
def DCRF_postprocess_2D(post_map, img_slice):
"""Dense-CRF applying on a 2D
binary posterior map
"""
d = dcrf.DenseCRF2D(img_slice.shape[0], img_slice.shape[1], 2)
# unary potentials
post_map[post_map == 0] += 1e-10
post_map = -np.log(post_map)
U = np.float32(np.array([1 - post_map, post_map]))
U = U.reshape((2, -1))
d.setUnaryEnergy(U)
# pairwise potentials
# ------------------
# smoothness kernel (considering only
# the spatial features)
feats = create_pairwise_gaussian(sdims=(3, 1), shape=img_slice.shape)
d.addPairwiseEnergy(feats,
compat=5,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# appearance kernel (considering spatial
# and intensity features)
# feats = create_pairwise_bilateral(
# sdims=(3, 3),
# schan=(1),
# img=img_slice,
# chdim=-1)
#
# d.addPairwiseEnergy(
# feats, compat=10,
# kernel=dcrf.DIAG_KERNEL,
# normalization=dcrf.NORMALIZE_SYMMETRIC)
# D-CRF's inference
niter = 5
Q = d.inference(niter)
# maximum probability as the label
MAP = np.argmax(Q, axis=0).reshape(
(img_slice.shape[0], img_slice.shape[1]))
return MAP
def CRF(prob, im):
height, width, _ = im.shape
nlabels = prob.shape[0]
d = dcrf.DenseCRF2D(width, height, nlabels)
# set Unary
U = -np.log(prob+1e-6)
U = U.astype('float32')
U = U.reshape(nlabels, -1) # needs to be flat
U = np.ascontiguousarray(U)
d.setUnaryEnergy(U)
# set Pairwise
im = np.ascontiguousarray(im).astype('uint8')
d.addPairwiseGaussian(sxy=(5,5), compat=3)
d.addPairwiseBilateral(sxy=(50,50), srgb=(20,20,20), rgbim=im, compat=10)
Q = d.inference(10)
map = np.argmax(Q, axis=0).reshape((height,width))
return map
def dense_crf(probs, img=None, n_classes=21, n_iters=1, scale_factor=1): c,h,w = probs.shape if img is not None: assert(img.shape[1:3] == (h, w)) img = np.transpose(img,(1,2,0)).copy(order='C') d = dcrf.DenseCRF2D(w, h, n_classes) # Define DenseCRF model. unary = unary_from_softmax(probs) unary = np.ascontiguousarray(unary) d.setUnaryEnergy(unary) d.addPairwiseGaussian(sxy=3/scale_factor, compat=3) d.addPairwiseBilateral(sxy=80/scale_factor, srgb=13, rgbim=np.copy(img), compat=10) Q = d.inference(n_iters) preds = np.array(Q, dtype=np.float32).reshape((n_classes, h, w)) return preds
def do_crf2_inference(image, annotation, t=5):
image = np.ascontiguousarray(image)
annotation = np.expand_dims(annotation, axis=0)
annotation = np.concatenate([annotation, 1 - annotation], axis=0)
h, w = image.shape[:2]
d = dcrf.DenseCRF2D(w, h, 2)
unary = unary_from_softmax(annotation)
unary = np.ascontiguousarray(unary)
d.setUnaryEnergy(unary)
d.addPairwiseGaussian(sxy=3, compat=3)
d.addPairwiseBilateral(sxy=80, srgb=13, rgbim=np.copy(image), compat=10)
q = d.inference(t)
result = np.array(q).reshape((2, h, w))
return result[0]
def dense_crf_process(self, images, outputs):
'''
Reference: https://github.com/kazuto1011/deeplab-pytorch/blob/master/libs/utils/crf.py
'''
# hyperparameters of the dense crf
# baseline = 79.5
# bi_xy_std = 67, 79.1
# bi_xy_std = 20, 79.6
# bi_xy_std = 10, 79.7
# bi_xy_std = 10, iter_max = 20, v4 79.7
# bi_xy_std = 10, iter_max = 5, v5 79.7
# bi_xy_std = 5, v3 79.7
iter_max = 10
pos_w = 3
pos_xy_std = 1
bi_w = 4
bi_xy_std = 10
bi_rgb_std = 3
b = images.size(0)
mean_vector = np.expand_dims(np.expand_dims(np.transpose(
np.array([102.9801, 115.9465, 122.7717])),
axis=1),
axis=2)
outputs = F.softmax(outputs, dim=1)
for i in range(b):
unary = outputs[i].data.cpu().numpy()
C, H, W = unary.shape
unary = dcrf_utils.unary_from_softmax(unary)
unary = np.ascontiguousarray(unary)
image = np.ascontiguousarray(images[i]) + mean_vector
image = image.astype(np.ubyte)
image = np.ascontiguousarray(image.transpose(1, 2, 0))
d = dcrf.DenseCRF2D(W, H, C)
d.setUnaryEnergy(unary)
d.addPairwiseGaussian(sxy=pos_xy_std, compat=pos_w)
d.addPairwiseBilateral(sxy=bi_xy_std,
srgb=bi_rgb_std,
rgbim=image,
compat=bi_w)
out_crf = np.array(d.inference(iter_max))
outputs[i] = torch.from_numpy(out_crf).cuda().view(C, H, W)
return outputs
def crf_inference(img, probs, t=10, scale_factor=1, n_label=201):
h, w = img.shape[:2]
d = dcrf.DenseCRF2D(w, h, n_label)
unary = unary_from_softmax(probs)
unary = np.ascontiguousarray(unary)
d.setUnaryEnergy(unary)
d.addPairwiseGaussian(sxy=3 / scale_factor, compat=3)
d.addPairwiseBilateral(sxy=80 / scale_factor,
srgb=13,
rgbim=np.copy(img),
compat=10)
Q = d.inference(t)
return np.array(Q).reshape((n_label, h, w))
def crf_fit_predict(softmax: np.ndarray, image: np.ndarray, niter: int = 150):
r"""Fits a Conditional Random Field (CRF) for image segmentation, given a mask of class probabilities (softmax)
from the WNet CNN and the raw image (image).
:param softmax: Softmax outputs from a CNN segmentation model. Shape: (nchan, nrow, ncol)
:param image: Raw image, containing any number of channels. Shape: (nchan, nrow, ncol)
:param niter: Number of iterations during CRF optimization
:return: Segmented class probabilities -- take argmax to get discrete class labels.
"""
unary = unary_from_softmax(softmax).reshape(softmax.shape[0], -1)
bilateral = create_pairwise_bilateral(sdims=(25, 25), schan=(0.05, 0.05), img=image, chdim=0)
crf = dcrf.DenseCRF2D(image.shape[2], image.shape[1], softmax.shape[0])
crf.setUnaryEnergy(unary)
crf.addPairwiseEnergy(bilateral, compat=100)
pred = crf.inference(niter)
return np.array(pred).reshape((-1, image.shape[1], image.shape[2]))
def crf(original_image, annotated_image, output_image, use_2d = True):
#Converting the annotations RGB color to single 32 bit integer
annotated_label = annotated_image[:,:,0] + (annotated_image[:,:,1]<<8) + (annotated_image[:,:,2]<<16)
# Convert the 32bit integer color to 0,1, 2, ... labels.
colors, labels = np.unique(annotated_label, return_inverse=True)
# colors = np.array([0,1,2,3,4,5,6,7,8])
# labels = annotated_label.flatten()
#Creating a mapping back to 32 bit colors
colorize = np.empty((len(colors), 3), np.uint8)
colorize[:,0] = (colors & 0x0000FF)
colorize[:,1] = (colors & 0x00FF00) >> 8
colorize[:,2] = (colors & 0xFF0000) >> 16
#Gives no of class labels in the annotated image
n_labels = len(set(labels.flat))
# print("No of labels in the Image are ")
# print(n_labels)
#Setting up the CRF model
if use_2d :
d = dcrf.DenseCRF2D(original_image.shape[1], original_image.shape[0], n_labels)
# get unary potentials (neg log probability)
zero_flag = True
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=zero_flag)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3, 3), compat=3, kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
d.addPairwiseBilateral(sxy=(10, 10), srgb=(13, 130, 13), rgbim=original_image,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
#Run Inference for 5 steps
Q = d.inference(5)
# Find out the most probable class for each pixel.
MAP = np.argmax(Q, axis=0)
# Convert the MAP (labels) back to the corresponding colors and save the image.
# Note that there is no "unknown" here anymore, no matter what we had at first.
if (zero_flag):
MAP = MAP + 1
MAP = colorize[MAP,:]
# imsave(output_image,MAP.reshape(original_image.shape))
return MAP.reshape(original_image.shape)
