def crf_proc(imgs, probs):
_, _, H, W = imgs.shape
imgs = imgs.transpose((0, 2, 3, 1))
lbls = []
for img, prob in zip(imgs, probs):
prob = np.concatenate((1 - prob[None, ...], prob[None, ...]), 0)
# Example using the DenseCRF class and the util functions
d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], 2)
# get unary potentials (neg log probability)
U = unary_from_softmax(prob)
d.setUnaryEnergy(U)
# This creates the color-dependent features and then add them to the CRF
feats = create_pairwise_bilateral(sdims=(80, 80),
schan=(13, 13, 13),
img=img,
chdim=2)
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# Run five inference steps.
Q = d.inference(5)
# Find out the most probable class for each pixel.
MAP = np.argmax(Q, axis=0).reshape((H, W))
lbls.append(MAP)
lbls = np.stack(lbls)
return lbls
def post_process_crf(predictions, base_images, clean=False):
#TODO CREATE FROM LABELS
new_predictions = []
images = base_images * 255
for prediction, base in tqdm(zip(predictions, images)):
base = base.astype(np.uint8)
d = dcrf.DenseCRF2D(SHAPE[0], SHAPE[1], 2)
probs = np.stack([1 - prediction, prediction])
unary = unary_from_softmax(probs)
d.setUnaryEnergy(unary)
d.addPairwiseGaussian(sxy=10, compat=3)
d.addPairwiseBilateral(sxy=30, srgb=15, rgbim=base, compat=10)
Q = d.inference(5)
res = np.argmax(Q, axis=0).reshape((SHAPE[0], SHAPE[1]))
if clean:
res = clean_crf(res)
new_predictions.append(res)
return new_predictions
def crf(softmax_outputs, inputs):
result = torch.zeros(softmax_outputs.shape)
idx = 0
for input in inputs:
# unary
u = unary_from_softmax(
softmax_outputs[idx].cpu().detach().numpy()).reshape(
softmax_outputs.shape[1], -1)
# pairwise
p = create_pairwise_bilateral(sdims=(25, 25),
schan=(0.05, 0.05),
img=input.cpu().detach().numpy(),
chdim=0)
crf = dcrf.DenseCRF2D(inputs.shape[3], inputs.shape[2],
softmax_outputs.shape[1])
# unary potential
crf.setUnaryEnergy(u)
# + pairwise potential
crf.addPairwiseEnergy(p, compat=100)
Q = crf.inference(10)
print(Q)
result[idx] = torch.tensor(
np.array(Q).reshape((-1, inputs.shape[2], inputs.shape[3])))
idx += 1
return result
def get_crf_seg(img, labels, n_labels, opts):
'''
crf_out_final = get_crf_seg(img, labels, n_labels)
'''
crf = dcrf.DenseCRF(img.shape[1] * img.shape[0], n_labels)
U = unary_from_softmax(labels)
crf.setUnaryEnergy(U)
# pairwise positional (gaussian) terms
feats = create_pairwise_gaussian(sdims=(opts.gaussian_sx, opts.gaussian_sx), shape=img.shape[:2])
crf.addPairwiseEnergy(feats, compat=opts.crf_gaussian_weight,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# pairwise bilateral (color + position) terms
feats = create_pairwise_bilateral(sdims=(opts.bilateral_sx, opts.bilateral_sx), \
schan=(opts.bilateral_color, opts.bilateral_color, opts.bilateral_color),
img=img, chdim=2)
crf.addPairwiseEnergy(feats, compat=opts.crf_bilateral_weight,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# run inference
Q = crf.inference(5)
return Q
def crf_post_process(image, prob, n_steps=5):
"""
Use CRF as a post processing technique.
:param image: np.array, the raw image with shape like(height, width, n_classes)
:param prob: np.array, same shape as `image`, giving the probabilities
:param n_steps: int, number of iterations for CRF inference.
:return:
result: np.array(dtype=np.int32), result after the CRF post-processing.
"""
height, width, n_classes = prob.shape
d = DenseCRF2D(width, height, n_classes)
# unary potential
unary = unary_from_softmax(prob.transpose((2, 0, 1)))
d.setUnaryEnergy(unary)
# pairwise potential
d.addPairwiseGaussian(sxy=3, compat=3)
d.addPairwiseBilateral(sxy=80, srgb=13, rgbim=image, compat=2)
# inference
Q = d.inference(n_steps)
result = np.argmax(Q, axis=0).reshape((height, width))
return result
def run_CRF(patient, sdimsB, sdimsG, schan):
path0 = path + 'patient_' + str(patient) +'_ProbMapClass0.nii.gz'
path1 = path + 'patient_' + str(patient) +'_ProbMapClass1.nii.gz'
ims, npoints, nlabels, affine = niiToNumpyCRFinput(path0, path1)
shape1 = ims.shape[1:]
name = 'patient_' + str(patient) + '_CRF'
d = dcrf.DenseCRF(npoints, nlabels) # npoints, nlabels
U = ims
shape = U.shape[1:]
# print (np.sum(U))
U = unary_from_softmax(ims)
d.setUnaryEnergy(U)
G = create_pairwise_gaussian(sdimsG, shape)
d.addPairwiseEnergy(w2*G, compat=compat1)
B = create_pairwise_bilateral(sdimsB, schan, ims, chdim=0)
d.addPairwiseEnergy(w1*B, compat=compat2)
# Q = d.inference(1)
Q, tmp1, tmp2 = d.startInference()
for i in range(5):
# print("KL-divergence at {}: {}".format(i, d.klDivergence(Q)))
d.stepInference(Q, tmp1, tmp2)
patientCRF = np.argmax(Q, axis=0).reshape(shape1).astype(np.float32)
#print (patientCRF.shape, patientCRF.dtype, np.sum(patientCRF))
# if patient == 48:
# im = nib.Nifti1Image(patientCRF, affine=affine)
# nib.save(im, os.path.join(CRFpath, name + '.nii.gz'))
# im1 = nib.load(CRFpath + name + '.nii.gz')
# print(im1.get_data().shape)
# print(im1.get_data().dtype)
# print(im1.header)
return patientCRF
def dense_crf(image_path: str, output_logits: torch.FloatTensor):
image = cv2.imread(image_path, cv2.IMREAD_COLOR).astype(np.float32)
image = image.astype(np.uint8)
H, W = image.shape[:2]
image = np.ascontiguousarray(image)
output_logits = F.interpolate(output_logits.unsqueeze(0),
size=(H, W),
mode="bilinear",
align_corners=False).squeeze()
output_probs = F.softmax(output_logits, dim=0).cpu().numpy()
c = output_probs.shape[0]
h = output_probs.shape[1]
w = output_probs.shape[2]
U = utils.unary_from_softmax(output_probs)
U = np.ascontiguousarray(U)
d = dcrf.DenseCRF2D(w, h, c)
d.setUnaryEnergy(U)
d.addPairwiseGaussian(sxy=POS_XY_STD, compat=POS_W)
d.addPairwiseBilateral(sxy=Bi_XY_STD,
srgb=Bi_RGB_STD,
rgbim=image,
compat=Bi_W)
Q = d.inference(MAX_ITER)
Q = np.array(Q).reshape((c, h, w))
return Q
def dense_crf(img, output_probs):
h = output_probs.shape[0]
w = output_probs.shape[1]
output_probs = np.expand_dims(output_probs, 0)
output_probs = np.append(1 - output_probs, output_probs, axis=0)
d = dcrf.DenseCRF2D(w, h, 2)
U = -np.log(output_probs)
U = U.reshape((2, -1))
U = np.ascontiguousarray(U)
img = np.ascontiguousarray(img)
U = unary_from_softmax(output_probs)
d.setUnaryEnergy(U)
# d.addPairwiseGaussian(sxy=10, compat=1)
# d.addPairwiseBilateral(sxy=10, srgb=13, rgbim=img, compat=10)
d.addPairwiseGaussian(sxy=10, compat=3)
d.addPairwiseBilateral(sxy=30, srgb=10, rgbim=img, compat=3)
Q = d.inference(5)
res = np.argmax(np.array(Q), axis=0).reshape((h, w))
# res = np.array(Q).reshape((h,w))
return res
def crf(labels,rgb,classes=None,ls=[0,1,2,3,4,5]):
if classes==None:
labels += np.amin(labels)
labels /= np.amax(labels)
classes = labels.shape[2]
c_img = np.rollaxis(labels,2)
U = unary_from_softmax(c_img)
else:
c_img = cl.classimg(labels,map=True,labels=ls)
c_img = np.reshape(c_img,(c_img.shape[0]*c_img.shape[1])).astype(int)
U = unary_from_labels(np.array([0,3]),2,.7,zero_unsure=True)
d = dcrf.DenseCRF2D(labels.shape[0], labels.shape[1], classes)
d.setUnaryEnergy(U)
PG = create_pairwise_gaussian((3,3),labels.shape[:2])
d.addPairwiseEnergy(PG,3)
PB = create_pairwise_bilateral((59,59), (13,13,13,13), rgb,chdim=2)
d.addPairwiseEnergy(PB,6)
Q = d.inference(5)
map = np.argmax(Q, axis=0)
map = np.reshape(map,labels.shape[:2])
return map
def dense_crf(img, output_probs):
"""
Conditional Random Field calculation. With the help of the following
sources:
1. https://github.com/zllrunning/deeplab-pytorch-crf/blob/master/libs/utils/crf.py
2. https://github.com/lucasb-eyer/pydensecrf
:param img: original input image
:type img: input matrix
:param output_probs: logits
:type output_probs: keras tensor or np nd_array
:return: results after crf
:rtype: numpy array
"""
w, h, c = output_probs
U = utils.unary_from_softmax(output_probs)
U = np.ascontiguousarray(U)
img = np.ascontiguousarray(img)
d = dcrf.DenseCRF2D(w, h, c)
d.setUnaryEnergy(U)
d.addPairwiseGaussian(sxy=POS_XY_STD, compat=POS_W)
d.addPairwiseBilateral(sxy=Bi_XY_STD,
srgb=Bi_RGB_STD,
rgbim=img,
compat=Bi_W)
Q = d.inference(MAX_ITER)
Q = np.array(Q).reshape((c, h, w))
return Q
def get_crf_img(inputs, outputs):
for i in range(outputs.shape[0]):
img = inputs[i]
softmax_prob = outputs[i]
unary = unary_from_softmax(softmax_prob)
unary = np.ascontiguousarray(unary)
d = dcrf.DenseCRF(img.shape[0] * img.shape[1], 2)
d.setUnaryEnergy(unary)
feats = create_pairwise_gaussian(sdims=(10, 10), shape=img.shape[:2])
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
feats = create_pairwise_bilateral(sdims=(50, 50),
schan=(20, 20, 20),
img=img,
chdim=2)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
res = np.argmax(Q, axis=0).reshape((img.shape[0], img.shape[1]))
if i == 0:
crf = np.expand_dims(res, axis=0)
else:
res = np.expand_dims(res, axis=0)
crf = np.concatenate((crf, res), axis=0)
return crf
def perform_crf(image=None, probabilities=None, number_class=2):
image = image.squeeze()
softmax = probabilities.squeeze().transpose((2, 0, 1))
# The input should be the negative of the logarithm of probability values
# Look up the definition of the softmax_to_unary for more information
unary = unary_from_softmax(softmax)
# The inputs should be C-continious -- we are using Cython wrapper
unary = np.ascontiguousarray(unary)
d = dcrf.DenseCRF(image.shape[0] * image.shape[1], number_class)
d.setUnaryEnergy(unary)
# This potential penalizes small pieces of segmentation that are
# spatially isolated -- enforces more spatially consistent segmentations
feats = create_pairwise_gaussian(sdims=(10, 10), shape=image.shape[:2])
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This creates the color-dependent features --
# because the segmentation that we get from CNN are too coarse
# and we can use local color features to refine them
feats = create_pairwise_bilateral(sdims=(50, 50),
schan=(20, 20, 20),
img=image,
chdim=2)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
res = np.argmax(Q, axis=0).reshape((image.shape[0], image.shape[1]))
return res
def crf_inference(img_path, softmax_path, output_path):
"""Run CRF inference (find MAP given unary and pairwise potentials)
Adapted from pydensecrf/inference.py example code.
"""
img = imread(img_path)
# Reintroduce softmax and process into unary potentials
data = loadmat(softmax_path)['data']
data = softmax(data.reshape(data.shape[0:3]))
data = np.transpose(data[0:img.shape[0], 0:img.shape[1], ...], (2, 0, 1))
U = unary_from_softmax(data)
# Should be two
n_labels = data.shape[0]
# Create DenseCRF object and set potentials (unary and pairwise)
d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], n_labels)
d.setUnaryEnergy(U)
d.addPairwiseGaussian(sxy=3, compat=3)
d.addPairwiseBilateral(sxy=80, srgb=13, rgbim=img, compat=10)
# Perform inference, get MAP prediction
Q = d.inference(5)
MAP = np.argmax(Q, axis=0)
# Convert back to color image and save
imwrite(output_path, MAP.reshape(img.shape[0], img.shape[1], 1))
def dense_crf(img,
output_probs,
compat_gaussian=3,
sxy_gaussian=1,
compat_bilateral=10,
sxy_bilateral=1,
srgb=50):
height = output_probs.shape[1]
width = output_probs.shape[2]
crf = DenseCRF2D(width, height, 2)
unary = unary_from_softmax(output_probs)
org_img = denormalize_img(img, mean=MEAN, std=STD) * 255.
org_img = org_img.transpose(1, 2, 0)
org_img = np.ascontiguousarray(org_img, dtype=np.uint8)
crf.setUnaryEnergy(unary)
crf.addPairwiseGaussian(sxy=sxy_gaussian, compat=compat_gaussian)
crf.addPairwiseBilateral(sxy=sxy_bilateral,
srgb=srgb,
rgbim=org_img,
compat=compat_bilateral)
crf_image = crf.inference(5)
crf_image = np.array(crf_image).reshape(output_probs.shape)
return crf_image
def CRF(img, prob):
prob = np.transpose(prob, [2, 0, 1])
d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], prob.shape[0])
unary = unary_from_softmax(prob)
unary = np.ascontiguousarray(unary)
unary = unary.astype(np.float32)
d.setUnaryEnergy(unary)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
d.addPairwiseBilateral(sxy=(80, 80),
srgb=(13, 13, 13),
rgbim=img,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(10)
crf_result = np.array(Q).reshape(
(prob.shape[0], img.shape[0], img.shape[1]))
result = np.argmax(crf_result, axis=0)
return CRF_label(img, result)
def get_unary_term(x, unary_from='prob', n_classes=None, gt_prob=None):
"""
Get unary potential either from probability or label guess. Basically, it can be used
as follows:
1) get_unary_term(prob); # from probability
2) get_unary_term(label_guess,
unary_from='label',
n_classes=2,
gt_prob=0.7) # from label guess
:param x: np.array, the unary source with shape(height, width) or (height, width, n_classes)
:param unary_from: str, either 'prob' or 'label'
:param n_classes: int, number of classes
:param gt_prob: float, between (0.0, 1.0), giving the confidence about the label.
:return:
"""
if unary_from == 'prob':
unary = unary_from_softmax(x.transpose((2, 0, 1)))
elif unary_from == 'label':
unary = unary_from_labels(x,
n_classes,
gt_prob=gt_prob,
zero_unsure=False)
else:
raise NotImplemented
return unary
def dense_crf(img, output_probs):
h = output_probs.shape[0]
w = output_probs.shape[1]
output_probs = np.expand_dims(output_probs, axis=0)
# print output_probs.shape # (1, 500, 500)
output_probs = np.append(1 - output_probs, output_probs, axis=0)
# print output_probs.shape
d = dcrf.DenseCRF2D(w, h, 2)
U = unary_from_softmax(output_probs)
# Return a contiguous array in memory
img = np.ascontiguousarray(img)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=1,
compat=1,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
d.addPairwiseBilateral(sxy=1,
srgb=1,
rgbim=im,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(10)
Q = np.argmax(np.array(Q), axis=0).reshape((h, w))
return Q
def apply_crf(self, softmax, image, **kwargs):
transposed_softmax = softmax.transpose((2, 0, 1))
height, width = image.shape[:2]
nlabels = kwargs.get("nlabels", 2)
infer_nums = kwargs.get("infer_nums", 3)
sxy = kwargs.get("sxy", (80, 80))
srgb = kwargs.get("srgb", (13, 13, 13))
compat = kwargs.get("compat", 2)
kernel = kwargs.get("kernel", dcrf.DIAG_KERNEL)
normalization = kwargs.get("normalization", dcrf.NORMALIZE_SYMMETRIC)
# DenseCRF 선언
dense_crf = dcrf.DenseCRF2D(width, height, nlabels)
# Unary Potential Setting
unary = unary_from_softmax(transposed_softmax)
dense_crf.setUnaryEnergy(unary)
# Pairwise Potential Setting
dense_crf.addPairwiseBilateral(sxy=sxy,
srgb=srgb,
rgbim=image,
compat=compat,
kernel=kernel,
normalization=normalization)
# Inferencing
Q = dense_crf.inference(infer_nums)
res = np.argmax(Q, axis=0).reshape((height, width))
return res
def post_processing(data, probas):
[h,w] = data.shape
n_labels = 2
pred_maps = np.zeros(data.shape)
#print 'postprocessing:', data.shape, probas.shape
img = np.uint8(data*255)
img = data[...,np.newaxis]
#proba = probas
proba = probas + 0.1
#proba[proba>1] = 1
labels = np.zeros((2,img.shape[0],img.shape[1]))
labels[0] = 1-proba
labels[1] = proba
#U=labels
U = unary_from_softmax(labels) # note: num classes is first dim
pairwise_energy = create_pairwise_bilateral(sdims=(7,7), schan=(0.001,), img=img, chdim=2)
pairwise_gaussian = create_pairwise_gaussian(sdims=(3,3), shape=img.shape[:2])
d = dcrf.DenseCRF2D(w, h, n_labels)
d.setUnaryEnergy(U)
d.addPairwiseEnergy(pairwise_gaussian, compat=1, kernel=dcrf.DIAG_KERNEL, normalization=dcrf.NORMALIZE_SYMMETRIC)
d.addPairwiseEnergy(pairwise_energy, compat=1, kernel=dcrf.DIAG_KERNEL, normalization=dcrf.NORMALIZE_SYMMETRIC) # `compat` is the "strength" of this potential.
Q = d.inference(5)
pred_maps = np.argmax(Q, axis=0).reshape((h,w))
return pred_maps
def dense_crf(t_img, t_output_probs):
"""dense crf적용
"""
res = []
for i in range(len(t_img)):
img = t_img[i].numpy()
img = np.uint8(255 * img)
output_probs = t_output_probs[i]
c = output_probs.shape[0]
h = output_probs.shape[1]
w = output_probs.shape[2]
U = utils.unary_from_softmax(output_probs)
U = np.ascontiguousarray(U)
img = np.ascontiguousarray(img)
d = dcrf.DenseCRF2D(w, h, c)
d.setUnaryEnergy(U)
d.addPairwiseGaussian(sxy=POS_XY_STD, compat=POS_W)
d.addPairwiseBilateral(sxy=Bi_XY_STD, srgb=Bi_RGB_STD, rgbim=img.reshape(512,512,3), compat=Bi_W)
Q = d.inference(MAX_ITER)
Q = np.array(Q).reshape((c, h, w))
res.append(Q)
return np.array(res)
def crf(original_image,
output,
nb_iterations=1,
sxy1=(3, 3),
sxy2=(80, 80),
compat=3,
srgb=(13, 13, 13)):
"""
Parameters explained https://github.com/lucasb-eyer/pydensecrf
Parameters
----------
original_image : H x W x RGB
[0:255]
output : C x H x W
float confidence of the network
Returns
-------
H x W
map of the selected labels, [0..C] where C is the number of classes
"""
import pydensecrf.densecrf as dcrf
from pydensecrf.utils import unary_from_softmax
original_image = original_image.astype(np.uint8)
# The output needs to be between 0 and 1
if np.max(output) > 1 or np.min(output) < 0:
output = softmax(output, axis=0)
# Make the array contiguous in memory
output = output.copy(order='C')
d = dcrf.DenseCRF2D(original_image.shape[1], original_image.shape[0],
output.shape[0])
U = unary_from_softmax(output)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=sxy1,
compat=compat,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
# im is an image-array, e.g. im.dtype == np.uint8 and im.shape == (640,480,3)
d.addPairwiseBilateral(sxy=sxy2,
srgb=srgb,
rgbim=original_image,
compat=compat * 3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(nb_iterations)
return np.argmax(Q, axis=0).reshape(original_image.shape[0],
original_image.shape[1])
def dense_crf(original_image, annotated_image, NUM_OF_CLASSESS, use_2d=True):
# Converting annotated image to RGB if it is Gray scale
print(original_image.shape, annotated_image.shape)
# Gives no of class labels in the annotated image
#n_labels = len(set(labels.flat))
n_labels = NUM_OF_CLASSESS
# Setting up the CRF model
d = dcrf.DenseCRF2D(original_image.shape[1], original_image.shape[0],
n_labels)
# get unary potentials (neg log probability)
processed_probabilities = annotated_image
softmax = processed_probabilities.transpose((2, 0, 1))
print(softmax.shape)
U = unary_from_softmax(softmax, scale=None, clip=1e-5)
U = np.ascontiguousarray(U)
#U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=False)
d.setUnaryEnergy(U)
# This potential penalizes small pieces of segmentation that are
# spatially isolated -- enforces more spatially consistent segmentations
feats = create_pairwise_gaussian(sdims=(3, 3),
shape=original_image.shape[:2])
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This creates the color-dependent features --
# because the segmentation that we get from CNN are too coarse
# and we can use local color features to refine them
feats = create_pairwise_bilateral(sdims=(80, 80),
schan=(13, 13, 13),
img=original_image,
chdim=2)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# Run Inference for 5 steps
Q = d.inference(5)
print(Q)
#print(">>>>>>>>Qshape: ", Q.shape)
# Find out the most probable class for each pixel.
output = np.argmax(Q, axis=0).reshape(
(original_image.shape[0], original_image.shape[1]))
print(output.shape)
plt.subplot(240 + 1)
plt.imshow(output, cmap=plt.get_cmap('nipy_spectral'))
plt.show()
return output
def refine_mask(rgbim, rawmask):
if len(rawmask.shape) == 2:
rawmask = rawmask[:, :, None]
mask_softmax = np.concatenate(
[cv2.bitwise_not(rawmask)[:, :, None], rawmask], axis=2)
mask_softmax = mask_softmax.astype(np.float32) / 255.0
n_classes = 2
feat_first = mask_softmax.transpose((2, 0, 1)).reshape((n_classes, -1))
unary = unary_from_softmax(feat_first)
unary = np.ascontiguousarray(unary)
d = dcrf.DenseCRF2D(rgbim.shape[1], rgbim.shape[0], n_classes)
d.setUnaryEnergy(unary)
d.addPairwiseGaussian(sxy=1,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NO_NORMALIZATION)
d.addPairwiseBilateral(sxy=23,
srgb=7,
rgbim=rgbim,
compat=20,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NO_NORMALIZATION)
Q = d.inference(5)
res = np.argmax(Q, axis=0).reshape((rgbim.shape[0], rgbim.shape[1]))
crf_mask = np.array(res * 255, dtype=np.uint8)
return crf_mask
def densecrf(logits):
"""
applies coditional random fields on predictions
The idea is consider the nbr voxels in making
class prediction of current pixel
refer CRF and MRF papers for more theoretical idea
args
logits: Nb_classes x Height x Width x Depth
returns
tensor of size Height x Width x Depth
"""
shape = logits.shape[1:]
new_image = np.empty(shape)
d = dcrf.DenseCRF(np.prod(shape), logits.shape[0])
U = unary_from_softmax(logits)
d.setUnaryEnergy(U)
feats = create_pairwise_gaussian(sdims=(1.0, 1.0, 1.0), shape=shape)
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
new_image = np.argmax(Q, axis=0).reshape((shape[0], shape[1], shape[2]))
return new_image
def crf(original, pred, steps=3, gauss_sxy=2, pairwise_sxy=40, rgb_sxy=11, pairwise_compat=20):
annotated = np.asarray([1 - pred, pred])
original = img_as_ubyte(original)
# annotated = np.moveaxis(annotated, -1, 0)
annotated = annotated.copy(order='C')
d = dcrf.DenseCRF2D(original.shape[1], original.shape[0], 2)
U = unary_from_softmax(annotated)
# print(U.shape)
d.setUnaryEnergy(U)
if isinstance(pairwise_compat, list):
pairwise_compat = np.asarray(pairwise_compat, dtype=np.float32)
d.addPairwiseGaussian(sxy=gauss_sxy, compat=20, kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
d.addPairwiseBilateral(sxy=pairwise_sxy, srgb=rgb_sxy, rgbim=original, compat=pairwise_compat,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(steps)
MAP = np.argmax(Q, axis=0).reshape(original.shape[0], original.shape[1])
return MAP
def crf(im_softmax, im_rgb):
n_classes = im_softmax.shape[2]
feat_first = im_softmax.transpose((2, 0, 1)).reshape(n_classes, -1)
unary = unary_from_softmax(feat_first)
unary = np.ascontiguousarray(unary)
im_rgb = np.ascontiguousarray(im_rgb)
d = dcrf.DenseCRF2D(im_rgb.shape[1], im_rgb.shape[0], n_classes)
d.setUnaryEnergy(unary)
d.addPairwiseGaussian(sxy=(5, 5),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
d.addPairwiseBilateral(sxy=(5, 5),
srgb=(13, 13, 13),
rgbim=im_rgb,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
res = np.argmax(Q, axis=0).reshape((im_rgb.shape[0], im_rgb.shape[1]))
if mode == 'binary':
return res * 255.0
if mode == 'multi':
res_hot = to_categorical(res) * 255.0
res_crf = add_masks(res_hot)
return res_crf
def pp_denseCRF(imag, y_pred,
pairwise_gauss=True, pairwise_bilateral=True,
pw_gauss_sdims=(10, 10), pw_gauss_compat=3,
pw_bilat_sdims=(20, 20), pw_bilat_schan=(0.005,),
pw_bilat_compat=10, inf_steps=5):
'''Dense CRF postprocessing using 2D image / softmax prediction matrix'''
height, width, channels = imag.shape
n_classes = y_pred.shape[-1]
# Set unary energy
d = dcrf.DenseCRF2D(width, height, n_classes)
U = unary_from_softmax(np.moveaxis(y_pred, -1, 0))
d.setUnaryEnergy(U)
# Create the (color)-independent features and add them to the CRF
if pairwise_gauss == True:
pw_gauss = create_pairwise_gaussian(sdims=pw_gauss_sdims,
shape=y_pred.shape[:2])
d.addPairwiseEnergy(pw_gauss, compat=pw_gauss_compat)
# Create the (color)-dependent features and add them to the CRF
if pairwise_bilateral == True:
pw_bilateral = create_pairwise_bilateral(sdims=pw_bilat_sdims,
schan=pw_bilat_schan,
img=imag, chdim=2)
d.addPairwiseEnergy(pw_bilateral, compat=pw_bilat_compat)
# Inference
Q = d.inference(inf_steps)
# Reshape eigen matrix and return prediction in original shape
pred_Q = np.reshape(Q, (n_classes, height, width))
pred_orig_shape = np.moveaxis(pred_Q, 0, -1)
return pred_orig_shape
def crf(sm_mask_one, img_one, num_class, input_size, mask_size, num_iter):
sm_u = np.transpose(
resize(np.transpose(sm_mask_one, [1, 2, 0]),
input_size,
mode='constant'), [2, 0, 1])
U = unary_from_softmax(sm_u)
d = dcrf.DenseCRF2D(input_size[0], input_size[1], num_class)
d.setUnaryEnergy(U)
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# d.addPairwiseBilateral(sxy=(30,30), srgb=(13,13,13), rgbim=img_one.astype(np.uint8), compat=20, kernel=dcrf.DIAG_KERNEL, normalization=dcrf.NORMALIZE_SYMMETRIC)
d.addPairwiseBilateral(sxy=(80, 80),
srgb=(13, 13, 13),
rgbim=img_one.astype(np.uint8),
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(num_iter)
return np.array(Q).reshape((num_class, input_size[0], input_size[1]))
def crf_inference(img, probs, t=10, scale_factor=1, labels=21):
"""
dense crf
- img: np_array [h,w,c]
- probs: prediction_score [c,h,w]
- t: number of iteration for inference
"""
h, w = img.shape[:2]
n_labels = labels
d = dcrf.DenseCRF2D(w, h, n_labels)
unary = unary_from_softmax(probs)
unary = np.ascontiguousarray(unary)
img_c = np.ascontiguousarray(img)
d.setUnaryEnergy(unary)
d.addPairwiseGaussian(sxy=4 / scale_factor, compat=3)
d.addPairwiseBilateral(sxy=20 / scale_factor,
srgb=3,
rgbim=np.copy(img_c),
compat=10)
Q = d.inference(t)
return np.array(Q).reshape((n_labels, h, w))
def CRF(img, probabilities, K):
#print("Probabilities shape : " , probabilities.shape)
processed_probabilities = probabilities.squeeze()
#print("Processed : " , processed_probabilities.shape)
softmax = processed_probabilities.transpose((2, 0, 1))
#print(softmax.shape)
unary = unary_from_softmax(softmax)
#print(unary.shape)
unary = np.ascontiguousarray(unary)
#print(unary.shape)
d = dcrf.DenseCRF(img.shape[0] * img.shape[1], K)
d.setUnaryEnergy(unary)
#d.addPairwiseGaussian(sxy=3, compat=3)
feats = create_pairwise_gaussian(sdims=(3, 3), shape=(img.shape[1], img.shape[0]))
#feats = create_pairwise_bilateral(sdims=(5, 5), schan=(10, 10, 10), img=img.reshape(img.shape[1], img.shape[0], 3), chdim=2)
#print("Feats : \n", feats)
d.addPairwiseEnergy(feats, compat=5, kernel=dcrf.DIAG_KERNEL, normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
res = np.argmax(Q, axis=0).reshape(img.shape[1], img.shape[0]).astype('float32')
res *= (255. / res.max())
res.reshape(img.shape[:2])
#print("Res \n", res)
return res
def perform_crf(image, probabilities):
image = image.squeeze()
softmax = probabilities.squeeze().transpose((2, 0, 1))
# The input should be the negative of the logarithm of probability values
# Look up the definition of the softmax_to_unary for more information
unary = unary_from_softmax(softmax)
# The inputs should be C-continious -- we are using Cython wrapper
unary = np.ascontiguousarray(unary)
d = dcrf.DenseCRF(image.shape[0] * image.shape[1], number_of_classes)
d.setUnaryEnergy(unary)
# This potential penalizes small pieces of segmentation that are
# spatially isolated -- enforces more spatially consistent segmentations
feats = create_pairwise_gaussian(sdims=(10, 10), shape=image.shape[:2])
d.addPairwiseEnergy(feats, compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This creates the color-dependent features --
# because the segmentation that we get from CNN are too coarse
# and we can use local color features to refine them
feats = create_pairwise_bilateral(sdims=(50, 50), schan=(20, 20, 20),
img=image, chdim=2)
d.addPairwiseEnergy(feats, compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
res = np.argmax(Q, axis=0).reshape((image.shape[0], image.shape[1]))
return res
