def dencecrf(x, y, nlabels, w1, w2, alpha, beta, gamma, iteration=5):
_x = np.array(x)
_y = np.array(y, np.float32)
assert len(_x.shape) == len(_y.shape), 'Shape of x and y should be (..., nchannels), (..., nlabels)'
assert _y.shape[-1] == nlabels, 'Expect y.shape (...,{}), got {}'.format(nlabels, _y.shape)
_y = _y.reshape((-1, nlabels))
_y = np.transpose(_y, (1,0))
d = dcrf.DenseCRF(np.prod(_x.shape[0:-1]), nlabels)
d.setUnaryEnergy(_y.copy(order='C'))
imgdim = len(_x.shape)-1
gamma = (gamma,) * imgdim
alpha = (alpha,) * imgdim
beta = (beta,) * _x.shape[-1]
featG = create_pairwise_gaussian(gamma, _x.shape[0:-1]) # g
featB = create_pairwise_bilateral(alpha, beta, _x, imgdim) #a b
d.addPairwiseEnergy(featG, w2) # w2
d.addPairwiseEnergy(featB, w1) # w1
out = d.inference(iteration)
out = np.transpose(out, (1,0))
out = np.reshape(out, y.shape)
return out
def get_crf_img(inputs, outputs):
for i in range(outputs.shape[0]):
img = inputs[i]
softmax_prob = outputs[i]
unary = unary_from_softmax(softmax_prob)
unary = np.ascontiguousarray(unary)
d = dcrf.DenseCRF(img.shape[0] * img.shape[1], 2)
d.setUnaryEnergy(unary)
feats = create_pairwise_gaussian(sdims=(10, 10), shape=img.shape[:2])
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
feats = create_pairwise_bilateral(sdims=(50, 50),
schan=(20, 20, 20),
img=img,
chdim=2)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
res = np.argmax(Q, axis=0).reshape((img.shape[0], img.shape[1]))
if i == 0:
crf = np.expand_dims(res, axis=0)
else:
res = np.expand_dims(res, axis=0)
crf = np.concatenate((crf, res), axis=0)
return crf
def process(final_probabilities,image,iters): softmax = final_probabilities.squeeze() print softmax.shape processed_probabilities = softmax.transpose((2, 0, 1)) # The input should be the negative of the logarithm of probability values # Look up the definition of the softmax_to_unary for more information unary = softmax_to_unary(processed_probabilities) # The inputs should be C-continious -- we are using Cython wrapper unary = np.ascontiguousarray(unary) d = dcrf.DenseCRF(image.shape[0] * image.shape[1], 2) d.setUnaryEnergy(unary) # This potential penalizes small pieces of segmentation that are # spatially isolated -- enforces more spatially consistent segmentations feats = create_pairwise_gaussian(sdims=(10, 10), shape=image.shape[:2]) d.addPairwiseEnergy(feats, compat=3, kernel=dcrf.DIAG_KERNEL, normalization=dcrf.NORMALIZE_SYMMETRIC) # This creates the color-dependent features -- # because the segmentation that we get from CNN are too coarse # and we can use local color features to refine them feats = create_pairwise_bilateral(sdims=(50, 50), schan=(20, 20, 20), img=image, chdim=2) d.addPairwiseEnergy(feats, compat=10, kernel=dcrf.DIAG_KERNEL, normalization=dcrf.NORMALIZE_SYMMETRIC) return d.inference(iters)
def test_complete_inference2():
dcrf = densecrf.DenseCRF(100, 2)
pcrf = pycrf.DenseCRF(100, 2)
unary = test_utils._get_simple_unary()
img = test_utils._get_simple_img()
dcrf.setUnaryEnergy(-np.log(unary))
pcrf.set_unary_energy(-np.log(unary))
feats = utils.create_pairwise_gaussian(sdims=(1.5, 1.5),
shape=img.shape[:2])
dcrf.addPairwiseEnergy(feats, compat=3)
pcrf.add_pairwise_energy(feats, compat=3)
feats = utils.create_pairwise_bilateral(sdims=(2, 2), schan=2,
img=img, chdim=2)
dcrf.addPairwiseEnergy(feats, compat=3)
pcrf.add_pairwise_energy(feats, compat=3)
# d.addPairwiseBilateral(2, 2, img, 3)
res1 = np.argmax(dcrf.inference(10), axis=0).reshape(10, 10)
res2 = np.argmax(pcrf.inference(10), axis=0).reshape(10, 10)
if False:
# Save the result for visual inspection
import scipy as scp
import scipy.misc
scp.misc.imsave("test.png", res1)
assert(np.all(res1 == res2))
def densecrf(logits):
"""
applies coditional random fields on predictions
The idea is consider the nbr voxels in making
class prediction of current pixel
refer CRF and MRF papers for more theoretical idea
args
logits: Nb_classes x Height x Width x Depth
returns
tensor of size Height x Width x Depth
"""
shape = logits.shape[1:]
new_image = np.empty(shape)
d = dcrf.DenseCRF(np.prod(shape), logits.shape[0])
U = unary_from_softmax(logits)
d.setUnaryEnergy(U)
feats = create_pairwise_gaussian(sdims=(1.0, 1.0, 1.0), shape=shape)
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
new_image = np.argmax(Q, axis=0).reshape((shape[0], shape[1], shape[2]))
return new_image
def CRF(x, image):
unary = unary_from_softmax(x) #C,H,W
unary = np.ascontiguousarray(unary)
d = dcrf.DenseCRF(image.shape[-2] * image.shape[-1], 10)
d.setUnaryEnergy(unary)
gau_feats = create_pairwise_gaussian(sdims=(3, 3), shape=image.shape[-2:])
d.addPairwiseEnergy(gau_feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
bil_feats = create_pairwise_bilateral(sdims=(10, 10),
schan=[5],
img=np.array(image).transpose(
1, 2, 0),
chdim=2)
d.addPairwiseEnergy(bil_feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
x = np.argmax(Q, axis=0).reshape(image.shape[-2], image.shape[-1])
Q = np.array(Q)
Q = Q.reshape(-1, image.shape[-2], image.shape[-1])
return Q, x
def CRF(self,
image,
softmax,
iterations=1,
sdims_g=(1, 1),
sdims_b=(3, 3),
schan=0.5):
softmax = softmax.transpose(2, 0, 1)
n_classes = softmax.shape[0]
unary = unary_from_softmax(softmax)
d = dcrf.DenseCRF(image.shape[1] * image.shape[0], n_classes)
d.setUnaryEnergy(unary)
feats = create_pairwise_gaussian(sdims=sdims_g, shape=image.shape[:2])
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This creates the color-dependent features and then add them to the CRF
feats = create_pairwise_bilateral(sdims=sdims_b,
schan=schan,
img=image,
chdim=2)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(iterations)
probabilities = np.array(Q).reshape((n_classes, image.shape[0], -1))
labels = np.argmax(Q, axis=0).reshape(image.shape[0:2])
return probabilities.transpose(1, 2, 0), labels
def crf_func(imgs, plabels):
outputs = []
for img, probs in zip(imgs, plabels):
# Example using the DenseCRF class and the util functions
d = dcrf.DenseCRF(img.shape[1] * img.shape[0], 2)
# get unary potentials (neg log probability)
U = unary_from_softmax(probs)
d.setUnaryEnergy(U)
# This creates the color-dependent features and then add them to the CRF
feats = create_pairwise_bilateral(sdims=(80, 80),
schan=(13, 13, 13),
img=img,
chdim=2)
d.addPairwiseEnergy(feats,
compat=1,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# Run five inference steps.
Q = d.inference(5)
# Find out the most probable class for each pixel.
# MAP = np.argmax(Q, axis=0)
# bb = 1-MAP.reshape(img.shape[:2])
bb = np.array(Q).reshape((2, img.shape[0], img.shape[1]))
outputs.append(bb)
outputs = np.array(outputs)
return outputs
def adjust_prediction(self, probability, image):
crf = dcrf.DenseCRF(np.prod(probability.shape), 2)
# crf = dcrf.DenseCRF(np.prod(probability.shape), 1)
binary_prob = np.stack((1 - probability, probability), axis=0)
unary = unary_from_softmax(binary_prob)
# unary = unary_from_softmax(np.expand_dims(probability, axis=0))
crf.setUnaryEnergy(unary)
# per dimension scale factors
sdims = [self.sdims] * 3
smooth = create_pairwise_gaussian(sdims=sdims, shape=probability.shape)
crf.addPairwiseEnergy(smooth, compat=2)
if self.schan:
# per channel scale factors
schan = [self.schan] * 6
appearance = create_pairwise_bilateral(sdims=sdims,
schan=schan,
img=image,
chdim=3)
crf.addPairwiseEnergy(appearance, compat=2)
result = crf.inference(self.iter)
crf_prediction = np.argmax(result, axis=0).reshape(
probability.shape).astype(np.float32)
return crf_prediction
def CRF(img, probabilities, K):
#print("Probabilities shape : " , probabilities.shape)
processed_probabilities = probabilities.squeeze()
#print("Processed : " , processed_probabilities.shape)
softmax = processed_probabilities.transpose((2, 0, 1))
#print(softmax.shape)
unary = unary_from_softmax(softmax)
#print(unary.shape)
unary = np.ascontiguousarray(unary)
#print(unary.shape)
d = dcrf.DenseCRF(img.shape[0] * img.shape[1], K)
d.setUnaryEnergy(unary)
#d.addPairwiseGaussian(sxy=3, compat=3)
feats = create_pairwise_gaussian(sdims=(3, 3), shape=(img.shape[1], img.shape[0]))
#feats = create_pairwise_bilateral(sdims=(5, 5), schan=(10, 10, 10), img=img.reshape(img.shape[1], img.shape[0], 3), chdim=2)
#print("Feats : \n", feats)
d.addPairwiseEnergy(feats, compat=5, kernel=dcrf.DIAG_KERNEL, normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
res = np.argmax(Q, axis=0).reshape(img.shape[1], img.shape[0]).astype('float32')
res *= (255. / res.max())
res.reshape(img.shape[:2])
#print("Res \n", res)
return res
def main():
print('Welcome to MIALab 2018!')
# --- scikit-learn
clf = sk_ensemble.RandomForestClassifier(max_depth=2, random_state=0)
# --- SimpleITK
image = sitk.Image(256, 128, 64, sitk.sitkInt16)
print('Image dimension:', image.GetDimension())
print('Voxel intensity before setting:', image.GetPixel(0, 0, 0))
image.SetPixel(0, 0, 0, 1)
print('Voxel intensity after setting:', image.GetPixel(0, 0, 0))
# --- numpy and matplotlib
array = np.array([1, 23, 2, 4])
plt.plot(array)
plt.ylabel('Some meaningful numbers')
plt.xlabel('The x-axis')
plt.title('Wohoo')
plt.show()
# --- pydensecrf
d = crf.DenseCRF(1000, 2)
# --- pymia
eval = pymia_eval.Evaluator()
print('Everything seems to work fine!')
def run_CRF(patient, sdimsB, sdimsG, schan):
path0 = path + 'patient_' + str(patient) +'_ProbMapClass0.nii.gz'
path1 = path + 'patient_' + str(patient) +'_ProbMapClass1.nii.gz'
ims, npoints, nlabels, affine = niiToNumpyCRFinput(path0, path1)
shape1 = ims.shape[1:]
name = 'patient_' + str(patient) + '_CRF'
d = dcrf.DenseCRF(npoints, nlabels) # npoints, nlabels
U = ims
shape = U.shape[1:]
# print (np.sum(U))
U = unary_from_softmax(ims)
d.setUnaryEnergy(U)
G = create_pairwise_gaussian(sdimsG, shape)
d.addPairwiseEnergy(w2*G, compat=compat1)
B = create_pairwise_bilateral(sdimsB, schan, ims, chdim=0)
d.addPairwiseEnergy(w1*B, compat=compat2)
# Q = d.inference(1)
Q, tmp1, tmp2 = d.startInference()
for i in range(5):
# print("KL-divergence at {}: {}".format(i, d.klDivergence(Q)))
d.stepInference(Q, tmp1, tmp2)
patientCRF = np.argmax(Q, axis=0).reshape(shape1).astype(np.float32)
#print (patientCRF.shape, patientCRF.dtype, np.sum(patientCRF))
# if patient == 48:
# im = nib.Nifti1Image(patientCRF, affine=affine)
# nib.save(im, os.path.join(CRFpath, name + '.nii.gz'))
# im1 = nib.load(CRFpath + name + '.nii.gz')
# print(im1.get_data().shape)
# print(im1.get_data().dtype)
# print(im1.header)
return patientCRF
def crf(self, input_vol, softmax_vol): # prob shape: classes, width, height, depth # vol shape: width, height, depth assert len(softmax_vol.shape) == 4 assert len(input_vol.shape) == 4 return_ = np.empty((input_vol.shape[1:])) for slice_idx in range(input_vol.shape[3]): image = input_vol[:, :, :, slice_idx] softmax = softmax_vol[:, :, :, slice_idx] # softmax = softmax.transpose(2, 0, 1) # image = image.transpose(2, 0, 1) n_classes = softmax.shape[0] unary = unary_from_softmax(softmax) d = dcrf.DenseCRF(image.shape[1]*image.shape[0], n_classes) d.setUnaryEnergy(unary) feats = create_pairwise_gaussian(sdims=(1., 1.), shape=image.shape[:2]) d.addPairwiseEnergy(feats, compat=3, kernel=dcrf.DIAG_KERNEL, normalization=dcrf.NORMALIZE_SYMMETRIC) # This creates the color-dependent features and then add them to the CRF feats = create_pairwise_bilateral(sdims=(3., 3.), schan=0.5, img=image, chdim=2) d.addPairwiseEnergy(feats, compat=10, kernel=dcrf.DIAG_KERNEL, normalization=dcrf.NORMALIZE_SYMMETRIC) Q = d.inference(iterations) probabilities = np.array(Q).reshape((n_classes,image.shape[0],-1)) labels = np.argmax(Q, axis=0).reshape(image.shape[0:2]) return_[:,:,slice_idx] = labels return return_
def get_crf_seg(img, labels, n_labels, opts):
'''
crf_out_final = get_crf_seg(img, labels, n_labels)
'''
crf = dcrf.DenseCRF(img.shape[1] * img.shape[0], n_labels)
U = unary_from_softmax(labels)
crf.setUnaryEnergy(U)
# pairwise positional (gaussian) terms
feats = create_pairwise_gaussian(sdims=(opts.gaussian_sx, opts.gaussian_sx), shape=img.shape[:2])
crf.addPairwiseEnergy(feats, compat=opts.crf_gaussian_weight,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# pairwise bilateral (color + position) terms
feats = create_pairwise_bilateral(sdims=(opts.bilateral_sx, opts.bilateral_sx), \
schan=(opts.bilateral_color, opts.bilateral_color, opts.bilateral_color),
img=img, chdim=2)
crf.addPairwiseEnergy(feats, compat=opts.crf_bilateral_weight,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# run inference
Q = crf.inference(5)
return Q
def compute_lattice(self, img, num_classes=None):
if num_classes is not None:
self.num_classes = num_classes
assert self.num_classes is not None
npixels = self.shape[0] * self.shape[1]
crf = dcrf.DenseCRF(npixels, self.num_classes)
sdims = self.conf['pos_feats']['sdims']
feats = utils.create_pairwise_gaussian(sdims=(sdims, sdims),
shape=img.shape[:2])
self.smooth_feats = feats
self.crf = crf
self.crf.addPairwiseEnergy(self.smooth_feats,
compat=self.conf['pos_feats']['compat'])
sdims = self.conf['col_feats']['sdims']
schan = self.conf['col_feats']['schan']
feats = utils.create_pairwise_bilateral(sdims=(sdims, sdims),
schan=(schan, schan, schan),
img=img,
chdim=2)
self.appear_feats = feats
self.crf.addPairwiseEnergy(self.appear_feats,
compat=self.conf['pos_feats']['compat'])
def __call__(self, image: np.ndarray, keypoints: np.ndarray) -> np.ndarray:
if not imageutils.is_in_range(image, [0, 255]):
raise imageutils.RangeError(image, "image", [0, 255])
dynamic_range = image.max() / image.min()
is_low_range = dynamic_range <= 2.0
if is_low_range:
import warnings
warnings.warn(
Warning(
"image has low dynamic range. Maybe convert to [0, 255]?"))
h, w, c = image.shape
keypoint_probs = imageutils.keypoints_to_heatmaps((h, w),
keypoints,
var=self.var)
keypoint_probs = np.rollaxis(keypoint_probs, 2, 0)
bg_prob = 1.0 - np.amax(keypoint_probs, axis=0, keepdims=True)
probs = np.concatenate([bg_prob, keypoint_probs], axis=0)
n_labels = probs.shape[0]
probs_flat = np.reshape(probs, (n_labels, -1))
d = dcrf.DenseCRF(h * w, n_labels)
# flatten everything for dense crf
# Set unary according to keypoints
U = -np.log(probs_flat + 1.0e-6).astype(np.float32)
d.setUnaryEnergy(U.copy(order="C"))
# This creates the color-independent features and then add them to the CRF
feats = create_pairwise_gaussian(sdims=(3, 3), shape=image.shape[:2])
d.addPairwiseEnergy(
feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC,
)
# This creates the color-dependent features and then add them to the CRF
feats = create_pairwise_bilateral(sdims=(80, 80),
schan=(13, 13, 13),
img=image,
chdim=2)
d.addPairwiseEnergy(
feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC,
)
# Run five inference steps.
Q = d.inference(self.n_steps)
# Find out the most probable class for each pixel.
MAP = np.argmax(Q, axis=0)
MAP = MAP.reshape((h, w))
return MAP
def calculate_crf_on_labels(frame_path: Union[str, np.ndarray], detection_path: Union[str, np.ndarray]) -> np.ndarray:
# if path of images is given
if isinstance(frame_path, str):
img = cv2.imread(frame_path)
else:
img = frame_path
# if probability path is given
if isinstance(detection_path, str):
anno_rgb = cv2.imread(detection_path)
else:
anno_rgb = detection_path
# convert to grayscale if it has 3 channels
if len(anno_rgb.shape) == 3 and anno_rgb.shape[2] == 3:
anno_rgb = cv2.cvtColor(anno_rgb, cv2.COLOR_BGR2GRAY)
# expanding labels a little bit to accommodate any missing area of the object
anno_rgb = expand_labels(anno_rgb, label_extra_pixels)
anno_rgb[anno_rgb > 0] = 1
anno_rgb = anno_rgb.astype(np.uint32)
n_labels = 2
labels = np.zeros((n_labels, img.shape[0], img.shape[1]), dtype=np.uint8)
labels[0, :, :] = 1 - anno_rgb
labels[1, :, :] = anno_rgb
colors = [0, 255]
colorize = np.empty((len(colors), 1), np.uint8)
colorize[:, 0] = colors
crf = dcrf.DenseCRF(img.shape[1] * img.shape[0], n_labels)
U = unary_from_labels(labels[1], n_labels, gt_prob=0.7, zero_unsure=False)
crf.setUnaryEnergy(U)
# feats = create_pairwise_gaussian(sdims=(2, 2), shape=img.shape[:2])
# crf.addPairwiseEnergy(feats, compat=10,
# kernel=dcrf.FULL_KERNEL,
# normalization=dcrf.NORMALIZE_SYMMETRIC)
# try different sdims 64*64 or 100*100 or larger on one single image and check which pramter gives better results
feats = create_pairwise_bilateral(sdims=(10, 10), schan=(13, 13, 13),
img=img, chdim=2)
crf.addPairwiseEnergy(feats, compat=10,
kernel=dcrf.FULL_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = crf.inference(20)
MAP = np.argmax(Q, axis=0)
MAP = colorize[MAP]
output_image = MAP.reshape(anno_rgb.shape)
return output_image
def crfInference(imageGT, fgMask, probs):
##################################
### Read images and annotation ###
##################################
# img = np.uint8(img*255)
img = skimage.color.rgb2lab(imageGT)
M = 3 # forground, background, occluding object.
###########################
### Setup the CRF model ###
###########################
# Example using the DenseCRF class and the util functions
crfmodel = dcrf.DenseCRF(img.shape[0] * img.shape[1], M)
# get unary potentials (neg log probability)
# U = compute_unary(labels, M)
U = unaryOcclusionModel(img, fgMask, probs)
U = superpixelUnary(imageGT, U, fgMask, 0.8)
crfmodel.setUnaryEnergy(U)
# This creates the color-independent features and then add them to the CRF
feats = create_pairwise_gaussian(sdims=(3, 3), shape=img.shape[:2])
crfmodel.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This creates the color-dependent features and then add them to the CRF
feats = create_pairwise_bilateral(sdims=(30, 30),
schan=(13, 13, 13),
img=img,
chdim=2)
crfmodel.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
####################################
### Do inference and compute map ###
####################################
Q = crfmodel.inference(5)
mapseg = np.argmax(Q, axis=0).reshape(img.shape[:2])
# res = map.astype('float32') * 255 / map.max()
# plt.imshow(res)
# plt.show()
# # Manually inference
# Q, tmp1, tmp2 = crfmodel.startInference()
# for i in range(5):
# print("KL-divergence at {}: {}".format(i, crfmodel.klDivergence(Q)))
# crfmodel.stepInference(Q, tmp1, tmp2)
return mapseg, np.array(Q)
def crf(img, prob):
func_start = time.time()
img = np.swapaxes(img, 0, 2)
# img.shape: (width, height, num of channels)
num_iter = 5
prob = np.swapaxes(prob, 1, 2) # shape: (1, width, height)
# preprocess prob to (num_classes, width, height) since we have 2 classes: car and background.
num_classes = 2
probs = np.tile(prob, (num_classes, 1, 1)) # shape: (2, width, height)
probs[0] = np.subtract(1, prob) # class 0 is background
probs[1] = prob # class 1 is car
d = dcrf.DenseCRF(img.shape[0] * img.shape[1], num_classes)
unary = unary_from_softmax(probs) # shape: (num_classes, width * height)
unary = np.ascontiguousarray(unary)
d.setUnaryEnergy(unary)
# This potential penalizes small pieces of segmentation that are
# spatially isolated -- enforces more spatially consistent segmentations
feats = create_pairwise_gaussian(sdims=(3, 3), shape=img.shape[:2])
d.addPairwiseEnergy(feats, compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# Note that this potential is not dependent on the image itself.
# This creates the color-dependent features --
# because the segmentation that we get from CNN are too coarse
# and we can use local color features to refine them
feats = create_pairwise_bilateral(sdims=(80, 80), schan=(13, 13, 13),
img=img, chdim=2)
d.addPairwiseEnergy(feats, compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(num_iter) # set the number of iterations
res = np.argmax(Q, axis=0).reshape((img.shape[0], img.shape[1]))
# res.shape: (width, height)
res = np.swapaxes(res, 0, 1) # res.shape: (height, width)
res = res[np.newaxis, :, :] # res.shape: (1, height, width)
func_end = time.time()
# print('{:.2f} sec spent on CRF with {} iterations'.format(func_end - func_start, num_iter))
# about 2 sec for a 1280 * 960 image with 5 iterations
return res
def postprocessing_pydensecrf(logits):
# probs of shape 3d image per class: Nb_classes x Height x Width x Depth
shape = logits.shape[1:]
new_image = np.empty(shape)
d = dcrf.DenseCRF(np.prod(shape), logits.shape[0])
U = unary_from_softmax(logits)
d.setUnaryEnergy(U)
feats = create_pairwise_gaussian(sdims=(1.0, 1.0, 1.0), shape=shape)
d.addPairwiseEnergy(feats, compat=3, kernel=dcrf.DIAG_KERNEL, normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
new_image = np.argmax(Q, axis=0).reshape((shape[0], shape[1],shape[2]))
return new_image
def test_unary_inference():
dcrf = densecrf.DenseCRF(100, 2)
pcrf = pycrf.DenseCRF(100, 2)
unary = test_utils._get_simple_unary()
dcrf.setUnaryEnergy(-np.log(unary))
pcrf.set_unary_energy(-np.log(unary))
cresult = dcrf.inference(5)
pyresult = pcrf.inference(5)
assert(np.all(np.array(cresult) == pyresult))
def crf(train_image, final_probabilities, train_annotation, number_class):
for index_image in xrange(1):
image = train_image
softmax = final_probabilities[0].squeeze()
#softmax_to_unary
softmax = softmax.transpose((2, 0, 1))
unary = unary_from_softmax(softmax)
#softmax_to_unary
unary = np.ascontiguousarray(unary)
d = dcrf.DenseCRF(image.shape[0] * image.shape[1], number_class)
d.setUnaryEnergy(unary)
# This potential penalizes small pieces of segmentation that are
# spatially isolated -- enforces more spatially consistent segmentations
feats = create_pairwise_gaussian(sdims=(10, 10), shape=image.shape[:2])
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This creates the color-dependent features --
# because the segmentation that we get from CNN are too coarse
# and we can use local color features to refine them
feats = create_pairwise_bilateral(sdims=(50, 50),
schan=(20, 20, 20),
img=image,
chdim=2)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
res = np.argmax(Q, axis=0).reshape((image.shape[0], image.shape[1]))
cmap = plt.get_cmap('bwr')
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True)
ax1.imshow(res, vmax=1.5, vmin=-0.4, cmap=cmap)
ax1.set_title('Segmentation with CRF post-processing')
probability_graph = ax2.imshow(
np.dstack((train_annotation, ) * 3) * 100)
ax2.set_title('Ground-Truth Annotation')
plt.savefig('annotation_%d.png' % index_image,
bbox_inches='tight',
pad_inches=0)
plt.gcf().clear()
plt.show()
def denseCRF_refine(imgName, annNum):
# Get arguments
#imgName = '2007_001526' #sys.argv[1]
#annNum = 5 #int(sys.argv[2])
# read images and probability map
img = mpimg.imread(IMG_DIR+imgName+'.jpg')
annImg = []
for k in range(annNum):
im = mpimg.imread(ANN_DIR+imgName+'_'+str(k)+'.png')
annImg.append(im[..., 0])
annImg = np.array(annImg)
# get label
labels = np.argmax(annImg, axis=0)
proImg = np.max(annImg, axis=0)
labels = (labels+1)*(proImg > 0.001)+1
# mpimg.imsave(imgName+'_out.png', labels)
# dence crf inference
annImg = annImg/max(np.max(annImg), 1e-10)
bgImg = (1 - np.max(annImg, axis=0))*0.05
U = np.concatenate((bgImg[np.newaxis, ...], annImg), axis=0)
U = -np.log(U+1e-10)
U = U.reshape([U.shape[0], U.shape[1]*U.shape[2]])
n_labels = annNum+1
d = dcrf.DenseCRF(img.shape[1]*img.shape[0], n_labels)
#U2 = unary_from_labels(labels, n_labels, gt_prob=0.5, zero_unsure=True)
#U = U*0.5 + U2*0.5
d.setUnaryEnergy(U)
feats = create_pairwise_gaussian(sdims=(9, 9), shape=img.shape[:2])
d.addPairwiseEnergy(feats, compat=3, kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
feats = create_pairwise_bilateral(sdims=(40, 40), schan=(17, 17, 17), img=img, chdim=2)
d.addPairwiseEnergy(feats, compat=10, kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(10)
fineLabel = np.argmax(Q, axis=0)
mpimg.imsave(ANN_DIR+imgName+'_out_fine.png',fineLabel.reshape(labels.shape[0:2]))
return fineLabel.reshape(labels.shape[0:2]), (labels-1)
def test(filename, save_name):
MEAN_VALUES = np.array([123.68, 116.78, 103.94])
MEAN_VALUES = MEAN_VALUES.reshape((1, 1, 1, 3))
image = scipy.misc.imread(filename, mode='RGB')
image = scipy.misc.imresize(image, (HEIGHT, WIDTH))
h, w, d = img.shape
timg = np.reshape(image, (1, h, w, 3)) - MEAN_VALUES
with tf.Session() as sess:
images = tf.placeholder(tf.float32, shape=[1, h, w, d])
genered = tf.nn.softmax(tf.squeeze(segMnet(images, multiplier),
axis=0))
saver = tf.train.Saver(tf.global_variables())
model_file = tf.train.latest_checkpoint(MODEL_SAVE_PATH)
if model_file:
saver.restore(sess, model_file)
else:
raise Exception('Testing needs pre-trained model!')
feed_dict = {images: timg}
start = time.time()
result = sess.run(genered, feed_dict=feed_dict)
end = time.time()
print("cost time:%f" % (end - start))
unary = unary_from_softmax(result.transpose((2, 0, 1)))
unary = np.ascontiguousarray(unary)
d = dcrf.DenseCRF(h * w, 2)
d.setUnaryEnergy(unary)
feats = create_pairwise_gaussian(sdims=(10, 10), shape=image.shape[:2])
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
feats = create_pairwise_bilateral(sdims=(50, 50),
schan=(20, 20, 20),
img=image,
chdim=2)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(50)
MAP = np.argmax(Q, axis=0).reshape(h, w)
img = np.zeros((h, w, 4), dtype=np.int)
img[:, :, 0:3] = image
img[:, :, 3] = MAP * 255
scipy.misc.imsave(save_name, img)
def apply_crf_on_output(img, out, n_classes):
"""
Given an image and its segmentation, applies a Conditional Random
Field (CRF) on the segmented image.
:param img: default image
:param out: segmented image
:param n_classes: number of classes in the segmentation
:return: `out` with a CRF applied on it
"""
# Remove single-dimensional entries
out = out.squeeze()
# Put the last dimension in the first place (otherwise we can't use unary_from_softmax)
out = out.transpose((2, 0, 1))
# Get unary potentials (neg log probability)
unary = unary_from_softmax(out)
# The inputs should be C-continuous -- we are using Cython wrapper
unary = np.ascontiguousarray(unary)
d = dcrf.DenseCRF(img.shape[0] * img.shape[1], n_classes)
d.setUnaryEnergy(unary)
# This potential penalizes small pieces of segmentation that are
# spatially isolated -- enforces more spatially consistent segmentations
# It creates the color-independent features and then adds them to the CRF
feats = create_pairwise_gaussian(sdims=(3, 3), shape=img.shape[:2])
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This creates the color-dependent features --
# because the segmentation that we get from CNN are too coarse
# and we can use local color features to refine them
feats = create_pairwise_bilateral(sdims=(15, 15),
schan=(20, 20, 20),
img=img,
chdim=2)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
q = d.inference(10)
res = np.argmax(q, axis=0).reshape((img.shape[0], img.shape[1]))
return res
def get_prediction(pred_after_cnn, image):
unary = pred_after_cnn.transpose((2, 0, 1))
unary = -unary.reshape((unary.shape[0],-1))
unary = np.ascontiguousarray(unary).astype(np.float32)
d = dcrf.DenseCRF(image.shape[0] * image.shape[1], pred_after_cnn.shape[2])
d.setUnaryEnergy(unary)
feats = create_pairwise_gaussian(sdims=(10, 10), shape=image.shape[:2])
d.addPairwiseEnergy(feats, compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
feats = create_pairwise_bilateral(sdims=(50, 50), schan=(20, 20, 20),
img=image, chdim=2)
d.addPairwiseEnergy(feats, compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
res = np.argmax(Q, axis=0).reshape((image.shape[0], image.shape[1]))
return res
# def get_prediction(pred_after_cnn, image):
# d = dcrf.DenseCRF2D(pred_after_cnn.shape[1], pred_after_cnn.shape[0], pred_after_cnn.shape[2])
#
# U = pred_after_cnn.transpose((2, 0, 1))
# U = -U.reshape((U.shape[0],-1))
#
# U = np.ascontiguousarray(U).astype(np.float32)
#
# d.setUnaryEnergy(U)
#
# d.addPairwiseGaussian(sxy=3, compat=3)
# d.addPairwiseBilateral(sxy=80, srgb=13, rgbim=image, compat=10)
#
# Q = d.inference(5)
#
# map = np.argmax(Q, axis=0).reshape(image.shape[0:2])
#
# return map
def test_call_dcrf():
d = dcrf.DenseCRF(100, 2)
unary = _get_simple_unary()
img = _get_simple_img()
d.setUnaryEnergy(-np.log(unary))
# d.setUnaryEnergy(PyConstUnary(-np.log(Up)))
feats = utils.create_pairwise_bilateral(sdims=(2, 2), schan=2,
img=img, chdim=2)
d.addPairwiseEnergy(feats, compat=3)
# d.addPairwiseBilateral(2, 2, img, 3)
np.argmax(d.inference(10), axis=0).reshape(10, 10)
def predict_image(self, image):
# image_flat_shape = image.shape[0] * image.shape[1]
image = image.astype(np.float32)
image -= NP_IMAGENET_MEAN
prob = self.sess.run(self.prob,
feed_dict={
self.x: np.expand_dims(image, axis=0),
self.keep_prob: 1.0
})
processed_probabilities = prob.squeeze()
softmax = processed_probabilities.transpose((2, 0, 1))
unary = unary_from_softmax(softmax)
unary = np.ascontiguousarray(unary)
d = dcrf.DenseCRF(image.shape[0] * image.shape[1], 22)
d.setUnaryEnergy(unary)
# This potential penalizes small pieces of segmentation that are
# spatially isolated -- enforces more spatially consistent segmentations
feats = create_pairwise_gaussian(sdims=(10, 10), shape=image.shape[:2])
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This creates the color-dependent features --
# because the segmentation that we get from CNN are too coarse
# and we can use local color features to refine them
feats = create_pairwise_bilateral(sdims=(50, 50),
schan=(20, 20, 20),
img=image,
chdim=2)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
res = np.argmax(Q, axis=0).reshape((image.shape[0], image.shape[1]))
return res
def im_label_crf(image_path,softmax_prob):
#add image path as argument here
image = image_path
#give softmax probabilities as argument
#softmax = final_probabilities.squeeze()
#softmax = processed_probabilities.transpose((2, 0, 1))
# The input should be the negative of the logarithm of probability values
# Look up the definition of the softmax_to_unary for more information
unary = unary_from_softmax(softmax_prob)
#figure out a way to add partial gt information here
#one trick could be making prob 1 of those of gts in softmax for GT labels
# The inputs should be C-continious -- we are using Cython wrapper
unary = np.ascontiguousarray(unary)
d = dcrf.DenseCRF(image.shape[0] * image.shape[1], 19)
d.setUnaryEnergy(unary)
# This potential penalizes small pieces of segmentation that are
# spatially isolated -- enforces more spatially consistent segmentations
feats = create_pairwise_gaussian(sdims=(10, 10), shape=image.shape[:2])
d.addPairwiseEnergy(feats, compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This creates the color-dependent features --
# because the segmentation that we get from CNN are too coarse
# and we can use local color features to refine them
feats = create_pairwise_bilateral(sdims=(50, 50), schan=(20, 20, 20),
img=image, chdim=2)
d.addPairwiseEnergy(feats, compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
res = np.argmax(Q, axis=0).reshape((image.shape[0], image.shape[1]))
def apply_crf(input_image, input_prob, theta_a, theta_b, theta_r, mu1, mu2):
n_slices = input_image.shape[2]
output = np.zeros(input_image.shape)
for slice_id in range(n_slices):
image = input_image[:, :, slice_id]
prob = input_prob[:, :, slice_id, :]
n_pixel = image.shape[0] * image.shape[1]
n_class = prob.shape[-1]
P = np.transpose(prob, axes=(2, 0, 1))
# Setup the CRF model
d = dcrf.DenseCRF(n_pixel, n_class)
# Set unary potentials (negative log probability)
U = -np.log(P + 1e-10)
U = np.ascontiguousarray(U.reshape((n_class, n_pixel)))
d.setUnaryEnergy(U)
# Set edge potential
# This creates the color-dependent features and then add them to the CRF
feats = create_pairwise_bilateral(sdims=(theta_a, theta_a),
schan=(theta_b, ),
img=image,
chdim=-1)
d.addPairwiseEnergy(feats,
compat=mu1,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This creates the color-independent features and then add them to the CRF
feats = create_pairwise_gaussian(sdims=(theta_r, theta_r),
shape=image.shape)
d.addPairwiseEnergy(feats,
compat=mu2,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# Perform the inference
Q = d.inference(5)
res = np.argmax(Q, axis=0).astype('float32')
res = np.reshape(res, image.shape).astype(dtype='int8')
output[:, :, slice_id] = res
return output
