def crf(original_image, mask_img):
# Converting annotated image to RGB if it is Gray scale
if(len(mask_img.shape)<3):
mask_img = gray2rgb(mask_img)
#Converting the annotations RGB color to single 32 bit integer
annotated_label = mask_img[:,:,0] + (mask_img[:,:,1]<<8) + (mask_img[:,:,2]<<16)
# Convert the 32bit integer color to 0,1, 2, ... labels.
colors, labels = np.unique(annotated_label, return_inverse=True)
n_labels = 2
#Setting up the CRF model
d = dcrf.DenseCRF2D(original_image.shape[1], original_image.shape[0], n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=False)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3, 3), compat=3, kernel=dcrf.DIAG_KERNEL,normalization=dcrf.NORMALIZE_SYMMETRIC)
#Run Inference for 10 steps
Q = d.inference(10)
# Find out the most probable class for each pixel.
MAP = np.argmax(Q, axis=0)
return MAP.reshape((original_image.shape[0],original_image.shape[1]))
def crf_label(image, annotation, t=5, n_label=2, a=0.05, b=0.25):
image = np.ascontiguousarray(image)
h, w = image.shape[:2]
annotation = np.squeeze(np.array(annotation))
a, b = (a * 255, b * 255) if np.max(annotation) > 1 else (a, b)
label_extend = np.zeros_like(annotation, dtype=np.int)
label_extend[annotation >= b] = 2
label_extend[annotation <= a] = 1
_, label = np.unique(label_extend, return_inverse=True)
d = dcrf.DenseCRF2D(w, h, n_label)
u = unary_from_labels(label, n_label, gt_prob=0.7, zero_unsure=True)
u = np.ascontiguousarray(u)
d.setUnaryEnergy(u)
d.addPairwiseGaussian(sxy=(3, 3), compat=3)
d.addPairwiseBilateral(sxy=(80, 80),
srgb=(13, 13, 13),
rgbim=np.copy(image),
compat=10)
q = d.inference(t)
map_result = np.argmax(q, axis=0)
result = map_result.reshape((h, w))
return result
def getCRF_justcol(img, Lc, theta, n_iter, label_lines, compat_col=40, scale=5, prob=0.5):
if np.ndim(img)==2:
img = np.dstack((img, img, img))
H = img.shape[0]
W = img.shape[1]
d = dcrf.DenseCRF2D(H, W, len(label_lines)+1)
U = unary_from_labels(Lc.astype('int'), len(label_lines)+1, gt_prob= prob)
d.setUnaryEnergy(U)
del U
# sdims = The scaling factors per dimension.
# schan = The scaling factors per channel in the image.
# This creates the color-dependent features and then add them to the CRF
feats = create_pairwise_bilateral(sdims=(theta, theta), schan=(scale, scale, scale), #11,11,11
img=img, chdim=2)
del img
d.addPairwiseEnergy(feats, compat=compat_col,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
del feats
Q = d.inference(n_iter)
preds = np.array(Q, dtype=np.float32).reshape(
(len(label_lines)+1, H, W)).transpose(1, 2, 0)
preds = np.expand_dims(preds, 0)
preds = np.squeeze(preds)
return np.argmax(Q, axis=0).reshape((H, W)), preds #, p, R, np.abs(d.klDivergence(Q)/ (H*W))
def seed_dense_crf(img, labels):
print('using --------------------- post crf')
h , w = labels.shape
#n_labels = len(np.unique(labels))
n_labels = 21
# Example using the DenseCRF2D code
d = dcrf.DenseCRF2D(w , h , n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=False)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only .
d.addPairwiseGaussian(sxy=3, compat=10, kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
d.addPairwiseBilateral(sxy=20, srgb=3, rgbim=img,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(10)
Q = np.argmax(np.array(Q), axis=0).reshape((h, w))
return Q
def crf(original_image, mask_img):
# Converting annotated image to RGB if it is Gray scale
if (len(mask_img.shape) < 3):
mask_img = gray2rgb(mask_img)
# #Converting the annotations RGB color to single 32 bit integer
annotated_label = mask_img[:, :, 0] + (mask_img[:, :, 1] << 8) + (
mask_img[:, :, 2] << 16)
# # Convert the 32bit integer color to 0,1, 2, ... labels.
colors, labels = np.unique(annotated_label, return_inverse=True)
n_labels = 2
#Setting up the CRF model
d = dcrf.DenseCRF2D(original_image.shape[1], original_image.shape[0],
n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=False)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
#Run Inference for 10 steps
Q = d.inference(10)
# Find out the most probable class for each pixel.
MAP = np.argmax(Q, axis=0)
return MAP.reshape((original_image.shape[0], original_image.shape[1]))
fn_im = os.path.join(img_dir, f)
fn_anno = os.path.join(anno_dir, f)
fn_output = os.path.join(output_dir, f)
# Read images and annotation
img = load_image(fn_im, 400, 600)
mask = load_mask(fn_anno, img.shape)
assert img.shape[:2] == mask.shape
labels = mask.flatten()
n_labels = 2
# Setup the CRF model
d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], n_labels)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=False)
d.setUnaryEnergy(U)
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
d.addPairwiseBilateral(sxy=(80, 80),
srgb=(13, 13, 13),
rgbim=img,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# Do inference and compute MAP
Q = d.inference(1)
MAP = np.argmax(Q, axis=0)
print(n_labels, " labels", (" plus \"unknown\" 0: " if HAS_UNK else ""),
set(labels.flat))
###########################
### Setup the CRF model ###
###########################
use_2d = False
# use_2d = True
if use_2d:
print("Using 2D specialized functions")
# Example using the DenseCRF2D code
d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=HAS_UNK)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
d.addPairwiseBilateral(sxy=(80, 80),
srgb=(13, 13, 13),
rgbim=img,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
def crf(fn_im,fn_anno,fn_output,colorful_fn_output):
##################################
### Read images and annotation ###
##################################
img = imread(fn_im)
#truth_img = imread(truth_image).astype(np.uint8)
# Convert the annotation's RGB color to a single 32-bit integer color 0xBBGGRR
anno_rgb = imread(fn_anno)
#anno_lbl = anno_rgb[:,:,0] + (anno_rgb[:,:,1] << 8) + (anno_rgb[:,:,2] << 16)
anno_lbl = anno_rgb
# Convert the 32bit integer color to 1, 2, ... labels.
# Note that all-black, i.e. the value 0 for background will stay 0.
colors, labels = np.unique(anno_lbl, return_inverse=True)
# And create a mapping back from the labels to 32bit integer colors.
# But remove the all-0 black, that won't exist in the MAP!
colors = colors[1:]
colorize = np.empty((len(colors), 1), np.uint8)
colorize[:,0] = (colors & 0x0000FF)
#colorize[:,1] = (colors & 0x00FF00) >> 8
#colorize[:,2] = (colors & 0xFF0000) >> 16
# Compute the number of classes in the label image.
# We subtract one because the number shouldn't include the value 0 which stands
# for "unknown" or "unsure".
n_labels = len(set(labels.flat)) - 1
print(n_labels, " labels and \"unknown\" 0: ", set(labels.flat))
###########################
### Setup the CRF model ###
###########################
use_2d = False
# use_2d = True
if use_2d:
print("Using 2D specialized functions")
# Example using the DenseCRF2D code
if n_labels==1:
n_labels+=1
d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=False)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3, 3), compat=3, kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
d.addPairwiseBilateral(sxy=(80, 80), srgb=(13, 13, 13), rgbim=img,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
else:
print("Using generic 2D functions")
# Example using the DenseCRF class and the util functions
d = dcrf.DenseCRF(img.shape[1] * img.shape[0], n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=True)
d.setUnaryEnergy(U)
# This creates the color-independent features and then add them to the CRF
feats = create_pairwise_gaussian(sdims=(3, 3), shape=img.shape[:2])
d.addPairwiseEnergy(feats, compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This creates the color-dependent features and then add them to the CRF
feats = create_pairwise_bilateral(sdims=(80, 80), schan=(13, 13, 13),
img=img, chdim=2)
d.addPairwiseEnergy(feats, compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
####################################
### Do inference and compute MAP ###
####################################
# Run five inference steps.
Q = d.inference(10)
# Find out the most probable class for each pixel.
MAP = np.argmax(Q, axis=0)
# Convert the MAP (labels) back to the corresponding colors and save the image.
MAP = colorize[MAP,:]
####################################
### Convert to greyscale ###
####################################
crf_img = MAP.reshape(anno_lbl.shape)
########change to rgb########
label = anno_lbl
ind = crf_img
r = ind.copy()
g = ind.copy()
b = ind.copy()
r_gt = label.copy()
g_gt = label.copy()
b_gt = label.copy()
wall = [139,181,248]
building = [251,209,244]
sky = [44,230,121]
floor = [156,40,149]
tree = [166,219,98]
ceiling = [35,229,138]
road = [143,56,194]
bed = [144,223,70]
windowpane = [200,162,57]
grass = [120,225,199]
cabinet = [87,203,13]
sidewalk = [185,1,136]
person = [16,167,16]
earth = [29,249,241]
door = [17,192,40]
table = [199,44,241]
mountain = [193,196,159]
plant = [241,172,78]
curtain = [56,94,128]
chair = [231,166,116]
car = [50,209,252]
water = [217,56,227]
painting = [168,198,178]
sofa = [77,179,188]
shelf = [236,191,103]
house = [248,138,151]
sea = [214,251,89]
mirror = [208,204,187]
rug = [115,104,49]
field = [29,202,113]
armchair = [159,160,95]
seat = [78,188,13]
fence = [83,203,82]
desk = [8,234,116]
rock = [80,159,200]
wardrobe = [124,194,2]
lamp = [192,146,237]
bathtub = [64,3,73]
railing = [17,213,58]
cushion = [106,54,105]
base = [125,72,155]
box = [202,36,231]
column = [79,144,4]
signboard = [118,185,128]
chest = [138,61,178]
counter = [23,182,182]
sand = [154,114,4]
sink = [201,0,83]
skyscraper = [21,134,53]
fireplace = [194,77,237]
refrigerator = [198,81,106]
grandstand = [37,222,181]
path = [203,185,14]
stairs = [134,140,113]
runway = [220,196,79]
case = [64,26,68]
pooltable = [128,89,2]
pillow = [199,228,65]
screen = [62,215,111]
stairway = [124,148,166]
river = [221,119,245]
bridge = [68,57,158]
bookcase = [80,47,26]
blind = [143,59,56]
coffeetable = [14,80,215]
toilet = [212,132,31]
flower = [2,234,129]
book = [134,179,44]
hill = [53,21,129]
bench = [80,176,236]
countertop = [154,39,168]
stove = [221,44,139]
palm = [103,56,185]
kitchenisland = [224,138,83]
computer = [243,93,235]
swivelchair = [80,158,63]
boat = [81,229,38]
bar = [116,215,38]
arcademachine = [103,69,182]
hovel = [66,81,5]
bus = [96,157,229]
towel = [164,49,170]
light = [14,42,146]
truck = [164,67,44]
tower = [108,116,151]
chandelier = [144,8,144]
awning = [85,68,228]
streetlight = [16,236,72]
booth = [108,7,86]
television = [172,27,94]
airplane = [119,247,193]
dirttrack = [155,240,152]
apparel = [49,158,204]
pole = [23,193,204]
land = [228,66,107]
bannister = [69,36,163]
escalator = [238,158,228]
ottoman = [202,226,35]
bottle = [194,243,151]
buffet = [192,56,76]
poster = [16,115,240]
stage = [61,190,185]
van = [7,134,32]
ship = [192,87,171]
fountain = [45,11,254]
conveyerbelt = [179,183,31]
canopy = [181,175,146]
washer = [13,187,133]
plaything = [12,1,2]
swimmingpool = [63,199,190]
stool = [221,248,32]
barrel = [183,221,51]
basket = [90,111,162]
waterfall = [82,0,6]
tent = [40,0,239]
bag = [252,81,54]
minibike = [110,245,152]
cradle = [0,187,93]
oven = [163,154,153]
ball = [134,66,99]
food = [123,150,242]
step = [38,144,137]
tank = [59,180,230]
tradename = [144,212,16]
microwave = [132,125,200]
pot = [26,3,35]
animal = [199,56,92]
bicycle = [83,223,224]
lake = [203,47,137]
dishwasher = [74,74,251]
screen = [246,81,197]
blanket = [168,130,178]
sculpture = [136,85,200]
hood = [186,147,103]
sconce = [170,21,85]
vase = [104,52,182]
trafficlight = [166,147,202]
tray = [103,119,71]
ashcan = [74,161,165]
fan = [14,9,83]
pier = [129,194,43]
crtscreen = [7,100,55]
plate = [13,12,170]
monitor = [30,21,22]
bulletinboard = [224,189,139]
shower = [40,77,25]
radiator = [194,14,94]
glass = [178,8,231]
clock = [234,166,8]
flag = [248,25,7]
unlabelled = [0,0,0]
label_colours = np.array([wall,building,sky,floor,tree,ceiling,road,bed ,windowpane,grass,cabinet,sidewalk,person,earth,door,table,mountain,plant,curtain,chair,car,water,painting,sofa,shelf,house,sea,mirror,rug,field,armchair,seat,fence,desk,rock,wardrobe,lamp,bathtub,railing,cushion,base,box,column,signboard,chest,counter,sand,sink,skyscraper,fireplace,refrigerator,grandstand,path,stairs,runway,case,pooltable,pillow,screen,stairway,river,bridge,bookcase,blind,coffeetable,toilet,flower,book,hill,bench,countertop,stove,palm,kitchenisland,computer,swivelchair,boat,bar,arcademachine,hovel,bus,towel,light,truck,tower,chandelier,awning,streetlight,booth,television,airplane,dirttrack,apparel,pole,land,bannister,escalator,ottoman,bottle,buffet,poster,stage,van,ship,fountain,conveyerbelt,canopy,washer,plaything,swimmingpool,stool,barrel,basket,waterfall,tent,bag,minibike,cradle,oven,ball,food,step,tank,tradename,microwave,pot,animal,bicycle,lake,dishwasher,screen,blanket,sculpture,hood,sconce,vase,trafficlight,tray,ashcan,fan,pier,crtscreen,plate,monitor,bulletinboard,shower,radiator,glass,clock,flag,unlabelled])
for l in range(0,150):
r[ind==l] = label_colours[l,0]
g[ind==l] = label_colours[l,1]
b[ind==l] = label_colours[l,2]
r_gt[label==l] = label_colours[l,0]
g_gt[label==l] = label_colours[l,1]
b_gt[label==l] = label_colours[l,2]
# r_truth[truth_img==l] = label_colours[l,0]
# g_truth[truth_img==l] = label_colours[l,1]
# b_truth[truth_img==l] = label_colours[l,2]
rgb = np.zeros((ind.shape[0], ind.shape[1], 3))
rgb[:,:,0] = r
rgb[:,:,1] = g
rgb[:,:,2] = b
rgb_gt = np.zeros((label.shape[0], label.shape[1], 3))
rgb_gt[:,:,0] = r_gt
rgb_gt[:,:,1] = g_gt
rgb_gt[:,:,2] = b_gt
cv2.imwrite(colorful_fn_output,rgb)
########change to rgb########
imsave(fn_output,crf_img)
# Just randomly manually run inference iterations
Q, tmp1, tmp2 = d.startInference()
for i in range(10):
print("KL-divergence at {}: {}".format(i, d.klDivergence(Q)))
d.stepInference(Q, tmp1, tmp2)
def crf(original_image, annotated_image, output_image, use_2d=True):
# Converting annotated image to RGB if it is Gray scale
if (len(annotated_image.shape) < 3):
annotated_image = gray2rgb(annotated_image)
imsave("testing5.png", annotated_image)
#Converting the annotations RGB color to single 32 bit integer
annotated_label = annotated_image[:, :, 0] + (
annotated_image[:, :, 1] << 8) + (annotated_image[:, :, 2] << 16)
# Convert the 32bit integer color to 0,1, 2, ... labels.
colors, labels = np.unique(annotated_label, return_inverse=True)
#Creating a mapping back to 32 bit colors
colorize = np.empty((len(colors), 3), np.uint8)
colorize[:, 0] = (colors & 0x0000FF)
colorize[:, 1] = (colors & 0x00FF00) >> 8
colorize[:, 2] = (colors & 0xFF0000) >> 16
#Gives no of class labels in the annotated image
n_labels = len(set(labels.flat))
print("No of labels in the Image are ")
print(n_labels)
#Setting up the CRF model
if use_2d:
d = dcrf.DenseCRF2D(original_image.shape[1], original_image.shape[0],
n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=False)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
d.addPairwiseBilateral(sxy=(80, 80),
srgb=(13, 13, 13),
rgbim=original_image,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
#Run Inference for 5 steps
Q = d.inference(5)
# Find out the most probable class for each pixel.
MAP = np.argmax(Q, axis=0)
# Convert the MAP (labels) back to the corresponding colors and save the image.
# Note that there is no "unknown" here anymore, no matter what we had at first.
MAP = colorize[MAP, :]
imsave(output_image, MAP.reshape(original_image.shape))
return MAP.reshape(original_image.shape)
def CRFs(original_image_path, predicted_image_path, CRF_image_path):
print("original_image_path: ", original_image_path)
img = imread(original_image_path)
anno_rgb = imread(predicted_image_path).astype(np.uint32)
anno_lbl = anno_rgb[:, :, 0] + (anno_rgb[:, :, 1] << 8) + (
anno_rgb[:, :, 2] << 16)
colors, labels = np.unique(anno_lbl, return_inverse=True)
colorize = np.empty((len(colors), 3), np.uint8)
colorize[:, 0] = (colors & 0x0000FF)
colorize[:, 1] = (colors & 0x00FF00) >> 8
colorize[:, 2] = (colors & 0xFF0000) >> 16
n_labels = len(set(labels.flat))
use_2d = True
if use_2d:
d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], n_labels)
# gt_prob: The certainty of the ground-truth (must be within (0,1)).
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=None)
d.setUnaryEnergy(U)
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
d.addPairwiseBilateral(sxy=(80, 80),
srgb=(13, 13, 13),
rgbim=img,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
else:
d = dcrf.DenseCRF(img.shape[1] * img.shape[0], n_labels)
U = unary_from_labels(labels, n_labels, gt_prob=0.5, zero_unsure=None)
d.setUnaryEnergy(U)
feats = create_pairwise_gaussian(sdims=(3, 3), shape=img.shape[:2])
d.addPairwiseEnergy(feats,
compat=8,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
feats = create_pairwise_bilateral(sdims=(80, 80),
schan=(13, 13, 13),
img=img,
chdim=2)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(10)
MAP = np.argmax(Q, axis=0)
MAP = colorize[MAP, :]
imwrite(CRF_image_path, MAP.reshape(img.shape))
print("saving CRF image in ", CRF_image_path, "!")
n_labels = len(set(labels.flat)) - 1
print(n_labels, " labels and \"unknown\" 0: ", set(labels.flat))
###########################
### Setup the CRF model ###
###########################
use_2d = False
# use_2d = True
if use_2d:
print("Using 2D specialized functions")
# Example using the DenseCRF2D code
d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=True)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
d.addPairwiseBilateral(sxy=(80, 80),
srgb=(13, 13, 13),
rgbim=img,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
def crf(fn_im, fn_anno, fn_output, colorful_fn_output):
##################################
### Read images and annotation ###
##################################
img = imread(fn_im)
#truth_img = imread(truth_image).astype(np.uint8)
# Convert the annotation's RGB color to a single 32-bit integer color 0xBBGGRR
anno_rgb = imread(fn_anno)
#anno_lbl = anno_rgb[:,:,0] + (anno_rgb[:,:,1] << 8) + (anno_rgb[:,:,2] << 16)
anno_lbl = anno_rgb
# Convert the 32bit integer color to 1, 2, ... labels.
# Note that all-black, i.e. the value 0 for background will stay 0.
colors, labels = np.unique(anno_lbl, return_inverse=True)
# And create a mapping back from the labels to 32bit integer colors.
# But remove the all-0 black, that won't exist in the MAP!
colors = colors[1:]
colorize = np.empty((len(colors), 1), np.uint8)
colorize[:, 0] = (colors & 0x0000FF)
#colorize[:,1] = (colors & 0x00FF00) >> 8
#colorize[:,2] = (colors & 0xFF0000) >> 16
# Compute the number of classes in the label image.
# We subtract one because the number shouldn't include the value 0 which stands
# for "unknown" or "unsure".
n_labels = len(set(labels.flat)) - 1
print(n_labels, " labels and \"unknown\" 0: ", set(labels.flat))
###########################
### Setup the CRF model ###
###########################
use_2d = False
# use_2d = True
if use_2d:
print("Using 2D specialized functions")
# Example using the DenseCRF2D code
if n_labels == 1:
n_labels += 1
d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=False)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
d.addPairwiseBilateral(sxy=(80, 80),
srgb=(13, 13, 13),
rgbim=img,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
else:
print("Using generic 2D functions")
# Example using the DenseCRF class and the util functions
d = dcrf.DenseCRF(img.shape[1] * img.shape[0], n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=True)
d.setUnaryEnergy(U)
# This creates the color-independent features and then add them to the CRF
feats = create_pairwise_gaussian(sdims=(3, 3), shape=img.shape[:2])
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This creates the color-dependent features and then add them to the CRF
feats = create_pairwise_bilateral(sdims=(80, 80),
schan=(13, 13, 13),
img=img,
chdim=2)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
####################################
### Do inference and compute MAP ###
####################################
# Run five inference steps.
Q = d.inference(5)
# Find out the most probable class for each pixel.
MAP = np.argmax(Q, axis=0)
# Convert the MAP (labels) back to the corresponding colors and save the image.
MAP = colorize[MAP, :]
####################################
### Convert to greyscale ###
####################################
crf_img = MAP.reshape(anno_lbl.shape)
########change to rgb########
label = anno_lbl
ind = crf_img
r = ind.copy()
g = ind.copy()
b = ind.copy()
r_gt = label.copy()
g_gt = label.copy()
b_gt = label.copy()
wall = [139, 181, 248]
building = [251, 209, 244]
sky = [44, 230, 121]
floor = [156, 40, 149]
tree = [166, 219, 98]
ceiling = [35, 229, 138]
road = [143, 56, 194]
bed = [144, 223, 70]
windowpane = [200, 162, 57]
grass = [120, 225, 199]
cabinet = [87, 203, 13]
sidewalk = [185, 1, 136]
person = [16, 167, 16]
earth = [29, 249, 241]
door = [17, 192, 40]
table = [199, 44, 241]
mountain = [193, 196, 159]
plant = [241, 172, 78]
curtain = [56, 94, 128]
chair = [231, 166, 116]
car = [50, 209, 252]
water = [217, 56, 227]
painting = [168, 198, 178]
sofa = [77, 179, 188]
shelf = [236, 191, 103]
house = [248, 138, 151]
sea = [214, 251, 89]
mirror = [208, 204, 187]
rug = [115, 104, 49]
field = [29, 202, 113]
armchair = [159, 160, 95]
seat = [78, 188, 13]
fence = [83, 203, 82]
desk = [8, 234, 116]
rock = [80, 159, 200]
wardrobe = [124, 194, 2]
lamp = [192, 146, 237]
bathtub = [64, 3, 73]
railing = [17, 213, 58]
cushion = [106, 54, 105]
base = [125, 72, 155]
box = [202, 36, 231]
column = [79, 144, 4]
signboard = [118, 185, 128]
chest = [138, 61, 178]
counter = [23, 182, 182]
sand = [154, 114, 4]
sink = [201, 0, 83]
skyscraper = [21, 134, 53]
fireplace = [194, 77, 237]
refrigerator = [198, 81, 106]
grandstand = [37, 222, 181]
path = [203, 185, 14]
stairs = [134, 140, 113]
runway = [220, 196, 79]
case = [64, 26, 68]
pooltable = [128, 89, 2]
pillow = [199, 228, 65]
screen = [62, 215, 111]
stairway = [124, 148, 166]
river = [221, 119, 245]
bridge = [68, 57, 158]
bookcase = [80, 47, 26]
blind = [143, 59, 56]
coffeetable = [14, 80, 215]
toilet = [212, 132, 31]
flower = [2, 234, 129]
book = [134, 179, 44]
hill = [53, 21, 129]
bench = [80, 176, 236]
countertop = [154, 39, 168]
stove = [221, 44, 139]
palm = [103, 56, 185]
kitchenisland = [224, 138, 83]
computer = [243, 93, 235]
swivelchair = [80, 158, 63]
boat = [81, 229, 38]
bar = [116, 215, 38]
arcademachine = [103, 69, 182]
hovel = [66, 81, 5]
bus = [96, 157, 229]
towel = [164, 49, 170]
light = [14, 42, 146]
truck = [164, 67, 44]
tower = [108, 116, 151]
chandelier = [144, 8, 144]
awning = [85, 68, 228]
streetlight = [16, 236, 72]
booth = [108, 7, 86]
television = [172, 27, 94]
airplane = [119, 247, 193]
dirttrack = [155, 240, 152]
apparel = [49, 158, 204]
pole = [23, 193, 204]
land = [228, 66, 107]
bannister = [69, 36, 163]
escalator = [238, 158, 228]
ottoman = [202, 226, 35]
bottle = [194, 243, 151]
buffet = [192, 56, 76]
poster = [16, 115, 240]
stage = [61, 190, 185]
van = [7, 134, 32]
ship = [192, 87, 171]
fountain = [45, 11, 254]
conveyerbelt = [179, 183, 31]
canopy = [181, 175, 146]
washer = [13, 187, 133]
plaything = [12, 1, 2]
swimmingpool = [63, 199, 190]
stool = [221, 248, 32]
barrel = [183, 221, 51]
basket = [90, 111, 162]
waterfall = [82, 0, 6]
tent = [40, 0, 239]
bag = [252, 81, 54]
minibike = [110, 245, 152]
cradle = [0, 187, 93]
oven = [163, 154, 153]
ball = [134, 66, 99]
food = [123, 150, 242]
step = [38, 144, 137]
tank = [59, 180, 230]
tradename = [144, 212, 16]
microwave = [132, 125, 200]
pot = [26, 3, 35]
animal = [199, 56, 92]
bicycle = [83, 223, 224]
lake = [203, 47, 137]
dishwasher = [74, 74, 251]
screen = [246, 81, 197]
blanket = [168, 130, 178]
sculpture = [136, 85, 200]
hood = [186, 147, 103]
sconce = [170, 21, 85]
vase = [104, 52, 182]
trafficlight = [166, 147, 202]
tray = [103, 119, 71]
ashcan = [74, 161, 165]
fan = [14, 9, 83]
pier = [129, 194, 43]
crtscreen = [7, 100, 55]
plate = [13, 12, 170]
monitor = [30, 21, 22]
bulletinboard = [224, 189, 139]
shower = [40, 77, 25]
radiator = [194, 14, 94]
glass = [178, 8, 231]
clock = [234, 166, 8]
flag = [248, 25, 7]
unlabelled = [0, 0, 0]
label_colours = np.array([
wall, building, sky, floor, tree, ceiling, road, bed, windowpane,
grass, cabinet, sidewalk, person, earth, door, table, mountain, plant,
curtain, chair, car, water, painting, sofa, shelf, house, sea, mirror,
rug, field, armchair, seat, fence, desk, rock, wardrobe, lamp, bathtub,
railing, cushion, base, box, column, signboard, chest, counter, sand,
sink, skyscraper, fireplace, refrigerator, grandstand, path, stairs,
runway, case, pooltable, pillow, screen, stairway, river, bridge,
bookcase, blind, coffeetable, toilet, flower, book, hill, bench,
countertop, stove, palm, kitchenisland, computer, swivelchair, boat,
bar, arcademachine, hovel, bus, towel, light, truck, tower, chandelier,
awning, streetlight, booth, television, airplane, dirttrack, apparel,
pole, land, bannister, escalator, ottoman, bottle, buffet, poster,
stage, van, ship, fountain, conveyerbelt, canopy, washer, plaything,
swimmingpool, stool, barrel, basket, waterfall, tent, bag, minibike,
cradle, oven, ball, food, step, tank, tradename, microwave, pot,
animal, bicycle, lake, dishwasher, screen, blanket, sculpture, hood,
sconce, vase, trafficlight, tray, ashcan, fan, pier, crtscreen, plate,
monitor, bulletinboard, shower, radiator, glass, clock, flag,
unlabelled
])
for l in range(0, 150):
r[ind == l] = label_colours[l, 0]
g[ind == l] = label_colours[l, 1]
b[ind == l] = label_colours[l, 2]
r_gt[label == l] = label_colours[l, 0]
g_gt[label == l] = label_colours[l, 1]
b_gt[label == l] = label_colours[l, 2]
# r_truth[truth_img==l] = label_colours[l,0]
# g_truth[truth_img==l] = label_colours[l,1]
# b_truth[truth_img==l] = label_colours[l,2]
rgb = np.zeros((ind.shape[0], ind.shape[1], 3))
rgb[:, :, 0] = r
rgb[:, :, 1] = g
rgb[:, :, 2] = b
rgb_gt = np.zeros((label.shape[0], label.shape[1], 3))
rgb_gt[:, :, 0] = r_gt
rgb_gt[:, :, 1] = g_gt
rgb_gt[:, :, 2] = b_gt
cv2.imwrite(colorful_fn_output, rgb)
########change to rgb########
imsave(fn_output, crf_img)
# Just randomly manually run inference iterations
Q, tmp1, tmp2 = d.startInference()
for i in range(5):
print("KL-divergence at {}: {}".format(i, d.klDivergence(Q)))
d.stepInference(Q, tmp1, tmp2)
def crf(inimage, img_anno):
fn_im = inimage
fn_anno = img_anno
img = inimage
anno_rgb = img_anno
rgb = anno_rgb
print "=========>>", anno_rgb.shape
#rgb= np.argmax(anno_rgb[0],axis=0)
print "=======>>", rgb.shape
print np.max(rgb), np.min(rgb)
anno_lbl = rgb
img = img[0]
img = img.transpose(1, 2, 0)
colors, labels = np.unique(anno_lbl, return_inverse=True)
colors = colors[1:]
colorize = np.empty((len(colors), 3), np.uint8)
colorize[:, 0] = (colors & 0x0000FF)
colorize[:, 1] = (colors & 0x00FF00) >> 8
colorize[:, 2] = (colors & 0xFF0000) >> 16
n_labels = len(set(labels.flat)) - 1
if n_labels <= 1:
return rgb
use_2d = True
if use_2d:
img = img.astype(int)
d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], n_labels)
print n_labels
U = unary_from_labels(labels, n_labels, gt_prob=0.99, zero_unsure=True)
d.setUnaryEnergy(U)
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NO_NORMALIZATION)
img = counts = np.copy(np.array(img, dtype=np.uint8), order='C')
d.addPairwiseBilateral(sxy=(201, 401),
srgb=(1, 1, 1),
rgbim=img,
compat=101,
kernel=dcrf.FULL_KERNEL,
normalization=dcrf.NO_NORMALIZATION)
else:
# Example using the DenseCRF class and the util functions
d = dcrf.DenseCRF(img.shape[1] * img.shape[0], n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=True)
d.setUnaryEnergy(U)
# This creates the color-independent features and then add them to the CRF
feats = create_pairwise_gaussian(sdims=(3, 3), shape=img.shape[:2])
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This creates the color-dependent features and then add them to the CRF
feats = create_pairwise_bilateral(sdims=(80, 80),
schan=(13, 13, 13),
img=img,
chdim=2)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
# Find out the most probable class for each pixel.
MAP = np.argmax(Q, axis=0)
return MAP.reshape(img.shape[:2])
def simple_example():
image_filepath = './driver_license.png'
#image_filepath = './driver_license_20190329.jpg'
#image_filepath = './passport_chaewanlee_20130402.jpg'
#image_filepath = './passport_chaewanlee_20170804.jpg'
#image_filepath = './passport_hyejoongkim_20140508.jpg'
#image_filepath = './passport_jihyunglee_20130402.jpg'
#image_filepath = './passport_jihyunglee_20170804.jpg'
#image_filepath = './passport_malnamkang_1.jpg'
#image_filepath = './passport_malnamkang_2.jpg'
#image_filepath = './passport_sangwooklee_20031211.jpg'
#image_filepath = './passport_sangwooklee_20130402.jpg'
#image_filepath = './rrn_malnamkang.jpg
#image_filepath = './rrn_sangwooklee_20190329.jpg'
anno_filepath = image_filepath
img = cv2.imread(image_filepath)
# Convert the annotation's RGB color to a single 32-bit integer color 0xBBGGRR.
anno_rgb = cv2.imread(anno_filepath).astype(np.uint32)
anno_lbl = anno_rgb[:, :, 0] + (anno_rgb[:, :, 1] << 8) + (
anno_rgb[:, :, 2] << 16)
# Convert the 32bit integer color to 1, 2, ..., labels.
# Note that all-black, i.e. the value 0 for background will stay 0.
colors, labels = np.unique(anno_lbl, return_inverse=True)
# But remove the all-0 black, that won't exist in the MAP!
HAS_UNK = 0 in colors
if HAS_UNK:
print(
'Found a full-black pixel in annotation image, assuming it means "unknown" label, and will thus not be present in the output!'
)
print(
'If 0 is an actual label for you, consider writing your own code, or simply giving your labels only non-zero values.'
)
colors = colors[1:]
#else:
# print('No single full-black pixel found in annotation image. Assuming there\'s no "unknown" label!')
# And create a mapping back from the labels to 32bit integer colors.
colorize = np.empty((len(colors), 3), np.uint8)
colorize[:, 0] = (colors & 0x0000FF)
colorize[:, 1] = (colors & 0x00FF00) >> 8
colorize[:, 2] = (colors & 0xFF0000) >> 16
# Compute the number of classes in the label image.
# We subtract one because the number shouldn't include the value 0 which stands for "unknown" or "unsure".
n_labels = len(set(labels.flat)) - int(HAS_UNK)
#print(n_labels, 'labels', ('plus "unknown" 0: ' if HAS_UNK else ''), set(labels.flat))
print(n_labels, 'labels', ('plus "unknown" 0: ' if HAS_UNK else ''))
#--------------------
# Setup the CRF model.
use_2d = False
#use_2d = True
print('Start building a CRF model...')
start_time = time.time()
if use_2d:
print('Using 2D specialized functions.')
# Example using the DenseCRF2D code.
d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], n_labels)
# Get unary potentials (neg log probability).
U = unary_from_labels(labels,
n_labels,
gt_prob=0.7,
zero_unsure=HAS_UNK)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This adds the color-dependent term, i.e. features are (x, y, r, g, b).
d.addPairwiseBilateral(sxy=(80, 80),
srgb=(13, 13, 13),
rgbim=img,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
else:
print('Using generic 2D functions.')
# Example using the DenseCRF class and the util functions.
d = dcrf.DenseCRF(img.shape[1] * img.shape[0], n_labels)
# Get unary potentials (neg log probability).
U = unary_from_labels(labels,
n_labels,
gt_prob=0.7,
zero_unsure=HAS_UNK)
d.setUnaryEnergy(U)
# This creates the color-independent features and then add them to the CRF.
feats = create_pairwise_gaussian(sdims=(3, 3), shape=img.shape[:2])
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This creates the color-dependent features and then add them to the CRF.
feats = create_pairwise_bilateral(sdims=(80, 80),
schan=(13, 13, 13),
img=img,
chdim=2)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
print('End building a CRF model: {} secs.'.format(time.time() -
start_time))
#--------------------
# Do inference and compute MAP.
print('Start inferring by the CRF model...')
start_time = time.time()
# Run five inference steps.
Q = d.inference(5)
# Find out the most probable class for each pixel.
MAP = np.argmax(Q, axis=0)
print('End inferring by the CRF model: {} secs.'.format(time.time() -
start_time))
# Convert the MAP (labels) back to the corresponding colors and save the image.
# Note that there is no "unknown" here anymore, no matter what we had at first.
MAP = colorize[MAP, :]
MAP = MAP.reshape(img.shape)
#cv2.imwrite(output_filepath, MAP)
cv2.imshow('Inference result', MAP)
cv2.waitKey(0)
#--------------------
# Just randomly manually run inference iterations.
print('Start manually inferring by the CRF model...')
start_time = time.time()
Q, tmp1, tmp2 = d.startInference()
for i in range(5):
print('KL-divergence at {}: {}.'.format(i, d.klDivergence(Q)))
d.stepInference(Q, tmp1, tmp2)
print('End manually inferring by the CRF model: {} secs.'.format(
time.time() - start_time))
def CRFs(original_image_path, predicted_image_path, CRF_image_path):
img = original_image_path
# 将predicted_image的RGB颜色转换为uint32颜色 0xbbggrr
prd = predicted_image_path
anno_rgb = prd.astype(np.uint32)
anno_lbl = anno_rgb[:, :, 0] + (anno_rgb[:, :, 1] << 8) + (
anno_rgb[:, :, 2] << 16) #<<表示移位符号，为左移
# 将uint32颜色转换为1,2,...
colors, labels = np.unique(anno_lbl, return_inverse=True)
# 如果你的predicted_image里的黑色（0值）不是待分类类别，表示不确定区域，即将分为其他类别
# 那么就取消注释以下代码
#HAS_UNK = 0 in colors
#if HAS_UNK:
#colors = colors[1:]
# 创建从predicted_image到32位整数颜色的映射。
colorize = np.empty((len(colors), 3), np.uint8)
colorize[:, 0] = (colors & 0x0000FF)
colorize[:, 1] = (colors & 0x00FF00) >> 8
colorize[:, 2] = (colors & 0xFF0000) >> 16
# 计算predicted_image中的类数。
n_labels = len(set(labels.flat))
#n_labels = len(set(labels.flat)) - int(HAS_UNK) ##如果有不确定区域，用这一行代码替换上一行
###########################
### 设置CRF模型 ###
###########################
use_2d = False
# use_2d = True
###########################################################
##不是很清楚什么情况用2D
##作者说“对于图像，使用此库的最简单方法是使用DenseCRF2D类”
##作者还说“DenseCRF类可用于通用（非二维）密集CRF”
##但是根据我的测试结果一般情况用DenseCRF比较对
#########################################################33
if use_2d:
# 使用densecrf2d类
d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], n_labels)
# 得到一元势（负对数概率）
U = unary_from_labels(labels, n_labels, gt_prob=0.2, zero_unsure=None)
#U = unary_from_labels(labels, n_labels, gt_prob=0.2, zero_unsure=HAS_UNK)## 如果有不确定区域，用这一行代码替换上一行
d.setUnaryEnergy(U)
# 增加了与颜色无关的术语，功能只是位置而已
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# 增加了颜色相关术语，即特征是(x,y,r,g,b)
d.addPairwiseBilateral(sxy=(80, 80),
srgb=(13, 13, 13),
rgbim=img,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
else:
# 使用densecrf类
d = dcrf.DenseCRF(img.shape[1] * img.shape[0], n_labels)
# 得到一元势（负对数概率）
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=None)
# U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=HAS_UNK)## 如果有不确定区域，用这一行代码替换上一行
d.setUnaryEnergy(U)
# 这将创建与颜色无关的功能，然后将它们添加到CRF中
feats = create_pairwise_gaussian(sdims=(3, 3), shape=img.shape[:2])
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# 这将创建与颜色相关的功能，然后将它们添加到CRF中
feats = create_pairwise_bilateral(sdims=(3, 3),
schan=(13, 13, 13),
img=img,
chdim=2)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
####################################
### 做推理和计算 ###
####################################
# 进行5次推理
Q = d.inference(20)
# Q = d.inference(100)
# 找出每个像素最可能的类
MAP = np.argmax(Q, axis=0)
# 将predicted_image转换回相应的颜色并保存图像
MAP = colorize[MAP, :]
imwrite(CRF_image_path, MAP.reshape(img.shape))
print("CRF图像保存在", CRF_image_path, "!")
def my_crf_1(fn_im, fn_anno, fn_output):
img = cv2.imread(fn_im)
anno_rgb = cv2.imread(fn_anno).astype(np.uint32)
anno_lbl = anno_rgb[:,:,0] + (anno_rgb[:,:,1] << 8) + (anno_rgb[:,:,2] << 16)
colors, labels = np.unique(anno_lbl, return_inverse=True)
# But remove the all-0 black, that won't exist in the MAP!
HAS_UNK = 0 in colors
if HAS_UNK:
print("Found a full-black pixel in annotation image, assuming it means 'unknown' label, and will thus not be present in the output!")
print("If 0 is an actual label for you, consider writing your own code, or simply giving your labels only non-zero values.")
colors = colors[1:]
#else:
# print("No single full-black pixel found in annotation image. Assuming there's no 'unknown' label!")
# And create a mapping back from the labels to 32bit integer colors.
colorize = np.empty((len(colors), 3), np.uint8)
colorize[:,0] = (colors & 0x0000FF)
colorize[:,1] = (colors & 0x00FF00) >> 8
colorize[:,2] = (colors & 0xFF0000) >> 16
# Compute the number of classes in the label image.
# We subtract one because the number shouldn't include the value 0 which stands
# for "unknown" or "unsure".
print(fn_im)
n_labels = len(set(labels.flat)) - int(HAS_UNK)
print(n_labels, " labels", (" plus \"unknown\" 0: " if HAS_UNK else ""), set(labels.flat))
###########################
### Setup the CRF model ###
###########################
use_2d = False
# use_2d = True
if use_2d:
print("Using 2D specialized functions")
# Example using the DenseCRF2D code
d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=HAS_UNK)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3, 3), compat=3, kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
d.addPairwiseBilateral(sxy=(80, 80), srgb=(13, 13, 13), rgbim=img,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
else:
print("Using generic 2D functions")
# Example using the DenseCRF class and the util functions
d = dcrf.DenseCRF(img.shape[1] * img.shape[0], n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=HAS_UNK)
d.setUnaryEnergy(U)
# This creates the color-independent features and then add them to the CRF
feats = create_pairwise_gaussian(sdims=(3, 3), shape=img.shape[:2])
d.addPairwiseEnergy(feats, compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This creates the color-dependent features and then add them to the CRF
feats = create_pairwise_bilateral(sdims=(80, 80), schan=(13, 13, 13),
img=img, chdim=2)
d.addPairwiseEnergy(feats, compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
MAP = np.argmax(Q, axis=0)
MAP = colorize[MAP,:]
cv2.imwrite(fn_output, MAP.reshape(img.shape))
# Just randomly manually run inference iterations
Q, tmp1, tmp2 = d.startInference()
for i in range(5):
print("KL-divergence at {}: {}".format(i, d.klDivergence(Q)))
d.stepInference(Q, tmp1, tmp2)
def detect_and_get_masks(model, config, args):
########################
# Load test images
########################
print("args.dataset_split", args.dataset_split)
dataset = Affordance.AffordanceDataset()
dataset.load_Affordance(args.dataset, args.dataset_split)
dataset.prepare()
config.display()
print("Num of Test Images: {}".format(len(dataset.image_ids)))
for image_id in range(len(dataset.image_ids)):
print("\nimage_file:", dataset.image_reference(image_id))
##############################
# Address for saving mask
##############################
image_file1 = dataset.image_reference(image_id)
image_file2 = image_file1.split(args.dataset)[1] # remove dataset path
idx = image_file2.split('_rgb')[0] # remove _rgb label
rgb_addr = args.dataset + idx + '_rgb.jpg'
depth_addr = args.dataset + idx + '_depth.png'
gt_mask_addr = args.dataset + idx + '_label.png'
# gt_mask_addr = args.dataset + idx + '_gt_affordance.png'
if os.path.isfile(rgb_addr) == False:
continue
if os.path.isfile(depth_addr) == False:
continue
if os.path.isfile(gt_mask_addr) == False:
continue
mask_addr = args.dataset + idx + '_mask_og.png'
color_mask_addr = args.dataset + idx + '_mask_color.png'
cropped_mask_addr = args.dataset + idx + '_mask_cropped.png'
print("mask_addr:", mask_addr)
##############################
### ground truth
##############################
rgb = np.array(skimage.io.imread(rgb_addr))
depth = np.array(skimage.io.imread(depth_addr))
gt_label = np.array(skimage.io.imread(gt_mask_addr))
### print("GT RGB SHAPE: ", rgb.shape)
print("GT Affordance Label:", np.unique(gt_label))
######################
# configure depth
######################
UMD_DEPTH_MAX = 3626
depth[np.isnan(depth)] = 0
depth[depth == -np.inf] = 0
depth[depth == np.inf] = 0
# convert to 8-bit image
# depth = depth * (2 ** 16 -1) / np.max(depth) ### 16 bit
depth = depth * (2**8 - 1) / UMD_DEPTH_MAX ### 8 bit
depth = np.array(depth, dtype=np.uint8)
# print("depth min: ", np.min(np.array(depth)))
# print("depth max: ", np.max(np.array(depth)))
#
# print("depth type: ", depth.dtype)
# print("depth shape: ", depth.shape)
##################################
# RGB has 4th channel - alpha
# depth to 3 channels
##################################
rgb, depth = rgb[..., :3], skimage.color.gray2rgb(depth)
##############################
# Resize
##############################
### rgb = cv2.resize(rgb, dsize=(config.IMAGE_MIN_DIM, config.IMAGE_MIN_DIM), interpolation=cv2.INTER_CUBIC)
### depth = cv2.resize(depth, dsize=(config.IMAGE_MIN_DIM, config.IMAGE_MIN_DIM), interpolation=cv2.INTER_CUBIC)
### gt_label = cv2.resize(gt_label, dsize=(1280, 1280), interpolation=cv2.INTER_CUBIC)
##############################
# CROP
##############################
if CROP == True:
# Pick a random crop
h, w = rgb.shape[:2]
x = (w - config.IMAGE_MIN_DIM) // 2
y = (h - config.IMAGE_MIN_DIM) // 2
rgb = rgb[y:y + config.IMAGE_MIN_DIM, x:x + config.IMAGE_MIN_DIM]
depth = depth[y:y + config.IMAGE_MIN_DIM,
x:x + config.IMAGE_MIN_DIM]
gt_label = gt_label[y:y + config.IMAGE_MIN_DIM,
x:x + config.IMAGE_MIN_DIM]
# print("gt_label: ", gt_label.shape)
##############################
# Detect
##############################
if args.detect == 'rgb':
# run detect
cur_detect = model.detect([rgb], verbose=0)[0]
elif args.detect == 'rgbd':
# run detect
cur_detect = model.detectWdepth([rgb], [depth], verbose=0)[0]
# get instance_masks
instance_mask, color_mask = seq_get_masks(rgb, cur_detect, gt_label,
args)
####################
cv2.imwrite(mask_addr, instance_mask)
cv2.imwrite(color_mask_addr, color_mask)
cv2.imwrite(cropped_mask_addr, gt_label)
####################
if args.post_process:
img = rgb
# labels, n_labels = instance_mask, config.NUM_CLASSES
labels, n_labels = gt_label, config.NUM_CLASSES
# Example using the DenseCRF2D code
d = dcrf.DenseCRF(img.shape[0] * img.shape[1], n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=0)
d.setUnaryEnergy(U)
# This potential penalizes small pieces of segmentation that are
# spatially isolated -- enforces more spatially consistent segmentations
feats = create_pairwise_gaussian(sdims=(30, 30),
shape=img.shape[:2])
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This creates the color-dependent features --
# because the segmentation that we get from CNN are too coarse
# and we can use local color features to refine them
feats = create_pairwise_bilateral(sdims=(3, 3),
schan=(3, 3, 3),
img=img,
chdim=2)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
crf = np.argmax(Q, axis=0).reshape((img.shape[0], img.shape[1]))
crf = np.array(crf, dtype=np.uint8)
cv2.imwrite(mask_addr, crf)
if args.show_plots:
print("GT shape:", gt_label.shape)
print("Pred shape:", instance_mask.shape)
print("resize_pred shape:", instance_mask.shape)
cv2.imshow("rgb", img)
cv2.imshow("gt", gt_label * 100)
cv2.imshow("pred", labels * 100)
cv2.imshow("crf", crf * 100)
cv2.waitKey(0)
####################
else:
if args.show_plots:
print("GT shape:", gt_label.shape)
print("Pred shape:", instance_mask.shape)
print("resize_pred shape:", instance_mask.shape)
cv2.imshow("gt", gt_label * 25)
cv2.imshow("resize pred", instance_mask * 25)
cv2.waitKey(0)
n_labels = len(set(labels.flat)) - 1
print(n_labels, " labels and \"unknown\" 0: ", set(labels.flat))
###########################
### Setup the CRF model ###
###########################
use_2d = False
# use_2d = True
if use_2d:
print("Using 2D specialized functions")
# Example using the DenseCRF2D code
d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=True)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3, 3), compat=3, kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
d.addPairwiseBilateral(sxy=(80, 80), srgb=(13, 13, 13), rgbim=img,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
else:
print("Using generic 2D functions")
# Example using the DenseCRF class and the util functions
def crf(fn_im, fn_anno, fn_output, NUM_OF_CLASSESS, use_2d=True):
##################################
### Read images and annotation ###
##################################
img = imread(fn_im)
#print(fn_anno.shape)
# Convert the annotation's RGB color to a single 32-bit integer color 0xBBGGRR
anno_rgb = imread(fn_anno).astype(np.uint32)
anno_lbl = anno_rgb[:, :, 0] + \
(anno_rgb[:, :, 1] << 8) + (anno_rgb[:, :, 2] << 16)
# Convert the 32bit integer color to 1, 2, ... labels.
# Note that all-black, i.e. the value 0 for background will stay 0.
colors, labels = np.unique(anno_lbl, return_inverse=True)
# labels = np.unique(fn_anno)
#print(colors, labels)
# But remove the all-0 black, that won't exist in the MAP!
# HAS_UNK = 0 in colors
HAS_UNK = False
# if HAS_UNK:
#print("Found a full-black pixel in annotation image, assuming it means 'unknown' label, and will thus not be present in the output!")
#print("If 0 is an actual label for you, consider writing your own code, or simply giving your labels only non-zero values.")
#colors = colors[1:]
# else:
# print("No single full-black pixel found in annotation image. Assuming there's no 'unknown' label!")
# And create a mapping back from the labels to 32bit integer colors.
colorize = np.empty((len(colors), 3), np.uint8)
colorize[:, 0] = (colors & 0x0000FF)
colorize[:, 1] = (colors & 0x00FF00) >> 8
colorize[:, 2] = (colors & 0xFF0000) >> 16
# Compute the number of classes in the label image.
# We subtract one because the number shouldn't include the value 0 which stands
# for "unknown" or "unsure".
#n_labels = len(set(labels.flat)) - int(HAS_UNK)
#print(n_labels, " labels", (" plus \"unknown\" 0: " if HAS_UNK else ""), set(labels.flat))
n_labels = NUM_OF_CLASSESS
###########################
### Setup the CRF model ###
###########################
#use_2d = False
#use_2d = True
if use_2d:
#print("Using 2D specialized functions")
# Example using the DenseCRF2D code
d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels,
n_labels,
gt_prob=0.7,
zero_unsure=HAS_UNK)
# get unary potentials (neg log probability)
# processed_probabilities = fn_anno
# softmax = processed_probabilities.transpose((2, 0, 1))
# print(softmax.shape)
# U = unary_from_softmax(softmax, scale=None, clip=None)
# U = np.ascontiguousarray(U)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
d.addPairwiseBilateral(sxy=(80, 80),
srgb=(13, 13, 13),
rgbim=img,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
else:
#print("Using generic 2D functions")
# Example using the DenseCRF class and the util functions
d = dcrf.DenseCRF(img.shape[1] * img.shape[0], n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels,
n_labels,
gt_prob=0.7,
zero_unsure=HAS_UNK)
d.setUnaryEnergy(U)
# This creates the color-independent features and then add them to the CRF
feats = create_pairwise_gaussian(sdims=(3, 3), shape=img.shape[:2])
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This creates the color-dependent features and then add them to the CRF
feats = create_pairwise_bilateral(sdims=(80, 80),
schan=(13, 13, 13),
img=img,
chdim=2)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
####################################
### Do inference and compute MAP ###
####################################
# Run five inference steps.
Q = d.inference(5)
# Find out the most probable class for each pixel.
MAP = np.argmax(Q, axis=0)
#print(MAP.shape)
crfoutput = MAP.reshape((img.shape[0], img.shape[1]))
#print(crfoutput.shape)
#print(np.unique(crfoutput))
# Convert the MAP (labels) back to the corresponding colors and save the image.
# Note that there is no "unknown" here anymore, no matter what we had at first.
MAP = colorize[MAP, :]
#print(MAP.shape)
imwrite(fn_output, MAP.reshape(img.shape))
crfimage = MAP.reshape(img.shape)
#print(crfimage.shape)
# Just randomly manually run inference iterations
# Q, tmp1, tmp2 = d.startInference()
# for i in range(5):
# print("KL-divergence at {}: {}".format(i, d.klDivergence(Q)))
# d.stepInference(Q, tmp1, tmp2)
return crfimage, crfoutput
def FC_CRF(pred,
im,
NLABELS,
RESIZE_FACTOR=5,
DELTA_COL=5,
DELTA_SIZE=5,
n_steps=10,
CERTAINTY=0.7,
mode="multilabel"):
"""
Fully Connected Conditional Random Fields
See:
1- https://github.com/lucasb-eyer/pydensecrf (./examples/Non RGB Example.ipynb)
2- http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/18/ ...
image-segmentation-with-tensorflow-using-cnns-and-conditional-random-fields/
Add (non-RGB) pairwise term
For example, in image processing, a popular pairwise relationship is the "bilateral" one,
which roughly says that pixels with either a similar color or a similar location are likely to
belong to the same class.
"""
import pydensecrf.densecrf as dcrf
from pydensecrf.utils import (unary_from_labels, unary_from_softmax,
create_pairwise_bilateral,
create_pairwise_gaussian)
assert mode in ["multilabel", "prob"]
(H0, W0) = im.shape[:2]
H, W = (H0 // RESIZE_FACTOR, W0 // RESIZE_FACTOR)
# regions RGB -- resize for efficiency
image = cv2.resize(im, (W, H))
# region labels -- note that cv2 is the only library I found
# that downsizes without changing range of labels (which is
# obviously important for segmentation masks where values have meaning)
labels0 = cv2.resize(pred, (W, H), interpolation=cv2.INTER_NEAREST)
labels0 = (np.array(labels0)).astype('uint8')
# The input should be the negative of the logarithm of probability values
# Look up the definition of the softmax_to_unary for more information
unary = unary_from_labels(labels=labels0,
n_labels=NLABELS - 1,
gt_prob=CERTAINTY,
zero_unsure=True)
# The inputs should be C-continious -- we are using Cython wrapper
unary = np.ascontiguousarray(unary)
d = dcrf.DenseCRF(H * W, NLABELS - 1)
d.setUnaryEnergy(unary)
# NOTE: THE ORDER OF THE FOLLOWING TWO OPS
# (create_pairwise_bilateral and create_pairwise_gaussian)
# MAKES A DIFFERENCE !!!
# This creates the color-dependent features --
# because the segmentation that we get from CNN are too coarse
# and we can use local color features to refine them
feats = create_pairwise_bilateral(sdims=(DELTA_COL, DELTA_COL),
schan=(20, 20, 20),
img=image,
chdim=2)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This potential penalizes small pieces of segmentation that are
# spatially isolated -- enforces more spatially consistent segmentations
feats = create_pairwise_gaussian(sdims=(DELTA_SIZE, DELTA_SIZE),
shape=(H, W))
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# do inference
Q = d.inference(n_steps)
res = 1 + np.argmax(Q, axis=0).reshape((H, W))
# resize back up to original
res = cv2.resize(res, (W0, H0), interpolation=cv2.INTER_NEAREST)
res = (np.array(res)).astype('uint8')
return res
def dense_crf(original_image, annotated_image, use_2d=True):
# 将由FCN等得到的预测结果图annotated_image转换为彩色图，并且统计彩色图中有多少种颜色类别存储在colorize中，并且
# Converting annotated image to RGB if it is Gray scale
if (len(annotated_image.shape) < 3):
annotated_image = gray2rgb(annotated_image)
# 转换数据类型
annotated_image = annotated_image.astype(np.int64)
# Converting the annotations RGB color to single 32 bit integer 即[00000000],[00000000],[00000000]---->[0000000 00000000 00000000]
annotated_label = annotated_image[:, :, 0] + (annotated_image[:, :, 1] << 8) + (annotated_image[:, :, 2] << 16)
# Convert the 32bit integer color to 0,1, 2, ... labels.
# np.unique该函数是去除数组中的重复数字，并进行排序之后输出到colors，
# return_inverse=True表示返回annotated_label列表元素在colors列表中的位置，并以列表形式储存在label中
# len(labels)=height*width
colors, labels = np.unique(annotated_label, return_inverse=True)
# Creating a mapping back to 32 bit colors ,最后colorize为[[r_1,g_1,b_1],...,[r_m,g_m,b_m],....,[r_n,g_n,b_n]]
colorize = np.empty((len(colors), 3), np.uint8)
colorize[:, 0] = (colors & 0x0000FF)
colorize[:, 1] = (colors & 0x00FF00) >> 8
colorize[:, 2] = (colors & 0xFF0000) >> 16
# print(annotated_image.dtype)
# print(colors)
# print(np.unique(annotated_image[:, :, 0]))
# print(np.unique(annotated_image[:, :, 1]))
# print(np.unique(annotated_image[:, :, 2]))
# print(np.unique(annotated_image[:, :, 1] << 8))
# print(np.unique(annotated_image[:, :, 2] << 16))
# Gives no of class labels in the annotated image
n_labels = len(set(labels.flat))
print("No of labels in the Image are ")
print(n_labels)
# Setting up the CRF model
d = dcrf.DenseCRF2D(original_image.shape[1], original_image.shape[0], n_labels) # width, height, nlabels
if use_2d:
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=False)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3, 3), compat=3, kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
d.addPairwiseBilateral(sxy=(80, 80), srgb=(13, 13, 13), rgbim=original_image,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# Run Inference for 5 steps
Q = d.inference(5)
# Find out the most probable class for each pixel.
MAP = np.argmax(Q, axis=0)
# Convert the MAP (labels) back to the corresponding colors and save the image.
# Note that there is no "unknown" here anymore, no matter what we had at first.
MAP = colorize[MAP, :]
# imsave(output_image, MAP.reshape(original_image.shape))
return MAP.reshape(original_image.shape)
def main(argv=None):
# dropout保留率
keep_probability = tf.placeholder(tf.float32, name="keep_probabilty")
# 图像占坑
image = tf.placeholder(tf.float32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3],
name="input_image")
# 标签占坑
annotation = tf.placeholder(tf.int32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1],
name="annotation")
# 预测一个batch图像 获得预测图[b,h,w,c=1] 结果特征图[b,h,w,c=151]
pred_annotation, logits = inference(image, keep_probability)
tf.summary.image("input_image", image, max_outputs=2)
tf.summary.image("ground_truth",
tf.cast(annotation, tf.uint8),
max_outputs=2)
tf.summary.image("pred_annotation",
tf.cast(pred_annotation, tf.uint8),
max_outputs=2)
# 空间交叉熵损失函数[b,h,w,c=151] 和labels[b,h,w] 每一张图分别对比
loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=tf.squeeze(annotation,
axis=[3]), name="entropy")))
tf.summary.scalar("entropy", loss)
raw_output_up = tf.image.resize_bilinear(logits, tf.shape(image)[0:2, ])
raw_output_up_squeeze = tf.squeeze(raw_output_up, axis=0)
raw_output_up_squeeze = tf.nn.softmax(raw_output_up_squeeze, )
probabilities = tf.nn.softmax(logits)
# 返回需要训练的变量列表
trainable_var = tf.trainable_variables()
if FLAGS.debug:
for var in trainable_var:
utils.add_to_regularization_and_summary(var)
# 传入损失函数和需要训练的变量列表
train_op = train(loss, trainable_var)
print("Setting up summary op...")
# 生成绘图数据
summary_op = tf.summary.merge_all()
if FLAGS.mode != "test":
print("Setting up image reader...")
# data_dir = Data_zoo/MIT_SceneParsing/
# training: [{image: 图片全路径， annotation:标签全路径， filename:图片名字}] [{}][{}]
test_records, valid_records = scene_parsing.read_dataset(
FLAGS.data_dir)
print(len(test_records)) # 长度
# print(len(valid_records))
print("Setting up dataset reader")
image_options = {'resize': True, 'resize_size': IMAGE_SIZE}
if FLAGS.mode == 'train':
# 读取图片 产生类对象 其中包含所有图片信息
train_dataset_reader = dataset.BatchDatset(test_records, image_options)
if FLAGS.mode != "test":
validation_dataset_reader = dataset.BatchDatset(
valid_records, image_options)
sess = tf.Session()
print("Setting up Saver...")
saver = tf.train.Saver()
summary_writer = tf.summary.FileWriter(FLAGS.logs_dir, sess.graph)
sess.run(tf.global_variables_initializer())
if fine_tuning:
ckpt = tf.train.get_checkpoint_state(FLAGS.logs_dir) # 训练断点回复
if ckpt and ckpt.model_checkpoint_path: # 如果存在checkpoint文件 则恢复sess
saver.restore(sess, ckpt.model_checkpoint_path)
print("Model restored...")
if FLAGS.mode == "train":
for itr in range(MAX_ITERATION):
# 读取下一batch
train_images, train_annotations = train_dataset_reader.next_batch(
FLAGS.batch_size)
feed_dict = {
image: train_images,
annotation: train_annotations,
keep_probability: 0.85
}
# 迭代优化需要训练的变量
sess.run(train_op, feed_dict=feed_dict)
if itr % 10 == 0:
# 迭代10次打印显示
train_loss, summary_str = sess.run([loss, summary_op],
feed_dict=feed_dict)
print("Step: %d, Train_loss:%g" % (itr, train_loss))
summary_writer.add_summary(
summary_str,
itr) #调用train_writer的add_summary方法将训练过程以及训练步数保存
if itr % 500 == 0:
# 迭代500 次验证
valid_images, valid_annotations = validation_dataset_reader.next_batch(
FLAGS.batch_size)
valid_loss = sess.run(loss,
feed_dict={
image: valid_images,
annotation: valid_annotations,
keep_probability: 1.0
})
print("%s ---> Validation_loss: %g" %
(datetime.datetime.now(), valid_loss))
# 保存模型
saver.save(sess, FLAGS.logs_dir + "model.ckpt", itr)
elif FLAGS.mode == "visualize":
# 可视化
valid_images, valid_annotations = validation_dataset_reader.get_random_batch(
FLAGS.batch_size)
# pred_annotation预测结果图
pred = sess.run(pred_annotation,
feed_dict={
image: valid_images,
annotation: valid_annotations,
keep_probability: 1.0
})
valid_annotations = np.squeeze(valid_annotations, axis=3)
pred = np.squeeze(pred, axis=3)
for itr in range(FLAGS.batch_size):
utils.save_image(valid_images[itr].astype(np.uint8),
FLAGS.logs_dir,
name="inp_" + str(5 + itr))
utils.save_image(valid_annotations[itr].astype(np.uint8),
FLAGS.logs_dir,
name="gt_" + str(5 + itr))
utils.save_image(pred[itr].astype(np.uint8),
FLAGS.logs_dir,
name="pred_" + str(5 + itr))
print("Saved image: %d" % itr)
elif FLAGS.mode == "test":
file_list = []
# ./ number_segment_2/img.jpg
# 加入文件列表 包含所有图片文件全路径+文件名字 如 Data_zoo/MIT_SceneParsing/ADEChallengeData2016/images/training/hi.jpg
file_glob = os.path.join(FLAGS.data_dir + 'ADEChallengeData2016/',
"images/", 'validation/', '*.' + 'jpg')
file_list.extend(glob.glob(file_glob))
print('Start Transport')
for f in file_list:
filename = os.path.splitext(f.split("/")[-1])[0]
filename = os.path.splitext(filename.split("_")[-1])[0]
image_orignal = misc.imread(f)
# resize_image = misc.imresize(image_orignal,
# [224, 224], interp='nearest')
image_final = np.array([image_orignal])
pred, probabilities_np = sess.run([pred_annotation, probabilities],
feed_dict={
image: image_final,
keep_probability: 1.0
})
image_pred = np.squeeze(pred)
image1 = image_orignal
# softmax = probabilities_np.squeeze()
softmax = probabilities_np.transpose((2, 0, 1, 3))
# softmax_to_unary函数的输入数据为概率值的负对数
unary = softmax_to_unary(softmax)
# 输入数据应该是 C-continious -- 这里采用 Cython 封装器
unary = np.ascontiguousarray(unary)
d = dcrf.DenseCRF(image1.shape[0] * image1.shape[1], 256)
n_labels = len(set(image_pred.flat))
unary = unary_from_labels(image_pred,
n_labels,
gt_prob=0.7,
zero_unsure=0)
d.setUnaryEnergy(unary)
# 对空间独立的小分割区域进行潜在地惩罚 - 促使生成更多连续的分割区域.
feats = create_pairwise_gaussian(sdims=(5, 5),
shape=image1.shape[:2])
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# 创建与颜色相关的特征
# 因为 CNN 的分割结果太粗糙，使用局部的颜色特征来进一步提升分割结果.
feats = create_pairwise_bilateral(sdims=(100, 100),
schan=(20, 20, 20),
img=image1,
chdim=2)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)
res = np.argmax(Q, axis=0).reshape(
(image1.shape[0], image1.shape[1]))
utils.save_image(res.astype(np.uint8),
FLAGS.pic_final_dir,
name='img' + '_' + filename)
def CRF_act( fn_im ,fn_anno , fn_output , fn_output2 ):
anno_rgb = imread(fn_anno).astype(np.uint8)
anno_lbl = anno_rgb[:,:,0] + (anno_rgb[:,:,1] << 8) + (anno_rgb[:,:,2] << 16)
colors_not_used, labels = np.unique(anno_lbl, return_inverse=True)
img = imread(fn_im)
img = cv2.resize(img , (320,192), interpolation =cv2.INTER_AREA)
n_labels = 4
colorize = np.empty((len(colors_not_used), 3), np.uint8)
colorize[:,0] = (colors_not_used & 0x0000FF)
colorize[:,1] = (colors_not_used & 0x0000FF) >> 8
colorize[:,2] = (colors_not_used & 0x0000FF) >> 8
d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], n_labels)
U = unary_from_labels(labels, n_labels, gt_prob=0.8, zero_unsure=False)
d.setUnaryEnergy(U)
feats = create_pairwise_gaussian(sdims=(5, 5), shape=img.shape[:2])
d.addPairwiseEnergy(feats, compat=2,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
feats = create_pairwise_bilateral(sdims=(100, 100), schan=(13, 13, 13),
img=img, chdim=2)
d.addPairwiseEnergy(feats, compat=7,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q, tmp1, tmp2 = d.startInference()
for i in range(6):
print("KL-divergence at {}: {}".format(i, d.klDivergence(Q)))
d.stepInference(Q, tmp1, tmp2)
MAP = np.argmax(Q, axis=0)
MAP = colorize[MAP,:]
MAP = MAP.reshape(img.shape)
MAP = cv2.morphologyEx(MAP, cv2.MORPH_CLOSE, kernel)
print(np.shape(MAP))
imwrite(fn_output, MAP)
for rows in range(np.shape(MAP)[0]):
for cols in range(np.shape(MAP)[1]):
if (MAP[rows][cols][0] > 0):
MAP[rows][cols] =255
if (MAP[rows][cols][0] == 0):
MAP[rows][cols] =0
imwrite(fn_output2, MAP)
def crf(fn_im, fn_anno, fn_output, num_classes=n_classes, use_2d=True):
img = imread(fn_im)
# Convert the annotation's RGB color to a single 32-bit integer color 0xBBGGRR
anno_rgb = imread(fn_anno).astype(np.uint32)
anno_lbl = anno_rgb[:, :, 0] + \
(anno_rgb[:, :, 1] << 8) + (anno_rgb[:, :, 2] << 16)
# Convert the 32bit integer color to 1, 2, ... labels.
# Note that all-black, i.e. the value 0 for background will stay 0.
colors, labels = np.unique(anno_lbl, return_inverse=True)
# But remove the all-0 black, that won't exist in the MAP!
# HAS_UNK = 0 in colors
HAS_UNK = False
# And create a mapping back from the labels to 32bit integer colors.
colorize = np.empty((len(colors), 3), np.uint8)
colorize[:, 0] = (colors & 0x0000FF)
colorize[:, 1] = (colors & 0x00FF00) >> 8
colorize[:, 2] = (colors & 0xFF0000) >> 16
# Compute the number of classes in the label image.
# We subtract one because the number shouldn't include the value 0 which stands
if use_2d:
# Setting up the CRF model
d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], num_classes)
# get unary potentials (neg log probability)
U = unary_from_labels(labels,
num_classes,
gt_prob=0.7,
zero_unsure=False)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
d.addPairwiseBilateral(sxy=(10, 10),
srgb=(13, 13, 13),
rgbim=img,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
else:
# print("Using generic 2D functions")
# Example using the DenseCRF class and the util functions
d = dcrf.DenseCRF(img.shape[1] * img.shape[0], num_classes)
# get unary potentials (neg log probability)
U = unary_from_labels(labels,
num_classes,
gt_prob=0.7,
zero_unsure=HAS_UNK)
d.setUnaryEnergy(U)
# This creates the color-independent features and then add them to the CRF
feats = create_pairwise_gaussian(sdims=(3, 3), shape=img.shape[:2])
d.addPairwiseEnergy(feats,
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This creates the color-dependent features and then add them to the CRF
feats = create_pairwise_bilateral(sdims=(10, 10),
schan=(13, 13, 13),
img=img,
chdim=2)
d.addPairwiseEnergy(feats,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# Run five inference steps.
Q = d.inference(5)
# Find out the most probable class for each pixel.
MAP = np.argmax(Q, axis=0)
# Convert the MAP (labels) back to the corresponding colors and save the image.
# Note that there is no "unknown" here anymore, no matter what we had at first.
MAP = colorize[MAP, :]
crfimage = MAP.reshape(img.shape)
msk = decode_labels(crfimage, num_classes=num_classes)
parsing_im = Image.fromarray(msk)
parsing_im.save(fn_output + '_vis.png')
cv2.imwrite(fn_output + '.png', crfimage[:, :, 0])
def crf_refine(label,
img,
crf_theta_slider_value,
crf_mu_slider_value,
crf_downsample_factor,
gt_prob):
"""
"crf_refine(label, img)"
This function refines a label image based on an input label image and the associated image
Uses a conditional random field algorithm using spatial and image features
INPUTS:
* label [ndarray]: label image 2D matrix of integers
* image [ndarray]: image 3D matrix of integers
OPTIONAL INPUTS: None
GLOBAL INPUTS: None
OUTPUTS: label [ndarray]: label image 2D matrix of integers
"""
Horig = label.shape[0]
Worig = label.shape[1]
l_unique = np.unique(label.flatten())#.tolist()
scale = 1+(5 * (np.array(img.shape).max() / 3000))
logging.info(datetime.now().strftime("%Y-%m-%d-%H-%M-%S"))
logging.info('CRF scale: %f' % (scale))
logging.info(datetime.now().strftime("%Y-%m-%d-%H-%M-%S"))
logging.info('CRF downsample factor: %f' % (crf_downsample_factor))
logging.info('CRF theta parameter: %f' % (crf_theta_slider_value))
logging.info('CRF mu parameter: %f' % (crf_mu_slider_value))
logging.info('CRF prior probability of labels: %f' % (gt_prob))
# decimate by factor by taking only every other row and column
img = img[::crf_downsample_factor,::crf_downsample_factor, :]
# do the same for the label image
label = label[::crf_downsample_factor,::crf_downsample_factor]
# yes, I know this aliases, but considering the task, it is ok; the objective is to
# make fast inference and resize the output
logging.info(datetime.now().strftime("%Y-%m-%d-%H-%M-%S"))
logging.info('Images downsampled by a factor os %f' % (crf_downsample_factor))
Hnew = label.shape[0]
Wnew = label.shape[1]
orig_mn = np.min(np.array(label).flatten())
orig_mx = np.max(np.array(label).flatten())
if l_unique[0]==0:
n = (orig_mx-orig_mn)#+1
else:
n = (orig_mx-orig_mn)+1
label = (label - orig_mn)+1
mn = np.min(np.array(label).flatten())
mx = np.max(np.array(label).flatten())
n = (mx-mn)+1
H = label.shape[0]
W = label.shape[1]
U = unary_from_labels(label.astype('int'), n, gt_prob=gt_prob)
d = dcrf.DenseCRF2D(H, W, n)
d.setUnaryEnergy(U)
# to add the color-independent term, where features are the locations only:
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
feats = create_pairwise_bilateral(
sdims=(crf_theta_slider_value, crf_theta_slider_value),
schan=(scale,scale,scale),
img=img,
chdim=2)
d.addPairwiseEnergy(feats, compat=crf_mu_slider_value, kernel=dcrf.DIAG_KERNEL,normalization=dcrf.NORMALIZE_SYMMETRIC) #260
logging.info(datetime.now().strftime("%Y-%m-%d-%H-%M-%S"))
logging.info('CRF feature extraction complete ... inference starting')
Q = d.inference(10)
result = np.argmax(Q, axis=0).reshape((H, W)).astype(np.uint8) +1
logging.info(datetime.now().strftime("%Y-%m-%d-%H-%M-%S"))
logging.info('CRF inference made')
uniq = np.unique(result.flatten())
result = resize(result, (Horig, Worig), order=0, anti_aliasing=False) #True)
result = rescale(result, orig_mn, orig_mx).astype(np.uint8)
logging.info(datetime.now().strftime("%Y-%m-%d-%H-%M-%S"))
logging.info('label resized and rescaled ... CRF post-processing complete')
return result, n
def apply_crf(original_image_path, output_image_path, final_result_path):
# Get im{read,write} from somewhere.
try:
from cv2 import imread, imwrite
except ImportError:
# Note that, sadly, skimage unconditionally import scipy and matplotlib,
# so you'll need them if you don't have OpenCV. But you probably have them.
from skimage.io import imread, imsave
imwrite = imsave
# TODO: Use scipy instead.
from pydensecrf.utils import unary_from_labels, create_pairwise_bilateral, create_pairwise_gaussian
fn_im = original_image_path
fn_anno = output_image_path
fn_output = final_result_path
# fn_im = "/data1/LJH/cvpppnet/A1/plant002_rgb.png"
# fn_anno = "/data1/LJH/cvpppnet/A1_predict/output_001.png"
# fn_output = "/data1/LJH/cvpppnet/semantic_segmentation_usinng_crf/output.png"
##################################
### Read images and annotation ###
##################################
img = imread(fn_im)
# Convert the annotation's RGB color to a single 32-bit integer color 0xBBGGRR
anno_rgb = imread(fn_anno).astype(np.uint32)
anno_lbl = anno_rgb[:, :, 0] + (anno_rgb[:, :, 1] << 8) + (anno_rgb[:, :, 2] << 16)
# Convert the 32bit integer color to 1, 2, ... labels.
# Note that all-black, i.e. the value 0 for background will stay 0.
colors, labels = np.unique(anno_lbl, return_inverse=True)
# But remove the all-0 black, that won't exist in the MAP!
HAS_UNK = 0 in colors
if HAS_UNK:
print(
"Found a full-black pixel in annotation image, assuming it means 'unknown' label, and will thus not be present in the output!")
print(
"If 0 is an actual label for you, consider writing your own code, or simply giving your labels only non-zero values.")
colors = colors[1:]
# else:
# print("No single full-black pixel found in annotation image. Assuming there's no 'unknown' label!")
# And create a mapping back from the labels to 32bit integer colors.
colorize = np.empty((len(colors), 3), np.uint8)
colorize[:, 0] = (colors & 0x0000FF)
colorize[:, 1] = (colors & 0x00FF00) >> 8
colorize[:, 2] = (colors & 0xFF0000) >> 16
# Compute the number of classes in the label image.
# We subtract one because the number shouldn't include the value 0 which stands
# for "unknown" or "unsure".
n_labels = len(set(labels.flat)) - int(HAS_UNK)
print(n_labels, " labels", (" plus \"unknown\" 0: " if HAS_UNK else ""), set(labels.flat))
###########################
### Setup the CRF model ###
###########################
use_2d = False
# use_2d = True
if use_2d:
print("Using 2D specialized functions")
# Example using the DenseCRF2D code
d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=HAS_UNK)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3, 3), compat=6, kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
d.addPairwiseBilateral(sxy=(95, 95), srgb=(13, 13, 13), rgbim=img,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
else:
print("Using generic 2D functions")
# Example using the DenseCRF class and the util functions
d = dcrf.DenseCRF(img.shape[1] * img.shape[0], n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=HAS_UNK)
d.setUnaryEnergy(U)
# This creates the color-independent features and then add them to the CRF
feats = create_pairwise_gaussian(sdims=(3, 3), shape=img.shape[:2])
d.addPairwiseEnergy(feats, compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This creates the color-dependent features and then add them to the CRF
feats = create_pairwise_bilateral(sdims=(80, 80), schan=(13, 13, 13),
img=img, chdim=2)
d.addPairwiseEnergy(feats, compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
####################################
### Do inference and compute MAP ###
####################################
# Run five inference steps.
Q = d.inference(10)
# Find out the most probable class for each pixel.
MAP = np.argmax(Q, axis=0)
# Convert the MAP (labels) back to the corresponding colors and save the image.
# Note that there is no "unknown" here anymore, no matter what we had at first.
MAP = colorize[MAP, :]
# TODO: save image
# Just randomly manually run inference iterations
Q, tmp1, tmp2 = d.startInference()
for i in range(3):
print("KL-divergence at {}: {}".format(i, d.klDivergence(Q)))
d.stepInference(Q, tmp1, tmp2)
# final result.
final_result = MAP.reshape(img.shape)
rt = np.zeros(final_result.shape)
rt[np.where(final_result > 30)] = 255
# save image.
imwrite(fn_output, rt)
fig = plt.figure()
fig.set_size_inches(10, 2) # 1800 x600
ax1 = fig.add_subplot(1, 5, 1)
ax2 = fig.add_subplot(1, 5, 2)
ax3 = fig.add_subplot(1, 5, 3)
ax4 = fig.add_subplot(1, 5, 4)
ax5 = fig.add_subplot(1, 5, 5)
ax1.set_title("origin")
ax2.set_title("output")
ax3.set_title("after CRF")
ax4.set_title("masking : over.{}".format(30))
ax5.set_title("final")
ax1.imshow(img)
ax2.imshow(anno_rgb)
final_result = MAP.reshape(img.shape)
ax3.imshow(final_result)
temp = np.zeros(final_result.shape)
temp[np.where(final_result > 30)] = 255
# to decide the dividing range, using iou
ax4.imshow(temp)
temp2 = np.copy(img)
temp2[np.where(final_result <= 30)] = 0
ax5.imshow(temp2)
# save image.
imwrite(fn_output, temp2)
plt.show()
return rt
def getCRF(image,
Lc,
theta,
n_iter,
label_lines,
compat_spat=12,
compat_col=40,
scale=5,
prob=0.5):
# n_iters: number of iterations of MAP inference.
# sxy_gaussian: standard deviations for the location component
# of the colour-independent term.
# compat_gaussian: label compatibilities for the colour-independent
# term (can be a number, a 1D array, or a 2D array).
# kernel_gaussian: kernel precision matrix for the colour-independent
# term (can take values CONST_KERNEL, DIAG_KERNEL, or FULL_KERNEL).
# normalisation_gaussian: normalisation for the colour-independent term
# (possible values are NO_NORMALIZATION, NORMALIZE_BEFORE, NORMALIZE_AFTER, NORMALIZE_SYMMETRIC).
# sxy_bilateral: standard deviations for the location component of the colour-dependent term.
# compat_bilateral: label compatibilities for the colour-dependent
# term (can be a number, a 1D array, or a 2D array).
# srgb_bilateral: standard deviations for the colour component
# of the colour-dependent term.
# kernel_bilateral: kernel precision matrix for the colour-dependent term
# (can take values CONST_KERNEL, DIAG_KERNEL, or FULL_KERNEL).
# normalisation_bilateral: normalisation for the colour-dependent term
# (possible values are NO_NORMALIZATION, NORMALIZE_BEFORE, NORMALIZE_AFTER, NORMALIZE_SYMMETRIC).
H = image.shape[0]
W = image.shape[1]
d = dcrf.DenseCRF2D(H, W, len(label_lines) + 1)
U = unary_from_labels(Lc.astype('int'), len(label_lines) + 1, gt_prob=prob)
d.setUnaryEnergy(U)
del U
# This potential penalizes small pieces of segmentation that are
# spatially isolated -- enforces more spatially consistent segmentations
# This adds the color-independent term, features are the locations only.
# sxy = The scaling factors per dimension.
d.addPairwiseGaussian(
sxy=(theta, theta),
compat=compat_spat,
kernel=dcrf.DIAG_KERNEL, #compat=6
normalization=dcrf.NORMALIZE_SYMMETRIC)
# sdims = The scaling factors per dimension.
# schan = The scaling factors per channel in the image.
# This creates the color-dependent features and then add them to the CRF
feats = create_pairwise_bilateral(
sdims=(theta, theta),
schan=(scale, scale, scale), #11,11,11
img=image,
chdim=2)
del image
d.addPairwiseEnergy(
feats,
compat=compat_col, #20
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
del feats
Q = d.inference(n_iter)
#preds = np.array(Q, dtype=np.float32).reshape(
# (len(label_lines)+1, nx, ny)).transpose(1, 2, 0)
#preds = np.expand_dims(preds, 0)
#preds = np.squeeze(preds)
return np.argmax(Q, axis=0).reshape(
(H, W)) #, preds#, p, R, d.klDivergence(Q),
def crf(original_image, annotated_image, output_image, use_2d=True):
##将注释RGB颜色转换为单个32位整数
#annotated_label = annotated_image[:,:,0] + (annotated_image[:,:,1]<<8) + (annotated_image[:,:,2]<<16)
#print(annotated_label.shape)
# 将32位整数颜色转换为0、1、2、…标签。
colors, labels = np.unique(annotated_label, return_inverse=True)
print(len(labels))
print(labels)
print(len(colors))
print(colors)
n_labels = len(set(labels.flat))
#创建一个32位颜色的映射
## colorize = np.empty((len(colors), 3), np.uint8)
## colorize[:,0] = (colors & 0x0000FF)
## colorize[:,1] = (colors & 0x00FF00) >> 8
## colorize[:,2] = (colors & 0xFF0000) >> 16
##
## #在带注释的图像中不给出任何类标签
## n_labels = len(set(labels.flat))
##
## print("图像中没有标签")
## print(n_labels)
#Setting up the CRF model
if use_2d:
d = dcrf.DenseCRF2D(original_image.shape[1] * original_image.shape[0],
n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=False)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
d.addPairwiseBilateral(sxy=(80, 80),
srgb=(13, 13, 13),
rgbim=original_image,
compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
#Run Inference for 5 steps
Q = d.inference(5)
# Find out the most probable class for each pixel.
MAP = np.argmax(Q, axis=0)
# C将地图(标签)转换回相应的颜色并保存图像。
# 注意，这里不再有“未知”，不管我们一开始拥有什么。
MAP = colorize[MAP, :]
MAP = MAP.reshape(original_image.shape)
MAP = array_to_img(MAP)
MAP.save(output_image)
return MAP
def test_img_fw():
img = cv2.imread('examples/im1.ppm')
anno_rgb = cv2.imread('examples/anno1.ppm').astype(np.uint32)
anno_lbl = anno_rgb[:, :, 0] + \
(anno_rgb[:, :, 1] << 8) + (anno_rgb[:, :, 2] << 16)
# Convert the 32bit integer color to 1, 2, ... labeling.
# Note that all-black, i.e. the value 0 for background will stay 0.
colors, labels = np.unique(anno_lbl, return_inverse=True)
# And create a mapping back from the labels to 32bit integer colors.
# But remove the all-0 black, that won't exist in the MAP!
colors = colors[1:]
colorize = np.empty((len(colors), 3), np.uint8)
colorize[:, 0] = (colors & 0x0000FF)
colorize[:, 1] = (colors & 0x00FF00) >> 8
colorize[:, 2] = (colors & 0xFF0000) >> 16
# Compute the number of classes in the label image.
# We subtract one because the number shouldn't include the value 0 which stands
# for "unknown" or "unsure".
n_labels = len(set(labels.flat)) - 1
print(n_labels, " labels and \"unknown\" 0: ", set(labels.flat))
# Example using the DenseCRF class and the util functions
d = DenseCRF(img.shape[1] * img.shape[0], n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(labels, n_labels, gt_prob=0.7, zero_unsure=True)
d.setUnaryEnergy(U)
# This creates the color-independent features and then add them to the CRF
feats = create_pairwise_gaussian(sdims=(3, 3),
shape=img.shape[:2]).astype(np.float32)
d.addPairwiseEnergy(feats, PottsCompatibility(3), KernelType.DIAG_KERNEL,
NormalizationType.NORMALIZE_SYMMETRIC)
# This creates the color-dependent features and then add them to the CRF
feats = create_pairwise_bilateral(sdims=(80, 80),
schan=(13, 13, 13),
img=img,
chdim=2).astype(np.float32)
d.addPairwiseEnergy(feats, PottsCompatibility(10), KernelType.DIAG_KERNEL,
NormalizationType.NORMALIZE_SYMMETRIC)
####################################
### Do inference and compute MAP ###
####################################
# Run five inference steps.
Q = d.inference(5)
# Find out the most probable class for each pixel.
MAP = np.argmax(Q, axis=0)
# Convert the MAP (labels) back to the corresponding colors and save the
# image.
MAP = colorize[MAP, :]
print MAP.shape
show_img(MAP.reshape(img.shape))
def getCRF(img, Lc, config): #, optim):
"""
Uses a dense CRF model to refine labels based on sparse labels and underlying image
Input:
img: 3D ndarray image
Lc: 2D ndarray label image (sparse or dense)
label_lines: list of class names
Hard-coded variables:
kernel_bilateral: DIAG_KERNEL kernel precision matrix for the colour-dependent term
(can take values CONST_KERNEL, DIAG_KERNEL, or FULL_KERNEL).
normalisation_bilateral: NORMALIZE_SYMMETRIC normalisation for the colour-dependent term
(possible values are NO_NORMALIZATION, NORMALIZE_BEFORE, NORMALIZE_AFTER, NORMALIZE_SYMMETRIC).
kernel_gaussian: DIAG_KERNEL kernel precision matrix for the colour-independent
term (can take values CONST_KERNEL, DIAG_KERNEL, or FULL_KERNEL).
normalisation_gaussian: NORMALIZE_SYMMETRIC normalisation for the colour-independent term
(possible values are NO_NORMALIZATION, NORMALIZE_BEFORE, NORMALIZE_AFTER, NORMALIZE_SYMMETRIC).
Output:
res : CRF-refined 2D label image
"""
# 1. if `optim` is `True`, the probability of your annotation is set to 0.8,
# and a set of 10 factors are defined that modify `theta_col` and `compat_col`.
# If `optim` is `False`, the probability of your annotation
# is set to 0.9, and a set of 5 factors are defined that modify `theta_col` and `compat_col`.
if config['optim'] is True:
search = [.5,.66,.75,1,1.25,1.33,2]
prob = 0.8
else:
search = [.5,.75,1,1.25,1.5] #[.25,.5,1,2,4]
prob = 0.9
label_lines = config['classes']
fact = config['fact']
# initial parameters
n_iter = 10
scale = 1+(5 * (np.array(img.shape).max() / 3681))
compat_col = config['compat_col'] #20
theta_col = config['theta_col'] #20
theta_spat = 3
prob = 0.9
compat_spat = 3
# 2. per-class weights are computed as inverse relative frequencies.
# The number of each pixel in each annotation class is computed and normalized.
# If the maximum relative frequency is more than 5 times the minimum, the maximum is halved
# and the relative frequencies re-normalized before their inverses are used as weights in the CRF model.
## relative frequency of annotations
#rel_freq = np.bincount(Lc.flatten())#, minlength=len(label_lines)+1)
if len(np.unique(Lc.flatten()))>1:
rel_freq = np.sqrt(np.bincount(Lc.flatten())) #sqrt does area --> length
rel_freq[0] = 0
rel_freq = rel_freq / rel_freq.sum()
rel_freq[rel_freq<1e-4] = 1e-4
rel_freq = rel_freq / rel_freq.sum()
else:
rel_freq = 1
if type(rel_freq) is not int:
if rel_freq.max() > 5*rel_freq.min():
print("Large class imbalance detected ... modifying weights accordingly")
rel_freq[np.argmax(rel_freq)] = rel_freq[np.argmax(rel_freq)]/2
rel_freq = rel_freq / rel_freq.sum()
if np.mean(img)<1:
H = img.shape[0]
W = img.shape[1]
res = np.zeros((H,W)) * np.mean(Lc)
else:
# if image is 2D, make it 3D by stacking bands
if np.ndim(img) == 2:
# decimate by factor first
img = img[::fact,::fact, :]
img = np.dstack((img, img, img))
# get original image shapes
Horig = img.shape[0]
Worig = img.shape[1]
# decimate by factor by taking only every other row and column
img = img[::fact,::fact, :]
# do the same for the label image
Lc = Lc[::fact,::fact]
if config['do_stack']=="true":
R = img[:,:,0]
G = img[:,:,1]
B = img[:,:,2]
R[R<1]=1; G[G<1]=1; B[B<1]=1;
VARI = (G-R)/(G+R-B)
NEXG = (2*G - R - B) / (G+R+B)
NGRDI = (G-R)/(G+R)
VARI[np.isinf(VARI)] = 1e-2
NEXG[np.isinf(NEXG)] = 1e-2
NGRDI[np.isinf(NGRDI)] = 1e-2
VARI[np.isnan(VARI)] = 1e-2
NEXG[np.isnan(NEXG)] = 1e-2
NGRDI[np.isnan(NGRDI)] = 1e-2
VARI[VARI==0] = 1e-2
NEXG[NEXG==0] = 1e-2
NGRDI[NGRDI==0] = 1e-2
VARI = rescale(np.log(VARI),0,255)
NEXG = rescale(np.log(NEXG),0,255)
NGRDI = rescale(np.log(NGRDI),0,255)
STACK = np.dstack((R,G,B,VARI,NEXG,NGRDI)).astype(np.int)
del R, G, B, VARI, NEXG, NGRDI
# plt.subplot(221); plt.imshow(img)
# plt.axis('off'); plt.title('RGB', fontsize=8)
# plt.subplot(222); plt.imshow(VARI)
# plt.axis('off'); plt.title('log(VARI) = log((G-R)/(G+R-B))', fontsize=8)
# plt.subplot(223); plt.imshow(NEXG)
# plt.axis('off'); plt.title('log(NEXG) = log((2G-R-B)/(G+R+B))', fontsize=8)
# plt.subplot(224); plt.imshow(NGRDI)
# plt.axis('off'); plt.title('log(NGRDI) = log((G-R)/(G+R))', fontsize=8)
# plt.savefig('ex_6band_RGB.png'); plt.close()
#
# plt.subplot(121)
# plt.imshow(img)
# plt.axis('off'); plt.title('RGB', fontsize=8)
# plt.subplot(122)
# plt.imshow(np.dstack((VARI,NEXG,NGRDI)).astype(np.int))
# plt.axis('off'); plt.title('VARI-NEXG-NGRDI false color', fontsize=8)
# plt.savefig('ex_3band_falsecolor.png'); plt.close()
# get the new shapes
H = img.shape[0]
W = img.shape[1]
U = unary_from_labels(Lc.astype('int'),
len(label_lines) + 1,
gt_prob=prob)
R = []; P = []
# 4. the hyperparameters `theta_col` and `compat_col` are modified by the factors described above.
# The 10 `optim` factors are .25, .33, .5, .75, 1, 1.33, 2, 3, and 4, therefore if `theta_col` is 100,
# the program would cycle through creating a CRF realization
# based on `theta_col` = `compat_col` = 25, then 33, 50, 75, 100, 133, 200, 300, and 400.
# The 5 `optim` factors are .25,.5,1,2, and 4.
## loop through the 'theta' values (half, given, and double)
for mult in tqdm(search):
d = dcrf.DenseCRF2D(H, W, len(label_lines) + 1)
d.setUnaryEnergy(U)
# to add the color-independent term, where features are the locations only:
d.addPairwiseGaussian(sxy=(theta_spat, theta_spat),
compat=compat_spat,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
if config['do_stack']=="true":
feats = create_pairwise_bilateral(
sdims=(theta_col*mult, theta_col*mult),
schan=(scale,
scale,
scale),
img=STACK, #img,
chdim=2)
else:
feats = create_pairwise_bilateral(
sdims=(theta_col*mult, theta_col*mult),
schan=(scale,
scale,
scale),
img=img,
chdim=2)
d.addPairwiseEnergy(feats, compat=compat_col,kernel=dcrf.DIAG_KERNEL,normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(n_iter)
#print("KL-divergence at {}: {}".format(i, d.klDivergence(Q)))
R.append(1+np.argmax(Q, axis=0).reshape((H, W)).astype(np.uint8))
preds = np.array(Q, dtype=np.float32).reshape((len(label_lines)+1, H, W)).transpose(1, 2, 0)
P.append(preds)
del Q
##res = np.round(np.median(R, axis=0))
R = list(R)
# 5. The predictions (softmax scores based on normalized logits)
# per class are computed as the weighted average of the per-class
# predictions compiled over the number of hyperparameter factors.
if len(label_lines)==2: #<len(search):
try:
preds = np.average(P, axis=-1, weights = 1/rel_freq)
except:
preds = np.average(P[1:], axis=0, weights = 1/rel_freq)
# finally:
# print("using unweighted average")
# preds = np.median(P, axis=-1)
probs_per_class = np.median(P, axis=0)
elif np.asarray(P).shape[0] > np.asarray(P).shape[-1]:
preds = np.median(np.asarray(P).T, axis=-1).T
probs_per_class = np.median(P, axis=0)
else:
try:
if np.asarray(P).shape[0]==len(rel_freq):
preds = np.average(np.asarray(P), axis=0, weights = 1/rel_freq)
else:
preds = np.average(np.asarray(P), axis=0, weights = search)
except:
preds = np.average(P[1:], axis=0, weights = 1/rel_freq)
probs_per_class = np.median(P, axis=-1)
del P
#class is the last channel
if 1+len(label_lines) != np.asarray(probs_per_class).shape[-1]:
probs_per_class = np.swapaxes(probs_per_class,0,-1)
if 1+len(label_lines) != np.asarray(preds).shape[-1]:
preds = np.swapaxes(preds,0,-1)
# 6. Each per-class prediction raster is then median-filtered using a disk-shaped
# structural element with a radius of 15*(M/(3681)) pixels
for k in range(len(label_lines)):
N = np.round(10*(Worig/(3681))).astype('int') #11 when ny=3681
if (len(label_lines)==2):
preds[k,:,:] = median(img_as_ubyte(preds[k,:,:]), disk(N))
preds[k,:,:] = preds[k,:,:]/np.max(preds[k,:,:])
else:
preds[:,:,k] = median(img_as_ubyte(preds[:,:,k]), disk(N))
preds[:,:,k] = preds[:,:,k]/np.max(preds[:,:,k])
# 7. To make the final classification, if the per-class prediction is > .5 (for binary)
# or > 1/N where N=number of classes (for multiclass), the pixel is encoded that label.
# This works in order of classes listed in the config file, so a latter class
# could override a former one. Where no grid cell is encoded
# with a label, such as where the above criteria are not met,
# the label is the argument maximum for for that cell in the stack.
if (len(label_lines)==2): #special case of two classes, we average the stack of classes
preds
res = np.zeros((H,W))
counter = 1
for k in range(len(label_lines)):
if (len(label_lines)==2):
res[preds[:,:,k]>=(1/len(label_lines) )] = counter
else:
res[preds[:,:,k]>= 2*(1/len(label_lines)) ] = counter #.5
counter += 1
try:
res[res==0] = np.argmax(preds, axis=-1)[res==0]
except:
res[res==0] = np.argmax(preds, axis=-1).T[res==0]
if len(label_lines)==2: #<len(search):
res = res.T #transpose binary images
p = 1-(np.std(preds, axis=0)/len(label_lines))
else:
#p = np.max(preds, axis=-1)
#s = np.max(preds, axis=-1) - np.min(preds, axis=-1)
#s = (s/np.max(s))
p = np.nanmax(preds, axis=-1) #1-s
p[p<1/len(label_lines)] = 1/len(label_lines)
del R, preds
if fact>1:
img = np.array(Image.fromarray(img).resize((Worig, Horig),
resample=1))-1
res = np.array(Image.fromarray(1+res.astype(np.uint8)).resize((Worig, Horig),
resample=1))-1
if len(label_lines)==2:
res[res>2] = 0
res[res==1] = 0
else:
res[res>len(label_lines)] = 0
res[res<0] = 0
p = np.array(Image.fromarray((100*p).astype(np.uint8)).resize((Worig, Horig),
resample=1))/100
p[p>1]=1
p[p<0]=0
tmp = np.zeros((Horig,Worig,probs_per_class.shape[-1]))
for k in range(probs_per_class.shape[-1]):
tmp[:,:,k] = np.array(
Image.fromarray((100*probs_per_class[:,:,k]).astype(np.uint8)).resize((Worig, Horig),
resample=1)
)/100
del probs_per_class
probs_per_class = tmp.copy().astype('float16')
del tmp
probs_per_class[probs_per_class>1] = 1
probs_per_class[probs_per_class<0]= 0
if (config['medfilt']=="true") or (len(label_lines)==2):
## median filter to remove remaining high-freq spatial noise (radius of N pixels)
N = np.round(5*(Worig/(3681))).astype('int') #11 when ny=3681
print("Applying median filter of size: %i" % (N))
res = median(res.astype(np.uint8), disk(N))
return res, p, probs_per_class.astype('float16')
#print(n_labels, " labels", set(anno_rgb.flat))
###########################
### Setup the CRF model ###
###########################
#use_2d = True
# use_2d = True
if use_2d:
#print("Using 2D specialized functions")
# Example using the DenseCRF2D code
d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], n_labels)
# get unary potentials (neg log probability)
U = unary_from_labels(anno_rgb,
n_labels,
gt_prob=gt_prob,
zero_unsure=False)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=Gaussian_sxy,
compat=Gaussian_compat,
kernel=Gaussian_kernel,
normalization=Gaussion_norm)
# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
d.addPairwiseBilateral(sxy=Bilateral_sxy,
srgb=Bilateral_srgb,
rgbim=img,
compat=Bilateral_compat,
kernel=Bilateral_kernel,
print(img_name)
img = imread(fn_im)
anno = imread(fn_anno).astype(np.uint32)
anno = anno[:, :, 1]
gt, labels = np.unique(anno, return_inverse=True)
n_labels = len(set(labels.flat))
NL = np.unique(labels)
if n_labels == 1:
anno_n = imread(fn_anno)
imwrite(fn_output, anno_n[:, :, 1])
continue
d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], n_labels)
U = unary_from_labels(labels,
n_labels,
gt_prob=args.gtprob,
zero_unsure=False)
d.setUnaryEnergy(U)
# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3, 3),
compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
d.addPairwiseBilateral(sxy=(121, 121),
srgb=(5, 5, 5),
rgbim=img,
compat=4,
kernel=dcrf.DIAG_KERNEL,
for b_schan in range(5, 22, 4):
print('g_window={}, b_s_dim={}, b_schan={}'.format(g_window, b_sdim, b_schan))
save_dir = r'/home/lab/Documents/bohao/data/deeplab_model/post_{}_{}_{}'.format(g_window, b_sdim, b_schan)
ersa_utils.make_dir_if_not_exist(save_dir)
for p_file, r_file in tqdm(zip(pred_files, rgb_files), total=len(pred_files)):
# take one sample file
pred = ersa_utils.load_file(p_file)
rgb = ersa_utils.load_file(r_file)
# define 2d class
d = dcrf.DenseCRF2D(pred.shape[0], pred.shape[1], 33)
# get unary
unary = unary_from_labels(pred, 33, 0.5)
d.setUnaryEnergy(unary)
# get pairwise potentials
feats = create_pairwise_gaussian(sdims=(g_window, g_window), shape=pred.shape[:2])
d.addPairwiseEnergy(feats, compat=3,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
feats = create_pairwise_bilateral(sdims=(b_sdim, b_sdim), schan=(13, 13, 13),
img=rgb, chdim=2)
d.addPairwiseEnergy(feats, compat=10,
kernel=dcrf.DIAG_KERNEL,
normalization=dcrf.NORMALIZE_SYMMETRIC)
Q = d.inference(5)