def test_enforce_connectivity():
img = np.array([[0, 0, 0, 1, 1, 1],
[1, 0, 0, 1, 1, 0],
[0, 0, 0, 1, 1, 0]], np.float)
segments_connected = slic(img, 2, compactness=0.0001,
enforce_connectivity=True,
convert2lab=False)
segments_disconnected = slic(img, 2, compactness=0.0001,
enforce_connectivity=False,
convert2lab=False)
# Make sure nothing fatal occurs (e.g. buffer overflow) at low values of
# max_size_factor
segments_connected_low_max = slic(img, 2, compactness=0.0001,
enforce_connectivity=True,
convert2lab=False, max_size_factor=0.8)
result_connected = np.array([[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1]], np.float)
result_disconnected = np.array([[0, 0, 0, 1, 1, 1],
[1, 0, 0, 1, 1, 0],
[0, 0, 0, 1, 1, 0]], np.float)
assert_equal(segments_connected, result_connected)
assert_equal(segments_disconnected, result_disconnected)
assert_equal(segments_connected_low_max, result_connected)
def slics_3D(im, pseudo_3D=True, n_segments=100, get_slicewise=False):
if im.ndim != 3:
raise Exception('3D image is needed.')
if not pseudo_3D:
# need to convert to RGB image
im_rgb = np.zeros((im.shape[0], im.shape[1], im.shape[2], 3))
im_rgb[:,:,:,0] = im
im_rgb[:,:,:,1] = im
im_rgb[:,:,:,2] = im
suppxls = skiseg.slic(im_rgb, n_segments=n_segments, spacing=(2,1,1))
else:
suppxls = np.zeros(im.shape)
if get_slicewise:
suppxls_slicewise = np.zeros(im.shape)
offset = 0
for i in range(im.shape[0]):
# suppxl = skiseg.slic(cv2.cvtColor(im[i,:,:], cv2.COLOR_GRAY2RGB), n_segments=n_segments)
suppxl = skiseg.slic(skicol.gray2rgb(im[i,:,:]), n_segments=n_segments)
suppxls[i,:,:] = suppxl + offset
if get_slicewise:
suppxls_slicewise[i,:,:] = suppxl
offset = suppxls.max() + 1
if get_slicewise:
return suppxls, suppxls_slicewise
else:
return suppxls
def test_spacing():
rnd = np.random.RandomState(0)
img = np.array([[1, 1, 1, 0, 0],
[1, 1, 0, 0, 0]], np.float)
result_non_spaced = np.array([[0, 0, 0, 1, 1],
[0, 0, 1, 1, 1]], np.int)
result_spaced = np.array([[0, 0, 0, 0, 0],
[1, 1, 1, 1, 1]], np.int)
img += 0.1 * rnd.normal(size=img.shape)
seg_non_spaced = slic(img, n_segments=2, sigma=0, multichannel=False,
compactness=1.0)
seg_spaced = slic(img, n_segments=2, sigma=0, spacing=[1, 500, 1],
compactness=1.0, multichannel=False)
assert_equal(seg_non_spaced, result_non_spaced)
assert_equal(seg_spaced, result_spaced)
def segment(self):
self.segments = slic(img_as_float(self.img), enforce_connectivity=True)
self.mask = np.zeros(self.img.shape[:2],dtype='int' )
self.mask = self.mask - 1
for (i, segVal) in enumerate(np.unique(self.segments)):
self.mask[segVal == self.segments] = i
self.pixel_list.append(self.Pixel(i))
def detectOpticDisc(image):
kernel = octagon(10, 10)
thresh = threshold_otsu(image[:,:,1])
binary = image > thresh
print binary.dtype
luminance = convertToHLS(image)[:,:,2]
t = threshold_otsu(luminance)
t = erosion(luminance, kernel)
labels = segmentation.slic(image[:,:,1], n_segments = 3)
out = color.label2rgb(labels, image[:,:,1], kind='avg')
skio.imshow(out)
x, y = computeCentroid(t)
print x, y
rows, cols, _ = image.shape
p1 = closing(image[:,:,1],kernel)
p2 = opening(p1, kernel)
p3 = reconstruction(p2, p1, 'dilation')
p3 = p3.astype(np.uint8)
#g = dilation(p3, kernel)-erosion(p3, kernel)
#g = rank.gradient(p3, disk(5))
g = cv2.morphologyEx(p3, cv2.MORPH_GRADIENT, kernel)
#markers = rank.gradient(p3, disk(5)) < 10
markers = drawCircle(rows, cols, x, y, 85)
#markers = ndimage.label(markers)[0]
#skio.imshow(markers)
g = g.astype(np.uint8)
#g = cv2.cvtColor(g, cv2.COLOR_GRAY2RGB)
w = watershed(g, markers)
print np.max(w), np.min(w)
w = w.astype(np.uint8)
#skio.imshow(w)
return w
def extract_roi(img, labels_to_keep=[1,2]):
'''
Given a wheat image, this method returns an image containing only the region
of interest.
Args:
img: input image.
labels_to_keep: cluster labels to be kept in image while pixels
belonging to clusters besides these ones are removed.
Return:
roi_img: Input image containing only the region
of interest.
'''
label_img = segmentation.slic(img, compactness=30, n_segments=6)
labels = np.unique(label_img);print(labels)
gray = rgb2gray(img);
for label in labels:
if(label not in labels_to_keep):
logicalIndex = (label_img == label)
gray[logicalIndex] = 0;
#Display.show_image(gray)
return gray
def remove_background(self): L_b = 3 self.i_original = self.i_original[self.sub[1]:self.sub[3],self.sub[0]:self.sub[2],] segments = slic(self.i_original, n_segments=2, compactness=0.1,enforce_connectivity=False) # segments += 1 temp = self.i_original if sum(sum(segments[:5,:5])) > 10: for ii in range(0,3): temp[:,:,ii] = (np.ones([self.i_original.shape[0],self.i_original.shape[1]])-segments)*self.i_original[:,:,ii] #Haut, Bas, Droite, Gauche temp[:L_b,:,ii] = 0 temp[self.i_original.shape[0]-L_b-1:self.i_original.shape[0]-1,:,ii]=0 temp[:,self.i_original.shape[1]-L_b-1:self.i_original.shape[1]-1,ii] = 0 temp[:,:L_b,ii] = 0 else: for ii in range(0,3): temp[:,:,ii] = segments*self.i_original[:,:,ii] # print "else" #Haut, Bas, Droite, Gauche temp[:L_b,:,ii] = 0 temp[self.i_original.shape[0]-L_b-1:self.i_original.shape[0]-1,:,ii]=0 temp[:,self.i_original.shape[1]-L_b-1:self.i_original.shape[1]-1,ii] = 0 temp[:,:L_b,ii] = 0 # pdb.set_trace() fig, ax = plt.subplots(1, 1) ax.imshow(mark_boundaries(self.i_original,segments)) ax.imshow(temp) plt.show() p2, p98 = np.percentile(temp, (2, 98)) temp = exposure.rescale_intensity(temp, in_range=(p2, p98)) return temp
def detectOpticDisc(image):
labels = segmentation.slic(image, n_segments = 70)
out = color.label2rgb(labels, image, kind='avg')
gray = cv2.cvtColor(out, cv2.COLOR_RGB2GRAY)
minimum = np.max(gray)
image[gray==minimum] = 255
return image
def compute(self, image, n_region, mask=None):
"""
Parameters
----------
image: numpy ndarray
image array to be represented as regions
n_region : int
the number of regions to be generated
mask: mask image to give region of interest
Returns
-------
regions : a list of regions
"""
gray_image = image.copy()
if mask is not None:
gray_image[mask > 0] = 0
# Convert the original image to the 0~255 gray image
gray_image = (gray_image - gray_image.min()) / (gray_image.max() - gray_image.min())
labels = segmentation.slic(gray_image.astype(np.float),
n_segments=n_region,
slic_zero=True, sigma=2,
multichannel=False,
enforce_connectivity=True)
# edge_map = filters.sobel(gray_image) # just for 2-D image
edge_map = filters.laplace(gray_image)
# Given an image's initial segmentation and its edge map this method constructs the
# corresponding Region Adjacency Graph (RAG). Each node in the RAG represents a set of
# pixels within the image with the same label in labels. The weight between two adjacent
# regions is the average value in edge_map along their boundary.
rag = graph.rag_boundary(labels, edge_map)
regions = []
n_labels = labels.max() + 1
for r in np.arange(n_labels):
# get vox_pos
position = np.transpose(np.nonzero(labels == r))
# get vox_feat
vox_feat = np.zeros((position.shape[0], 1))
n_D = position.shape[1]
for i in range(position.shape[0]):
if n_D == 2:
vox_feat[i][0] = image[position[i][0], position[i][1]]
elif n_D == 3:
vox_feat[i][0] = image[position[i][0], position[i][1], position[i][2]]
else:
raise RuntimeError("We just consider 2_D and 3_D images at present!")
regions.append(Region(position, vox_feat=vox_feat, r_id=r))
for r in range(n_labels):
for r_key in rag.edge[r].keys():
regions[r].add_neighbor(regions[r_key])
self.regions = regions
self.image_shape = image.shape
return regions
def getSlic(self):
self.slic = slic(self.depth_image,
n_segments=50,
compactness=.001,
sigma=1,
multichannel=False)
return self.slic
def color_histogram(image_roi, attrs={}, debug=False):
# segment image using kmeans clustering
#segments = slic(image_roi[...,:3], sigma=1, n_segments=10, max_iter=30)
segments = slic(image_roi[...,:3], sigma=1, n_segments=100, max_iter=10)
nsegments = segments.max()
CLASS_COLORS = np.array([ c[1] for c in RGB_COLOR_CLASSES ])/255.
totals = [ 0. for c in RGB_COLOR_CLASSES ]
lesion_mask = image_roi[...,3]
for i in range(nsegments+1):
area = np.logical_and(segments == i, lesion_mask)
segment_size = area.sum()
if segment_size == 0: continue
rgb_average = image_roi[area,:3].mean(axis=0)
delta = CLASS_COLORS - rgb_average
distances = np.sqrt((delta * delta).sum(axis=1))
closest = np.argmin(distances)
totals[closest] += segment_size
totals = np.array(totals) / sum(totals)
for i, v in enumerate(totals):
name = RGB_COLOR_CLASSES[i][0]
attrs["Color Histogram %s" % name] = v
def superpixels(img, slide_magnif='x40'):
"""
SUPERPIXELS: produces a super-pixel representation of the image, with the new
super-pixels being the average (separate by channel) of the pixels in the
original image falling in the same "cell".
:param img: numpy.ndarray
RGB image
:param slide_magnif: string
Indicates the microscope magnification at which the image was acquired.
It is used to set some parameters, depending on the magnification.
:return: numpy.ndarray
The RGB super-pixel image.
"""
params = dict([('x40', dict([('n_segments', int(10*np.log2(img.size/3))), ('compactness', 50), ('sigma', 2.0)])),
('x20', dict([('n_segments', int(100*np.log2(img.size/3))), ('compactness', 50), ('sigma', 1.5)]))])
p = params[slide_magnif]
sp = slic(img, n_segments=p['n_segments'], compactness=p['compactness'], sigma=p['sigma'],
multichannel=True, convert2lab=True)
n_sp = sp.max() + 1
img_res = np.ndarray(img.shape, dtype=img.dtype)
for i in np.arange(n_sp):
img_res[sp == i, 0] = int(np.mean(img[sp == i, 0]))
img_res[sp == i, 1] = int(np.mean(img[sp == i, 1]))
img_res[sp == i, 2] = int(np.mean(img[sp == i, 2]))
return img_res
def get_segmentation(img):
segmentation = slic(img, n_segments=140, compactness=13, sigma=4, enforce_connectivity=True)
# segmentation_img = mark_boundaries(img, segmentation)
# io.imsave('slic.jpg', segmentation_img)
return segmentation
def main():
filename = 'mesquitesFloat.png'
img = io.imread(filename)
#CONVERSIONS:
#convert to lab color space
#img = skimage.color.rgb2lab(img)
#img = img_as_float(img)
#io.imsave('mesquitesFloat.png', img)
#print(img.shape)
plt.figure(1)
plt.imshow(img, cmap='gray')
plt.axis('off')
# loop over the number of segments
for numSegments in (100, 200, 300):
# apply SLIC and extract (approximately) the supplied number
# of segments
segments = slic(img, n_segments = numSegments, sigma = 1)
# show the output of SLIC
fig = plt.figure("Superpixels of -- %d segments" % (numSegments))
subplot = fig.add_subplot(1, 1, 1)
subplot.imshow(mark_boundaries(img, segments))
plt.axis("off")
plt.show()
def SuperPixel(self, Image):
segments = slic(Image, n_segments=20, sigma=5)
# show the output of SLIC
segments = segments + 1
# So that no labelled region is 0 and ignored by regionprops
label_rgb = color.label2rgb(segments, Image, kind='avg')
return label_rgb
def SLIC( Input_Image,ratio, n_segments, sigma):
'''
Description: Segments image using k-means clustering in Color space.
source: skimage, openCv python
parameters: Input_Image : ndarray
Input image, which can be 2D or 3D, and grayscale or multi-channel (see multichannel parameter).
n_segments : int
The (approximate) number of labels in the segmented output image.
ratio: float
Balances color-space proximity and image-space proximity. Higher values give more weight to color-space and yields more square regions
sigma : float
Width of Gaussian smoothing kernel for preprocessing. Zero means no smoothing.
return: Output_mask : ndarray
Integer mask indicating segment labels.
'''
if ratio == 0:
ratio = 0.5
if n_segments == 0:
n_segments = 3
if sigma ==0:
sigma = 1
img = cv2.imread(Input_Image)
segments_slic = slic(img, ratio=0.5, n_segments=3, sigma=1)
print("Slic number of segments: %d" % len(np.unique(segments_slic)))
return segments_slic
def updateParametros(val):
global p_segmentos, p_sigma, p_compactness, segments, image, cuda_python
if(val == "Python SLIC"):
cuda_python = 0
elif(val == "CUDA gSLICr"):
cuda_python = 1
p_segmentos = int("%d" % (slider_segmentos.val))
p_sigma = slider_sigma.val
p_compactness = slider_compactness.val
image = c_image.copy()
if(cuda_python == 0):
start_time = time.time()
segments = slic(img_as_float(image), n_segments=p_segmentos, sigma=p_sigma, compactness=p_compactness)
print("--- Tempo Python skikit-image SLIC: %s segundos ---" % (time.time() - start_time))
else:
start_time = time.time()
gSLICrInterface.process( p_segmentos)
print("--- Tempo C++/CUDA gSLICr: %s segundos ---" % (time.time() - start_time))
segments = cuda_seg
obj.set_data(mark_boundaries(img_as_float(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)), segments, outline_color=p_outline))
draw()
def update_slic(self):
sec = self.auto_submasks_gscene.active_section
t = time.time()
self.slic_labelmaps[sec] = slic(self.contrast_stretched_images[sec].astype(np.float),
sigma=SLIC_SIGMA, compactness=SLIC_COMPACTNESS,
n_segments=SLIC_N_SEGMENTS, multichannel=False, max_iter=SLIC_MAXITER)
sys.stderr.write('SLIC: %.2f seconds.\n' % (time.time() - t)) # 10 seconds, iter=100, nseg=1000;
self.slic_boundary_images[sec] = img_as_ubyte(mark_boundaries(self.contrast_stretched_images[sec],
label_img=self.slic_labelmaps[sec],
background_label=-1, color=(1,0,0)))
self.slic_image_feeder.set_image(sec=sec, numpy_image=self.slic_boundary_images[sec])
self.gscene_slic.update_image(sec=sec)
####
# self.ncut_labelmaps[sec] = normalized_cut_superpixels(self.contrast_stretched_images[sec], self.slic_labelmaps[sec])
self.ncut_labelmaps[sec] = self.slic_labelmaps[sec]
self.sp_dissim_maps[sec] = compute_sp_dissims_to_border(self.contrast_stretched_images[sec], self.ncut_labelmaps[sec])
# self.sp_dissim_maps[sec] = compute_sp_dissims_to_border(self.thresholded_images[sec], self.ncut_labelmaps[sec])
# self.border_dissim_images[sec] = generate_dissim_viz(self.sp_dissim_maps[sec], self.ncut_labelmaps[sec])
# self.dissim_image_feeder.set_image(sec=sec, numpy_image=self.border_dissim_images[sec])
# self.gscene_dissimmap.update_image(sec=sec)
self.selected_dissim_thresholds[sec] = determine_dissim_threshold(self.sp_dissim_maps[sec], self.ncut_labelmaps[sec])
self.ui.slider_dissimThresh.setValue(int(self.selected_dissim_thresholds[sec]/0.01))
######################################################
self.update_init_submasks_image()
def __build_region__(self,q_frame): start_time = time.time(); regions = segmentation.slic(q_frame,self.props.num_superpixels, self.props.compactness, convert2lab=self.props.useLAB,multichannel=True) num_regions = len(np.unique(regions)); s_frame = color.label2rgb(regions,q_frame, kind='avg') mean = np.array([region['centroid'] for region in regionprops(regions+1)]) freq = np.array([np.sum(regions==region) for region in range(num_regions)]) region_props = (mean,freq); if self.props.useColor: color_data = self.__extract_color__(q_frame,regions,region_props); if self.props.useTexture: texture_data = self.__extract_texture__(q_frame,regions,region_props); if self.props.useTexture and self.props.useColor: data = np.hstack((color_data,texture_data)) elif self.props.useTexture: data = texture_data else : data = color_data if self.props.doProfile: cv2.imwrite(self.PROFILE_PATH+self.method+'_s.png',s_frame); print "Build region (preprocess) : ",time.time()-start_time return (num_regions,regions,region_props,data);
def build_region(self):
start_time = time.time();
labels = segmentation.slic(self.q_frame,self.num_superpixels, self.compactness,convert2lab=True,multichannel=True)
_num_superpixels = np.max(labels) + 1;
self.s_frame = color.label2rgb(labels,self.q_frame, kind='avg')
self.freq = np.array([np.sum(labels==label) for label in range(_num_superpixels)])
self.mean = np.array([region['centroid'] for region in regionprops(labels+1)],dtype=np.int16);
self.color_data = np.array([np.sum(self.q_frame[np.where(labels==label)],0) for label in range(_num_superpixels)])
_inv_freq = 1/(self.freq+0.0000001); self.color_data = self.color_data*_inv_freq[:,None]
gray_frame = cv2.cvtColor(self.q_frame,cv2.COLOR_RGB2GRAY)
def texture_prop(label,patch_size = 5):
_mean_min = self.mean[label]-patch_size;
_mean_max = self.mean[label]+patch_size;
glcm = greycomatrix(gray_frame[_mean_min[0]:_mean_max[0],_mean_min[1]:_mean_max[1]],
[3], [0], 256, symmetric=True, normed=True)
_dis = greycoprops(glcm, 'dissimilarity')[0, 0];
_cor = greycoprops(glcm, 'correlation')[0, 0];
return (_dis,_cor);
self.texture_data = np.array([texture_prop(label) for label in range(_num_superpixels)])
self.data = np.hstack((self.color_data,self.texture_data))
cv2.imwrite('outs.png',self.s_frame);
print "Build region (preprocess) : ",time.time()-start_time
return (labels,_num_superpixels);
def superpixels(image):
""" given an input image, create super pixels on it
"""
# we could try to limit the problem of holes in boundary by first over segmenting the image
import matplotlib.pyplot as plt
from skimage.segmentation import felzenszwalb, slic, quickshift
from skimage.segmentation import mark_boundaries
from skimage.util import img_as_float
jac_float = img_as_float(image)
plt.imshow(jac_float)
#segments_fz = felzenszwalb(jac_float, scale=100, sigma=0.5, min_size=50)
segments_slic = slic(jac_float, n_segments=600, compactness=0.01, sigma=0.001
#, multichannel = False
, max_iter=50)
fig, ax = plt.subplots(1, 1, sharex=True, sharey=True, subplot_kw={'adjustable':'box-forced'})
fig.set_size_inches(8, 3, forward=True)
fig.subplots_adjust(0.05, 0.05, 0.95, 0.95, 0.05, 0.05)
#ax[0].imshow(mark_boundaries(jac, segments_fz))
#ax[0].set_title("Felzenszwalbs's method")
ax.imshow(mark_boundaries(jac_float, segments_slic))
ax.set_title("SLIC")
return segments_slic
def slic_data():
for i in range(uu_num_train+uu_num_test):
print "data %d" %(i+1)
img_name = ''
if i < 10:
img_name = '0' + str(i)
else:
img_name = str(i)
#Read first 70 images as floats
img = img_as_float(io.imread('..\data\\training\image_2\uu_0000' + img_name + '.png'))
img_hsv = color.rgb2hsv(img)
gt_img = img_as_float(io.imread('..\data\\training\gt_image_2\uu_road_0000' + img_name + '.png'))
#Create superpixels for training images
image_segment = slic(img, n_segments = numSegments, sigma = 5)
t, train_indices = np.unique(image_segment, return_index=True)
images_train_indices.append(train_indices)
image = np.reshape(img,(1,(img.shape[0]*img.shape[1]),3))
image_hsv = np.reshape(img_hsv,(1,(img_hsv.shape[0]*img_hsv.shape[1]),3))
#images_train.append([image[0][i] for i in train_indices])
images_train_hsv.append([image_hsv[0][i] for i in train_indices])
#Create gt training image values index at train_indices and converted to 1 or 0
gt_image = np.reshape(gt_img, (1,(gt_img.shape[0]*gt_img.shape[1]),3))
gt_image = [1 if gt_image[0][i][2] > 0 else 0 for i in train_indices]
gt_images_train.append(gt_image)
def fun_compare_colorsegmentation_and_display(image_data, number_segments=250, compactness_factor=10):
"""
The function is a copy of what does this link http://scikit-image.org/docs/dev/auto_examples/plot_segmentations.html
"""
segments_fz = felzenszwalb(image_data, scale=100, sigma=0.5, min_size=50)
segments_slic = slic(image_data, n_segments=number_segments, compactness=compactness_factor, sigma=1)
segments_quick = quickshift(image_data, kernel_size=3, max_dist=6, ratio=0.5)
print ("Felzenszwalb's number of segments: %d" % len(np.unique(segments_fz)))
print ("Slic number of segments: %d" % len(np.unique(segments_slic)))
print ("Quickshift number of segments: %d" % len(np.unique(segments_quick)))
fig, ax = plt.subplots(1, 3, sharex=True, sharey=True, subplot_kw={"adjustable": "box-forced"})
fig.set_size_inches(8, 3, forward=True)
fig.subplots_adjust(0.05, 0.05, 0.95, 0.95, 0.05, 0.05)
ax[0].imshow(mark_boundaries(image_data, segments_fz, color=(1, 0, 0)))
ax[0].set_title("Felzenszwalbs's method")
ax[1].imshow(mark_boundaries(image_data, segments_slic, color=(1, 0, 0)))
ax[1].set_title("SLIC")
ax[2].imshow(mark_boundaries(image_data, segments_quick, color=(1, 0, 0)))
ax[2].set_title("Quickshift")
for a in ax:
a.set_xticks(())
a.set_yticks(())
plt.show()
def svm_predict(case,svm_classifier):
print "svm predict"
A = []
b = []
if case == 1:
for i in range(uu_num_test):
img_name = i + uu_num_train
if img_name < 10:
img_name = '0' + str(img_name)
else:
img_name = str(img_name)
print "data %d " %(i+uu_num_train)
#Read test images as floats
img = img_as_float(io.imread('..\data\\training\image_2\uu_0000' + img_name + '.png'))
gt_img = img_as_float(io.imread('..\data\\training\gt_image_2\uu_road_0000' + img_name + '.png'))
#Create superpixels for test images
image_segment = slic(img, n_segments = numSegments, sigma = 5)
t, train_indices = np.unique(image_segment, return_index=True)
images_train_indices.append(train_indices)
image = np.reshape(img,(1,(img.shape[0]*img.shape[1]),3))
images_train.append([image[0][i] for i in train_indices])
#Create gt test image values index at train_indices and converted to 1 or 0
gt_image = np.reshape(gt_img, (1,(gt_img.shape[0]*gt_img.shape[1]),3))
gt_image = [1 if gt_image[0][i][2] > 0 else 0 for i in train_indices]
print "len of gt_image: %d with ones: %d" %(len(gt_image),gt_image.count(1))
gt_images_train.append(gt_image)
for i in range(uu_num_test):
for j in range(len(images_train[i])):
A.append(images_train[i][j])
b.append(gt_images_train[i][j])
else:
for i in range(uu_num_train,uu_num_train+uu_num_test):
for j in range(len(images_train_hsv[i])):
#val = np.zeros(6)
#val[0:3] = images_train[i][j]
#val[3:6] = images_train_hsv[i][j]
#A.append(val)
#A.append(images_train[i][j])
A.append(images_train_hsv[i][j])
b.append(gt_images_train[i][j])
A = np.asarray(A)
b = np.asarray(b)
print "A.shape = %s, b.shape = %s" %(A.shape,b.shape)
predicted = svm_classifier.predict(A)
print svm_classifier.score(A,b)
#for i in range(len(gt_images_train)):
# for j in range(len(gt_images_train[i])):
# print "%d, %d" %(gt_images_train[i][j], predicted[j])
print("Classification report for classifier %s:\n%s\n" %(svm_classifier,metrics.classification_report(b,predicted)))
def generate_features(self):
# prepare variables
img_lab = rgb2lab(self._img)
segments = slic(img_lab, n_segments=500, compactness=30.0, convert2lab=False)
max_segments = segments.max() + 1
# create x,y feather
shape = self._img.shape
a = shape[0]
b = shape[1]
x_axis = np.linspace(0, b - 1, num=b)
y_axis = np.linspace(0, a - 1, num=a)
x_coordinate = np.tile(x_axis, (a, 1,)) # 创建X轴的坐标表
y_coordinate = np.tile(y_axis, (b, 1,)) # 创建y轴的坐标表
y_coordinate = np.transpose(y_coordinate)
coordinate_segments_mean = np.zeros((max_segments, 2))
# create lab feather
img_l = img_lab[:, :, 0]
img_a = img_lab[:, :, 1]
img_b = img_lab[:, :, 2]
img_segments_mean = np.zeros((max_segments, 3))
for i in xrange(max_segments):
segments_i = segments == i
coordinate_segments_mean[i, 0] = x_coordinate[segments_i].mean()
coordinate_segments_mean[i, 1] = y_coordinate[segments_i].mean()
img_segments_mean[i, 0] = img_l[segments_i].mean()
img_segments_mean[i, 1] = img_a[segments_i].mean()
img_segments_mean[i, 2] = img_b[segments_i].mean()
# element distribution
wc_ij = np.exp(-cdist(img_segments_mean, img_segments_mean) ** 2 / (2 * self._sigma_distribution ** 2))
wc_ij = wc_ij / wc_ij.sum(axis=1)[:, None]
mu_i = np.dot(wc_ij, coordinate_segments_mean)
distribution = np.dot(wc_ij, np.linalg.norm(coordinate_segments_mean - mu_i, axis=1) ** 2)
distribution = normalize(distribution)
distribution = np.array([distribution]).T
# element uniqueness feature
wp_ij = np.exp(
-cdist(coordinate_segments_mean, coordinate_segments_mean) ** 2 / (2 * self._sigma_uniqueness ** 2))
wp_ij = wp_ij / wp_ij.sum(axis=1)[:, None]
uniqueness = np.sum(cdist(img_segments_mean, img_segments_mean) ** 2 * wp_ij, axis=1)
uniqueness = normalize(uniqueness)
uniqueness = np.array([uniqueness]).T
# save features and variables
self.img_lab = img_lab
self.segments = segments
self.img_segments_mean = img_segments_mean
self.coordinate_segments_mean = coordinate_segments_mean
self.uniqueness = uniqueness
self.distribution = distribution
def obtain_cand(Initial,Initial2,Nsme,user,action,Total):
TT = Initial[1].copy()
Rg = Initial[0].copy()
Output = []
kernel = np.ones((7, 7), np.uint8)
Mask2 = cv2.dilate(Initial2[1][:,:,0], kernel, 1)
kernel = np.ones((4, 4), np.uint8)
Mask2 = cv2.erode(Mask2, kernel, 1)
Mask2 = cv2.bitwise_not(Mask2)
kernel = np.ones((7, 7), np.uint8)
Mask1 = cv2.dilate(Initial2[0][:,:,0], kernel, 1)
kernel = np.ones((4, 4), np.uint8)
Mask1 = cv2.erode(Mask1, kernel, 1)
Mask1 = cv2.bitwise_not(Mask1)
Rg1 = cv2.bitwise_and(Rg,Rg,mask=Mask1)
Sup1 = cv2.bitwise_and(Initial[1],Initial[1],mask=Mask2)
Sup = cv2.cvtColor(Sup1, cv2.COLOR_BGR2RGB)
segments_fz = slic(Sup, n_segments=250, compactness=20, sigma=5)
segments_fz[Mask2 < 1] = -1
segments_fz += 2
# Img_Slic = label2rgb(segments_fz, Sup, kind='avg')
# Img_Slic_TT = cv2.cvtColor(Img_Slic, cv2.COLOR_RGB2BGR)
# Img_Slic = cv2.cvtColor(Img_Slic, cv2.COLOR_RGB2BGR)
for i in xrange(len(Total)):
col = [random.randint(0, 255), random.randint(0, 255), random.randint(0, 255)]
T= Total[i][0][0]
x,y,x2,y2 = T[0],T[1],T[2],T[3]
cv2.rectangle(Rg1, (T[4], T[5]), (T[6],T[7]), col, 2)
P1 = Obj_segment.Rect.Point(T[4], T[5])
P2 = Obj_segment.Rect.Point(T[6],T[7])
Rec_top = Obj_segment.Rect.Rect(P1,P2)
sp = np.array(segments_fz[y:y2,x:x2])
sp = np.unique(sp)
if len(sp) == 0:
# Output =Img_Slic[y:y2,x:x2]
P1 = Obj_segment.Rect.Point(x,y)
P2 = Obj_segment.Rect.Point(x2,y2)
rec = Obj_segment.Rect.Rect(P1,P2)
elif sp[0] ==[1] and len(sp)==1:
# Output = Img_Slic[y:y2, x:x2]
P1 = Obj_segment.Rect.Point(x, y)
P2 = Obj_segment.Rect.Point(x2, y2)
rec = Obj_segment.Rect.Rect(P1, P2)
else:
rmin, rmax,cmin, cmax = bbox2(segments_fz, sp,(x,y),(x2,y2))
if rmin is None:
continue
# Output = TT[cmin:cmax,rmin:rmax]
P1 = Obj_segment.Rect.Point(rmin, cmin)
P2 = Obj_segment.Rect.Point(rmax, cmax)
rec = Obj_segment.Rect.Rect(P1, P2)
Ouput_Top = Rg[T[5]:T[7],T[4]:T[6]]
Output.append((rec,Rec_top))
# cv2.imwrite("Morphed/Patches_Front/"+user+"_"+action+"_"+Nsme[:-4]+"_"+i.__str__()+"_Front.jpg",Output)
# cv2.imwrite("Morphed/Patches_Top/" + user + "_" + action + "_" + Nsme[:-4] + "_" + i.__str__() + "_Top.jpg", Ouput_Top)
# cv2.rectangle(Img_Slic_TT,(x,y),(x2,y2),col,3)
# cv2.imwrite("Morphed/Top/" + user + "_" + action + "_" + Nsme[:-4] + "_v2" + "_Top.jpg", Rg1)
# cv2.imwrite("Morphed/Front/"+user+"_"+action+"_"+Nsme[:-4]+"_v2"+ "_Front.jpg",Img_Slic_TT)
return Output
def mask_slic(image, config):
#constants for slic
n_segments = config[':slic'][':n_segments']
compactness = config[':slic'][':compactness']
sigma = config[':slic'][':sigma']
segments = slic(image, n_segments, compactness, sigma)
return segments
def forward(self, bottom, top):
# bottom[0]: images N*3*W*H
# bottom[1]: prediction N*1*W*H
n = bottom[0].data.shape[0]
for i in range(n):
labels = segmentation.slic( bottom[0].data[i].transpose((1,2,0)),
compactness=self.compactness, n_segments=self.n_segs)
top[0].data[i, ...] = color.label2rgb(labels, bottom[1].data[i].transpose((1,2,0)), kind='avg').transpose((2,0,1)) #.reshape(top[0].data[i].shape)
def doSegment(param, img):
if param[0] == 'slic':
segments_res = slic(img, n_segments=int(param[1]), compactness=int(param[2]), sigma=int(param[3]), convert2lab=True)
elif param[0] == 'pff':
segments_res = felzenszwalb(img, scale=int(param[1]), sigma=float(param[2]), min_size=int(param[3]))
elif param[0] == 'quick':
segments_res = quickshift(img, kernel_size=int(param[1]), max_dist=int(param[2]), ratio=float(param[3]), convert2lab=True)
return segments_res
def masked_slic(img, mask, n_segments, compactness):
labels = slic(img, n_segments=n_segments, compactness=compactness)
labels += 1
n_labels = len(np.unique(labels))
try:
mask = ndi.binary_closing(mask, structure=np.ones((3, 3)), iterations=1)
except IndexError, e:
rospy.logerr(e)
return
def run(self):
segment = segmentation.slic(self.image_data,
n_segments=self.super_pixel_num,
sigma=self.slic_sigma,
max_iter=self.slic_max_iter)
super_pixel_info = {}
for i in range(segment.max() + 1):
_now_i = segment == i
_now_where = np.argwhere(_now_i)
x_min, x_max = _now_where[:, 0].min(), _now_where[:, 0].max()
y_min, y_max = _now_where[:, 1].min(), _now_where[:, 1].max()
# 大小
super_pixel_size = len(_now_where)
assert super_pixel_size > 0
# 坐标
super_pixel_area = (x_min, x_max, y_min, y_max)
# 是否属于超像素
super_pixel_label = np.asarray(_now_i[x_min:x_max + 1,
y_min:y_max + 1],
dtype=np.int)
super_pixel_label = np.expand_dims(super_pixel_label, axis=-1)
# 属于超像素所在矩形区域的值
super_pixel_data = self.image_data[x_min:x_max + 1,
y_min:y_max + 1]
# 属于超像素的值
_super_pixel_label_3 = np.concatenate(
[super_pixel_label, super_pixel_label, super_pixel_label],
axis=-1)
super_pixel_data2 = super_pixel_data * _super_pixel_label_3
super_pixel_data3 = super_pixel_data.copy()
super_pixel_data3[_super_pixel_label_3 == 0] = -255
# 计算邻接矩阵
_x_min_a = x_min - (1 if x_min > 0 else 0)
_y_min_a = y_min - (1 if y_min > 0 else 0)
_x_max_a = x_max + 1 + (1 if x_max < len(segment) else 0)
_y_max_a = y_max + 1 + (1 if y_max < len(segment[0]) else 0)
super_pixel_area_large = segment[_x_min_a:_x_max_a,
_y_min_a:_y_max_a]
super_pixel_unique_id = np.unique(super_pixel_area_large)
super_pixel_adjacency = [
sp_id for sp_id in super_pixel_unique_id if sp_id != i
]
# 结果
super_pixel_info[i] = {
"size": super_pixel_size,
"area": super_pixel_area,
"label": super_pixel_label,
"data": super_pixel_data,
"data2": super_pixel_data2,
"data3": super_pixel_data3,
"adj": super_pixel_adjacency
}
pass
adjacency_info = []
for super_pixel_id in super_pixel_info:
now_adj = super_pixel_info[super_pixel_id]["adj"]
now_area = super_pixel_info[super_pixel_id]["area"]
_adjacency_area = [
super_pixel_info[sp_id]["area"] for sp_id in now_adj
]
_now_center = ((now_area[0] + now_area[1]) / 2,
(now_area[2] + now_area[3]) / 2)
_adjacency_center = [((area[0] + area[1]) / 2,
(area[2] + area[3]) / 2)
for area in _adjacency_area]
adjacency_dis = [
np.sqrt((_now_center[0] - center[0])**2 +
(_now_center[1] - center[1])**2)
for center in _adjacency_center
]
softmax_w = self._softmax_of_distance(adjacency_dis)
adjacency_w = [(super_pixel_id, adj_id, softmax_w[ind])
for ind, adj_id in enumerate(now_adj)]
adjacency_info.extend(adjacency_w)
pass
return segment, super_pixel_info, adjacency_info
# 10. Emboss
kernerl = np.array([
[-1,-1,0],
[-1,0,1],
[0,1,1]
])
for x in range(len(one)):
temp = convolve2d(one[x,:,:],kernerl,mode='same') + 128
plt.axis('off')
plt.imshow(temp,cmap='gray')
plt.show()
# 11. Super Pixel
for x in range(len(one)):
segments = slic(one[x,:,:], n_segments = 50, sigma = 10)
plt.axis('off')
plt.imshow(mark_boundaries(one[x,:,:], segments))
plt.show()
print('done')
# -- end code --
def pixel_report(y_pred,
gt_numerical,
n_segments,
max_n_superpxiel,
rownum,
colnum,
train_patch_ind,
test_patch_ind,
target_name=['urban', 'farmland']):
#得到预测结果共29个out_img,即(rowheight,colwidth,29)
iput_img_original = image.imread('SAR.jpg')
h_ipt = iput_img_original.shape[0]
w_ipt = iput_img_original.shape[1]
rowheight = h_ipt // rownum
colwidth = w_ipt // colnum
#得到每个test块的预测值矩阵
out_img_all = np.zeros(rowheight * colwidth * len(y_pred),
dtype="uint8") #存储29个testpatch的预测输出
#然后得到对应的gt的块的预测值矩阵
gt_all = np.zeros(rowheight * colwidth * len(y_pred), dtype="uint8")
index_1 = 0
for img_i in range(len(y_pred)):
print(test_patch_ind[img_i])
ind_img = test_patch_ind[img_i]
r = ind_img // colnum
c = ind_img % colnum
iput_img = iput_img_original[r * rowheight:(r + 1) * rowheight,
c * colwidth:(c + 1) * colwidth]
pred_out = y_pred[img_i]
segments = slic(iput_img, n_segments=n_segments, compactness=0.5)
segments[segments > (max_n_superpxiel - 1)] = max_n_superpxiel - 1
np_segments = np.array(segments)
out_img = np.zeros([rowheight, colwidth], dtype="uint8")
region_fea = measure.regionprops(segments + 1)
#函数draw_fun，在输出out_img中相应的像素点赋予其标签值
index_2 = 0 #位于图像区域的super pixel的序号
#对所有的super pixel开始循环，包括那些没有在成像区域的超像素
for ind_pixel in range(segments.max() + 1):
#根据我们之前给出的标签，赋予相应的像素值对应的标签值
# black ---river
if pred_out[index_2] == 1:
draw_fun(out_img, 1, ind_pixel, region_fea)
index_2 = index_2 + 1
# red ---urban area
elif pred_out[index_2] == 0:
draw_fun(out_img, 0, ind_pixel, region_fea)
index_2 = index_2 + 1
#得到该小块的预测值
out_img_all[img_i * rowheight * colwidth:(img_i + 1) * rowheight *
colwidth] = out_img.flatten()
#得到测试块对应的r/c
ind_img = test_patch_ind[img_i]
r = ind_img // colnum
c = ind_img % colnum
gt_all[img_i * rowheight * colwidth:(img_i + 1) * rowheight *
colwidth] = gt_numerical[r * rowheight:(r + 1) * rowheight, c *
colwidth:(c + 1) * colwidth].flatten()
print(sklearn.metrics.cohen_kappa_score(gt_all, out_img_all))
print(
classification_report(gt_all,
out_img_all,
target_names=target_name,
digits=4))
print(confusion_matrix(gt_all, out_img_all))
def run_k_means(im, compactness=0.01, n_segments=1000):
return segmentation.slic(im,
compactness=compactness,
n_segments=n_segments)
bm_mask = bm[:, :, 0] == 1
# make a copy of the image (to preserve original data)
img_copy = img.copy()
# in copy image, where bm_mask is True, convert to nan
img_copy[bm_mask] = np.nan #--> could make function to convert to a color
return img_copy
# use filtered, fast scan
img = NBL31.scan341[0, :, :-2]
#img = NBL33.scan264[0,:,:-2]
img_stand = standardize_map(img)
# prepare for SLIC segmentation; float32 to float64
img_cln = img_stand.astype('float64')
# simple linear iterative clustering (SLIC) segmentation
labels = slic(img_cln, n_segments=75, compactness=1, sigma=1)
plt.imshow(mark_superpixels(img_cln, labels))
# replacing with average #
# non-zero labels for regionprops,color
labels = labels + 1
# replace each segment with its average
sup_pix_avgs = color.label2rgb(labels, img_cln, kind='avg')
plt.imshow(sup_pix_avgs)
x = sup_pix_avgs.ravel()
plt.figure()
plt.hist(x, bins=50)
# these superpixel images are not returning bimodal distributions...
# tried an XBIC image from both NBL33 and NBL31
def train(self):
"""
Optimize a patch to generate an adversarial example.
:return: Nothing
"""
img_size = self.darknet_model.height
batch_size = self.config.batch_size
n_epochs = 5000
max_lab = 14
time_str = time.strftime("%Y%m%d-%H%M%S")
# Generate stating point
adv_patch_cpu = self.generate_patch("gray")
#adv_patch_cpu = self.read_image("saved_patches/patchnew0.jpg")
adv_patch_cpu.requires_grad_(True)
# zzj: position set
patch_position_bias_cpu = torch.full((2, 1), 0)
patch_position_bias_cpu[0] = 0.0
patch_position_bias_cpu[1] = 0.01
patch_position_bias_cpu.requires_grad_(True)
# zzj: optimizer = optim.Adam([adv_patch_cpu, patch_position_bias], lr=self.config.start_learning_rate, amsgrad=True)
optimizer = optim.Adam([{
'params': adv_patch_cpu,
'lr': self.config.start_learning_rate
}],
amsgrad=True)
scheduler = self.config.scheduler_factory(optimizer)
et0 = time.time()
# import csv
# with open(csv_name, 'w') as f:
# f_csv = csv.writer(f)
# f_csv.writerow([0,float(patch_position_bias_cpu[0]), float(patch_position_bias_cpu[1])])
print(optimizer.param_groups[0]["lr"])
####### IMG ########
img_dir = '/disk2/mycode/0511models/pytorch-YOLOv4-master-unofficial/select_from_test_500_0615'
img_list = os.listdir(img_dir)
img_list.sort()
for img_name in img_list:
print('------------------------')
print('------------------------')
print('Now testing', img_name)
img_path = os.path.join(img_dir, img_name)
img_batch = Image.open(img_path).convert('RGB')
img_size = 608
tf = transforms.Resize((img_size, img_size))
img_batch_pil = tf(img_batch)
tf = transforms.ToTensor()
img_batch = tf(img_batch)
import matplotlib.pyplot as plt
image = img_as_float(io.imread(img_path))
numSegments = 1000
img_tensor_for_slic = img_batch.squeeze().permute(1, 2, 0)
segments = slic(
img_tensor_for_slic, n_segments=numSegments, sigma=3) + 1
# fig = plt.figure("Superpixels -- %d segments" % (numSegments))
# ax = fig.add_subplot(1, 1, 1)
# ax.imshow(mark_boundaries(image, segments))
# img = torch.from_numpy(mark_boundaries(image, segments)).permute(2, 0, 1).float() # cpu [3,500,500]
img = torch.from_numpy(segments).float() # cpu [3,500,500]
img = transforms.ToPILImage()(img.detach().cpu())
img.save('seg.png')
seg_result_num = np.max(segments)
boxes = do_detect(self.darknet_model, img_batch_pil, 0.4, 0.4,
True)
print('obj num begin:', len(boxes), float(boxes[0][4]),
float(boxes[0][5]), float(boxes[0][6]))
mask_detected = torch.Tensor(500, 500).fill_(0)
for box in boxes:
bx_center = box[0] * 500
by_center = box[1] * 500
bw = box[2] * 500
bh = box[3] * 500
x1 = int(by_center - bh / 2)
x2 = int(by_center + bh / 2)
y1 = int(bx_center - bw / 2)
y2 = int(bx_center + bw / 2)
x1 = max(0, min(500, x1))
x2 = max(0, min(500, x2))
y1 = max(0, min(500, y1))
y2 = max(0, min(500, y2))
mask_detected[x1:x2, y1:y2] = 1
segments_tensor = torch.from_numpy(segments).float()
old_boxes = boxes.copy()
old_boxes_tensor = torch.Tensor(old_boxes)
# img = mask_detected/torch.max(mask_detected)
# img = transforms.ToPILImage()(img.detach().cpu())
# img.show()
segments_cover = torch.where((mask_detected == 1), segments_tensor,
torch.FloatTensor(500, 500).fill_(0))
# segments_cover = mask_detected.cpu()*torch.from_numpy(segments)
segments_cover = segments_cover.numpy().astype(int)
# img = torch.from_numpy(segments_cover).float() # cpu [3,500,500]
# img = img/torch.max(img)
# img = transforms.ToPILImage()(img.detach().cpu())
# img.show()
unique_segments_cover = np.unique(segments_cover)
unique_segments_cover = unique_segments_cover[1:]
black_img = torch.Tensor(3, 500, 500).fill_(0)
white_img = torch.Tensor(3, 500, 500).fill_(1)
white_img_single_layer = torch.Tensor(500, 500).fill_(1)
black_img_single_layer = torch.Tensor(500, 500).fill_(0)
noise_img = torch.Tensor(3, 500, 500).uniform_(0, 1)
# compute each super-pixel's attack ability
resize_small = transforms.Compose([
transforms.ToPILImage(),
transforms.Resize((608, 608)),
transforms.ToTensor()
])
img_now = img_batch.clone()
with torch.no_grad():
area_sum = 0
patch_single_layer = torch.Tensor(500, 500).fill_(0)
unique_segments_num = len(unique_segments_cover)
print('list_len:', len(unique_segments_cover))
# set a graph for super-pixel (region)
from graph_test1 import Graph
from itertools import combinations
from skimage import measure
c_2_n = list(combinations(unique_segments_cover, 2))
graph_0 = Graph()
for ver in unique_segments_cover:
graph_0.addVertex(int(ver))
# reg_img_134 = torch.where((segments_tensor == 134).mul(mask_detected == 1), white_img*0.1, black_img)
# reg_img_139= torch.where((segments_tensor == 139).mul(mask_detected == 1), white_img*0.3, black_img)
# reg_img_144= torch.where((segments_tensor == 144).mul(mask_detected == 1), white_img*0.5, black_img)
# reg_img_150= torch.where((segments_tensor == 150).mul(mask_detected == 1), white_img*0.7, black_img)
# reg_img_157= torch.where((segments_tensor == 157).mul(mask_detected == 1), white_img*0.8, black_img)
# reg_img_151 = torch.where((segments_tensor == 151).mul(mask_detected == 1), white_img * 0.9, black_img)
# reg_img_test = reg_img_134 + reg_img_139 + reg_img_144 + reg_img_150 + reg_img_157 + reg_img_151
# img = reg_img_test # cpu [500,500]
# img = transforms.ToPILImage()(img.detach().cpu())
# img.save('reg_img_test.png')
# img.show()
# find the neighborhood
'''
for reg_combin in tqdm(c_2_n):
reg_num_0 = reg_combin[0]
reg_num_1 = reg_combin[1]
reg_img_0 = torch.where((segments_tensor == reg_num_0).mul(mask_detected == 1), white_img, black_img)
reg_img_1 = torch.where((segments_tensor == reg_num_1).mul(mask_detected == 1), white_img, black_img)
reg_img_sum = reg_img_0 + reg_img_1
labels_0 = measure.label(reg_img_0, background=0, connectivity=2)
labels_1 = measure.label(reg_img_1, background=0, connectivity=2)
labels_sum = measure.label(reg_img_sum, background=0, connectivity=2)
label_max_number_0 = np.max(labels_0)
label_max_number_1 = np.max(labels_1)
label_max_number_sum = np.max(labels_sum)
if label_max_number_sum < label_max_number_0 + label_max_number_1:
graph_0.addEdge(reg_num_0, reg_num_1, 1)
graph_0.addEdge(reg_num_1, reg_num_0, 1)
rw = graph_0
output_hal = open("graph.pkl", 'wb')
str = pickle.dumps(rw)
output_hal.write(str)
output_hal.close()'''
with open("graph.pkl", 'rb') as file:
rq = pickle.loads(file.read())
graph_0 = rq
'''
# print('list_len:', len(unique_segments_cover))
osp_img_gpu_batch = torch.Tensor().cuda()
osp_area_list_tensor = torch.Tensor()
batch_size_0 = 8
max_prob_list_tensor = torch.Tensor()
obj_min_score_list_tensor = torch.Tensor()
do_boxes_list = []
output_all = torch.Tensor()
for reg_num_index in range(len(unique_segments_cover)):
reg_num = unique_segments_cover[reg_num_index]
# one super-pixel image
osp_img = torch.where((segments_tensor.repeat(3, 1, 1) == reg_num).mul(mask_detected.repeat(3, 1, 1) == 1), noise_img, img_now)
# show the img
# if reg_num_index == 2:
# osp_img_gpu = resize_small(osp_img).cuda().unsqueeze(0)
# boxes = do_detect(self.darknet_model, osp_img_gpu, 0.4, 0.4, True)
# class_names = load_class_names('data/coco.names')
# plot_boxes(transforms.ToPILImage()(osp_img), boxes, 'predictions0.jpg', class_names)
# print()
# compute the area
osp_layer = torch.where(
(segments_tensor == reg_num).mul(mask_detected == 1),
white_img_single_layer, black_img_single_layer)
osp_area = torch.sum(osp_layer).unsqueeze(0)
osp_area_list_tensor = torch.cat((osp_area_list_tensor, osp_area))
# img = osp_img_count_area # cpu [3,500,500]
# img = transforms.ToPILImage()(img.detach().cpu())
# img.show()
osp_img_gpu = resize_small(osp_img).cuda().unsqueeze(0)
osp_img_gpu_batch = torch.cat((osp_img_gpu_batch, osp_img_gpu), dim=0)
if osp_img_gpu_batch.shape[0] >= batch_size_0 or reg_num_index+1 == len(unique_segments_cover):
## alternative method 1
# output_ = self.darknet_model(osp_img_gpu_batch)img_batch
old_boxes222 = do_detect(self.darknet_model, osp_img_gpu_batch,0.4,0.4,True)
output_ = do_detect_all(self.darknet_model, osp_img_gpu_batch,0.0,0.1,True)
output_l = [torch.tensor(output_[0]),torch.tensor(output_[1]),torch.tensor(output_[2])]
# output_2 = self.darknet_model(osp_img_gpu_batch)
## get yolo output batch
YOLOoutput = output_l
num_anchors = 3 # zzj! you should change here
num_classes = 80 # zzj! you should change here
output_batch = torch.Tensor()
output_batch = output_batch
for i in range(len(YOLOoutput)):
YOLOoutput_t = YOLOoutput[i]
if YOLOoutput_t.dim() == 3:
YOLOoutput_t = YOLOoutput_t.float().cpu()
batch = YOLOoutput_t.size(0)
output = YOLOoutput_t
# output = output.transpose(1, 2).contiguous() # [batch, 85, 3, 361] -- [7, batch, 17328]
# output = output.reshape(7, -1) # [batch, 85, 3*361] -- [batch
output_batch = torch.cat([output_batch, output], dim=1)
output_all = torch.cat([output_all, output_batch], dim=0)
# boxes = do_detect(self.darknet_model, osp_img_gpu_batch, 0.4, 0.4, True)
# do_boxes_list = do_boxes_list + boxes
# max_prob = self.prob_extractor(output_)
# max_prob_list_tensor = torch.cat((max_prob_list_tensor, max_prob.cpu()))
## alternative method 2
# boxes = do_detect(self.darknet_model, osp_img_gpu_batch, 0.4, 0.4, True)
# obj_min_score = torch.from_numpy(get_obj_min_score(boxes)).float()
# obj_min_score_list_tensor = torch.cat((obj_min_score_list_tensor, obj_min_score.cpu()))
osp_img_gpu_batch = torch.Tensor().cuda()
## no nms
# output_all = output_all # [46, 22743, 7]
# output_all = output_all.permute(2, 0, 1).contiguous().reshape(7, -1)[:5]
confirm_confidence_list = iou_all_tensor(old_boxes_tensor, output_all, threshold=0.8)
confirm_confidence_tensor = torch.tensor(confirm_confidence_list)
old_boxes_confidence_list = [torch.tensor(x[4]) for x in old_boxes]
old_boxes_confidence_tensor = torch.tensor(old_boxes_confidence_list)
old_boxes_confidence_tensor_extend = old_boxes_confidence_tensor.repeat(confirm_confidence_tensor.shape[0],1)
confidence_reduce_tensor = old_boxes_confidence_tensor_extend - confirm_confidence_tensor
# reduce_max_index = torch.argmax(confidence_reduce_tensor, dim=0)
reduce_max, reduce_max_index = torch.max(confidence_reduce_tensor, dim=0)
select_superpixel = unique_segments_cover[reduce_max_index]'''
## apply init select sp
sp_layer_now = torch.Tensor(500, 500).fill_(0)
img_now = img_batch.clone()
# select_list = list(select_superpixel)
select_list = [586, 661, 661]
for selected_node in select_list:
sp_layer_now = torch.where(
(segments_tensor == selected_node).mul(
mask_detected == 1), white_img_single_layer,
sp_layer_now)
# compute the area
sp_layer_area = torch.sum(sp_layer_now)
## iteration search
batch_size_0 = 8
do_boxes_list = []
for step in range(unique_segments_num):
## get all neighborhood
now_neighbor_list = []
output_all = torch.Tensor()
osp_area_list_tensor = torch.Tensor()
osp_img_gpu_batch = torch.Tensor().cuda()
for selected_node in select_list:
neighbor_list = list(
graph_0.getVertex(
selected_node).connectedTo.keys())
for nei in neighbor_list:
now_neighbor_list.append(nei.id)
now_neighbor_np = np.array(now_neighbor_list)
now_neighbor_unique_np = np.unique(now_neighbor_np)
now_neighbor_unique_list = list(now_neighbor_unique_np)
for ite1 in select_list:
if now_neighbor_unique_list.count(ite1) > 0:
now_neighbor_unique_list.remove(ite1)
now_neighbor_unique_np = np.array(now_neighbor_unique_list)
img_tmp = torch.where((sp_layer_now.repeat(
3, 1, 1) == 1).mul(mask_detected.repeat(3, 1, 1) == 1),
noise_img, img_batch.clone())
boxes222 = do_detect(self.darknet_model,
resize_small(img_tmp).cuda(), 0.4,
0.4, True)
class_names = load_class_names('data/coco.names')
plot_boxes(transforms.ToPILImage()(img_tmp.detach().cpu()),
boxes222,
'test2/predictions' + str(step) + '.jpg',
class_names)
print()
## iteration with all neighborhood
noise_repeat_times = 7
for i1 in range(now_neighbor_unique_np.size):
for nz_t in range(noise_repeat_times):
now_node = now_neighbor_unique_np[i1]
noise_img = torch.Tensor(3, 500,
500).uniform_(0, 1)
sp_layer_tmp = torch.where(
(segments_tensor == now_node).mul(
mask_detected == 1),
white_img_single_layer, sp_layer_now)
osp_img = torch.where(
(sp_layer_tmp.repeat(3, 1, 1) == 1).mul(
mask_detected.repeat(3, 1, 1) == 1),
noise_img, img_batch.clone())
osp_img_gpu = resize_small(
osp_img).cuda().unsqueeze(0)
osp_img_gpu_batch = torch.cat(
(osp_img_gpu_batch, osp_img_gpu), dim=0)
# compute the area
osp_layer = torch.where(
(segments_tensor == now_node).mul(
mask_detected == 1), white_img_single_layer,
black_img_single_layer)
osp_area = torch.sum(osp_layer).unsqueeze(0)
osp_area_list_tensor = torch.cat(
(osp_area_list_tensor, osp_area))
output_ = do_detect_all(self.darknet_model,
osp_img_gpu_batch, 0.0, 0.1,
True)
output_l = [
torch.tensor(output_[0]),
torch.tensor(output_[1]),
torch.tensor(output_[2])
]
## get yolo output batch
YOLOoutput = output_l
num_anchors = 3 # zzj! you should change here
num_classes = 80 # zzj! you should change here
output_batch = torch.Tensor()
output_batch = output_batch
for i in range(len(YOLOoutput)):
YOLOoutput_t = YOLOoutput[i]
if YOLOoutput_t.dim() == 3:
YOLOoutput_t = YOLOoutput_t.float().cpu()
output = YOLOoutput_t
if YOLOoutput_t.dim() != 3:
pass
output_batch = torch.cat([output_batch, output],
dim=1)
output_all = torch.cat([output_all, output_batch],
dim=0)
osp_img_gpu_batch = torch.Tensor().cuda()
print()
## no nms
# output_all = output_all # [46, 22743, 7]
# output_all = output_all.permute(2, 0, 1).contiguous().reshape(7, -1)[:5]
confirm_confidence_list = iou_all_tensor(old_boxes_tensor,
output_all,
threshold=0.4)
confirm_confidence_tensor = torch.tensor(
confirm_confidence_list)
## mean of dif noise
confirm_confidence_tensor_new = torch.Tensor()
for i3 in range(
int(
confirm_confidence_tensor.size(0) /
noise_repeat_times)):
num_t = i3 * noise_repeat_times
confirm_confidence_tensor_tmp = torch.mean(
confirm_confidence_tensor[num_t:num_t +
noise_repeat_times],
dim=0)
confirm_confidence_tensor_new = torch.cat([
confirm_confidence_tensor_new,
confirm_confidence_tensor_tmp.unsqueeze(0)
],
dim=0)
confirm_confidence_tensor = confirm_confidence_tensor_new
old_boxes_confidence_list = [
torch.tensor(x[4]) for x in old_boxes
]
old_boxes_confidence_tensor = torch.tensor(
old_boxes_confidence_list)
old_boxes_confidence_tensor_extend = old_boxes_confidence_tensor.repeat(
confirm_confidence_tensor.shape[0], 1)
confidence_reduce_tensor = old_boxes_confidence_tensor_extend - confirm_confidence_tensor
## delete too low confidence anchor
zeros_tensor = torch.Tensor(
confirm_confidence_tensor.size()).fill_(0)
confidence_reduce_tensor = torch.where(
confirm_confidence_tensor < 0.3, zeros_tensor,
confidence_reduce_tensor)
## div area
confidence_reduce_div_area_tensor = confidence_reduce_tensor / osp_area_list_tensor.repeat(
3, 1).permute(1, 0)
reduce_max, reduce_max_index = torch.max(
confidence_reduce_tensor, dim=0)
# reduce_max, reduce_max_index = torch.max(confidence_reduce_div_area_tensor, dim=0)
## find max of 1
# select_superpixel_index = torch.argmax(confidence_reduce_tensor) / confidence_reduce_tensor.shape[1]
## find max of average 3
select_superpixel_index = torch.argmax(
torch.sum(confidence_reduce_tensor, dim=1))
select_superpixel_no = now_neighbor_unique_np[
select_superpixel_index]
select_list.append(select_superpixel_no)
sp_layer_now = torch.where(
(segments_tensor == select_superpixel_no).mul(
mask_detected == 1), white_img_single_layer,
sp_layer_now)
print()
'''
def update(self, bbox, frame, mask, color):
"""
Takes the current frame and the bbox predicted by the rigid-tracker and returns a binary mask representing the target object
"""
enlarge_bbox = 20 #20 pixels in all directions
bbox = (max(bbox[0] - enlarge_bbox, 0), max(bbox[1] - enlarge_bbox, 0),
min(bbox[0] + bbox[2] + enlarge_bbox, frame.shape[1]) -
bbox[0] + enlarge_bbox,
min(bbox[1] + bbox[3] + enlarge_bbox, frame.shape[0]) -
bbox[1] + enlarge_bbox) #enlarge the box
crop_frame = frame[bbox[1]:bbox[1] + bbox[3],
bbox[0]:bbox[0] + bbox[2]]
frames, params = self.buildFramesParameter(crop_frame)
X, _ = self.getFeatures(bbox,
frames,
np.array([], ndmin=2, dtype=np.uint8),
int(params[0]),
params,
train=False)
X = X / 255
if self.config["params"]["novelty_detection"]:
X_pca = self.novelty_det[self.current_model]["model"].transform(X)
X_inv = self.novelty_det[
self.current_model]["model"].inverse_transform(X_pca)
error = np.sum(np.sqrt(np.power(X - X_inv, 2)), axis=1)
sa = error.reshape(crop_frame.shape[0], crop_frame.shape[1]).copy()
else:
sa = np.zeros((crop_frame.shape[0], crop_frame.shape[1]),
dtype=np.uint8)
if self.config["params"]["novelty_detection"] and self.config.get(
"show_novelty_detection"):
error[
error <= self.novelty_det[self.current_model]["threshold"]] = 0
error[error > self.novelty_det[self.current_model]
["threshold"]] = 255
cv.imshow("Novelty_detection",
error.reshape(crop_frame.shape[0], crop_frame.shape[1]))
if self.config["params"]["over_segmentation"] == "quickshift":
segments = quickshift(crop_frame,
kernel_size=3,
max_dist=6,
ratio=0.5,
random_seed=42)
elif self.config["params"]["over_segmentation"] == "felzenszwalb":
segments = felzenszwalb(crop_frame,
scale=100,
sigma=0.5,
min_size=50)
elif self.config["params"]["over_segmentation"] == "SLIC":
segments = slic(crop_frame,
n_segments=250,
compactness=10,
sigma=1,
start_label=0)
else:
segments = None
#predict with the discriminative model
probs_curr_model = self.models[
self.current_model]["model"].predict_proba(X)
if self.multi_selection and len(
self.models
) > self.current_model + 1: #there is a future model
probs_future_model = self.models[self.current_model +
1]["model"].predict_proba(X)
span = self.models[self.current_model +
1]['n_frame'] - self.models[
self.current_model]['n_frame']
tmp = self.index - self.models[self.current_model]['n_frame']
probs = np.average([probs_curr_model, probs_future_model],
axis=0,
weights=[1 - (tmp / span),
tmp / span]) #weighted average
if self.config["params"]["novelty_detection"]:
X_pca = self.novelty_det[self.current_model +
1]["model"].transform(X)
X_inv = self.novelty_det[self.current_model +
1]["model"].inverse_transform(X_pca)
error_future_model = np.sum(np.sqrt(np.power(X - X_inv, 2)),
axis=1)
sa_future_model = error_future_model.reshape(
crop_frame.shape[0], crop_frame.shape[1]).copy()
sa = np.average([sa, sa_future_model],
axis=0,
weights=[1 - (tmp / span), tmp / span])
else:
probs = probs_curr_model
labels, areas = np.unique(segments, return_counts=True)
priors = self.computePriors(crop_frame, segments, labels)
self.compileSaliencyMap(
probs=probs,
mask=mask,
segments=segments,
outlier_scores=sa,
bbox=bbox,
labels=labels,
areas=areas,
priors=priors,
prior_weight=self.config["params"]["prior_weight"],
outlier_threshold=self.novelty_det[
self.current_model]["threshold"],
crop_frame_shape=crop_frame.shape)
#apply dilation to enlarge the shape and fill holes
mask[bbox[1]:bbox[1] + bbox[3], bbox[0]:bbox[0] + bbox[2],
2] = cv.dilate(mask[bbox[1]:bbox[1] + bbox[3],
bbox[0]:bbox[0] + bbox[2], 2],
np.ones((self.config["params"]["dilation_kernel"],
self.config["params"]["dilation_kernel"]),
np.uint8),
iterations=1)
self.index += 1
self.prevFrame = crop_frame
self.prevForegroundMask = mask[bbox[1]:bbox[1] + bbox[3],
bbox[0]:bbox[0] + bbox[2], 2]
if self.multi_selection and len(
self.models
) > self.current_model + 1 and self.index >= self.models[
self.current_model + 1]['n_frame']:
self.current_model += 1
print("\n \n CHANGE OF MODEL \n \n")
return self.current_model #to flag the re-initialization also of the tracker
if self.debug:
pass
#cv.imshow("Prob. map superpixels", cv.dilate(mask[bbox[1]:bbox[1] + bbox[3], bbox[0]:bbox[0] + bbox[2], 2], np.ones((7,7),np.uint8), iterations = 1))
#prob_map = (probs[:, 1].reshape(crop_frame.shape[:2])*255).astype(np.uint8)
#cv.imshow("Salicency map", prob_map)
return None
from skimage import data, color
hub = color.rgb2gray(data.hubble_deep_field()[350:450, 90:190])
# Plocka bort allt "brus" dvs små saker, men ha kvar den stora
compare2plots(hub, morphology.dilation(morphology.erosion(hub, disk), disk))
compare2plots(hub, morphology.opening(hub, disk))
# Segmentation
# Two different versions contrast-based and boundary-based
# SLIC K-means clustering
from skimage import io, segmentation as seg, color
#url = 'images/spice_1.jpg'
url = 'mynt.jpg'
image = io.imread(url)
labels = seg.slic(image,
n_segments=10,
compactness=20,
sigma=2,
enforce_connectivity=True)
compare2plots(image, labels.astype(float) / labels.max())
from skimage.morphology import watershed
# Watershed är ett annat sätt att segmentera, men vet inte om det tillför så mycket?
# De hävdar också att både SLIC och watershed är för enkla för att använda som final segmentation ouputs.
# Resultaten från dem kallas vanligen superpixels och används sedan för further processing.
## Part 4 of Tutorial
from skimage import io
image = io.imread('chromosomes.jpg')
protein, centromeres, chromosomes = image.transpose((2, 0, 1))
from skimage.filters import threshold_otsu
centromeres_binary = centromeres > threshold_otsu(centromeres)
'count': count,
'weight': (count_src * weight_src + count_dst * weight_dst) / count
}
def merge_boundary(graph, src, dst):
"""Call back called before merging 2 nodes.
In this case we don't need to do any computation here.
"""
pass
img = cv2.imread("catplusbrian.jpg")
edges = filters.sobel(color.rgb2gray(img))
labels = segmentation.slic(img, compactness=30, n_segments=400)
g = graph.rag_boundary(labels, edges)
graph.show_rag(labels, g, img)
plt.title('Initial RAG')
labels2 = graph.merge_hierarchical(labels,
g,
thresh=0.08,
rag_copy=False,
in_place_merge=True,
merge_func=merge_boundary,
weight_func=weight_boundary)
graph.show_rag(labels, g, img)
plt.title('RAG after hierarchical merging')
def segment_with(I, seg_method):
# PARAMETERS
# Felzenswalb
scale_ = 400
sigma_ = .5
min_size_ = 500
# SLIC
n_segments_ = 100
#15
compactness_ = 10
sigma_ = 1
# Quickshift
kernel_size_ = 20
max_dist_ = 45
ratio_ = 0.5
# Watershed
markers_ = 10
compactness_ = 0.001
# SEGMENTATION METHODS
if seg_method == 'Otsu Thresholding':
I = rgb2gray(I)
thd = threshold_otsu(I)
Ib = I < thd
plt.imshow(Ib, cmap='gray', interpolation='nearest')
plt.axis('off')
plt.show()
hist, bins_center = exposure.histogram(I)
plt.plot(bins_center, hist, lw=2)
plt.axvline(thd, color='r', ls='--')
plt.show()
return Ib
# FELZENSWALB'S METHOD
if seg_method == 'Felzenswalb':
plt.axis('off')
plt.title('Felzenswald\'s method')
segments_fz = felzenszwalb(I, scale_, sigma_, min_size_)
plt.imshow(mark_boundaries(I, segments_fz))
plt.show()
J = I
for l in range(np.max(segments_fz) + 1):
J[segments_fz == l] = np.mean(I[segments_fz == l], axis=0)
if len(I.shape) == 2:
plt.imshow(J, cmap='gray', interpolation='nearest')
else:
plt.imshow(J)
plt.axis('off')
plt.show()
return segments_fz
# SLIC'S METHOD
if seg_method == 'SLIC':
plt.axis('off')
plt.title('SLIC\'s method')
segments_slic = slic(I,
n_segments=n_segments_,
compactness=compactness_,
sigma=sigma_)
plt.imshow(mark_boundaries(I, segments_slic))
plt.show()
J = I
for l in range(np.max(segments_slic) + 1):
J[segments_slic == l] = np.mean(I[segments_slic == l], axis=0)
if len(I.shape) == 2:
plt.imshow(J, cmap='gray', interpolation='nearest')
else:
plt.imshow(J)
plt.axis('off')
plt.show()
return segments_slic
# QUICKSHIFT'S METHOD
if seg_method == 'Quickshift':
plt.axis('off')
plt.title('Quickshift\'s method')
segments_quick = quickshift(I,
kernel_size=kernel_size_,
max_dist=max_dist_,
ratio=ratio_)
plt.imshow(mark_boundaries(I, segments_quick))
plt.show()
return segments_quick
# WATERSHED'S METHOD
if seg_method == 'Watershed':
plt.axis('off')
plt.title('Watershed\'s method')
gradient = sobel(rgb2gray(I))
segments_watershed = watershed(gradient,
markers=markers_,
compactness=compactness_)
plt.imshow(mark_boundaries(I, segments_watershed))
plt.show()
return segments_watershed
ax[2].axis('off')
plt.show()
########################################################################################################################
# Segmentation into Super-Pixels
########################################################################################################################
#get leaf size in pixels
leaf_size = np.sum(img_gs != 255)
#adjust segments to leaf size
nsegs = leaf_size / 150
segments_slic_ok = slic(Color_Masked,
n_segments=nsegs,
compactness=10,
sigma=1,
max_size_factor=3)
fig, ax = plt.subplots()
ax.plot()
ax.imshow(mark_boundaries(Color_Masked, segments_slic_ok))
ax.set_title('qs')
for a in ax.ravel():
a.set_axis_off()
plt.tight_layout()
plt.show()
###########################################
crop_img = Color_Masked[1250:1600, 200:3000]
def get_saliency_rbd(img_path):
# Saliency map calculation based on:
# Saliency Optimization from Robust Background Detection, Wangjiang Zhu, Shuang Liang, Yichen Wei and Jian Sun, IEEE Conference on Computer Vision and Pattern Recognition (CVPR), 2014
img = skimage.io.imread(img_path)
if len(img.shape) != 3: # got a grayscale image
img = skimage.color.gray2rgb(img)
img_lab = img_as_float(skimage.color.rgb2lab(img))
img_rgb = img_as_float(img)
img_gray = img_as_float(skimage.color.rgb2gray(img))
segments_slic = slic(img_rgb,
n_segments=250,
compactness=10,
sigma=1,
enforce_connectivity=False)
num_segments = len(np.unique(segments_slic))
nrows, ncols = segments_slic.shape
max_dist = math.sqrt(nrows * nrows + ncols * ncols)
grid = segments_slic
(vertices, edges) = make_graph(grid)
gridx, gridy = np.mgrid[:grid.shape[0], :grid.shape[1]]
centers = dict()
colors = dict()
distances = dict()
boundary = dict()
for v in vertices:
centers[v] = [gridy[grid == v].mean(), gridx[grid == v].mean()]
colors[v] = np.mean(img_lab[grid == v], axis=0)
x_pix = gridx[grid == v]
y_pix = gridy[grid == v]
if np.any(x_pix == 0) or np.any(y_pix == 0) or np.any(
x_pix == nrows - 1) or np.any(y_pix == ncols - 1):
boundary[v] = 1
else:
boundary[v] = 0
G = nx.Graph()
#buid the graph
for edge in edges:
pt1 = edge[0]
pt2 = edge[1]
color_distance = scipy.spatial.distance.euclidean(
colors[pt1], colors[pt2])
G.add_edge(pt1, pt2, weight=color_distance)
#add a new edge in graph if edges are both on boundary
for v1 in vertices:
if boundary[v1] == 1:
for v2 in vertices:
if boundary[v2] == 1:
color_distance = scipy.spatial.distance.euclidean(
colors[v1], colors[v2])
G.add_edge(v1, v2, weight=color_distance)
geodesic = np.zeros((len(vertices), len(vertices)), dtype=float)
spatial = np.zeros((len(vertices), len(vertices)), dtype=float)
smoothness = np.zeros((len(vertices), len(vertices)), dtype=float)
adjacency = np.zeros((len(vertices), len(vertices)), dtype=float)
sigma_clr = 10.0
sigma_bndcon = 1.0
sigma_spa = 0.25
mu = 0.1
all_shortest_paths_color = nx.shortest_path(G,
source=None,
target=None,
weight='weight')
for v1 in vertices:
for v2 in vertices:
if v1 == v2:
geodesic[v1, v2] = 0
spatial[v1, v2] = 0
smoothness[v1, v2] = 0
else:
geodesic[v1,
v2] = path_length(all_shortest_paths_color[v1][v2], G)
spatial[v1, v2] = scipy.spatial.distance.euclidean(
centers[v1], centers[v2]) / max_dist
smoothness[v1, v2] = math.exp(
-(geodesic[v1, v2] * geodesic[v1, v2]) /
(2.0 * sigma_clr * sigma_clr)) + mu
for edge in edges:
pt1 = edge[0]
pt2 = edge[1]
adjacency[pt1, pt2] = 1
adjacency[pt2, pt1] = 1
for v1 in vertices:
for v2 in vertices:
smoothness[v1, v2] = adjacency[v1, v2] * smoothness[v1, v2]
area = dict()
len_bnd = dict()
bnd_con = dict()
w_bg = dict()
ctr = dict()
wCtr = dict()
for v1 in vertices:
area[v1] = 0
len_bnd[v1] = 0
ctr[v1] = 0
for v2 in vertices:
d_app = geodesic[v1, v2]
d_spa = spatial[v1, v2]
w_spa = math.exp(-((d_spa) * (d_spa)) /
(2.0 * sigma_spa * sigma_spa))
area_i = S(v1, v2, geodesic)
area[v1] += area_i
len_bnd[v1] += area_i * boundary[v2]
ctr[v1] += d_app * w_spa
bnd_con[v1] = len_bnd[v1] / math.sqrt(area[v1])
w_bg[v1] = 1.0 - math.exp(-(bnd_con[v1] * bnd_con[v1]) /
(2 * sigma_bndcon * sigma_bndcon))
for v1 in vertices:
wCtr[v1] = 0
for v2 in vertices:
d_app = geodesic[v1, v2]
d_spa = spatial[v1, v2]
w_spa = math.exp(-(d_spa * d_spa) / (2.0 * sigma_spa * sigma_spa))
wCtr[v1] += d_app * w_spa * w_bg[v2]
# normalise value for wCtr
min_value = min(wCtr.values())
max_value = max(wCtr.values())
minVal = [key for key, value in wCtr.iteritems() if value == min_value]
maxVal = [key for key, value in wCtr.iteritems() if value == max_value]
for v in vertices:
wCtr[v] = (wCtr[v] - min_value) / (max_value - min_value)
img_disp1 = img_gray.copy()
img_disp2 = img_gray.copy()
x = compute_saliency_cost(smoothness, w_bg, wCtr)
for v in vertices:
img_disp1[grid == v] = x[v]
img_disp2 = img_disp1.copy()
sal = np.zeros((img_disp1.shape[0], img_disp1.shape[1], 3))
sal = img_disp2
sal_max = np.max(sal)
sal_min = np.min(sal)
sal = 255 * ((sal - sal_min) / (sal_max - sal_min))
return sal
"""
Display a labels layer above of an image layer using the add_labels and
add_image APIs
"""
from skimage import data
from skimage.color import rgb2gray
from skimage.segmentation import slic
from napari import ViewerApp
from napari.util import app_context
with app_context():
astro = data.astronaut()
# initialise viewer with astro image
viewer = ViewerApp(astronaut=rgb2gray(astro), multichannel=False)
viewer.layers[0].colormap = 'gray'
# add the labels
# we add 1 because SLIC returns labels from 0, which we consider background
labels = slic(astro, multichannel=True, compactness=20) + 1
label_layer = viewer.add_labels(labels, name='segmentation')
print(f'The color of label 5 is {label_layer.label_color(5)}')
# Identify some background and foreground pixels from the intensity values.
# These pixels are used as seeds for watershed.
markers = np.zeros_like(coins)
foreground, background = 1, 2
markers[coins < 30.0] = background
markers[coins > 150.0] = foreground
ws = watershed(edges, markers)
seg1 = label(ws == foreground)
# Make segmentation using SLIC superpixels.
seg2 = slic(coins,
n_segments=117,
max_iter=160,
sigma=1,
compactness=0.75,
multichannel=False,
start_label=0)
# Combine the two.
segj = join_segmentations(seg1, seg2)
# Show the segmentations.
fig, axes = plt.subplots(ncols=2,
nrows=2,
figsize=(9, 5),
sharex=True,
sharey=True)
ax = axes.ravel()
ax[0].imshow(coins, cmap='gray')
truth = truth - pre
truth[truth > 0] = 1
truth[truth < 0] = 0
numer = np.sum(np.sum(np.sum(truth)))
ratio = (numer) / denom
print ratio
serial = '0000051'
image = img_as_float(io.imread('./release/images/' + serial + '.jpg'))
region = np.loadtxt('./release/labels/' + serial + '.regions.txt')
slics = slic(image, n_segments=10, sigma=5)
felzens = felzenszwalb(image, scale=3.0, sigma=0.95, min_size=50)
quicks = quickshift(image, ratio=1.0, kernel_size=10)
fig = plt.figure()
ax1 = fig.add_subplot(2, 2, 1)
ax1.imshow(mark_boundaries(image, region))
ax2 = fig.add_subplot(2, 2, 2)
ax2.imshow(mark_boundaries(image, slics))
print 'SLIC'
eval(slics, region, image)
plt.show()
'''
if __name__ == '__main__':
rgb_Dirpath=cfg.rgb_Dirpath
dep_Dirpath=cfg.dep_Dirpath
gt_Dirpath=cfg.gt_Dirpath
file_index=10001
rgb_filenm="%05d_rgb.*"%(file_index)
dep_filenm="%05d_depth.*"%(file_index)
gt_filenm="%05d_gt.*"%(file_index)
img_path=glob.glob(os.path.join(rgb_Dirpath,rgb_filenm))[0]
print(img_path)
gt_path=glob.glob(os.path.join(gt_Dirpath,gt_filenm))[0]
print(gt_path)
spx=slic(imread(img_path)) # SLICProcessor(img_path)
img=PIL.Image.open(img_path).convert('RGB')
gt=PIL.Image.open(gt_path).convert('L')
img_width, img_height = img.size
trans_compose = transforms.Compose([transforms.ToTensor()])
in_tensor = trans_compose(img)
suppool_obj=Superpixel_Pool()
pooled_out=suppool_obj.spx_pooling(in_tensor, spx, gt)
unpooled_out=suppool_obj.superpixel_unpool(pooled_out,spx)
def superpixels(img=None,
n_segments: int = 200,
cs=None,
n_iters: int = 10,
algo: str = 'slic',
kind: str = 'mix',
reduction=None,
replace_samples=(True, ),
max_size=None,
interpolation='BILINEAR') -> np.ndarray:
"""
Superpixel segmentation algorithms. Can use either cv2 (default)
or skimage algorithms if available.
Args:
img: Image to segment
cs: color space conversion, from: 'lab', 'hsv' or None
n_segments: approximate number of segments to produce
n_iters: number of iterations for cv2 algorithms and `sk_slic`
algo: Chooses the algorithm variant to use, from: 'seeds', 'slic',
'slico', 'mslic', 'sk_slic', 'sk_felzenszwalb'
kind: select how segments will be filled with RGB colors.
Average ('avg') is the original option, 'mix' is an
adaptive version using mean and average.
reduction: additional color reduction strategies. May be required
for algorithms that produce more segments than what is
defined in 'n_segments', particularly 'sk_felzenszwalb'
"""
if not np.any(replace_samples):
return img
orig_shape = None
if max_size is not None:
orig_shape = img.shape
size = max(img.shape[:2])
if size > max_size:
scale = max_size / size
height, width = img.shape[:2]
new_height, new_width = int(height * scale), int(width * scale)
resize_fn = _maybe_process_in_chunks(
cv2.resize,
dsize=(new_width, new_height),
interpolation=_cv2_str2interpolation[interpolation])
img = resize_fn(img)
img_sp = img.copy()
if 'sk' not in algo:
g_sigma = 3
g_ksize = 0
img_sp = cv2.GaussianBlur(img_sp, (g_ksize, g_ksize), g_sigma)
if not cs:
# TODO: may only be needed for real photos, not cartoon, test
cs = 'hsv'
else:
if not sk_available:
raise Exception('skimage package is not available.')
if not cs:
# TODO: better results with 'sk_slic', test
cs = 'lab'
if cs == 'lab':
img_sp = cv2.cvtColor(img_sp, cv2.COLOR_BGR2LAB)
elif cs == 'hsv':
img_sp = cv2.cvtColor(img_sp, cv2.COLOR_BGR2HSV)
h, w, c = img_sp.shape
if 'seeds' in algo:
prior = 2
double_step = False
num_levels = 4
num_histogram_bins = 5
elif 'slic' in algo or 'felzenszwalb' in algo:
# regionsize: Chooses an average superpixel size measured in pixels (50)
regionsize = int(np.sqrt(h * w / n_segments))
# ruler: Chooses the enforcement of superpixel smoothness factor
ruler = 10.0
if algo == 'seeds':
# SEEDS algorithm
ss = cv2.ximgproc.createSuperpixelSEEDS(w, h, c, n_segments,
num_levels, prior,
num_histogram_bins)
ss.iterate(img_sp, n_iters)
elif algo == 'slic':
# SLIC algorithm:
# cv2.ximgproc.SLICType.SLIC segments image using a desired region_size (value: 100)
ss = cv2.ximgproc.createSuperpixelSLIC(img_sp, 100, regionsize, ruler)
ss.iterate(n_iters)
elif algo == 'slico':
# SLICO algorithm:
# cv2.ximgproc.SLICType.SLICO will optimize using adaptive compactness factor (value: 101)
ss = cv2.ximgproc.createSuperpixelSLIC(img_sp, 101, regionsize, ruler)
ss.iterate(n_iters)
elif algo == 'mslic':
# MSLIC algorithm:
# cv2.ximgproc.SLICType.MSLIC will optimize using manifold methods
# resulting in more content-sensitive superpixels (value: 102).
ss = cv2.ximgproc.createSuperpixelSLIC(img_sp, 102, regionsize, ruler)
ss.iterate(n_iters)
elif algo == 'sk_slic':
# skimage SLIC algorithm:
labels = slic(img_sp,
n_segments=n_segments,
compactness=ruler,
max_iter=n_iters,
sigma=1) # sigma=0
elif algo == 'sk_felzenszwalb':
# skimage Felzenszwalb algorithm:
min_size = int(
0.5 * (h + w) /
2.5) # 2.5 is the rough empirical estimate factor, needs testing
k = 10 # a larger k causes a preference for larger components
# Note: can make k relative to image size with:
# k = int(regionsize/1.5)
# and this k calculation produces about the correct number of
# segments, but they don't look too good. Probably better to
# leave k=10 and apply a reduction afterwards
labels = felzenszwalb(img_sp, scale=k, sigma=0.8, min_size=min_size)
if 'sk' not in algo:
# retrieve the segmentation result
labels = ss.getLabels()
# img_sp = np.copy(img_sp)
# if img_sp.ndim == 2: # TODO: test with single channel images
# img_sp = img_sp.reshape(*img_sp.shape, 1)
if len(np.unique(labels)) > n_segments and reduction:
# reduce segments/colors and aggregate colors
rgbmap = segmentation_reduction(img,
labels,
n_segments,
reduction,
kind,
cs='lab')
else:
# aggregate (average/mix) colors in each of the labels and output
rgbmap = label2rgb(labels,
img,
kind=kind,
bg_label=-1,
bg_color=(0, 0, 0),
replace_samples=replace_samples)
if orig_shape and orig_shape != rgbmap.shape:
resize_fn = _maybe_process_in_chunks(
cv2.resize,
dsize=(orig_shape[1], orig_shape[0]),
interpolation=_cv2_str2interpolation[interpolation])
rgbmap = resize_fn(rgbmap)
return rgbmap
def _augment_images(self, images, random_state, parents, hooks):
#import time
nb_images = len(images)
#p_replace_samples = self.p_replace.draw_samples((nb_images,), random_state=random_state)
n_segments_samples = self.n_segments.draw_samples(
(nb_images, ), random_state=random_state)
seeds = random_state.randint(0, 10**6, size=(nb_images, ))
for i in sm.xrange(nb_images):
#replace_samples = ia.new_random_state(seeds[i]).binomial(1, p_replace_samples[i], size=(n_segments_samples[i],))
# TODO this results in an error when n_segments is 0
replace_samples = self.p_replace.draw_samples(
(n_segments_samples[i], ),
random_state=ia.new_random_state(seeds[i]))
#print("n_segments", n_segments_samples[i], "replace_samples.shape", replace_samples.shape)
#print("p", p_replace_samples[i])
#print("replace_samples", replace_samples)
if np.max(replace_samples) == 0:
# not a single superpixel would be replaced by its average color,
# i.e. the image would not be changed, so just keep it
pass
else:
image = images[i]
orig_shape = image.shape
if self.max_size is not None:
size = max(image.shape[0], image.shape[1])
if size > self.max_size:
resize_factor = self.max_size / size
new_height, new_width = int(
image.shape[0] * resize_factor), int(
image.shape[1] * resize_factor)
image = ia.imresize_single_image(
image, (new_height, new_width),
interpolation=self.interpolation)
#image_sp = np.random.randint(0, 255, size=image.shape).astype(np.uint8)
image_sp = np.copy(image)
#time_start = time.time()
segments = segmentation.slic(image,
n_segments=n_segments_samples[i],
compactness=10)
#print("seg", np.min(segments), np.max(segments), n_segments_samples[i])
#print("segmented in %.4fs" % (time.time() - time_start))
#print(np.bincount(segments.flatten()))
#time_start = time.time()
nb_channels = image.shape[2]
for c in sm.xrange(nb_channels):
# segments+1 here because otherwise regionprops always misses
# the last label
regions = measure.regionprops(segments + 1,
intensity_image=image[...,
c])
for ridx, region in enumerate(regions):
# with mod here, because slic can sometimes create more superpixel
# than requested. replace_samples then does not have enough
# values, so we just start over with the first one again.
if replace_samples[ridx % len(replace_samples)] == 1:
#print("changing region %d of %d, channel %d, #indices %d" % (ridx, np.max(segments), c, len(np.where(segments == ridx)[0])))
mean_intensity = region.mean_intensity
image_sp_c = image_sp[..., c]
image_sp_c[segments == ridx] = mean_intensity
#print("colored in %.4fs" % (time.time() - time_start))
if orig_shape != image.shape:
image_sp = ia.imresize_single_image(
image_sp,
orig_shape[0:2],
interpolation=self.interpolation)
images[i] = image_sp
return images
%matplotlib qt5
row=df_4.iloc[0]
file_selected=os.path.join(r'e:\OneDrive\KS-XR\X-ray képek\Test\roi',
# row['Date'].replace('.',''),
str(row['Rotor_ID'])+'-'+row['Orientation']+'.jpg')
im_orig=io.imread(file_selected)
im_adj=exposure.rescale_intensity(im_orig,in_range=(100, 200))
im_bilat=restoration.denoise_bilateral(im_adj,multichannel=False,sigma_spatial=2)
im_frangi=filters.frangi(im_bilat,scale_range=(1, 5),scale_step=1)
im_laplace=filters.laplace(im_bilat,10)
#im_hessian=filters.hessian(im_bilat,scale_range=(1, 5),scale_step=1)
im_ent = filters.rank.entropy(im_bilat, morphology.disk(5))
im_mask = segmentation.felzenszwalb(im_adj, scale=200, sigma=2,min_size=10)
im_bound=segmentation.mark_boundaries(im_orig,im_mask)
# ToDO: Watershed using DoG centers
segments = segmentation.slic(im_adj, n_segments=200, compactness=1)
im_bound=segmentation.mark_boundaries(im_orig,segments)
plt.imshow(im_mask,cmap='gray')
# K-means
k = 10
criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 100, 0.2)
_, labels, (centers) = cv.kmeans(pixels, k, None, criteria, 10, cv.KMEANS_RANDOM_CENTERS)
centers = np.uint8(centers)
labels = labels.flatten()
segmented_image = centers[labels.flatten()]
segmented_image = segmented_image.reshape(img.shape)
# Opening image
kernel = np.ones((3, 3), np.uint8)
segmented_image = cv.dilate(cv.erode(segmented_image, kernel), kernel)
# Now RAG
img = cv.resize(img, size)
labels = segmentation.slic(img, compactness=25, n_segments=250, start_label=1)
g = graph.rag_mean_color(img, labels)
labels2 = graph.merge_hierarchical(labels, g, thresh=45, rag_copy=False,
in_place_merge=True,
merge_func=merge_mean_color,
weight_func=_weight_mean_color)
for ii in range(np.max(labels2)):
if np.logical_or(np.sum(labels2 == ii) > 1000, np.sum(labels2 == ii) < 150):
labels2[labels2 == ii] = 0
out = color.label2rgb(labels2, img, kind='avg', bg_label=0)
out = segmentation.mark_boundaries(out, labels2, (0, 0, 0))
out = np.uint8(out * 255)
centers = []
for ii in range(np.max(labels2) - 1):
xInd, yInd = np.where(labels2 == ii + 1)
if len(xInd) == 0:
def segmentation(input):
# 利用 skimage 提供的 segmentation 將圖片分成 100 塊
return slic(input, n_segments=100, compactness=1, sigma=1)
def slicSeg(srcFiles_img,srcFiles_labels,img_mat,lbl_mat):
#suerpixel extraction
# Define hyperparameters
dist_sigma = 10; #sigma for gaussian distance weight in Part 1
gauss_weight = 1;
numSegments=200
# srcFiles_img = dir('G:\SPFExtarct\MSRA10K_Imgs_GT\MSRA10K_Imgs_GT\Imgs\*.jpg');
# srcFiles_labels = dir('G:\SPFExtarct\MSRA10K_Imgs_GT\MSRA10K_Imgs_GT\Imgs\*.png');
# may add some normalization to distance weight
#path, dirs, files = next(os.walk(srcFiles_img+'/*.jpg'))
#file_count = len(files)
dir_img=glob.glob(srcFiles_img)
dir_lbl=glob.glob(srcFiles_labels)
filenum=3
# filenum=len(dir_img)
all_Q={}
all_superpixel_labels={}
#for img in glob.glob(srcFiles_img):
for a in range(0,filenum):
sp_tx = '--superpixel segmentation for image: %s--' % (a+1)
print(sp_tx)
# read image
#print(dir_img[i])
#print(dir_lbl[i])
print(a)
path_img=dir_img[a]
path_lbl=dir_lbl[a]
im_image = io.imread(path_img)
#print(im_image)
im_label = io.imread(path_lbl)
#print(im_label)
#[L,N] = superpixels(im_lab,200,'IsInputLab',1);
# L= label numbers, N = superpixel numbers
# im_lab = rgb2lab(im);
L = slic(img_as_float(im_image), n_segments = numSegments) #include the lab convert
L=L+1 # start from 1
N=np.amax(L)
print('superpixel segment number: ', N)
# Vectorize superpixels in R and make mean color vector for each r
print('----mean color calculation----')
#im_size = io.imread(path_img).size;
C = np.zeros((N,3));
#r_val=im_image[:,:,0]
#g_val=im_image[:,:,1]
#b_val=im_image[:,:,2]
for i in range(1,N):
#print(np.where(L==i,1,0))
#r_val=im_image(:,:,0)
#g_val=im_image(:,:,1)
#b_valim_image(:,:,2)
red_spi_value=np.mean(im_image[(np.where(L==i,1,0)==1),0])
green_spi_value=np.mean(im_image[(np.where(L==i,1,0)==1),1])
blue_spi_value=np.mean(im_image[(np.where(L==i,1,0)==1),2])
#r_val_txt= 'sp:%s, r_val: %s'% (i,red_spi_value)
#print(r_val_txt)
C[i,:]=[red_spi_value, green_spi_value, blue_spi_value];
#np.append(C,[red_spi_value, green_spi_value, blue_spi_value], axis=0)
#print(C)
print('----mean color calculation: done!----')
# Find the superpixel center for each region r
print('----center position calculation----')
#P = np.zeros((N,1));
segments_ids = np.unique(L)
#print('sp segments id: ', segments_ids)
# centers
#label_idx = label2idx(L,N);
for i in range(1,N):
centers = np.round(np.array([np.mean(np.nonzero(L==i),axis=1) for i in segments_ids]))
#P(i,1) = round(mean(label_idx{i}));
#print(centers)
print('----center position calculation: done!----')
# Make contrast separation vector Q by comparing each superpixel
print('----mat obtaining----')
Q_color = np.zeros((N,N,3))
Q = np.zeros((N,N,3))
dist = np.zeros((N,N))
for i in range(1,N):
for j in range(1,N):
p_i=centers[i]
p_j=centers[j]
#dist(i,j) = norm(p_i - p_j);
dist[i,j] = np.linalg.norm(p_i - p_j);
#dist_txt='i: %s, j: %s, Euc distance: %s'% (p_i,p_j,dist[i,j])
#print(dist_txt)
#print('----distance of inter-superpixel: finished----')
#count of unit number in each superpixel
t_j = np.sum((L==j).astype(int)) #np.sum([np.nonzero(L==j)]) #numel(label_idx{j});
dist_weight = gaussian_weight(dist[i,j],0,dist_sigma);
#print(t_j)
Q[i,j,0] = t_j*abs(C[i,0]-C[j,0])*gauss_weight*dist_weight;
Q[i,j,1] = t_j*abs(C[i,1]-C[j,1])*gauss_weight*dist_weight;
Q[i,j,2] = t_j*abs(C[i,2]-C[j,2])*gauss_weight*dist_weight;
#print('----Q weighted by distance: finished----')
#print(dist[i,:])
#print(np.argsort(dist[i,:],axis=0))
#[~,I] = sort(dist(i,:)];
I=np.argsort(dist[i,:],axis=0)
Q_color[i,:,:] = Q[i,I,:]
#print('----Q_color weighted by distance: finished----')
#all_Q(1,a) = {Q_color};
#print(Q_color)
all_Q = dict(zip([1,a], Q_color)) #{Q_color}
print('------all_Q obtaining: done!------')
#label
superpixel_label = np.zeros((1,N))
im_bw=im_label #binary
for j in range(1,N): #1:size(label_idx,1)
#label_idx_j = label_idx{j};
label_region = L==j
if ( np.count_nonzero(label_region)>np.count_nonzero(~label_region) ):
superpixel_label[1,j]= 1;
all_superpixel_labels = dict(zip([a,1], superpixel_label)) #transpose
print('------all_superpixel_labels obtaining: done!------')
#save imagelists and segmentation labels
#save('all_Q.mat','all_Q');
sio.savemat(img_mat,{'all_Q':all_Q});
print('--save mat: All_Q.mat done!--')
sio.savemat(lbl_mat,{'all_superpixel_labels':all_superpixel_labels});
print('--save mat: all_superpixel_labels.mat done!--')
def _return_superpixels(self,
img,
img_mask,
method='slic',
param_dict=None):
"""Returns all patches for one image.
Given an image, calculates superpixels for each of the parameter lists in
param_dict and returns a set of unique superpixels by
removing duplicates. If two patches have Jaccard similarity more than 0.5,
they are concidered duplicates.
Args:
img: The input image
img_mask: The mask for the input image. Can be None.
method: superpixel method, one of slic, watershed, quichsift, or
felzenszwalb
param_dict: Contains parameters of the superpixel method used in the form
of {'param1':[a,b,...], 'param2':[z,y,x,...], ...}. For instance
{'n_segments':[15,50,80], 'compactness':[10,10,10]} for slic
method.
Raises:
ValueError: if the segementation method is invaled.
"""
if param_dict is None:
param_dict = {}
if img_mask is not None and method != 'slic':
raise ValueError('Invalid superpixel method!')
if method == 'slic':
n_segmentss = param_dict.get('n_segments', [15, 50, 80])
reference_area = param_dict.get('n_segments_reference_area')
if reference_area:
area = img.shape[0] * img.shape[1]
scale = area / reference_area
for i in range(len(n_segmentss)):
n_segmentss[i] = round(n_segmentss[i] * scale) or 1
n_params = len(n_segmentss)
compactnesses = param_dict.get('compactness', [20] * n_params)
sigmas = param_dict.get('sigma', [1.] * n_params)
elif method == 'watershed':
markerss = param_dict.get('marker', [15, 50, 80])
n_params = len(markerss)
compactnesses = param_dict.get('compactness', [0.] * n_params)
elif method == 'quickshift':
max_dists = param_dict.get('max_dist', [20, 15, 10])
n_params = len(max_dists)
ratios = param_dict.get('ratio', [1.0] * n_params)
kernel_sizes = param_dict.get('kernel_size', [10] * n_params)
elif method == 'felzenszwalb':
scales = param_dict.get('scale', [1200, 500, 250])
n_params = len(scales)
sigmas = param_dict.get('sigma', [0.8] * n_params)
min_sizes = param_dict.get('min_size', [20] * n_params)
else:
raise ValueError('Invalid superpixel method!')
unique_masks = []
for i in range(n_params):
param_masks = []
if method == 'slic':
if img_mask is None:
segments = segmentation.slic(img,
n_segments=n_segmentss[i],
compactness=compactnesses[i],
sigma=sigmas[i])
else:
segments = ace.slic.slic(img,
img_mask,
n_segments=n_segmentss[i],
compactness=compactnesses[i],
sigma=sigmas[i])
elif method == 'watershed':
segments = segmentation.watershed(img,
markers=markerss[i],
compactness=compactnesses[i])
elif method == 'quickshift':
segments = segmentation.quickshift(img,
kernel_size=kernel_sizes[i],
max_dist=max_dists[i],
ratio=ratios[i])
elif method == 'felzenszwalb':
segments = segmentation.felzenszwalb(img,
scale=scales[i],
sigma=sigmas[i],
min_size=min_sizes[i])
logger.debug('n_segment: {}'.format(segments.max() + 1))
for s in range(segments.max() + 1):
mask = (segments == s).astype(np.float32)
if np.mean(mask) > 0.001:
unique = True
for seen_mask in unique_masks:
jaccard = np.sum(seen_mask * mask) / np.sum(
(seen_mask + mask) > 0)
if jaccard > 0.5:
unique = False
break
if unique:
param_masks.append(mask)
unique_masks.extend(param_masks)
superpixels, patches = [], []
while unique_masks:
superpixel, patch = self._extract_patch(img, unique_masks.pop())
superpixels.append(superpixel)
patches.append(patch)
return superpixels, patches
import numpy as np
import os
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--image", required=True, help="Path to the image")
ap.add_argument("-s",
"--nbSegments",
required=True,
help="Nbr of areas to split the image")
args = vars(ap.parse_args())
image = img_as_float(io.imread(args["image"]))
print("Doing SLIC")
numSegments = int(args["nbSegments"])
segments = slic(image, start_label=1, n_segments=numSegments, sigma=5)
print("SLIC done")
rectedImage = np.zeros(image.shape, dtype=float)
xs = []
ys = []
reds = []
greens = []
blues = []
for progress, i in zip(
tqdm(range(numSegments),
desc="Step 1: Lists creation",
ascii=False,
ncols=125), range(numSegments)):
xs.append([])
ys.append([])
import numpy as np
frames = glob('./frames/*')
framesTotal = np.size(frames)
proposals = np.load('./5proposals.npy') #.round()
proposals = proposals[:, 0:4, :].astype(int)
proposalsTotal = proposals.shape[0]
bgdMasks = np.load('./bgdMasks.npy')
fgdMasks = np.load('./fgdMasks.npy')
for t in range(framesTotal):
frame = img_as_float(io.imread(frames[t]))
frameLAB = color.rgb2lab(frame)
segments = slic(frame, n_segments = 1000, sigma = 8.89, \
convert2lab = "True")+1
labAvg = np.zeros((segments.max(), 3))
for n in range(segments.max()):
maskLAB = segments.copy()
maskLAB[maskLAB != n + 1] = 0
labValTemp = frameLAB * maskLAB[:, :, np.newaxis]
labAvg[n]=[labValTemp[:,:,0].sum(),labValTemp[:,:,1].sum(),\
labValTemp[:,:,2].sum()]
labAvg[n] /= np.count_nonzero(maskLAB)
kMeans = KMeans(n_clusters=100).fit(labAvg)
codeWords = kMeans.labels_ + 1
maskCW = segments.copy()
for r in range(np.size(codeWords)):
maskCW = np.where(maskCW == r + 1, -codeWords[r], maskCW)
def transform(img, source, target, **kwargs):
op = kwargs['op'] if 'op' in kwargs else 'felzenszwalb'
denoise_img = denoise_tv_bregman(numpy.asarray(img), weight=0.4)
denoise_img = (denoise_img * 255).astype('uint8')
gray = cv2.cvtColor(denoise_img, cv2.COLOR_BGR2GRAY)
gray = cv2.equalizeHist(gray)
imgsize = denoise_img.shape[0] * denoise_img.shape[1]
max_area = imgsize / 100
dims = (math.ceil(denoise_img.shape[0] / 500.0) * 500.0,
math.ceil(denoise_img.shape[1] / 500.0) * 500.0)
sigma = max(0.75, math.log10(dims[0] * dims[1] / 10000.0) - 0.5)
min_size = max(0.05 * denoise_img.shape[0] * denoise_img.shape[1],
math.ceil(sigma * 10.0) * 10)
if op == 'felzenszwalb':
segment_labels = felzenszwalb(gray,
scale=min_size,
sigma=sigma,
min_size=int(min_size))
unique_labels, label_counts = numpy.unique(segment_labels,
return_counts=True)
else:
numsegments = max(16, int(imgsize * 0.000025))
segment_labels = slic(img, compactness=5, n_segments=numsegments)
unique_labels, label_counts = numpy.unique(segment_labels,
return_counts=True)
cnts = []
for label in unique_labels:
mask = numpy.zeros(gray.shape, dtype="uint8")
mask[segment_labels == label] = 255
cnts.extend(
cv2api.findContours(mask.copy(), cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)[-2])
areas = [(cnt, cv2.contourArea(cnt)) for cnt in cnts]
areas = [area for area in areas if area[1] > 4.0 and area[1] <= max_area]
cnts = sorted(areas, key=lambda cnt: cnt[1], reverse=True)
cnts = [cnt for cnt in cnts]
# top 15 largest
cnts = cnts[0:min(15, len(cnts))]
if len(cnts) == 0:
cnt = build_box_contour(denoise_img.shape)
else:
cnt = random.choice(cnts)[0]
mask = numpy.zeros((denoise_img.shape[0], denoise_img.shape[1]),
numpy.uint8)
cv2.fillPoly(mask, pts=[cnt], color=255)
if 'alpha' not in kwargs or kwargs['alpha'] == 'yes':
rgba = numpy.asarray(img.convert('RGBA'))
rgba = numpy.copy(rgba)
rgba[mask != 255] = 0
ImageWrapper(rgba).save(target)
else:
ImageWrapper(mask.astype('uint8')).save(target)
shape = mask.shape
x, y, w, h = cv2.boundingRect(cnt)
trial_boxes = [[0, 0, x, shape[0] - h], [0, 0, shape[1] - w, y],
[x + w, 0, shape[1] - w, shape[0] - h],
[0, y + h, shape[1] - w, shape[0] - h]]
boxes = [box for box in trial_boxes \
if (box[2] - box[0]) > 0 and (box[3] - box[1]) > 0]
if len(boxes) == 0:
box = [1, 1, shape[1] - w, shape[0] - h]
else:
box = choice(boxes)
new_position_x = randint(box[0], box[2])
new_position_y = randint(box[1], box[3])
return {'paste_x': new_position_x, 'paste_y': new_position_y}, None
def read_img(rownum, colnum, iput_img_original, gt_original, iput_img, gt,
n_segments, max_n_superpxiel, patch_size):
def crop_fun(new_input_padding, wi_ipt, hi_ipt, patch_size):
patch_out = new_input_padding[wi_ipt:wi_ipt + patch_size,
hi_ipt:hi_ipt + patch_size]
return patch_out
h_ipt = gt_original.shape[0]
w_ipt = gt_original.shape[1]
rowheight = h_ipt // rownum
colwidth = w_ipt // colnum
train_adj = []
train_features = []
train_labels = []
train_patch = []
test_adj = []
test_features = []
test_labels = []
test_patch = []
for r in range(1): #
for c in range(1): #
ALL_DATA_X_L = []
ALL_DATA_Y_L = []
segments = slic(iput_img, n_segments=n_segments, compactness=0.5)
out = mark_boundaries(gt, segments)
segments[segments > (max_n_superpxiel - 1)] = max_n_superpxiel - 1
#得到每个super pixel的质心
segments_label = segments + 1 #这里+1，是因为regionprops函数的输入要求是label之后的图像，而label的图像的区域编号是从1开始的
region_fea = measure.regionprops(segments_label)
#定义一个边界扩大的patch_size的空矩阵，主要是为了当super pixel位于图像边缘时，
new_input_padding = np.zeros(
(rowheight + patch_size, colwidth + patch_size))
#把这个iput_img放到new_input_padding中
new_input_padding[int(patch_size / 2):-int(patch_size / 2),
int(patch_size /
2):-int(patch_size / 2)] = iput_img
# Create graph of superpixels
from segraph import create_graph
vertices, edges = create_graph(segments)
#print( r*rownum+c ,len(vertices))
# settings for LBP
radius = 3
n_points = 8 * radius
METHOD = 'uniform'
lbp = local_binary_pattern(iput_img, n_points, radius, METHOD)
img_feature = []
#对所有的super pixel开始循环
for ind_pixel in range(segments_label.max()):
#计算当前superpixel的质心，为了生成切片，切片以这个质心为中心
centriod = np.array(
region_fea[ind_pixel].centroid).astype("int32")
wi_ipt = centriod[0]
hi_ipt = centriod[1]
#得到这个超像素的所有像素的坐标，根据坐标能够知道这个超像素在GT图中的所有像素值all_pixels_gt
#根据所有的像素，得到哪一个像素值最多，例如【0,0,0】最多，那这个超像素的标签就是“河流”
all_pixels_gt = gt[region_fea[ind_pixel].coords[:, 0],
region_fea[ind_pixel].coords[:, 1]]
n0 = np.bincount(all_pixels_gt[:, 0])
n1 = np.bincount(all_pixels_gt[:, 1])
n2 = np.bincount(all_pixels_gt[:, 2])
gt_of_superp = [n0.argmax(),
n1.argmax(),
n2.argmax()] #gt_of_superp这个超像素中出现最多次的像素值
# red ---urban
if gt_of_superp[0] >= 200 and gt_of_superp[
1] <= 50 and gt_of_superp[2] <= 50:
ALL_DATA_X_L.append(
crop_fun(new_input_padding, wi_ipt, hi_ipt,
patch_size))
ALL_DATA_Y_L.append(0)
# yellow ---farmland
elif gt_of_superp[0] >= 200 and gt_of_superp[
1] >= 200 and gt_of_superp[2] <= 50:
ALL_DATA_X_L.append(
crop_fun(new_input_padding, wi_ipt, hi_ipt,
patch_size))
ALL_DATA_Y_L.append(1)
else:
ALL_DATA_X_L.append(
crop_fun(new_input_padding, wi_ipt, hi_ipt,
patch_size))
ALL_DATA_Y_L.append(1)
#计算每个超像素的color difference（cd）， color histogram difference (hd） 和 texture disparity （lbpd）
pixels_img = iput_img[
region_fea[ind_pixel].coords[:, 0],
region_fea[ind_pixel].coords[:, 1]] #超像素内所有的像素
cd = np.mean(pixels_img)
hd, a = np.histogram(pixels_img, bins=64, range=[0, 255])
lbp_a_supixel = lbp[region_fea[ind_pixel].coords[:, 0],
region_fea[ind_pixel].coords[:, 1]]
lbp_d, a = np.histogram(lbp_a_supixel.astype(np.int64),
bins=n_points,
range=[0, n_points])
#所有超像素的featu10
img_feature.append(np.concatenate(([cd], hd, lbp_d)))
img_feature = np.array(img_feature)
edges = np.array(edges)
labels = np.array(ALL_DATA_Y_L)
adj = sp.coo_matrix(
(np.ones(edges.shape[0]), (edges[:, 0], edges[:, 1])),
shape=(labels.shape[0], labels.shape[0]),
dtype=np.float32)
# build symmetric adjacency matrix
adj = adj + adj.T.multiply(adj.T > adj) - adj.multiply(adj.T > adj)
features = normalize_features(img_feature)
adj_propg = np.linalg.inv(
sp.eye(adj.shape[0]).todense() -
0.5 * normalize_adj(adj).todense())
adj = normalize_adj(adj + sp.eye(adj.shape[0]))
labels = torch.LongTensor(labels)
adj = torch.FloatTensor(np.array(adj.todense()))
adj_propg = torch.FloatTensor(np.array(adj_propg))
#features = torch.FloatTensor(np.array(features.todense()))
ALL_DATA_X_L = torch.FloatTensor(ALL_DATA_X_L)
ALL_DATA_X_L = ALL_DATA_X_L.reshape(ALL_DATA_X_L.shape[0], 1,
ALL_DATA_X_L.shape[1],
ALL_DATA_X_L.shape[2])
return adj, ALL_DATA_X_L, labels, adj_propg
image2 = resize(image2, (200, 200)) # resize image to fit axis
image2 = swirl(image2, rotation=0, strength=10, radius=90) # swirling image2
fig, ax1 = plt.subplots() # creates plot
ax1.imshow(image2, cmap=plt.cm.gray,
interpolation='none') # displaying image on plot
ax1.axis('off') # removing axis and labels
plt.show() # displaying whole piece
#-------------------------------------------------------------------------------------------------
# Example 3 - Creating region boundries around a photo of a cat
from skimage.future import graph
from skimage import segmentation, color, filters, io
image3 = mpimg.imread("cat.jpg") # import photo as NumPy array
imagegray = color.rgb2gray(image3) # converts photo to grayscale
outline = segmentation.slic(
image3, compactness=50,
n_segments=1000) # takes grayscaled image and outlines it
imagecolor = color.gray2rgb(
imagegray
) # converts grayscale image to colored (based on intensity of brightness and darkness)
g = graph.rag_boundary(outline, imagegray) # combined graphic broundries
lc = graph.show_rag(
outline, g, imagecolor, img_cmap=None, edge_cmap='viridis', edge_width=1
) # graph itself combining color with outline, creating colored outline
io.show() # display
def mask_to_superpixel_co_occurence(img,
seg_mask,
tile_size=5000,
num_pixels_per_seg=3000,
viz_fname=None):
"""
Get SuperpixelFrequency features and SuperpixelCo-occurrence features for
an image with semantic segmentation mask.
The process follows these procedures:
- Step 1: Get neighbours for each superpixel
- Step 2: Get the class label (by majority voting) for each pxiel
- Step 3: Get the co-occurence features by using the information
extracted above.
Args:
img (np.array): size [h, w, 3]
seg_mask (np.array): size [h, w]
tile_size (int): the size (width and height) for each tile.
num_pixels_per_seg (int): the (average) number of pixels in each
superpixel.
viz_fname (str): a image filename, whose image will store the visualization
of the final semantic segmentation based on the superpixel majority
voting.
Output:
freq_features (list): an array of frequency features
cooccr_features (list): an array of co-occurence features
"""
assert (img.shape[0] == seg_mask.shape[0])
assert (img.shape[1] == seg_mask.shape[1])
# the following two arrays will store results for each tile
freqs = []
cooccrs = []
h, w, _ = img.shape
print(time.ctime(), "Running Superpixel Segmentation ...")
if viz_fname is not None:
viz_img = np.zeros(seg_mask.shape, dtype=np.uint8)
for row_i in tqdm.tqdm(range(0, h, tile_size)):
row_end = min(row_i + tile_size, h)
for col_i in range(0, w, tile_size):
col_end = min(col_i + tile_size, w)
tile_img = img[row_i:row_end, col_i:col_end, :]
tile_seg = seg_mask[row_i:row_end, col_i:col_end]
num_segments_in_tile = tile_img.shape[0] * tile_img.shape[
1] // num_pixels_per_seg
tile_sp_labels = slic(tile_img, n_segments=num_segments_in_tile)
tile_neigh = neighbours(tile_sp_labels)
tile_sp_cls = assign_sp_cls(tile_sp_labels, tile_seg)
tile_cooccr = co_occurence(tile_sp_labels,
tile_sp_cls,
tile_neigh,
k=8)
tile_sp_freq = sp_cls_count(tile_sp_cls, n_seg_cls=8)
freqs.append(tile_sp_freq)
cooccrs.append(tile_cooccr.reshape(-1))
if viz_fname is not None:
viz_img_tile = viz_img[row_i:row_end, col_i:col_end]
for sp_id in range(np.max(tile_sp_labels)):
sp_mask = tile_sp_labels == sp_id
viz_img_tile[sp_mask] = tile_sp_cls[sp_id]
# add border for each superpixel
viz_img_tile = viz_segmentation_countour(
viz_img_tile,
tile_sp_labels,
border_color=8, # class ID
border_width=5,
output_dir=None)
# Place the viz back
viz_img[row_i:row_end, col_i:col_end] = viz_img_tile
if viz_fname is not None:
outpil = Image.fromarray(np.uint8(viz_img))
outpil.putpalette(pallete)
outpil.save(viz_fname)
freqs = np.array(freqs)
cooccrs = np.array(cooccrs)
freq_features = np.sum(freqs, axis=0)
cooccr_features = np.sum(cooccrs, axis=0)
return freq_features.tolist(), cooccr_features.tolist()
def calculate_predictions(img,
img_orig,
original_class_position,
explainer_name,
num_features,
strategy,
sigma,
verbose=0):
# segment the image so we don't have to explain every pixel
segments_slic = slic(img, n_segments=49, compactness=1000, sigma=3)
def f(z):
return model.predict(
preprocess_input(mask_image(z, segments_slic, img_orig.copy(),
255)))
new_class_lime = None
new_prediction_lime = None
global prev_shap_values
global prev_img_orig
global prev_explanation
if ((explainer_name == 'shap' or explainer_name == 'random'
or explainer_name == 'grad') & (img_orig == prev_img_orig).all()):
print("Hitting cache, returning prev values")
shap_values = prev_shap_values
elif explainer_name == 'shap':
# use Kernel SHAP to explain the network's predictions
explainer = shap.KernelExplainer(f, np.zeros((1, 49)), start_label=1)
shap_values = explainer.shap_values(
np.ones((1, 49)), nsamples=1000,
start_label=1) # runs model 1000 times
elif explainer_name == 'grad':
heatmap_orig = grad.make_gradcam_heatmap(
img_orig.copy().reshape(1, 224, 224, 3), model)
heatmap = cv2.resize(heatmap_orig, (7, 7))
if verbose:
plt.imshow(heatmap_orig)
plt.show()
plt.imshow(heatmap)
plt.show()
grad.test_drive_grad_original(img_orig, model)
shap_values = []
for i in range(1000):
shap_values.append(
[[item for sublist in heatmap for item in sublist]])
shap_values = np.array(shap_values)
shap_values = shap_values / np.max(shap_values)
elif explainer_name == 'random':
shap_values = [
numpy.asarray([[random.uniform(0, 1) for iter in range(50)]])
for i in range(1000)
]
elif explainer_name == 'lime':
if ((img_orig == prev_img_orig).all()):
explanation = prev_explanation
print("Hitting LIME cache, returning prev values")
else:
explainer = lime_image.LimeImageExplainer()
explanation = explainer.explain_instance(img_orig.astype("double"),
f_lime,
num_samples=1000)
prev_explanation = explanation
prev_img_orig = img_orig
print(explanation.top_labels)
print(original_class_position)
# if strategy == "top":
# lime_img, _ = explanation.get_image_and_mask(explanation.top_labels[0] , positive_only=True, negative_only=False, hide_rest=True, num_features = num_features, min_weight=0)
# else:
# lime_img, _ = explanation.get_image_and_mask(explanation.top_labels[0], positive_only=False, negative_only=True, num_features=1000, hide_rest=True)
# new_class_lime = model.predict_classes(lime_img.reshape(1,224,224,3))[0]
# new_prediction_lime = model.predict(preprocess_input(mask_image(z, segments_slic, img_orig.copy(), 255)))
lime_img, mask = explanation.get_image_and_mask(
explanation.top_labels[0],
positive_only=True,
negative_only=False,
hide_rest=True,
num_features=num_features,
min_weight=0)
heatmap = cv2.resize(mask / 255, (7, 7))
if verbose:
plt.matshow(lime_img / 255)
plt.show()
plt.matshow(mask)
plt.show()
plt.matshow(heatmap)
plt.show()
shap_values = []
for i in range(1000):
shap_values.append(
[[item for sublist in heatmap for item in sublist]])
shap_values = np.array(shap_values)
shap_values = shap_values
prev_shap_values = shap_values.copy()
prev_img_orig = img_orig.copy()
# get the top predictions from the model
preds = model.predict(
preprocess_input(np.expand_dims(img_orig.copy(), axis=0)))
top_preds = np.argsort(-preds)
inds = top_preds[0]
# if verbose:
# show_explanation(shap_values, img, inds)
shap_values = [np.where(a < 0, 0, a) for a in shap_values]
shap_values = extract_top_ten(shap_values, num_features)
if strategy == 'rest':
shap_values = (shap_values - 1) * -1
masked_image = mask_image_with_noise(shap_values[inds[0]], segments_slic,
img_orig.copy(), sigma)[0]
if verbose:
plt.imshow((masked_image).astype(np.uint8))
plt.show()
prediction = model.predict(
preprocess_input(np.expand_dims(masked_image.copy(), axis=0)))
new_class = decode_predictions(prediction)[0][0][1]
new_prediction = prediction[0][original_class_position]
if verbose:
for pred in decode_predictions(prediction, 1000)[0]:
if (pred[2] == new_prediction):
print(pred)
return new_class, new_prediction, new_class_lime, new_prediction_lime
