def load_images(self, path):
'''
loads images from given path into an array list
'''
self.log.start("Load Images", 1, 1)
path_list = path_to_subfolder_pathlist(path, filter=".jpg")
self.image_path_list = path_list
self.image_list = np.array([
os.path.splitext(basename(filepath))[0] for filepath in path_list
])
if not self.seq_reading:
if os.path.isfile(self.image_array_list_path):
self.image_array_list = read_arr_from_matfile(
self.image_array_list_path,
os.path.splitext(basename(self.image_array_list_path))[0])
else:
self.image_array_list = np.array(
[np.array(Image.open(filepath)) for filepath in path_list])
write_arr_to_matfile(
self.image_array_list, self.image_array_list_path,
os.path.splitext(basename(self.image_array_list_path))[0])
self.log.update()
def _slic_thread(self, i, seg_folder, region_size, regularizer):
'''
thread for the vl_slic segmentation
'''
if not os.path.isfile(seg_folder + "/" + os.path.basename(os.path.dirname(self.image_path_list[i])) +\
os.path.basename(os.path.splitext(self.image_path_list[i])[0]) +".mat"):
if self.seq_reading:
image = np.array(Image.open(self.image_path_list[i]))
else:
image = self.image_array_list[i]
if len(image.shape) == 2: #check if a grayscale image is given
im = np.zeros((image.shape[0], image.shape[1], 3))
for k in [0, 1, 2]:
im[:, :, k] = image
image = im
labels = segmentation.slic(image, region_size, regularizer)
unique = np.unique(labels)
if min(unique) == 0:
labels = labels + 1
unique = np.unique(labels)
if unique[-1] > len(unique):
lab = labels.copy()
for j in range(len(unique)):
labels[lab == unique[j]] = j + 1
write_arr_to_matfile(
labels, seg_folder + "/" +
os.path.basename(os.path.dirname(self.image_path_list[i])) +
'/' + self.image_list[i] + ".mat", "superPixels")
self.max_thread.release()
self.log_lock.acquire()
try:
self.log.update()
if self.log.process_percentage >= 100:
try:
self.sp_thread_lock.release()
except:
print 'here'
finally:
self.log_lock.release()
def _qs_thread(self, i, seg_folder, ratio, kernelsize, maxdist):
'''
quick shift thread
'''
if not os.path.isfile(
seg_folder + "/" +
os.path.basename(os.path.dirname(self.image_path_list[i])) +
'/' + self.image_list[i] + ".mat"):
if self.seq_reading:
image = np.array(Image.open(self.image_path_list[i]))
else:
image = self.image_array_list[i]
if len(image.shape) == 2: #check if a grayscale image is given
im = np.zeros((image.shape[0], image.shape[1], 3))
for k in [0, 1, 2]:
im[:, :, k] = image
image = im
iseg = vlfeat.vl_quickseg(image, ratio, kernelsize, maxdist)
labels = iseg[1]
if min(np.unique(labels)) == 0:
labels = labels + 1
write_arr_to_matfile(
labels, seg_folder + "/" +
os.path.basename(os.path.dirname(self.image_path_list[i])) +
'/' + self.image_list[i] + ".mat", "superPixels")
gc.collect()
self.max_thread.release()
self.log_lock.acquire()
try:
self.log.update()
if self.log.process_percentage >= 100:
try:
self.sp_thread_lock.release()
except:
print 'here'
finally:
self.log_lock.release()
def load_super_pixel(self, path, seg_method):
'''
loads super pixels from given path or calculates them if not created yet
@param path: Path to superpixels
@param seg_method: segmantation method and parameters to calculate superpixels
'''
#calculate superpixels which are not created
sp_calculator = SPCalculator(self)
sp_calculator.calculate_sp(seg_method, path)
#load superpixels if seq_reading is set to False
self.log.start("Load SuperPixels", 1, 1)
path_list = path_to_subfolder_pathlist(path, filter=".mat")
self.sp_array_path_list = path_list
if not self.seq_reading:
if os.path.isfile(self.sp_array_list_path):
self.sp_array_list = read_arr_from_matfile(
self.sp_array_list_path,
os.path.splitext(basename(self.sp_array_list_path))[0])
else:
for i in range(len(path_list)):
try:
sp = read_arr_from_matfile(path_list[i], 'superPixels')
except:
print "Failure in File {0}".format(i)
print self.image_list[i]
raise
self.sp_array_list = np.array([
read_arr_from_matfile(filepath, 'superPixels')
for filepath in path_list
])
if self.save_mode:
write_arr_to_matfile(
self.sp_array_list, self.sp_array_list_path,
os.path.splitext(basename(self.sp_array_list_path))[0])
self.log.update()
def ground_truth(self, seg_folder, label_folder):
'''
calculates GroundTruth Superpixels
@param seg_folder: superpixel folder
'''
self.log.start("Calculating GroundTruth SuperPixels",
len(self.image_list), 1)
self.log.verbose = True
path_list = path_to_subfolder_pathlist(label_folder, filter=".mat")
for i in range(len(path_list)):
self.label_sp(i, seg_folder, path_list)
write_arr_to_matfile(self.sp_labeled,
seg_folder + "/" + \
os.path.splitext(basename(path_list[i]))[0] +".mat",
"superPixels")
self.log.update()
def saliency(self, seg_folder, saliency_folder, k):
'''
clusters and relabels saliency maps of images
@param seg_folder: superpixel folder
@param saliency_folder: folder to salience-images
'''
self.log.start("Recomputing Saliency SuperPixels",
len(self.image_array_list), 1)
self.log.verbose = True
path_list = path_to_subfolder_pathlist(saliency_folder, filter=".mat")
for i in range(len(path_list)):
self.sp_array = read_arr_from_matfile(path_list[i], "S")
self.sp_labeled = np.zeros(self.sp_array.shape) - 1
pixel = np.reshape(self.sp_array, (self.sp_array.shape[0]*\
self.sp_array.shape[1]))
#clustering
centroids, _ = kmeans(pixel, 60)
# quantization
qnt, _ = vq(pixel, centroids)
centers_idx = np.reshape(
qnt, (self.sp_array.shape[0], self.sp_array.shape[1]))
self.sp_array = centroids[centers_idx]
self.label_ffw()
write_arr_to_matfile(self.sp_labeled,
seg_folder + "/" + \
os.path.splitext(basename(path_list[i]))[0] +".mat",
"superPixels")
self.log.update()
def label_images(self, label_path, sp_label_path): #Deprecated
'''
label superpixels according to their image labels
'''
self.log.start("Load Image Labels", 1, 1)
path_list = path_to_subfolder_pathlist(label_path, filter=".mat")
if os.path.isfile(self.image_label_array_list_path):
self.image_label_array_list = read_arr_from_matfile(
self.image_label_array_list_path,
os.path.splitext(basename(
self.image_label_array_list_path))[0])
else:
self.image_label_array_list = np.array([
read_arr_from_matfile(filepath, 'S') for filepath in path_list
])
write_arr_to_matfile(
self.image_label_array_list, self.image_label_array_list_path,
os.path.splitext(basename(
self.image_label_array_list_path))[0])
self.log.update()
self.log.start("Labeling SuperPixels", len(self.image_array_list), 1)
if os.path.isfile(sp_label_path + "/segIndex.mat"):
self.segment_label_index = read_dict_from_matfile(sp_label_path +
"/segIndex.mat")
self.log.end()
else:
self.segment_label_index = {}
self.segment_label_index['label'] = np.array([])
self.segment_label_index['sp'] = np.array([])
self.segment_label_index['image'] = np.array([])
self.segment_label_index['spSize'] = np.array([])
self.label_thread_lock.acquire()
for i in range(len(self.image_label_array_list)):
#print "Active Threads: {0}".format(8-self.max_thread)
self.max_thread.acquire()
try:
t = threading.Thread(target=self.label_thread,
args=(
sp_label_path,
i,
))
t.start()
except:
pass
print "Acquire label_thread_lock"
self.label_thread_lock.acquire()
self.log.start("Writing Segment Label File",
len(self.image_label_array_list), 1)
try:
for i in range(len(self.image_label_array_list)):
self.segment_label_index['label'] = \
np.concatenate((self.segment_label_index['label'],
self.label_thread_index[i]['label']))
self.segment_label_index['sp'] = \
np.concatenate((self.segment_label_index['sp'],
self.label_thread_index[i]['sp']))
self.segment_label_index['image'] = \
np.concatenate((self.segment_label_index['image'],
np.ones(self.label_thread_index[i]['sp'].shape)*i))
self.segment_label_index['spSize'] = \
np.concatenate((self.segment_label_index['spSize'],
self.label_thread_index[i]['spSize']))
self.log.update()
finally:
try:
self.label_thread_lock.release()
except:
pass
write_dict_to_matfile(self.segment_label_index,
sp_label_path + "/segIndex.mat")
def calculate_segment_features(self, path): #Deprecated
'''
calculates the segment feature segments
'''
print path
self.log.start("Calculate Segment Features", len(self.image_list), 1)
for feature in self.feature_list:
if not os.path.exists(path + "/" + feature + "/"):
os.makedirs(path + "/" + feature + "/")
for im_num in range(len(self.image_list)):
if not os.path.isfile(path + "/" + self.image_list[im_num]):
if not os.path.isfile(path + "/busy_" +
self.image_list[im_num]):
write_arr_to_file(
np.array([1]),
path + "/busy_" + self.image_list[im_num])
sp_array = self.sp_array_list[im_num]
sp_ind = np.unique(sp_array)
image = self.image_array_list[im_num]
descs = {}
for desc in self.feature_list:
descs[desc] = np.array([[]])
log = Logger()
for sp_num in sp_ind:
mask = sp_array == sp_num
(borders, bb) = get_border_bounding_box(mask)
borders = borders[bb[0]:bb[1] + 1, bb[2]:bb[3] + 1, :]
mask_crop = mask[bb[0]:bb[1] + 1, bb[2]:bb[3] + 1]
im_crop = image[bb[0]:bb[1] + 1, bb[2]:bb[3] + 1, :]
textons_crop = {}
textons_crop['mr_filter'] = \
self.global_descriptor_dict['mr_filter'][im_num][bb[0]:bb[1]+1,
bb[2]:bb[3]+1]
textons_crop['sift_textons'] = \
self.global_descriptor_dict['sift_textons'][im_num][bb[0]:bb[1]+1,
bb[2]:bb[3]+1]
centers = self.centers
#Calculating Descriptors
#log.start("sift_hist_int",1,1)
sift_hist_int_ = self.sfe.sift_hist_int(
mask_crop, centers, textons_crop).flatten(1)
sift_hist_int_ = sift_hist_int_.\
reshape((sift_hist_int_.shape[0],1)).astype(np.float32)
#log.update()
#log.start("sift_hist_left",1,1)
sift_hist_left = self.sfe.sift_hist_left(
centers, textons_crop, borders).flatten(1)
sift_hist_left = sift_hist_left.\
reshape((sift_hist_left.shape[0],1)).astype(np.float32)
#log.update()
#log.start("sift_hist_right",1,1)
sift_hist_right = self.sfe.sift_hist_right(
centers, textons_crop, borders).flatten(1)
sift_hist_right = sift_hist_right.\
reshape((sift_hist_right.shape[0],1)).astype(np.float32)
#log.update()
#log.start("sift_hist_top",1,1)
sift_hist_top = self.sfe.sift_hist_top(
centers, textons_crop, borders).flatten(1)
sift_hist_top = sift_hist_top.\
reshape((sift_hist_top.shape[0],1)).astype(np.float32)
#log.update()
#log.start("top_height",1,1)
top_height = self.sfe.top_height(mask, mask_crop,
bb).flatten(1)
top_height = top_height.\
reshape((top_height.shape[0],1)).astype(np.float32)
#log.update()
#log.start("absolute_mask",1,1)
absolute_mask = self.sfe.absolute_mask(im_crop,
mask).flatten(1)
absolute_mask = absolute_mask.\
reshape((absolute_mask.shape[0],1)).astype(np.float32).astype(np.float32)
#log.update()
#log.start("bb_extent",1,1)
bb_extent = self.sfe.bb_extent(mask, mask_crop,
bb).flatten(1)
bb_extent = bb_extent.\
reshape((bb_extent.shape[0],1)).astype(np.float32)
#log.update()
#log.start("color_hist",1,1)
color_hist = self.sfe.color_hist(im_crop,
mask_crop).flatten(1)
color_hist = color_hist.\
reshape((color_hist.shape[0],1)).astype(np.float32)
#log.update()
#log.start("centered_mask_sp",1,1)
centered_mask_sp = self.sfe.centered_mask_sp(
im_crop, mask, mask_crop).flatten(1)
centered_mask_sp = centered_mask_sp.\
reshape((centered_mask_sp.shape[0],1)).astype(np.float32)
#log.update()
#log.start("color_std",1,1)
color_std = self.sfe.color_std(im_crop,
mask_crop).flatten(1)
color_std = color_std.\
reshape((color_std.shape[0],1)).astype(np.float32)
#log.update()
#log.start("color_thumb",1,1)
color_thumb = self.sfe.color_thumb(
im_crop, mask_crop).flatten(1)
color_thumb = color_thumb.\
reshape((color_thumb.shape[0],1)).astype(np.float32)
#log.update()
#log.start("color_thumb_mask",1,1)
color_thumb_mask = self.sfe.color_thumb_mask(
im_crop, mask_crop).flatten(1)
color_thumb_mask = color_thumb_mask.\
reshape((color_thumb_mask.shape[0],1)).astype(np.float32)
#log.update()
#log.start("dial_color_hist",1,1)
dial_color_hist = self.sfe.dial_color_hist(
im_crop, borders).flatten(1)
dial_color_hist = dial_color_hist.\
reshape((dial_color_hist.shape[0],1)).astype(np.float32)
#log.update()
#log.start("dial_text_hist_mr",1,1)
dial_text_hist_mr = self.sfe.dial_text_hist_mr(
centers, textons_crop, borders).flatten(1)
dial_text_hist_mr = dial_text_hist_mr.\
reshape((dial_text_hist_mr.shape[0],1)).astype(np.float32)
#log.update()
#log.start("gist_int",1,1)
# import profile
#profile.run('self.sfe.gist_int(im_crop, mask, centers, image).flatten(1)')
gist_int = self.sfe.gist_int(im_crop, mask, centers,
image).flatten(1)
gist_int = gist_int.\
reshape((gist_int.shape[0],1)).astype(np.float32)
#log.update()
#log.start("int_text_hist_mr",1,1)
int_text_hist_mr = self.sfe.int_text_hist_mr(
mask_crop, centers, textons_crop).flatten(1)
int_text_hist_mr = int_text_hist_mr.\
reshape((int_text_hist_mr.shape[0],1)).astype(np.float32)
#log.update()
#log.start("mean_color",1,1)
mean_color = self.sfe.mean_color(im_crop,
mask_crop).flatten(1)
mean_color = mean_color.\
reshape((mean_color.shape[0],1)).astype(np.float32)
#log.update()
#log.start("pixel_area",1,1)
pixel_area = self.sfe.pixel_area(mask,
mask_crop).flatten(1)
pixel_area = pixel_area.\
reshape((pixel_area.shape[0],1)).astype(np.float32)
#log.update()
#log.start("sift_hist_bottom",1,1)
sift_hist_bottom = self.sfe.sift_hist_bottom(
centers, textons_crop, borders).flatten(1)
sift_hist_bottom = sift_hist_bottom.\
reshape((sift_hist_bottom.shape[0],1)).astype(np.float32)
#log.update()
#log.start("sift_hist_dial",1,1)
sift_hist_dial = self.sfe.sift_hist_dial(
centers, textons_crop, borders).flatten(1)
sift_hist_dial = sift_hist_dial.\
reshape((sift_hist_dial.shape[0],1)).astype(np.float32)
#log.update()
#Writing Descriptors into dictionary
if sp_num != sp_ind[0]:
descs['sift_hist_int_'] = \
np.concatenate((descs['sift_hist_int_'],
sift_hist_int_),axis=1)
descs['sift_hist_left'] = \
np.concatenate((descs['sift_hist_left'],
sift_hist_left),axis=1)
descs['sift_hist_right'] = \
np.concatenate((descs['sift_hist_right'],
sift_hist_right),axis=1)
descs['sift_hist_top'] = \
np.concatenate((descs['sift_hist_top'],sift_hist_top),
axis=1)
descs['top_height'] = \
np.concatenate((descs['top_height'],top_height),axis=1)
descs['absolute_mask'] = \
np.concatenate((descs['absolute_mask'],absolute_mask),
axis=1)
descs['bb_extent'] = \
np.concatenate((descs['bb_extent'],bb_extent),axis=1)
descs['centered_mask_sp'] = \
np.concatenate((descs['centered_mask_sp'],
centered_mask_sp),axis=1)
descs['color_hist'] = \
np.concatenate((descs['color_hist'],color_hist),axis=1)
descs['color_std'] = \
np.concatenate((descs['color_std'],color_std),axis=1)
descs['color_thumb'] = \
np.concatenate((descs['color_thumb'],color_thumb),
axis=1)
descs['color_thumb_mask'] = \
np.concatenate((descs['color_thumb_mask'],
color_thumb_mask),axis=1)
descs['dial_color_hist'] = \
np.concatenate((descs['dial_color_hist'],
dial_color_hist),axis=1)
descs['dial_text_hist_mr'] = \
np.concatenate((descs['dial_text_hist_mr'],
dial_text_hist_mr),axis=1)
descs['gist_int'] = \
np.concatenate((descs['gist_int'],gist_int),axis=1)
descs['int_text_hist_mr'] = \
np.concatenate((descs['int_text_hist_mr'],
int_text_hist_mr),axis=1)
descs['mean_color'] = \
np.concatenate((descs['mean_color'],mean_color),axis=1)
descs['pixel_area'] = \
np.concatenate((descs['pixel_area'],pixel_area),axis=1)
descs['sift_hist_bottom'] = \
np.concatenate((descs['sift_hist_bottom'],
sift_hist_bottom),axis=1)
descs['sift_hist_dial'] = \
np.concatenate((descs['sift_hist_dial'],sift_hist_dial),
axis=1)
#create a new array in dictionary if it is empty
else:
descs['sift_hist_int_'] = sift_hist_int_
descs['sift_hist_left'] = sift_hist_left
descs['sift_hist_right'] = sift_hist_right
descs['sift_hist_top'] = sift_hist_top
descs['top_height'] = top_height
descs['absolute_mask'] = absolute_mask
descs['bb_extent'] = bb_extent
descs['centered_mask_sp'] = centered_mask_sp
descs['color_hist'] = color_hist
descs['color_std'] = color_std
descs['color_thumb'] = color_thumb
descs['color_thumb_mask'] = color_thumb_mask
descs['dial_color_hist'] = dial_color_hist
descs['dial_text_hist_mr'] = dial_text_hist_mr
descs['gist_int'] = gist_int
descs['int_text_hist_mr'] = int_text_hist_mr
descs['mean_color'] = mean_color
descs['pixel_area'] = pixel_area
descs['sift_hist_bottom'] = sift_hist_bottom
descs['sift_hist_dial'] = sift_hist_dial
if not os.path.isfile(path + "/" +
self.image_list[im_num]):
#store descriptor into files
for feature in self.feature_list:
write_arr_to_matfile(
descs[feature], path + "/" + feature + "/" +
self.image_list[im_num] + ".mat", "desc")
#if matrices are written, write a file for proove
write_arr_to_file(np.array([1]),
path + "/" + self.image_list[im_num])
os.remove(path + "/busy_" + self.image_list[im_num])
self.log.update()
def _gt_thread(self, i, seg_folder, path_list):
'''
Ground Truth Thread
@param i: index to current image
@param seg_folder: superpixel folder
@param path_list: path to images
'''
if not os.path.isfile(seg_folder + "/" + \
os.path.splitext(basename(path_list[i]))[0] +".mat"):
segments = read_arr_from_matfile(path_list[i], "S")
segments_new = segments.copy()
labels = np.unique(segments)
label_new = 1
for l in labels:
segment = (segments == l) * 1
segment_arg = np.argwhere(segments == l)
mask = np.array([[1, 1, 1, 1, 1], [1, 0, 1, 0, 1],
[1, 1, 1, 1, 1], [1, 0, 1, 0, 1],
[1, 1, 1, 1, 1]])
cluster = {}
points = range(len(segment_arg))
c = 0
for p in range(len(segment_arg)):
q = queue()
if p in points:
points.remove(p)
cluster[c] = [p]
q.put(p)
mod_arr = segments_new == -5
while (not q.empty()):
i = q.get()
seg = segment_arg[i]
row_up = max(0, seg[0] - 1)
row_down = min(segment.shape[0], seg[0] + 2)
col_left = max(0, seg[1] - 1)
col_right = min(segment.shape[1], seg[1] + 2)
if seg[0] - 1 < 0:
mask_up = 0
mask_down = 2
elif seg[0] + 1 >= segment.shape[0]:
mask_up = 3
mask_down = 5
else:
mask_up = 1
mask_down = 4
if seg[1] - 1 < 0:
mask_left = 0
mask_right = 2
elif seg[1] + 1 >= segment.shape[1]:
mask_left = 3
mask_right = 5
else:
mask_left = 1
mask_right = 4
segment[row_up:row_down, col_left:col_right] += \
mask[mask_up:mask_down,mask_left:mask_right]
[(cluster[c].append(j), q.put(j), points.remove(j))
for j in points if segment[segment_arg[j][0],
segment_arg[j][1]] > 1]
segments_new[segment > 1] = label_new
segment[row_up:row_down, col_left:col_right] -= \
mask[mask_up:mask_down,mask_left:mask_right]
label_new += 1
c += 1
print l
write_arr_to_matfile(segments_new,
seg_folder + "/" + \
os.path.splitext(basename(path_list[i]))[0] +".mat",
"superPixels")
def grid(self, seg_folder, k):
'''
calculates grid-wise segmentation
@param seg_folder: superpixel folder
@param k: segmentation factor, constructs a k x k grid
'''
self.log.start("Calculating GRID SuperPixels", len(self.image_list), 1)
for i in range(len(self.image_list)):
if not os.path.isdir(seg_folder + "/" + os.path.basename(
os.path.dirname(self.image_path_list[i]))):
os.makedirs(
seg_folder + "/" +
os.path.basename(os.path.dirname(self.image_path_list[i])))
if not os.path.isfile(seg_folder + "/" + os.path.basename(os.path.dirname(self.image_path_list[i])) +\
os.path.basename(os.path.splitext(self.image_path_list[i])[0]) +".mat"):
if self.seq_reading:
image = np.array(Image.open(self.image_path_list[i]))
else:
image = self.image_array_list[i]
y_max = image.shape[1]
x_max = image.shape[0]
y_step = y_max / k
x_step = x_max / k
y_rest = y_max % k
x_rest = x_max % k
if x_rest >= 0.75 * x_step:
x_range = range(0, x_max, x_step)
else:
x_steps = [x_step for j in range(k - 1)]
for j in range(x_rest):
x_steps[j%len(x_steps)] =\
x_steps[j%len(x_steps)]+1
x_range = [0] + x_steps
for j in range(len(x_range) - 1):
x_range[j + 1] = x_range[j] + x_range[j + 1]
if y_rest >= 0.75 * y_step:
y_range = range(0, y_max, y_step)
else:
y_steps = [y_step for j in range(k - 1)]
for j in range(y_rest):
y_steps[j%len(y_steps)] =\
y_steps[j%len(y_steps)]+1
y_range = [0] + y_steps
for j in range(len(y_range) - 1):
y_range[j + 1] = y_range[j] + y_range[j + 1]
x_steps = [(a, min(a + x_step + 1, x_max)) for a in x_range]
y_steps = [(a, min(a + y_step + 1, y_max)) for a in y_range]
super_pixels = np.zeros((image.shape[0], image.shape[1]))
label = 1
for x in x_steps:
for y in y_steps:
super_pixels[x[0]:x[1], y[0]:y[1]] = label
label += 1
write_arr_to_matfile(
super_pixels, seg_folder + "/" + os.path.basename(
os.path.dirname(self.image_path_list[i])) + '/' +
self.image_list[i] + ".mat", "superPixels")
self.log.update()