/
mdf_preprocessing.py
executable file
·368 lines (329 loc) · 15.2 KB
/
mdf_preprocessing.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
from skimage import io as sio
import dill
import scipy as sp
import os
import numpy as np
import sys
import picutils as pu
felzenpath = '/home/nyarbel/felzenswalb'
sys.path.insert(0,felzenpath)
from felseg import felseg
from skimage.transform import resize
from adjmat import adjmat
from skimage.segmentation import slic as slic_wrap
from skimage.segmentation import felzenszwalb as fseg
NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000
NUM_EXAMPLES_PER_EPOCH_FOR_EVAL = 10000
BOUNDING_BOX_WIDTH = 227
BOUNDING_BOX_HEIGHT = 227
DEPTH = 9
class MDFInRecord():
SP_Region = np.zeros([BOUNDING_BOX_WIDTH,BOUNDING_BOX_HEIGHT,3],dtype = np.uint8)
SP_Neighbor = np.zeros([BOUNDING_BOX_WIDTH,BOUNDING_BOX_HEIGHT,3],dtype = np.uint8)
Pic = np.zeros([BOUNDING_BOX_WIDTH,BOUNDING_BOX_HEIGHT,3],dtype = np.uint8)
SP_mask= np.zeros([BOUNDING_BOX_WIDTH,BOUNDING_BOX_HEIGHT],dtype = np.uint0)
class MDFInData(object):
segments = []
width = 227
height = 227
NSP = 0
depth = 9
def dirtomdfbatchmsra(dirpath):
image_ext = 'jpg'
images = [fn for fn in os.listdir(dirpath) if fn.endswith(image_ext)]
images.sort()
gt_ext = 'png'
gt_maps = [fn for fn in os.listdir(dirpath) if fn.endswith(gt_ext)]
gt_maps.sort()
return gt_maps,images
def save_SLIC_segmentations_MSRA(images,in_dir,out_dir,NSP):
if 'SLIC_Segs' not in os.listdir(out_dir):
os.mkdir(out_dir+'/SLIC_Segs')
if str(NSP) not in os.listdir(out_dir+'SLIC_Segs'):
os.mkdir(out_dir+'/SLIC_Segs/'+str(NSP))
for fimg in images :
img = sio.imread(in_dir+fimg)
gt = sio.imread(in_dir+fimg[0:-3]+'png')/255
SLIC_seg = np.uint16(slic_wrap(img,NSP, 10, sigma=1, enforce_connectivity=True))
saliency = []
segments = []
segments_temp = np.unique(SLIC_seg)
for segment in segments_temp:
sal_temp = calc_saliency_score(segment,SLIC_seg,gt)
if sal_temp >= 0 :
segments.append(segment)
saliency.append(np.uint0(sal_temp))
fslic = open(out_dir+'/SLIC_Segs/'+str(NSP)+'/'+fimg[0:-4]+'.slic','wb')
dill.dump(SLIC_seg,fslic)
dill.dump(segments,fslic)
dill.dump(saliency,fslic)
fslic.close()
def mult_seg(image,param_path):
segs = {}
fparams = np.load(param_path).item()
for g in range(0,15):
f_seg = np.zeros(image.shape[0:2],dtype=np.int32)
felseg(image,f_seg,fparams['sigma'][g],np.float(fparams['scale'][g]),np.int(fparams['min_size'][g]))
segments = np.unique(f_seg)
neighbour_mat = np.zeros([segments.__len__(),segments.__len__()],dtype=np.int32)
adjmat(f_seg,neighbour_mat)
f_seg+=1
segments+=1
# w = f_seg.shape[0]
# h = f_seg.shape[1]
# for i in range(0,w):
# for j in range(0,h):
# sp = f_seg[i][j]-1
# if i>0 :
# neighbour_mat[sp][f_seg[i-1][j]-1]=1
# if j>0 :
# neighbour_mat[sp][f_seg[i-1][j-1]-1]=1
# neighbour_mat[sp][f_seg[i][j-1]-1]=1
# if j< h-1 :
# neighbour_mat[sp][f_seg[i-1][j+1]-1]=1
# neighbour_mat[sp][f_seg[i][j+1]-1]=1
# if i<w-1 :
# neighbour_mat[sp][f_seg[i+1][j]-1]=1
# if j>0 :
# neighbour_mat[sp][f_seg[i+1][j-1]-1]=1
# if j< h-1 :
# neighbour_mat[sp][f_seg[i+1][j+1]-1]=1
segs[str(g)]={}
segs[str(g)]['segmap']= f_seg
segs[str(g)]['seglist']= segments
segs[str(g)]['neighbour_mat']=neighbour_mat
return segs
def save_fseg_segmentations_MSRA(images,in_dir,out_dir,param_path,train = False):
fparams = np.load(param_path).item()
if 'f_Segs' not in os.listdir(out_dir):
os.mkdir(out_dir+'/f_Segs')
for fimg in images :
img = sio.imread(in_dir+fimg)
gt = sio.imread(in_dir+fimg[0:-3]+'png')/255
segs = {}
for g in range(0,15):
f_seg = np.zeros(img.shape[0:2],dtype=np.int32)
felseg(img,f_seg,fparams['sigma'][g],np.float(fparams['scale'][g]),np.int(fparams['min_size'][g]))
saliency = []
segments = []
f_seg+=1
segments_temp = np.unique(f_seg)
for segment in segments_temp:
sal_temp = calc_saliency_score(segment,f_seg,gt)
if (not(train) or (sal_temp >= 0)) :
segments.append(segment)
saliency.append(np.uint0(sal_temp))
segs[str(g)]={}
segs[str(g)]['segmap']= f_seg
segs[str(g)]['seglist']= segments
segs[str(g)]['labels']= saliency
np.save(out_dir+'/f_Segs/'+fimg[0:-4],segs)
def trainable_segmentations_from_batch(segs):
result = {}
for g in range(0,segs.__len__()):
result[str(g)]={}
result[str(g)]['segmap']= segs[str(g)]['segmap']
saliency = []
segments = []
for s in segs[str(g)]['seglist']:
if ((segs[str(g)]['labels'][s] in [0,1])) :
segments.append(segs[str(g)]['seglist'][s])
sal_temp = [0,0]
sal_temp[segs[str(g)]['labels'][s]]=1
saliency.append(sal_temp)
result[str(g)]['seglist']= segments
result[str(g)]['labels']= saliency
return result
#calculates a segment's saliency score - binary label 0 or 1
#if saliency is undecided(not enough pixels of 1 class )
def calc_saliency_score(segment,slic,gt):
mask = np.uint0(slic == segment)
pixels = np.sum(mask)
sal = -1
sal_temp = np.sum(mask*gt)/pixels
sal = -1
if sal_temp > 0.7 :
sal = 1
elif sal_temp < 0.3:
sal = 0
return sal
#returning a list of MDF records for each segment containing sets of training examples and input of the following form:
#SP_Region - [227x227x3] bounding box of the segment with the area around it set to mean image values
#SP_Neighbour - [227x227x3] bounding box of the resized segment and it's immediate neighbouring segments
#Pic - [227x227x3] bounding box of the resized image with the segment blackened
#SP_mask - 227x227x3 mask for the segement location in the original image
#saliency - a saliency score for the segment if one can be decided upon.
def im2mdfin(img,mean,segmap,segments):
result = MDFInData()
mean_image = sp.misc.imresize(mean,img.shape)
#Superpixel segmentation - to be replaced by other segmentation if necessary
#SLIC_seg = slic_wrap(img, nsp, 10, sigma=1, enforce_connectivity=True)
#segments = np.unique(SLIC_seg)
#numSP = 0
for SPi in range(0,segments.__len__()):
pair = MDFInRecord()
curr_sp = segments[SPi]
sp_mask= np.uint0(segmap == curr_sp)
indices = np.where((segmap == curr_sp)!=0)
bb = np.array([[np.min(indices[0]),np.max(indices[0])],[np.min(indices[1]),np.max(indices[1])]])
#extracting only the superpixel
seg_img = np.copy(img[bb[0,0]:bb[0,1],bb[1,0]:bb[1,1]])
mean_seg = np.copy(mean_image[bb[0,0]:bb[0,1],bb[1,0]:bb[1,1]])
local_seg = segmap[bb[0,0]:bb[0,1],bb[1,0]:bb[1,1]]
#zeroing area around superpixel
seg_img[local_seg != curr_sp,:]=0
mean_seg[local_seg != curr_sp,:]=0
#num_pixels = np.sum(local_seg == curr_sp)
seg_img = seg_img-mean_seg
#GT_label = np.copy(gt[bb[0,0]:bb[0,1],bb[1,0]:bb[1,1]])
#GT_label[local_seg != curr_sp]=0
#saliency_score = np.sum(GT_label/255)/num_pixels
#Saliency score is deemed reliant so we can add it here
#if saliency_score > 0.7 or saliency_score < 0.3:
#numSP = numSP+1
#finding the neighbor segments
neighbors = np.unique(local_seg)
#extracting locations of neighbor segments in image
ix = np.where(np.in1d(segmap.ravel(),neighbors).reshape(segmap.shape))
#calculating a bounding box over neghbor superpixels
bb_mid= np.array([[np.min(ix[0]),np.max(ix[0])],[np.min(ix[1]),np.max(ix[1])]])
#cropping the bounding box - this is the input to the 2nd mini CNN
bounding_box_second = np.copy(img[bb_mid[0,0]:bb_mid[0,1],bb_mid[1,0]:bb_mid[1,1]])
#mean subtraction on region B
bounding_box_second = bounding_box_second - mean_image[bb_mid[0,0]:bb_mid[0,1],bb_mid[1,0]:bb_mid[1,1]]
#resizing superpixel to net input size
pair.SP_Region= np.array(sp.misc.imresize(seg_img,[227,227,3]))#,dtype = np.uint8)
#resizing neighborhood to net input size
pair.SP_Neighbor = np.array(sp.misc.imresize(bounding_box_second,[227,227,3]))#,dtype = np.uint8)
#picture with segment masked
picture = np.copy(img)-mean_image
picture[segmap == curr_sp,:]=0
pair.Pic = np.array(sp.misc.imresize(picture,[227,227,3]))#,dtype = np.uint8)
#pair.saliency = round(saliency_score)
pair.SP_mask = sp.misc.imresize(sp_mask,[227,227,3])
result.segments.append(pair)
return result
def msradirtomdfin(dir_path,NSP):
#extracting names of images in dataset
[gt_maps,images] = dirtomdfbatchmsra(dir_path)
#creating mean image of dataset
# mean_image = sp.zeros([227,227,3],dtype = sp.uint64)
# for i in range(0,images.__len__()):#(images.__len__()-1)):
# mean_image = mean_image + sp.misc.imresize(io.imread(dir_path+images[i]),[227,227,3])
# mean_image = mean_image/images.__len__()
# sp.misc.imsave(os.getcwd()+'/mean_image.jpg',mean_image)
mean_image = io.imread('mean_image.jpg')
out_dir = './mdfinputs/'
for i in range(0, (images.__len__())):
img = io.imread(dir_path+images[i])
gt = io.imread(dir_path+gt_maps[i])
record = im2mdfin(img,NSP,mean_image,gt)
outfile = out_dir+images[i][0:-4]+'mdf.out'
filehandle = open(outfile,'wb')
dill.dump(record,filehandle)
filehandle.close()
# -*- coding: utf-8 -*-
def _generate_image_segments_and_label_batch(images,img_dir,seg_dir,mean_img):
#choosing 5 images at random
test_batch = []
batch_labels = []
for i in range(0,images.__len__()):
img = io.imread(img_dir+images[i])
segf = open(seg_dir+images[i][0:-3]+'slic','rb')
segmap = dill.load(segf)
segments_l = dill.load(segf)
sal_l = dill.load(segf)
segf.close()
data = im2mdfin(img,mean_img,segmap,segments_l)
for j in range(0,segments_l.__len__()):
x = data.segments[j]
dat = np.zeros([227,227,9],dtype = np.uint8)
dat[:,:,0:3]= x.SP_Region
dat[:,:,3:6]=x.SP_Neighbor
dat[:,:,6:9]=x.Pic
test_batch.append(sal_l[j])
test_batch.append(x.SP_Region)
test_batch.append(x.SP_Neighbor)
test_batch.append(x.Pic)
return test_batch
#returning a list of MDF records for each segment containing sets of training examples and input of the following form:
#SP_Region - [227x227x3] bounding box of the segment with the area around it set to mean image values
#SP_Neighbour - [227x227x3] bounding box of the resized segment and it's immediate neighbouring segments
#Pic - [227x227x3] bounding box of the resized image with the segment blackened
#SP_mask - 227x227x3 mask for the segement location in the original image
#saliency - a saliency score for the segment if one can be decided upon.
def im2mdfin2(img,mean,segmap,segments,neighbormat):
SP_Region = []
SP_Neighbor = []
Pic = []
result = []
xdim = img.shape
mean_image = pu.imresize(mean,xdim[1],xdim[0])
#Superpixel segmentation - to be replaced by other segmentation if necessary
#SLIC_seg = slic_wrap(img, nsp, 10, sigma=1, enforce_connectivity=True)
#segments = np.unique(SLIC_seg)
#numSP = 0
for SPi in range(0,segments.__len__()):
pair = MDFInRecord()
curr_sp = segments[SPi]
indices = np.where((segmap == curr_sp))
bb = np.array([[np.min(indices[0]),np.max(indices[0])],[np.min(indices[1]),np.max(indices[1])]])
#extracting only the superpixel
seg_img = np.copy(img[bb[0,0]:bb[0,1]+1,bb[1,0]:bb[1,1]+1])
mean_seg = np.copy(mean_image[bb[0,0]:bb[0,1]+1,bb[1,0]:bb[1,1]+1,:])
local_seg = segmap[bb[0,0]:bb[0,1]+1,bb[1,0]:bb[1,1]+1]
#zeroing area around superpixel
seg_img[local_seg != curr_sp,:]=mean_seg[local_seg != curr_sp,:]
#num_pixels = np.sum(local_seg == curr_sp)
#resizing superpixel to net input size and mean subtraction
SP_Region.append(np.transpose(np.array(pu.imresize(seg_img,227,227))-mean,(1,0,2)))#,dtype = np.uint8)
#GT_label = np.copy(gt[bb[0,0]:bb[0,1],bb[1,0]:bb[1,1]])
#GT_label[local_seg != curr_sp]=0
#saliency_score = np.sum(GT_label/255)/num_pixels
#Saliency score is deemed reliant so we can add it here
#if saliency_score > 0.7 or saliency_score < 0.3:
#numSP = numSP+1
#finding the neighbor segments
neighbors = np.nonzero(neighbormat[curr_sp-1])
#if (neighbors.__len__() < 2) :
# if bb[0,0] >0 :
# bb[0,0]= bb[0,0]-1
# if bb[0,1] < img.shape[0]-1 :
# bb[0,1]= bb[0,1]+1
# if bb[1,0] >0 :
# bb[1,0]= bb[1,0]-1
# if bb[1,1] < img.shape[1]-1 :
# bb[1,1]= bb[1,1]+1
#local_seg = segmap[bb[0,0]:bb[0,1]+1,bb[1,0]:bb[1,1]+1]
#neighbors = np.unique(local_seg)
#extracting locations of neighbor segments in image
ix = np.where(np.in1d(segmap.ravel(),neighbors).reshape(segmap.shape))
#calculating a bounding box over neghbor superpixels
bb_mid= np.array([[np.min(ix[0]),np.max(ix[0])],[np.min(ix[1]),np.max(ix[1])]])
#cropping the bounding box - this is the input to the 2nd mini CNN
bounding_box_second = np.copy(img[bb_mid[0,0]:bb_mid[0,1]+1,bb_mid[1,0]:bb_mid[1,1]+1])
#mean subtraction on region B
#resizing neighborhood to net input size and mean subtraction
SP_Neighbor.append(np.transpose(np.array(pu.imresize(bounding_box_second,227,227))-mean,(1,0,2)))#,dtype = np.uint8)
#picture with segment masked
picture = np.copy(img)
picture[segmap == curr_sp,:]=mean_image[segmap == curr_sp,:]
Pic.append(np.transpose(np.array(pu.imresize(picture,227,227))-mean,(1,0,2)))#,dtype = np.uint8)
#pair.saliency = round(saliency_score)
#result.append(pair.SP_Region)
#result.append(pair.SP_Neighbor)
#result.append(pair.Pic)
return SP_Region,SP_Neighbor,Pic
def dill_file_to_shuffle_batch(file_path):
with open(file_path,'rb') as f:
batch = dill.load(f)
f.close()
def write_batch_to_file(path,images,i,batch_size,img_dir,seg_dir,mean_img):
with open(path,'wb') as f:
batch = _generate_image_segments_and_label_batch(images[i*batch_size:(i+1)*batch_size],img_dir,seg_dir,mean_img)
dill.dump(batch,f)
f.close()
#def _generate_image_segments_and_label_batch(image, label, min_queue_examples,
#batch_size, shuffle):
def generate_s3cnn_label_batch():
pass