def __init__(self,
x,
y,
image_data_generator,
batch_size=32,
shuffle=False,
seed=None,
data_format=None,
save_to_dir=None,
save_prefix='',
save_format='png',
crop_to=-1):
NumpyArrayIterator.__init__(self,
x,
y,
image_data_generator,
batch_size=batch_size,
shuffle=shuffle,
seed=seed,
data_format=data_format,
save_to_dir=save_to_dir,
save_prefix=save_prefix,
save_format=save_format)
self.crop_to = crop_to
def augment_images(width_shift=0.2,
height_shift=0.2,
zoom=0.2,
rotation=30,
vertical_flip=False,
horizontal_flip=False):
transformer = ImageDataGenerator(
width_shift_range=width_shift,
height_shift_range=height_shift,
zoom_range=zoom,
rotation_range=rotation,
vertical_flip=vertical_flip,
horizontal_flip=horizontal_flip)
iterator = None
while True:
x, y = yield
if iterator is None:
iterator = NumpyArrayIterator(
x, y, transformer,
batch_size=len(x),
shuffle=False,
seed=None,
data_format=transformer.data_format)
else:
iterator.n = x.shape[0]
iterator.x = x
iterator.y = y
transformed = next(iterator)
yield transformed
def flow(self, X, y=None, batch_size=32, shuffle=True, seed=None,
save_to_dir=None, save_prefix='', save_format='jpeg'):
return NumpyArrayIterator(
X, y, self,
batch_size=batch_size, shuffle=shuffle, seed=seed,
dim_ordering=self.dim_ordering,
save_to_dir=save_to_dir, save_prefix=save_prefix,
save_format=save_format)
def flow(self,
X,
y,
batch_size=32,
shuffle=True,
seed=None,
save_to_dir=None,
save_prefix='',
save_format='jpeg'):
"""
Basic flow (haha) is inside the NumpyArrayIterator, which calls
random_transform followed by standardize.
"""
return NumpyArrayIterator(X,
y,
self,
batch_size=batch_size,
shuffle=shuffle,
seed=seed,
dim_ordering=self.dim_ordering,
save_to_dir=save_to_dir,
save_prefix=save_prefix,
save_format=save_format)
def image_salience(model,
img_data,
tile_stride,
output_dir,
output_prefix,
preprocess=None,
img_map=None,
backend='tensorflow',
lab_mask=[],
verbose=0,
transpose=False,
print_status=True,
pred_thr=0.0,
do_show=False):
from skimage.util.shape import view_as_windows
input_shape = model.layers[0].input_shape
if len(input_shape) == 1:
input_shape = input_shape[0]
print('input_shape: "%s"' % str((input_shape)))
if backend == 'tensorflow' or backend.startswith('plaidml.keras'):
# channels last
tile_dim, tile_bands = input_shape[-2], input_shape[-1]
tile_transpose = [0, 1, 2]
elif backend == 'theano':
# channels first
tile_dim, tile_bands = input_shape[2], input_shape[1]
tile_transpose = [2, 0, 1]
assert (tile_bands == img_data.shape[-1])
if tile_stride >= 1:
stride = tile_stride
else:
stride = max(1, (tile_stride * tile_dim))
stride = int(stride)
img_test = img_data.transpose((1, 0, 2)) if transpose else img_data
img_test, radd, cadd = image_pad(img_test, tile_dim, stride)
rows, cols, bands = img_test.shape
# need to copy the rgb bands so view_as_windows will work(?)
img_rgb = img_test[..., :3].copy()
img_mask = np.zeros([rows, cols], dtype=np.bool8)
if bands == 4:
# add zero alphas to mask, drop alpha band
img_mask[img_test[..., -1] == 0] = 1
bands = 3
#ridx,cidx = map(lambda a: a.ravel(),np.meshgrid(range(cols),range(rows)))
img_pred = np.zeros([rows, cols, 2], dtype=np.int32)
img_prob = np.zeros([rows, cols, 2], dtype=np.float32)
tdims = [tile_dim, tile_dim]
tshape = tdims + [bands]
rshape = [-1] + tshape
rtranspose = [0] + [d + 1 for d in tile_transpose]
rrange = np.arange(0, rows - tile_dim + 1, stride)
crange = np.arange(0, cols - tile_dim + 1, stride)
n_rb, n_cb = len(rrange), len(crange)
cb_den = max(1, n_cb // 500)
batch_size = max(2 * (n_cb // 2), int(np.sqrt(n_cb))**2) // cb_den
batch_size = min(batch_size, batch_max)
pmsg = 'Collecting predictions'
print(pmsg + ', size = %d x %d tiles (tile_dim=%d, stride=%d)' %
(n_rb, n_cb, tile_dim, stride))
print('batch_size,cb_den: "%s"' % str((batch_size, cb_den)))
pred_list = []
cidx = np.arange(n_cb, dtype=np.int32)
rmask = np.ones(n_cb, dtype=np.bool8)
inittime = gettime()
preptime, predtime, posttime = 0., 0., 0.
pbar = progressbar(pmsg, n_rb)
sanity_check = True
img_u8 = 255 * np.ones([1, tile_dim, tile_dim, 3], dtype=np.uint8)
gray_thr = [0.0, 0.1, 0.5, 1.0] if sanity_check else [0.0]
for thr in gray_thr:
imgthr = preprocess_img_u8(np.uint8(thr * img_u8))
probthr = model.predict(imgthr.transpose(rtranspose))
if thr == 0.0:
prob0p0 = probthr
pred0p0 = to_binary(prob0p0)
print('prob(X = [%.2f]_%dx%d) = ' % (thr, tile_dim, tile_dim), probthr)
for l in model.layers:
l.trainable = False
from keras import backend as K
from keras.preprocessing.image import ImageDataGenerator, NumpyArrayIterator
#model_predict = K.function([model.input, K.learning_phase()],
# [model.output])
#predict_generator(generator, steps=None, max_queue_size=10, workers=1, use_multiprocessing=False, verbose=0)
@threadsafe_generator
def tile_gen(img_rgb, tile_dim, stride, rtranspose):
nrows, ncols, _ = img_rgb.shape
nr = nrows - tile_dim + 1
nc = ncols - tile_dim + 1
rows = np.arange(0, nr, stride, dtype=np.int32)
cols = np.arange(0, nc, stride, dtype=np.int32)
yout = np.zeros([1, 2])
for ij in range(len(rows) * len(cols)):
i = rows[ij // len(cols)]
j = cols[ij - i]
print('(row,col)', (i, j), 'of', (nr, nc))
tile = img_rgb[i:i + tile_dim, j:j + tile_dim, :][np.newaxis]
yield preprocess(tile).transpose(rtranspose), yout
@threadsafe_generator
def win_gen(img_rgb, tile_dim, stride, rtranspose):
nrows, ncols, _ = img_rgb.shape
img_win = view_as_windows(img_rgb, tshape)
print('img_win.shape: "%s"' % str((img_win.shape)))
img_win = img_win[::stride, ::stride]
nr = nrows - tile_dim + 1
nc = ncols - tile_dim + 1
rows = np.arange(0, nr, stride, dtype=np.int32)
cols = np.arange(0, nc, stride, dtype=np.int32)
print('img_win.shape (strided): "%s"' % str((img_win.shape)))
print('len(rows),len(cols): "%s"' % str((len(rows), len(cols))))
yout = np.zeros([1, 2])
nwin = img_win.shape[0] * img_win.shape[1]
for ij in range(nwin):
i = ij // img_win.shape[0]
j = ij - i
tileij = img_win[i, j].reshape([1] + tshape)
print('tileij.shape: "%s"' % str((tileij.shape)))
print('(row,col)', (i, j), 'of', (nr, nc), 'nwin', nwin)
yield preprocess(tileij).transpose(rtranspose)
use_mp = False
n_workers = 5
predictkw = dict(max_queue_size=5000,
workers=n_workers,
use_multiprocessing=use_mp)
#ntiles = len(rrange)*len(crange)
#ntgen = tile_gen(img_rgb,tile_dim,stride,rtranspose)
#ntgen = win_gen(img_rgb,tile_dim,stride,rtranspose)
#rprob = model.predict_generator(ntgen,ntiles,**predictkw)
#print(rprob), raw_input()
use_npi = False
use_seq = True
if use_npi:
_imggen = ImageDataGenerator()
for rbeg in pbar(rrange):
#preptime = gettime()
rend = rbeg + tile_dim
#img_row = img_rgb[rbeg:rend]
rwin = view_as_windows(img_rgb[rbeg:rend], tshape)
rwin = rwin.reshape(rshape)
rwin = rwin[::stride]
# only keep tiles that contain nonzero values
rmask[:] = rwin.any(axis=tuple(range(1, rwin.ndim)))
nkeep = np.count_nonzero(rmask)
nbatch = nkeep // batch_size
rpad = batch_size - (nkeep % batch_size)
if nkeep != 0:
#nkeepb = batch_size*((nkeep//batch_size)+1
rshapei = [nkeep, tile_dim, tile_dim, 3]
rinput = preprocess(
rwin[rmask].copy().reshape(rshapei)) # @profile = 14%
rinput = rinput.transpose(rtranspose)
if rpad != 0:
rpadimg = np.zeros([rpad, tile_dim, tile_dim, 3],
dtype=rinput.dtype)
rinput = np.r_[rinput, rpadimg]
nbatch += 1
#print('rinput.shape: "%s"'%str((rinput.shape)))
#print('nkeep mod batch_size: "%s"'%str((rpad)))
# preptime += (gettime()-preptime)
# predtime = gettime()
# probi = [nkeep,2] softmax class probabilities
rprob = np.zeros([nkeep, 2])
#print('nkeep,nbatch: "%s"'%str((nkeep,nbatch)))
if use_npi:
ry = np.zeros(rinput.shape[0])
rinput_npi = NumpyArrayIterator(rinput,
ry,
_imggen,
batch_size=batch_size,
shuffle=False,
seed=42)
rprob[:] = model.predict_generator(rinput_npi, nbatch,
**predictkw)[:nkeep]
elif use_seq:
rinput_seq = WindowSequence(rinput, batch_size)
rprob[:] = model.predict_generator(rinput_seq, nbatch,
**predictkw)[:nkeep]
else:
for bj in range(nbatch + 1):
js, je = bj * batch_size, (bj + 1) * batch_size
js, je = min(nkeep - 1, js), min(nkeep, je)
rprob[js:je] = model.predict([rinput[js:je],
0])[0] # @profile = 68%
rpred = np.int8(to_binary(rprob))
# predtime += (gettime()-predtime)
# predi = [nkeep,1] softmax class predictions
# if scale_probs:
# ckeep = cidx[:nkeep]
# probi[ckeep,predi] = 2*(probi[ckeep,predi]-0.5)
# probi[ckeep,1-predi] = 1-probi[ckeep,predi]
# probwin = view_as_windows(img_prob[rbeg:rend], tdims+[2], step=stride).squeeze()
# predwin = view_as_windows(img_pred[rbeg:rend], tdims+[1], step=stride).squeeze()
# probwin = probwin.swapaxes(1,3)
# predwin = predwin.swapaxes(1,3)
# probwin[cidx[rmask]] = probi
# if print_state and (i%10)==0:
# print('prediction time: %0.3f seconds'%(gettime()-begtime))
# print('row block %d of %d'%(i,n_rb))
# posttime = gettime()
pj = 0
tile_off = (tile_dim // 2)
for j, cbeg in enumerate(crange):
cend = cbeg + tile_dim
if rmask[j]: # @profile = 11%
img_prob[rbeg:rend, cbeg:cend, :] += rprob[pj]
img_pred[rbeg:rend, cbeg:cend, rpred[pj]] += 1
if pred_thr == 0:
roff, coff = rbeg + tile_off, cbeg + tile_off
pred_list.append([roff, coff, rprob[pj, 1]])
pj += 1
else:
# already know pred + prob for null entries
img_prob[rbeg:rend, cbeg:cend, :] += prob0p0[0]
img_pred[rbeg:rend, cbeg:cend, pred0p0] += 1
if pred_thr == 0:
roff, coff = rbeg + tile_off, cbeg + tile_off
pred_list.append([roff, coff, prob0p0[0, 1]])
#posttime += (gettime()-posttime)
pred_list = np.float32(pred_list)
print('total prediction time (%d tiles): %0.3f seconds' %
(pred_list.shape[0], gettime() - inittime))
if print_status:
#print('preprocessing time: %0.3f seconds'%(preptime/n_rb))
print('prediction time: %0.3f seconds' % (predtime / n_rb))
#print('postprocessing time: %0.3f seconds'%(posttime/n_rb))
# get average probabilities for each class by normalizing by pred counts
img_total = img_pred.sum(axis=2)
# find valid pixels with at least 1 prediction
img_haspred = img_total != 0
#img_prob[img_haspred,0] /= img_total[img_haspred]
#img_prob[img_haspred,1] /= img_total[img_haspred]
img_amax = np.argmax(img_prob, axis=2)
img_pos, img_neg = (img_amax == 1), ((img_amax == 0) & img_haspred)
#if scale_probs:
# print('img_prob before scale_probs:')
if img_pos.any():
print('img_prob[img_pos,0].min(): "%s"' % str(
(img_prob[img_pos, 0].min())))
print('img_prob[img_pos,0].max(): "%s"' % str(
(img_prob[img_pos, 0].max())))
print('img_prob[img_pos,1].min(): "%s"' % str(
(img_prob[img_pos, 1].min())))
print('img_prob[img_pos,1].max(): "%s"' % str(
(img_prob[img_pos, 1].max())))
if img_neg.any():
print('img_prob[img_neg,0].min(): "%s"' % str(
(img_prob[img_neg, 0].min())))
print('img_prob[img_neg,0].max(): "%s"' % str(
(img_prob[img_neg, 0].max())))
print('img_prob[img_neg,1].min(): "%s"' % str(
(img_prob[img_neg, 1].min())))
print('img_prob[img_neg,1].max(): "%s"' % str(
(img_prob[img_neg, 1].max())))
print('img_prob.sum(axis=2).max(): "%s"' % str(
(img_prob.sum(axis=2).max())))
# if scale_probs:
# img_prob[img_pos,1] = 2*(img_prob[img_pos,1]-0.5)
# img_prob[img_pos,0] = 1-img_prob[img_pos,1]
# img_prob[img_neg,0] = 2*(img_prob[img_neg,0]-0.5)
# img_prob[img_neg,1] = 1-img_prob[img_neg,0]
# img_prob[~img_valid] = 0
# print('img_prob after scale_probs:')
# if img_pos.any():
# print('img_prob[img_pos,0].min(): "%s"'%str((img_prob[img_pos,0].min())))
# print('img_prob[img_pos,0].max(): "%s"'%str((img_prob[img_pos,0].max())))
# print('img_prob[img_pos,1].min(): "%s"'%str((img_prob[img_pos,1].min())))
# print('img_prob[img_pos,1].max(): "%s"'%str((img_prob[img_pos,1].max())))
# if img_neg.any():
# print('img_prob[img_neg,0].min(): "%s"'%str((img_prob[img_neg,0].min())))
# print('img_prob[img_neg,0].max(): "%s"'%str((img_prob[img_neg,0].max())))
# print('img_prob[img_neg,1].min(): "%s"'%str((img_prob[img_neg,1].min())))
# print('img_prob[img_neg,1].max(): "%s"'%str((img_prob[img_neg,1].max())))
# print('img_prob.sum(axis=2).max(): "%s"'%str((img_prob.sum(axis=2).max())))
# raw_input()
# crop row,col buffers
rbeg, rend = tile_dim, rows - radd
cbeg, cend = tile_dim, cols - cadd
img_rgb = img_rgb[rbeg:rend, cbeg:cend, :]
img_prob = img_prob[rbeg:rend, cbeg:cend, :]
img_pred = img_pred[rbeg:rend, cbeg:cend, :]
img_mask = img_mask[rbeg:rend, cbeg:cend]
if transpose:
# transpose back to original shape
img_rgb = img_rgb.transpose((1, 0, 2))
img_pred = img_pred.transpose((1, 0, 2))
img_prob = img_prob.transpose((1, 0, 2))
img_mask = img_mask.transpose((1, 0))
prob_total = img_prob.sum(axis=2)
prob_pos = np.where(prob_total != 0, img_prob[..., 1] / prob_total, 0)
pred_out = dict(img_prob=img_prob,
img_pred=img_pred,
img_mask=img_mask,
prob_pos=prob_pos,
prob_total=prob_total,
pred_list=pred_list)
if output_dir:
print('img_map: "%s"' % str((img_map)))
save_pred_images(img_rgb,
pred_out,
mapinfo=img_map,
lab_mask=lab_mask,
output_dir=output_dir,
output_prefix=output_prefix,
do_show=do_show)
return pred_out
def __init__(self, *args, **kwargs):
NumpyArrayIterator.__init__(self, *args, **kwargs)