def main(args):
total_start = timeit.default_timer()
print('Starting reconstruction of volume %s ...'%(args.substack_id))
substack = SubStack(args.indir,args.substack_id)
np_tensor_3d, minz = imtensor.load_nearby(args.tensorimage, substack, args.extramargin)
if not args.local_mean_std:
print('Reading standardization data from', args.trainfile)
h5 = tables.openFile(args.trainfile)
Xmean = h5.root.Xmean[:].astype(np.float32)
Xstd = h5.root.Xstd[:].astype(np.float32)
h5.close()
else:
Xmean=None
Xstd=None
print('Starting semantic devonvolution of volume', args.substack_id)
model = pickle.load(open(args.model))
minz = int(re.split('[a-zA-z0-9]*_',substack.info['Files'][0])[1].split('.tif')[0])
reconstruction = deconvolver.filter_volume(np_tensor_3d, Xmean, Xstd,
args.extramargin, model, args.speedup, do_cython=args.do_cython, trainfile=args.trainfile)
imtensor.save_tensor_as_tif(reconstruction, args.outdir+'/'+args.substack_id, minz)
print ("total time reconstruction: %s" %(str(timeit.default_timer() - total_start)))
def main(args):
total_start = timeit.default_timer()
print('Starting Preibisch fusion', args.substack_id)
ss = SubStack(args.first_view_dir, args.substack_id)
minz = int(ss.info['Files'][0].split("/")[-1].split('_')[-1].split('.tif')[0])
prefix = '_'.join(ss.info['Files'][0].split("/")[-1].split('_')[0:-1])+'_'
np_tensor_3d_first_view,_ = imtensor.load_nearby(args.tensorimage_first_view, ss, args.size_patch)
sc_in=np_tensor_3d_first_view.shape
if args.transformation_file is not None:
R, t = parse_transformation_file(args.transformation_file)
np_tensor_3d_second_view = transform_substack(args.second_view_dir, args.tensorimage_second_view, args.substack_id, R, t, args.size_patch, invert=True)
else:
np_tensor_3d_second_view,_ = imtensor.load_nearby(args.tensorimage_second_view, ss, args.size_patch)
fused_image,entropy_mask__view,entropy_mask_second_view = do_content_based_fusion(np_tensor_3d_first_view,np_tensor_3d_second_view,args.size_patch, args.size_patch, speedup=1,fast_computation=True)
if args.extramargin>args.size_patch:
args.extramargin=args.size_patch
offset_margin=args.size_patch - args.extramargin
fused_image_output=fused_image[offset_margin:sc_in[0]-offset_margin,offset_margin:sc_in[1]-offset_margin,offset_margin:sc_in[2]-offset_margin]
atom = tables.UInt8Atom()
mkdir_p(args.outdir)
h5f = tables.openFile(args.outdir + '/' + args.substack_id + '.h5', 'w')
sc_out=fused_image_output.shape
ca = h5f.createCArray(h5f.root, 'full_image', atom, sc_out)
for z in xrange(0, sc_out[0], 1):
ca[z, :, :] = fused_image_output[z,:,:]
h5f.close()
imtensor.save_tensor_as_tif(fused_image_output, args.outdir+'/'+args.substack_id, minz,prefix=prefix)
print ("total time Preibisch fusion: %s" %(str(timeit.default_timer() - total_start)))
def main(args):
total_start = timeit.default_timer()
print('Starting Preibisch fusion', args.substack_id)
ss = SubStack(args.first_view_dir, args.substack_id)
minz = int(
ss.info['Files'][0].split("/")[-1].split('_')[-1].split('.tif')[0])
prefix = '_'.join(
ss.info['Files'][0].split("/")[-1].split('_')[0:-1]) + '_'
np_tensor_3d_first_view, _ = imtensor.load_nearby(
args.tensorimage_first_view, ss, args.size_patch)
sc_in = np_tensor_3d_first_view.shape
if args.transformation_file is not None:
R, t = parse_transformation_file(args.transformation_file)
np_tensor_3d_second_view = transform_substack(
args.second_view_dir,
args.tensorimage_second_view,
args.substack_id,
R,
t,
args.size_patch,
invert=True)
else:
np_tensor_3d_second_view, _ = imtensor.load_nearby(
args.tensorimage_second_view, ss, args.size_patch)
fused_image, entropy_mask__view, entropy_mask_second_view = do_content_based_fusion(
np_tensor_3d_first_view,
np_tensor_3d_second_view,
args.size_patch,
args.size_patch,
speedup=1,
fast_computation=True)
if args.extramargin > args.size_patch:
args.extramargin = args.size_patch
offset_margin = args.size_patch - args.extramargin
fused_image_output = fused_image[offset_margin:sc_in[0] - offset_margin,
offset_margin:sc_in[1] - offset_margin,
offset_margin:sc_in[2] - offset_margin]
atom = tables.UInt8Atom()
mkdir_p(args.outdir)
h5f = tables.openFile(args.outdir + '/' + args.substack_id + '.h5', 'w')
sc_out = fused_image_output.shape
ca = h5f.createCArray(h5f.root, 'full_image', atom, sc_out)
for z in xrange(0, sc_out[0], 1):
ca[z, :, :] = fused_image_output[z, :, :]
h5f.close()
imtensor.save_tensor_as_tif(fused_image_output,
args.outdir + '/' + args.substack_id,
minz,
prefix=prefix)
print("total time Preibisch fusion: %s" %
(str(timeit.default_timer() - total_start)))
def main(args):
total_start = timeit.default_timer()
print('Starting reconstruction of volume %s ...' % (args.substack_id))
ss = SubStack(args.first_view_dir, args.substack_id)
minz = int(
ss.info['Files'][0].split("/")[-1].split('_')[-1].split('.tif')[0])
prefix = '_'.join(
ss.info['Files'][0].split("/")[-1].split('_')[0:-1]) + '_'
np_tensor_3d_first_view, _ = imtensor.load_nearby(
args.tensorimage_first_view, ss, args.extramargin)
if args.transformation_file is not None:
R, t = parse_transformation_file(args.transformation_file)
np_tensor_3d_second_view = transform_substack(
args.second_view_dir,
args.tensorimage_second_view,
args.substack_id,
R,
t,
args.extramargin,
invert=True)
else:
np_tensor_3d_second_view, _ = imtensor.load_nearby(
args.tensorimage_second_view, ss, args.extramargin)
print('Loading model...')
model = pickle.load(open(args.model))
if not args.local_mean_std:
h5 = tables.openFile(args.trainfile)
Xmean = h5.root.Xmean[:].astype(np.float32)
Xstd = h5.root.Xstd[:].astype(np.float32)
h5.close()
else:
Xmean = None
Xstd = None
reconstruction = deconvolver.filter_volume(
[np_tensor_3d_first_view, np_tensor_3d_second_view],
Xmean,
Xstd,
args.extramargin,
model,
args.speedup,
do_cython=args.do_cython,
trainfile=args.trainfile)
pair_id = basename(args.first_view_dir) + '_' + basename(
args.second_view_dir)
outdir = args.outdir + '/' + args.substack_id + '/' + pair_id
imtensor.save_tensor_as_tif(reconstruction, outdir, minz, prefix='slice_')
print("total time reconstruction: %s" %
(str(timeit.default_timer() - total_start)))
def main(args):
substack = SubStack(args.indir,args.substack_id)
np_tensor_3d, minz = imtensor.load_nearby(args.tensorimage, substack, args.extramargin)
# Standardize volume according to mean and std found in the training set
print('Reading standardization data from', args.trainfile)
h5=tables.openFile(args.trainfile)
Xmean = h5.root.Xmean[:]
Xstd = h5.root.Xstd[:]
h5.close()
print('Starting semantic devonvolution of volume', args.substack_id)
model = pickle.load(open(args.model))
minz = int(substack.info['Files'][0].split('full_')[1].split('.tif')[0])
reconstruction = deconvolver.filter_volume(np_tensor_3d, Xmean, Xstd,
args.extramargin, model, args.speedup)
imtensor.save_tensor_as_tif(reconstruction, args.outdir+'/'+args.substack_id, minz)
def main(args):
total_start = timeit.default_timer()
print('Starting reconstruction of volume %s ...'%(args.substack_id))
ss = SubStack(args.first_view_dir, args.substack_id)
minz = int(ss.info['Files'][0].split("/")[-1].split('_')[-1].split('.tif')[0])
prefix = '_'.join(ss.info['Files'][0].split("/")[-1].split('_')[0:-1])+'_'
np_tensor_3d_first_view,_ = imtensor.load_nearby(args.tensorimage_first_view, ss, args.extramargin)
if args.transformation_file is not None:
R, t = parse_transformation_file(args.transformation_file)
np_tensor_3d_second_view = transform_substack(args.second_view_dir, args.tensorimage_second_view, args.substack_id, R, t, args.extramargin, invert=True)
else:
np_tensor_3d_second_view,_ = imtensor.load_nearby(args.tensorimage_second_view, ss, args.extramargin)
print('Loading model...')
model = pickle.load(open(args.model))
if not args.local_mean_std:
h5 = tables.openFile(args.trainfile)
Xmean = h5.root.Xmean[:].astype(np.float32)
Xstd = h5.root.Xstd[:].astype(np.float32)
h5.close()
else:
Xmean=None
Xstd=None
reconstruction = deconvolver.filter_volume([np_tensor_3d_first_view,np_tensor_3d_second_view], Xmean, Xstd,
args.extramargin, model, args.speedup, do_cython=args.do_cython,trainfile=args.trainfile)
pair_id = basename(args.first_view_dir)+ '_' +basename(args.second_view_dir)
outdir=args.outdir+'/'+args.substack_id+'/'+pair_id
imtensor.save_tensor_as_tif(reconstruction, outdir, minz, prefix='slice_')
print ("total time reconstruction: %s" %(str(timeit.default_timer() - total_start)))
def main(args):
total_start = timeit.default_timer()
print('Starting reconstruction of volume %s ...'%(args.substack_id))
substack = SubStack(args.indir,args.substack_id)
substack.load_volume()
tensor = substack.get_volume()
# Changing the tensor so that it has a 6 pixel black padding. Hopefully it won't frick things up too much. Else, mail time.
print("The shape of the tensor before padding: " + str(np.shape(tensor)))
tensor = pad(tensor, 6)
print("The shape of the tensor after padding: " + str(np.shape(tensor)))
if not args.local_mean_std:
print('Reading standardization data from', args.trainfile)
h5 = tables.openFile(args.trainfile)
Xmean = h5.root.Xmean[:].astype(np.float32)
Xstd = h5.root.Xstd[:].astype(np.float32)
h5.close()
else:
Xmean=None
Xstd=None
print('Starting semantic devonvolution of volume', args.substack_id)
# Importing here to have a clean --help
from keras.models import model_from_json
model = model_from_json(open(args.model + '/architecture.json').read())
model.load_weights(args.model + '/weights.h5')
minz = int(re.split('[a-zA-z0-9]*_',substack.info['Files'][0])[1].split('.tif')[0])
# Remove the margin, I have changed deconvolver to use a fized number instead of the extramargin. Hope it works.
reconstruction = deconvolver.filter_volume(tensor, Xmean, Xstd,
args.extramargin, model, args.speedup, do_cython=args.do_cython, trainfile=args.trainfile)
imtensor.save_tensor_as_tif(reconstruction, args.outdir+'/'+args.substack_id, minz)
print ("total time reconstruction: %s" %(str(timeit.default_timer() - total_start)))
def make_dataset(tensorimage, ss, C, L=12, size=None, save_tiff_files=False, negatives=False, margin=None):
hf5 = tables.openFile(tensorimage, 'r')
X0,Y0,Z0 = ss.info['X0'], ss.info['Y0'], ss.info['Z0']
origin = (Z0, Y0, X0)
H,W,D = ss.info['Height'], ss.info['Width'], ss.info['Depth']
ss_shape = (D,H,W)
print('Loading data for substack', ss.substack_id)
np_tensor_3d = hf5.root.full_image[origin[0]:origin[0]+ss_shape[0],
origin[1]:origin[1]+ss_shape[1],
origin[2]:origin[2]+ss_shape[2]]
X = []
y = []
patchlen = (1+2*size)**3
print('Preparing..')
kdtree = cKDTree(np.array([[c.x,c.y,c.z] for c in C]))
nrej_intensity = 0
nrej_near = 0
target_tensor_3d = np.zeros(np_tensor_3d.shape)
for c in C:
if inside_margin(c,ss) > 0:
target_tensor_3d[c.z, c.y, c.x] = 1
target_tensor_3d = gfilter.gaussian_filter(target_tensor_3d, sigma=1.0,
mode='constant', cval=0.0,
truncate=1.5)
# undo scipy normalization of the gaussian filter
target_tensor_3d = (target_tensor_3d / np.max(target_tensor_3d))
save_tiff_files = False
if save_tiff_files:
minz = int(ss.info['Files'][0].split('full_')[1].split('.tif')[0])
target_tensor_3d = np.array(target_tensor_3d*255.0, dtype=np.uint8)
imtensor.save_tensor_as_tif(target_tensor_3d,'/tmp/show_target/'+ss.substack_id, minz)
print('Patches of size', 2*size+1, 'patchlen', patchlen)
print('Negatives will','' if negatives else 'not', 'be included')
pbar = pb.ProgressBar(widgets=['Making patches for %d points: ' % len(C), pb.Percentage()],
maxval=len(C)).start()
nrej_intensity = 0
nrej_near = 0
for pbi,c in enumerate(C):
if inside_margin(c,ss) > 0:
n_neg = 0
cx = int(c.x)
cy = int(c.y)
cz = int(c.z)
print(cx,cy,cz,margin)
for x0 in range(cx-L,cx+L+1,3):
for y0 in range(cy-L,cy+L+1,3):
for z0 in range(cz-L,cz+L+1,3):
if np.random.rand(1)[0] > 0: # 0.5: #0.8:
print(z0,y0,x0)
patch = np_tensor_3d[z0-size:z0+size+1,y0-size:y0+size+1,x0-size:x0+size+1]
print(patch.shape)
X.append(np.reshape(patch, (patchlen,)))
ypatch = target_tensor_3d[z0-size:z0+size+1,y0-size:y0+size+1,x0-size:x0+size+1]
y.append(np.reshape(ypatch, (patchlen,)))
n_neg += 1 # Sample as many negatives as positives
if negatives:
while n_neg > 0:
g = [margin/2,margin/2,margin/2] + np.random.rand(3) * [D-margin,H-margin,W-margin]
g = g.astype(int)
nbrs = kdtree.query_ball_point(g, r=30)
if len(nbrs) == 0:
patch = np_tensor_3d[g[0]-size:g[0]+size+1,g[1]-size:g[1]+size+1,g[2]-size:g[2]+size+1]
if np.mean(patch) > 10: # or np.random.rand(1)[0] > 0.95:
X.append(np.reshape(patch, (patchlen,)))
ypatch = target_tensor_3d[z0-size:z0+size+1,y0-size:y0+size+1,x0-size:x0+size+1]
y.append(np.reshape(ypatch, (patchlen,)))
# y.append(np.zeros(patchlen))
n_neg -= 1
else:
nrej_intensity += 1
else:
nrej_near += 1
else:
pass
pbar.update(pbi+1)
pbar.finish()
print('Rejected:', nrej_intensity, 'by intensity and', nrej_near, 'by distance')
hf5.close()
return X, y
def make_pos_neg_dataset(tensor_view, ss, C, size=5, save_tiff_files=False, default_sigma=0.8, find_negative=True):
tensor_view = tensor_view.astype(np.float32)
H, W, D = ss.info['Height'], ss.info['Width'], ss.info['Depth']
patchlen = (1 + 2 * size) ** 3
c_array = np.array([[c.x, c.y, c.z] for c in C])
target_tensor_3d = np.zeros(tensor_view.shape)
margin = ss.plist['Margin'] / 2
trunc=1.5
fixed_radiiassigned={c:default_sigma for c in C if inside_margin(c,ss) > 0 }
kdt = cKDTree(c_array)
dist,ind = kdt.query(c_array,k=c_array.shape[0],distance_upper_bound=(2*default_sigma*trunc)+2.)
for c,id_c in zip(C,xrange(c_array.shape[0])):
if inside_margin(c, ss) > 0:
for j in xrange(2,c_array.shape[0]):
if not np.isinf(dist[id_c][j]):
if inside_margin(C[ind[id_c][j]], ss) > 0:
if trunc*(fixed_radiiassigned[c]+fixed_radiiassigned[C[ind[id_c][j]]]) > dist[id_c][j] - 2.:
new_sigma=((dist[id_c][j]-1.)/trunc)*fixed_radiiassigned[c]/(fixed_radiiassigned[c]+fixed_radiiassigned[C[ind[id_c][j]]])
fixed_radiiassigned[C[ind[id_c][j]]]=(new_sigma*fixed_radiiassigned[C[ind[id_c][j]]])/fixed_radiiassigned[c]
fixed_radiiassigned[c]=new_sigma
for c in C:
if inside_margin(c, ss) > 0:
target_tensor_3d_tmp = np.zeros(tensor_view.shape)
target_tensor_3d_tmp[c.z, c.y, c.x] = 1
sigma=fixed_radiiassigned[c]
target_tensor_3d_tmp = gfilter.gaussian_filter(target_tensor_3d_tmp, sigma,
mode='constant', cval=0.0,
truncate=trunc)
target_tensor_3d_tmp = (target_tensor_3d_tmp.astype(np.float32) / np.max(target_tensor_3d_tmp))
target_tensor_3d = np.maximum(np.array(target_tensor_3d_tmp * 255.0, dtype=np.uint8), target_tensor_3d)
if save_tiff_files:
debug_path='/mnt/data/marco/experiments/markers/debug_single_view_on_fused_image'
mkdir_p(debug_path)
minz = 0
target_tensor_3d = np.array(target_tensor_3d, dtype=np.uint8)
imtensor.save_tensor_as_tif(target_tensor_3d, debug_path+'/'+ss.substack_id, minz)
target_tensor_3d = (target_tensor_3d.astype(np.float32) / np.max(target_tensor_3d))
tensor_view = tensor_view.astype(np.float32) / 255.0
nrej_intensity = 0
num_negative_targets = 0
num_positive_targets = 0
step_x = 1
step_y = 1
step_z = 1
num_iterations = (W - 2*margin) * (H- 2*margin) * (D - 2*margin) / (step_x * step_y * step_z)
pbar = ProgressBar(widgets=['Making positive and negative examples for %d points: ' % num_iterations, Percentage()],
maxval=num_iterations).start()
X_positive = np.zeros((num_iterations, patchlen), dtype=np.float32)
y_positive = np.zeros((num_iterations, patchlen), dtype=np.float32)
X_negative = []
y_negative = []
if find_negative == True:
X_negative = np.zeros((num_iterations, patchlen), dtype=np.float32)
y_negative = np.zeros((num_iterations, patchlen), dtype=np.float32)
pbi = 0
for x0 in range(margin, W - margin, step_x):
for y0 in range(margin, H - margin, step_y):
for z0 in range(margin, D - margin, step_z):
if inside_patch(c_array, x0, y0, z0, size,offset=2.0): #found a marker inside margin
patch = tensor_view[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]
X_positive[num_positive_targets, :] = np.reshape(patch, (patchlen,))
ypatch = target_tensor_3d[z0 - size:z0 + size + 1, y0 - size:y0 + size + 1,
x0 - size:x0 + size + 1]
y_positive[num_positive_targets, :] = np.reshape(ypatch, (patchlen,))
num_positive_targets += 1 # Sample as many negatives as positives
elif find_negative == True: #min_distance > (size * (3**(1/2.0))) + 2.0
patch = tensor_view[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]
if np.mean(patch * 255) > 5:
X_negative[num_negative_targets, :] = np.reshape(patch, (patchlen,))
ypatch = target_tensor_3d[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]
y_negative[num_negative_targets, :] = np.reshape(ypatch, (patchlen,))
num_negative_targets += 1
else:
nrej_intensity += 1
pbar.update(pbi + 1)
pbi += 1
pbar.finish()
X_positive = X_positive[0: num_positive_targets]
y_positive = y_positive[0: num_positive_targets]
print('Total positive examples for substack (', ss.substack_id, '):', num_positive_targets)
if find_negative == True:
print('Total negative examples for substack (', ss.substack_id, '):', num_negative_targets)
print('Rejected by intensity ', nrej_intensity)
X_negative = X_negative[0: num_negative_targets]
y_negative = y_negative[0: num_negative_targets]
if (num_negative_targets > num_positive_targets):
print('Shuffling negative examples...')
perm = np.random.permutation(len(X_negative)).astype(int)
X_negative = X_negative[perm]
y_negative = y_negative[perm]
print('Shuffling done')
X_negative = X_negative[0: num_positive_targets]
y_negative = y_negative[0: num_positive_targets]
return X_positive, y_positive, X_negative, y_negative
def make_dataset(ss, C, L=12, size=None, save_tiff_files=False, negatives=False, margin=None):
X0,Y0,Z0 = ss.info['X0'], ss.info['Y0'], ss.info['Z0']
origin = (Z0, Y0, X0)
H,W,D = ss.info['Height'], ss.info['Width'], ss.info['Depth']
ss_shape = (D,H,W)
print('Loading data for substack', ss.substack_id)
ss.load_volume()
np_tensor_3d = ss.get_volume()
# Changing the tensor so that it has a 6 pixel black padding.
# np_tensor_3d = pad(np_tensor_3d, 6)
X = []
y = []
patchlen = (1+2*size)**3
print('Preparing..')
kdtree = cKDTree(np.array([[c.x,c.y,c.z] for c in C]))
nrej_intensity = 0
nrej_near = 0
target_tensor_3d = np.zeros(np_tensor_3d.shape)
for c in C:
if inside_margin(c,ss) > 0:
target_tensor_3d[c.z, c.y, c.x] = 1
target_tensor_3d = gfilter.gaussian_filter(target_tensor_3d, sigma=3.5,
mode='constant', cval=0.0,
truncate=1.5)
# undo scipy normalization of the gaussian filter
target_tensor_3d = (target_tensor_3d / np.max(target_tensor_3d))
save_tiff_files = False
if save_tiff_files:
minz = int(ss.info['Files'][0].split('full_')[1].split('.tif')[0])
target_tensor_3d = np.array(target_tensor_3d*255.0, dtype=np.uint8)
imtensor.save_tensor_as_tif(target_tensor_3d,'/tmp/show_target/'+ss.substack_id, minz)
print('Patches of size', 2*size+1, 'patchlen', patchlen)
print('Negatives will','' if negatives else 'not', 'be included')
pbar = pb.ProgressBar(widgets=['Making patches for %d points: ' % len(C), pb.Percentage()],
maxval=len(C)).start()
nrej_intensity = 0
nrej_near = 0
for pbi,c in enumerate(C):
if inside_margin(c,ss) > 0:
n_neg = 0
cx = int(c.x)
cy = int(c.y)
cz = int(c.z)
# print(cx,cy,cz,margin)
for x0 in range(cx-L,cx+L+1,3):
for y0 in range(cy-L,cy+L+1,3):
for z0 in range(cz-L,cz+L+1,3):
if np.random.rand(1)[0] > 0: # 0.5: #0.8:
# print(z0,y0,x0)
patch = np_tensor_3d[z0-size:z0+size+1,y0-size:y0+size+1,x0-size:x0+size+1]
# print(patch.shape)
X.append(np.reshape(patch, (patchlen,)))
ypatch = target_tensor_3d[z0-size:z0+size+1,y0-size:y0+size+1,x0-size:x0+size+1]
y.append(np.reshape(ypatch, (patchlen,)))
n_neg += 1 # Sample as many negatives as positives
if negatives:
while n_neg > 0:
g = [margin/2,margin/2,margin/2] + np.random.rand(3) * [D-margin,H-margin,W-margin]
g = g.astype(int)
nbrs = kdtree.query_ball_point(g, r=30)
if len(nbrs) == 0:
patch = np_tensor_3d[g[0]-size:g[0]+size+1,g[1]-size:g[1]+size+1,g[2]-size:g[2]+size+1]
if np.mean(patch) > 10: # or np.random.rand(1)[0] > 0.95:
X.append(np.reshape(patch, (patchlen,)))
ypatch = target_tensor_3d[z0-size:z0+size+1,y0-size:y0+size+1,x0-size:x0+size+1]
y.append(np.reshape(ypatch, (patchlen,)))
# y.append(np.zeros(patchlen))
n_neg -= 1
else:
nrej_intensity += 1
else:
nrej_near += 1
else:
pass
pbar.update(pbi+1)
pbar.finish()
print('Rejected:', nrej_intensity, 'by intensity and', nrej_near, 'by distance')
return X, y
def transform_substack(indir, tensorimage, substack_id, R, t, extramargin, outdir=None, invert=False, save_tiff=False, save_hdf5=False):
"""
Method that applies a previously estimated rigid transformation to a specific substack.
Parameters
----------
indir : str
substack dir of the input view
tensorimage : str
the whole tensor in hdf5 format
substack_id : str
id of the substack that will be transformed
R : numpy array of shape (3, 3)
rotational component of the estimated rigid transformation
t : numpy array of shape (3)
translational component of the estimated rigid transformation
extramargin : int
extramargin used to extract the transformed substack from tensorimage
outdir : str
output directory where the transformed substack will be saved (Default: None)
invert : boolean
if True the tranformation is inverted (Default: False)
save_tiff : boolean
save the transformed substack in a stack of tiff slices (Default: False)
save_hdf5 : boolean
save the transformed substack as a hdf5 tensor (Default: False)
Returns
-------
pixels_transformed_input: numpy tensor
tensor of the transformed substack
"""
ss = SubStack(indir, substack_id)
input_stack_file = tensorimage
hf5 = tables.openFile(input_stack_file, 'r')
full_D, full_H, full_W = hf5.root.full_image.shape
X0,Y0,Z0 = ss.info['X0'], ss.info['Y0'], ss.info['Z0']
origin = (Z0, Y0, X0)
H,W,D = ss.info['Height'], ss.info['Width'], ss.info['Depth']
or_ss_shape = (D,H,W)
offset_W=int((3**(1/2.0)*W + W/2.0 - W)/2.0)
offset_H=int((3**(1/2.0)*H + H/2.0 - H)/2.0)
offset_D=int((3**(1/2.0)*D + D/2.0 - D)/2.0)
if offset_W < extramargin:
offset_W = extramargin
if offset_H < extramargin:
offset_H = extramargin
if offset_D < extramargin:
offset_D = extramargin
offset_D_left = offset_D if int(origin[0] - offset_D) > 0 else origin[0]
offset_H_left = offset_H if int(origin[1] - offset_H) > 0 else origin[1]
offset_W_left = offset_W if int(origin[2] - offset_W) > 0 else origin[2]
offset_D_right = offset_D if int(origin[0] + or_ss_shape[0] + offset_D) <= full_D else full_D - (origin[0] + or_ss_shape[0])
offset_H_right = offset_H if int(origin[1] + or_ss_shape[1] + offset_H) <= full_H else full_H - (origin[1] + or_ss_shape[1])
offset_W_right = offset_W if int(origin[2] + or_ss_shape[2] + offset_W) <= full_W else full_W - (origin[2] + or_ss_shape[2])
pixels_input = hf5.root.full_image[origin[0] - offset_D_left:origin[0] + or_ss_shape[0] + offset_D_right,
origin[1] - offset_H_left:origin[1] + or_ss_shape[1] + offset_H_right,
origin[2] - offset_W_left:origin[2] + or_ss_shape[2] + offset_W_right]
exmar_D_left = 0 if offset_D_left == origin[0] else extramargin
exmar_H_left = 0 if offset_H_left == origin[1] else extramargin
exmar_W_left = 0 if offset_W_left == origin[2] else extramargin
depth_target, height_target, width_target = or_ss_shape[0] + 2 * extramargin, or_ss_shape[1] + 2 * extramargin, or_ss_shape[2] + 2 * extramargin #new
depth_input, height_input, width_input = pixels_input.shape[0], pixels_input.shape[1], pixels_input.shape[2]
pixels_transformed_input = np.zeros((depth_target,height_target,width_target), dtype=np.uint8)
total_start = timeit.default_timer()
coords_2d_target = np.vstack(np.indices((width_target,height_target)).swapaxes(0,2).swapaxes(0,1))
invR = R.T
if invert:
t = -np.dot(invR, t)
invR = R
invR_2d_transpose = np.transpose(np.dot(invR[:, 0:2], np.transpose(coords_2d_target - t[0:2])))
offset_coords = np.array([[offset_W_left - exmar_W_left, offset_H_left - exmar_H_left, offset_D_left - exmar_D_left]]*invR_2d_transpose.shape[0])#new
for z in xrange(0, depth_target, 1):
R_t_3d = np.transpose(invR_2d_transpose + invR[:, 2] * (z - t[2]) + offset_coords)
good_indices = np.array(range(R_t_3d.shape[1]))
good_indices = good_indices[(R_t_3d[0, :] > 0) * (R_t_3d[1, :] > 0) * (R_t_3d[2, :] > 0) * (R_t_3d[0, :] < (width_input - 1)) * (R_t_3d[1, :] < (height_input - 1)) * (R_t_3d[2, :] < (depth_input - 1))]
R_t_3d = R_t_3d.take(good_indices,axis=1)
R_t_3d = np.round(R_t_3d).astype(int)
coords_2d_target_tmp = coords_2d_target.take(good_indices, axis=0)
coords_3d_target_tmp = np.hstack((coords_2d_target_tmp, np.ones((coords_2d_target_tmp.shape[0], 1)).astype(int)*z))
pixels_transformed_input[coords_3d_target_tmp[:, 2], coords_3d_target_tmp[:, 1], coords_3d_target_tmp[:, 0]] = pixels_input[R_t_3d[2, :], R_t_3d[1, :], R_t_3d[0, :]]
total_stop = timeit.default_timer()
print ("total time transformation stack:%s "%(str(total_stop - total_start)))
pixels_transformed_input = np.array(pixels_transformed_input, dtype=np.uint8)
if save_tiff or save_hdf5:
mkdir_p(outdir)
if save_tiff:
minz = int(ss.info['Files'][0].split("/")[-1].split('_')[-1].split('.tif')[0])
_prefix = '_'.join(ss.info['Files'][0].split("/")[-1].split('_')[0:-1])+'_'
substack_outdir = outdir + '/' + substack_id
imtensor.save_tensor_as_tif(pixels_transformed_input, substack_outdir, minz, prefix=_prefix)
if save_hdf5:
target_shape = (depth_target, height_target, width_target)
atom = tables.UInt8Atom()
h5f = tables.openFile(outdir + '/' + ss.substack_id + '.h5', 'w')
ca = h5f.createCArray(h5f.root, 'full_image', atom, target_shape)
for z in xrange(0, depth_target, 1):
ca[z, :, :] = pixels_transformed_input[z,:,:]
h5f.close()
return pixels_transformed_input
def make_pos_neg_dataset(tensor_first_view, tensor_second_view, ss, C, view1_id, view2_id, default_sigma=0.8,size=5, save_tiff_files=False, find_negative=True):
tensor_first_view = tensor_first_view.astype(np.float32)
tensor_second_view = tensor_second_view.astype(np.float32)
H, W, D = ss.info['Height'], ss.info['Width'], ss.info['Depth']
patchlen = (1 + 2 * size) ** 3
margin = ss.plist['Margin'] / 2
c_array = np.array([[c.x, c.y, c.z] for c in C])
trunc=1.5
fixed_radiiassigned={c:default_sigma for c in C if inside_margin(c,ss) > 0 }
cccc=fixed_radiiassigned.copy()
kdt = cKDTree(c_array)
dist,ind = kdt.query(c_array,k=c_array.shape[0],distance_upper_bound=(2*default_sigma*trunc)+2.)
for c,id_c in zip(C,xrange(c_array.shape[0])):
if inside_margin(c, ss) > 0:
for j in xrange(2,c_array.shape[0]):
if not np.isinf(dist[id_c][j]):
if inside_margin(C[ind[id_c][j]], ss) > 0:
if trunc*(fixed_radiiassigned[c]+fixed_radiiassigned[C[ind[id_c][j]]]) > dist[id_c][j] - 2.:
new_sigma=((dist[id_c][j]-1.)/trunc)*fixed_radiiassigned[c]/(fixed_radiiassigned[c]+fixed_radiiassigned[C[ind[id_c][j]]])
fixed_radiiassigned[C[ind[id_c][j]]]=(new_sigma*fixed_radiiassigned[C[ind[id_c][j]]])/fixed_radiiassigned[c]
fixed_radiiassigned[c]=new_sigma
target_tensor_3d = np.zeros(tensor_first_view.shape,dtype=np.uint8)
for c in C:
if inside_margin(c, ss) > 0:
target_tensor_3d_tmp = np.zeros(tensor_first_view.shape)
target_tensor_3d_tmp[c.z, c.y, c.x] = 1
sigma=fixed_radiiassigned[c]
target_tensor_3d_tmp = gfilter.gaussian_filter(target_tensor_3d_tmp, sigma,
mode='constant', cval=0.0,
truncate=trunc) #sigma=3.5,mode='constant',cval=0.0,truncate=1.5
target_tensor_3d_tmp = (target_tensor_3d_tmp.astype(np.float32) / np.max(target_tensor_3d_tmp))
target_tensor_3d = np.maximum(np.array(target_tensor_3d_tmp * 255.0, dtype=np.uint8), target_tensor_3d)
if save_tiff_files:
debug_path='/tmp/debug/sigma_'+str(default_sigma)
mkdir_p(debug_path)
minz = 0
imtensor.save_tensor_as_tif(target_tensor_3d, debug_path+'/'+ss.substack_id+'_'+view1_id+'_'+view2_id, minz)
print('num markers =',len(C))
target_tensor_3d = (target_tensor_3d.astype(np.float32) / np.max(target_tensor_3d))
tensor_first_view = tensor_first_view.astype(np.float32) / 255.0
tensor_second_view = tensor_second_view.astype(np.float32) / 255.0
nrej_intensity = 0
num_negative_targets = 0
num_positive_targets = 0
step_x = 1
step_y = 1
step_z = 1
num_iterations = (W - 2*margin +1) * (H- 2*margin) * (D - 2*margin) / (step_x * step_y * step_z)
pbar = ProgressBar(widgets=['Making positive and negative examples for %d points: ' % num_iterations, Percentage()],
maxval=num_iterations).start()
X_positive = np.zeros((num_iterations, patchlen*2), dtype=np.float32)
y_positive = np.zeros((num_iterations, patchlen), dtype=np.float32)
X_negative = []
y_negative = []
if find_negative == True:
X_negative = np.zeros((num_iterations, patchlen*2), dtype=np.float32)
y_negative = np.zeros((num_iterations, patchlen), dtype=np.float32)
pbi = 0
for x0 in range(margin, W - margin, step_x):
for y0 in range(margin, H - margin, step_y):
for z0 in range(margin, D - margin, step_z):
if inside_patch(c_array, x0, y0, z0, size, offset=2.0):
patch_left = tensor_first_view[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]
patch_right = tensor_second_view[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]
X_positive[num_positive_targets, 0:patchlen] = np.reshape(patch_left, (patchlen,))
X_positive[num_positive_targets, patchlen:2*patchlen] = np.reshape(patch_right, (patchlen,))
ypatch = target_tensor_3d[z0 - size:z0 + size + 1, y0 - size:y0 + size + 1,
x0 - size:x0 + size + 1]
y_positive[num_positive_targets, :] = np.reshape(ypatch, (patchlen,))
num_positive_targets += 1
elif find_negative == True:
patch_left = tensor_first_view[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]
patch_right = tensor_second_view[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]
if np.mean(patch_left * 255) > 5 or np.mean(patch_right * 255) > 5:
X_negative[num_negative_targets, 0:patchlen] = np.reshape(patch_left, (patchlen,))
X_negative[num_negative_targets, patchlen:2*patchlen] = np.reshape(patch_right, (patchlen,))
ypatch = target_tensor_3d[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]
y_negative[num_negative_targets, :] = np.reshape(ypatch, (patchlen,))
num_negative_targets += 1
else:
nrej_intensity += 1
pbar.update(pbi+ 1)
pbi += 1
pbar.finish()
X_positive = X_positive[0: num_positive_targets]
y_positive = y_positive[0: num_positive_targets]
print('Total positive examples for substack (', ss.substack_id, '):', num_positive_targets)
if find_negative == True:
print('Total negative examples for substack (', ss.substack_id, '):', num_negative_targets)
print('Rejected by intensity ', nrej_intensity)
X_negative = X_negative[0: num_negative_targets]
y_negative = y_negative[0: num_negative_targets]
if (num_negative_targets > num_positive_targets):
print('Shuffling negative examples...')
perm = np.random.permutation(len(X_negative)).astype(int)
X_negative = X_negative[perm]
y_negative = y_negative[perm]
print('Shuffling done')
X_negative = X_negative[0: num_positive_targets]
y_negative = y_negative[0: num_positive_targets]
return X_positive, y_positive, X_negative, y_negative
