def processSingleConnection(args):
global temporary_folder
i = args
#steps_done += 1;
#if steps_done % 100 == 0:
# print('Processing step %d / %d...' % (steps_done, steps_total));
if i % 100 == 0:
print('Processing step %d...' % i)
try:
fid = mp.current_process()._identity[0]
except: # mostlikely sequential mode
fid = 0
data = smm.get(0)
mask = smm.get(1)
skel = smm.get(2)
spts = smm.get(3)
res = connectPoint(data,
mask,
spts,
i,
skeleton=skel,
radius=20,
tubeness=None,
remove_local_mask=True,
min_quality=15.0,
verbose=False,
maxSteps=12000,
costPerDistance=1.0)
#return res;
path, score = res
if len(path) > 0:
#convert path to indices
path_flat = np.ravel_multi_index(path.T, data.shape, order=order(data))
#append to file
try:
fid = mp.current_process()._identity[0]
except:
fid = 0
# sequential mode
fn = '%s/path_%03d.tmp' % (temporary_folder, fid)
#print fn;
fb = open(fn, "ab")
path_flat.tofile(fb)
fb.close()
return
def mesh_tube_from_coordinates_and_radii(coordinates,
radii,
indices,
n_tube_points=None,
edge_colors=None,
processes=None,
verbose=False):
"""Construct a mesh from the edge geometry of a graph."""
coordinates_hdl = smm.insert(coordinates)
radii_hdl = smm.insert(radii)
indices_hdl = smm.insert(indices)
if n_tube_points is None:
n_tube_points = 8
func = ft.partial(_parallel_mesh,
coordinates_hdl=coordinates_hdl,
radii_hdl=radii_hdl,
indices_hdl=indices_hdl,
n_tube_points=n_tube_points,
verbose=verbose)
argdata = np.arange(len(indices))
# process in parallel
pool = smm.mp.Pool(processes=processes)
results = pool.map(func, argdata)
pool.close()
pool.join()
smm.free(coordinates_hdl)
smm.free(radii_hdl)
smm.free(indices_hdl)
n_results = len(results)
vertices = [np.reshape(r[0], (-1, 3)) for r in results]
n_indices = [len(v) for v in vertices]
n_indices = np.cumsum(n_indices)
n_indices = np.hstack([[0], n_indices])
if edge_colors is not None:
colors = [len(v) * [c] for v, c in zip(vertices, edge_colors)]
colors = np.concatenate(colors)
#print(colors.shape)
else:
colors = None
vertices = np.concatenate(vertices)
#print(grid.shape)
faces = np.concatenate(
[results[i][1] + n_indices[i] for i in range(n_results)])
return vertices, faces, colors
def propagate(t):
i, hdl = t
a = smm.get(hdl)
#if i % 100000 == 0:
# print('i=%d' % i)
for j in range(1):
a[i] = i
def _parallel_mesh(i,
coordinates_hdl,
radii_hdl,
indices_hdl,
n_tube_points=15,
verbose=False):
coordinates = smm.get(coordinates_hdl)
radii = smm.get(radii_hdl)
start, end = smm.get(indices_hdl)[i]
coordinates = coordinates[start:end]
radii = radii[start:end]
if verbose:
if i % 1000 == 0:
print('Mesh calculation %d / %d.' % (i, len(smm.get(indices_hdl))))
return _mesh(coordinates, radii, n_tube_points)
def _test():
#from importlib import reload
import ClearMap.ParallelProcessing.SharedMemoryManager as smm
reload(smm)
def propagate(t):
i, hdl = t
a = smm.get(hdl)
#if i % 100000 == 0:
# print('i=%d' % i)
for j in range(1):
a[i] = i
n = 5000000
hdl = smm.zeros(n, dtype=float)
print(hdl)
pool = smm.mp.Pool(processes=2)
smm.mp.Process()
pp = pool.map_async(propagate, zip(range(n), [hdl] * n))
#analysis:ignore
pool.close()
pool.join()
result = smm.get(hdl)
print(result)
smm.sma.is_shared(result)
smm.free(hdl)
smm.clean()
def _parallel_interpolate(
i,
coordinates_hdl,
radii_hdl,
indices_hdl,
# coordinates_interp_hdl, radii_interp_hdl, indices_interp_hdl,
smooth=5,
order=2,
points_per_pixel=0.5,
verbose=False):
coordinates = smm.get(coordinates_hdl)
radii = smm.get(radii_hdl)
start, end = smm.get(indices_hdl)[i]
# coordinates_interp = smm.get(coordinates_interp_hdl);
# radii_interp = smm.get(radii_interp_hdl);
# start_interp,end_interp = smm.get(indices_interp_hdl)[i];
coordinates = coordinates[start:end]
radii = radii[start:end]
if verbose:
if i % 1000 == 0:
print('Mesh interpolation %d / %d.' %
(i, len(smm.get(indices_hdl))))
#coordinates_interp[start:end], radii_interp[start:end]=
n_points = _n_points_per_edge(end - start,
points_per_pixel=points_per_pixel)
#n_points = end_interp - start_interp;
#coordinates_interp[start_interp:end_interp], radii_interp[start_interp:end_interp] =
return _interpolate_edge(coordinates,
radii,
n_points=n_points,
smooth=smooth,
order=order)
def _shared(shape=None, dtype=None, order=None, array=None, handle=None):
if handle is not None:
array = smm.get(handle)
if array is None:
return sma.array(shape=shape, dtype=dtype, order=order)
elif is_shared(array):
if shape is None and dtype is None and order is None:
return array
shape = shape if shape is not None else array.shape
dtype = dtype if dtype is not None else array.dtype
order = order if order is not None else npy.order(array)
if shape != array.shape:
raise ValueError('Shapes do not match!')
if np.dtype(dtype) == array.dtype and order == npy.order(array):
return array
else:
new = sma.array(shape=shape, dtype=dtype, order=order)
new[:] = array
return new
elif isinstance(array, (np.ndarray, list, tuple)):
array = np.asarray(array)
shape = shape if shape is not None else array.shape
dtype = dtype if dtype is not None else array.dtype
order = order if order is not None else npy.order(array)
if shape != array.shape:
raise ValueError('Shapes do not match!')
new = sma.array(shape=shape, dtype=dtype, order=order)
new[:] = array
return new
else:
raise ValueError('Cannot create shared array from array %r!' % array)
def free(self):
if self._handle is not None:
smm.free(self._handle)
self._handle = None
def handle(self):
if self._handle is None:
self._handle = smm.insert(self.array)
return self._handle
def interpolate_edge_geometry(graph,
smooth=5,
order=2,
points_per_pixel=0.5,
processes=None,
verbose=False):
"""Smooth center lines and radii of the edge geometry."""
if not graph.has_edge_geometry():
raise ValueError('Graph has no edge geometry!')
coordinates, indices = graph.edge_geometry('coordinates',
return_indices=True,
as_list=False)
radii = graph.edge_geometry('radii', as_list=False)
# prepare result arrays
#indices_interp = np.array([_n_points_per_edge(i[1]-i[0], points_per_pixel=points_per_pixel) for i in indices]);
#indices_interp = np.cumsum(indices_interp);
#indices_interp = np.hstack([[0], indices_interp])
#indices_interp = np.array([indices_interp[:-1], indices_interp[1:]]).T;
#coordinates_interp = np.zeros((indices_interp[-1,1], coordinates.shape[1]), dtype=float);
#radii_interp = np.zeros(indices_interp[-1,1], dtype=float);
# process in parallel
coordinates_hdl = smm.insert(coordinates)
radii_hdl = smm.insert(radii)
indices_hdl = smm.insert(indices)
# coordinates_interp_hdl = smm.insert(coordinates_interp);
# radii_interp_hdl = smm.insert(radii_interp);
# indices_interp_hdl = smm.insert(indices_interp);
func = ft.partial(
_parallel_interpolate,
coordinates_hdl=coordinates_hdl,
radii_hdl=radii_hdl,
indices_hdl=indices_hdl,
# coordinates_interp_hdl=coordinates_interp_hdl, radii_interp_hdl=radii_interp_hdl, indices_interp_hdl=indices_interp_hdl,
smooth=smooth,
order=order,
points_per_pixel=points_per_pixel,
verbose=verbose)
argdata = np.arange(len(indices))
pool = smm.mp.Pool(processes=processes)
results = pool.map(func, argdata)
pool.close()
pool.join()
smm.free(coordinates_hdl)
smm.free(radii_hdl)
smm.free(indices_hdl)
# smm.free(coordinates_interp_hdl);
# smm.free(radii_interp_hdl);
# smm.free(indices_interp_hdl);
indices_interp = np.array([len(r[1]) for r in results])
indices_interp = np.cumsum(indices_interp)
indices_interp = np.hstack([[0], indices_interp])
indices_interp = np.array([indices_interp[:-1], indices_interp[1:]]).T
coordinates_interp = np.vstack([r[0] for r in results])
radii_interp = np.hstack([r[1] for r in results])
return coordinates_interp, radii_interp, indices_interp
def _test():
#%%
import numpy as np
import scipy.ndimage as ndi
import ClearMap.DataProcessing.LargeData as ld
import ClearMap.Visualization.Plot3d as p3d
import ClearMap.DataProcessing.ConvolvePointList as cpl
import ClearMap.ImageProcessing.Skeletonization.Topology3d as t3d
import ClearMap.ImageProcessing.Skeletonization.SkeletonCleanUp as scu
import ClearMap.ImageProcessing.Tracing.Connect as con
reload(con)
data = np.load('/home/ckirst/Desktop/data.npy')
binary = np.load('/home/ckirst/Desktop/binarized.npy')
skel = np.load('/home/ckirst/Desktop/skel.npy')
#points = np.load('/home/ckirst/Desktop/pts.npy');
data = np.copy(data, order='F')
binary = np.copy(binary, order='F')
skel = np.copy(skel, order='F')
skel_copy = np.copy(skel, order='F')
points = np.ravel_multi_index(np.where(skel), skel.shape, order='F')
skel, points = scu.cleanOpenBranches(skel,
skel_copy,
points,
length=3,
clean=True)
deg = cpl.convolve3DIndex(skel, t3d.n26, points)
ends, isolated = con.findEndpoints(skel, points, border=25)
special = np.sort(np.hstack([ends, isolated]))
ends_xyz = np.array(np.unravel_index(ends, data.shape, order='F')).T
isolated_xyz = np.array(np.unravel_index(isolated, data.shape,
order='F')).T
special_xyz = np.vstack([ends_xyz, isolated_xyz])
#%%
import ClearMap.ParallelProcessing.SharedMemoryManager as smm
data_s = smm.asShared(data, order='F')
binary_s = smm.asShared(binary.view('uint8'), order='F')
skel_s = smm.asShared(skel.view('uint8'), order='F')
smm.clean()
res = con.addConnections(data_s,
binary_s,
skel_s,
points,
radius=20,
start_points=None,
add_to_skeleton=True,
add_to_mask=True,
verbose=True,
processes=4,
debug=False,
block_size=10)
skel_s = skel_s.view(bool)
binary_s = binary_s.view(bool)
#%%
mask_img = np.asarray(binary, dtype=int, order='A')
mask_img[:] = mask_img + binary_s
mask_img[:] = mask_img + skel
data_img = np.copy(data, order='A')
data_img[skel] = 120
mask_img_f = np.reshape(mask_img, -1, order='A')
data_img_f = np.reshape(data_img, -1, order='A')
mask_img_f[res] = 7
data_img_f[res] = 512
mask_img_f[special] = 8
data_img_f[special] = 150
for d in [3, 4, 5]:
mask_img_f[points[deg == d]] = d + 1
try:
con.viewer[0].setSource(mask_img)
con.viewer[1].setSource(data_img)
except:
con.viewer = p3d.plot([mask_img, data_img])
con.viewer[0].setMinMax([0, 8])
con.viewer[1].setMinMax([24, 160])
#%%
mask = binary
data_new = np.copy(data, order='A')
data_new[skel] = 120
skel_new = np.asarray(skel, dtype=int, order='A')
skel_new[:] = skel_new + binary
binary_new = np.copy(binary, order='A')
qs = []
for i, e in enumerate(special):
print('------')
print('%d / %d' % (i, len(special)))
path, quality = con.connectPoint(data,
mask,
special,
i,
radius=25,
skeleton=skel,
tubeness=None,
remove_local_mask=True,
min_quality=15.0,
verbose=True,
maxSteps=15000,
costPerDistance=1.0)
#print path, quality
if len(path) > 0:
qs.append(quality * 1.0 / len(path))
q = con.addPathToMask(skel_new, path, value=7)
q = con.addPathToMask(data_new, path, value=512)
binary_new = con.addDilatedPathToMask(binary_new,
path,
iterations=1)
skel_new[:] = skel_new + binary_new
q = con.addPathToMask(skel_new, special_xyz, value=6)
for d in [3, 4, 5]:
xyz = np.array(
np.unravel_index(points[deg == d], data.shape, order='F')).T
q = con.addPathToMask(skel_new, xyz, value=d)
q = con.addPathToMask(data_new, special_xyz, value=150)
try:
con.viewer[0].setSource(skel_new)
con.viewer[1].setSource(data_new)
except:
con.viewer = p3d.plot([skel_new, data_new])
con.viewer[0].setMinMax([0, 8])
con.viewer[1].setMinMax([24, 160])
#%%
import matplotlib.pyplot as plt
plt.figure(1)
plt.clf()
#plt.plot(qs);
plt.hist(qs)
#%%
i = 20
i = 21
i = 30
i = 40
r = 25
center = np.unravel_index(ends[i], data.shape)
print(center, data.shape)
mask = binary
path = con.tracePointToMask(data,
mask,
center,
radius=r,
points=special_xyz,
plot=True,
skel=skel,
binary=binary,
tubeness=None,
removeLocalMask=True,
maxSteps=None,
verbose=False,
costPerDistance=0.0)
#%%
nbs = ap.findNeighbours(ends, i, skel.shape, skel.strides, r)
center = np.unravel_index(ends[i], skel.shape)
nbs_xyz = np.array(np.unravel_index(nbs, skel.shape)).T
dists = nbs_xyz - center
dists = np.sum(dists * dists, axis=1)
nb = np.argmin(dists)
center = np.unravel_index(ends[i], data.shape)
print(center, data.shape)
mask = binary
path = con.tracePointToNeighbor(data,
mask,
center,
nbs_xyz[nb],
radius=r,
points=special_xyz,
plot=True,
skel=skel,
binary=binary,
tubeness=None,
removeLocalMask=True,
maxSteps=None,
verbose=False,
costPerDistance=0.0)
#%%
import ClearMap.ImageProcessing.Filter.FilterKernel as fkr
dog = fkr.filterKernel('DoG', size=(13, 13, 13))
dv.plot(dog)
data_filter = ndi.correlate(np.asarray(data, dtype=float), dog)
data_filter -= data_filter.min()
data_filter = data_filter / 3.0
#dv.dualPlot(data, data_filter);
#%%add all paths
reload(con)
r = 25
mask = binary
data_new = data.copy()
data_new[skel] = 120
skel_new = np.asarray(skel, dtype=int)
skel_new = skel_new + binary
binary_new = binary.copy()
for i, e in enumerate(special):
center = np.unravel_index(e, data.shape)
print(i, e, center)
path = con.tracePointToMask(data,
mask,
center,
radius=r,
points=special_xyz,
plot=False,
skel=skel,
binary=binary,
tubeness=None,
removeLocalMask=True,
maxSteps=15000,
costPerDistance=1.0)
q = con.addPathToMask(skel_new, path, value=7)
q = con.addPathToMask(data_new, path, value=512)
binary_new = con.addDilatedPathToMask(binary_new, path, iterations=1)
q = con.addPathToMask(skel_new, special_xyz, value=6)
for d in [3, 4, 5]:
xyz = np.array(np.unravel_index(points[deg == d], data.shape)).T
q = con.addPathToMask(skel_new, xyz, value=d)
q = con.addPathToMask(data_new, special_xyz, value=150)
skel_new = skel_new + binary_new
try:
con.viewer[0].setSource(skel_new)
con.viewer[1].setSource(data_new)
except:
con.viewer = dv.dualPlot(skel_new, data_new)
con.viewer[0].setMinMax([0, 8])
con.viewer[1].setMinMax([24, 160])
#%%
import ClearMap.ImageProcessing.Skeletonization.Skeletonize as skl
skel_2 = skl.skeletonize3D(binary_new.copy())
#%%
np.save('/home/ckirst/Desktop/binarized_con.npy', binary_new)
#%%
# write image
import ClearMap.IO.IO as io
#r = np.asarray(128 * binary_new, dtype = 'uint8');
#g = r.copy(); b = r.copy();
#r[:] = r + 127 * skel_2[0];
#g[:] = g - 128 * skel_2[0];
#b[:] = b - 128 * skel_2[0];
#img = np.stack((r,g,b), axis = 3)
img = np.asarray(128 * binary_new, dtype='uint8')
img[:] = img + 127 * skel_2[0]
io.writeData('/home/ckirst/Desktop/3d.tif', img)
def addConnections(data,
mask,
skeleton,
points,
radius=20,
start_points=None,
remove_local_mask=True,
min_quality=15.0,
add_to_skeleton=True,
add_to_mask=False,
verbose=True,
processes=mp.cpu_count(),
block_size=5000,
debug=False):
global temporary_folder
timer = tmr.Timer()
timer_total = tmr.Timer()
if start_points is None:
ends, isolated = findEndpoints(skeleton, points, border=20)
start_points = np.hstack([ends, isolated])
start_points = smm.asShared(start_points)
npts = len(start_points)
if verbose:
timer.printElapsedTime('Found %d endpoints' % (npts, ))
timer.reset()
assert smm.isShared(data)
assert smm.isShared(mask)
assert smm.isShared(skeleton)
#steps_total = npts / processes;
#steps_done = 0;
smm.clean()
data_hdl = smm.insert(data)
mask_hdl = smm.insert(mask)
skel_hdl = smm.insert(skeleton)
spts_hdl = smm.insert(start_points)
if not (data_hdl == 0 and mask_hdl == 1 and skel_hdl == 2
and spts_hdl == 3):
raise RuntimeError(
'The shared memory handles are invalid: %d, %d, %d, %d' %
(data_hdl, mask_hdl, skel_hdl, spts_hdl))
#generate temporary folder to write path too
temporary_folder = tmpf.mkdtemp()
if verbose:
timer.printElapsedTime('Preparation of %d connections' % (npts, ))
timer.reset()
# process in parallel / block processing is to clean up memory leaks for now
nblocks = max(1, int(np.ceil(1.0 * npts / processes / block_size)))
ranges = np.asarray(np.linspace(0, npts, nblocks + 1), dtype=int)
#nblocks = 1;
#ranges = [0, 100];
for b in range(nblocks):
argdata = np.arange(ranges[b], ranges[b + 1])
if debug:
result = [processSingleConnection(a) for a in argdata]
else:
#mp.process.current_process()._counter = mp.process.itertools.count(1); #reset worker counter
pool = mp.Pool(processes=processes)
#for i,_ in enumerate(pool.imap_unordered(processSingleConnection, argdata)):
# if i % 100 == 0:
# timer.printElapsedTime('Iteration %d / %d' % (i + ranges[b], npts));
pool.map(processSingleConnection, argdata)
pool.close()
pool.join()
gc.collect()
smm.free(data_hdl)
smm.free(mask_hdl)
smm.free(skel_hdl)
smm.free(spts_hdl)
if verbose:
timer.printElapsedTime('Processing of %d connections' % (npts, ))
timer.reset()
#return result;
#get paths from temporay files
result = []
for f in os.listdir(temporary_folder):
result.append(np.fromfile(os.path.join(temporary_folder, f),
dtype=int))
result = np.hstack(result)
#clean up
shutil.rmtree(temporary_folder)
temporary_folder = None
if verbose:
timer.printElapsedTime('Loading paths with %d points' %
(len(result), ))
timer.reset()
#add dilated version to skeleton
if add_to_skeleton:
skeleton_f = np.reshape(skeleton, -1, order='A')
strides = skeleton.strides
skeleton_f_len = len(skeleton_f)
for dx in [-1, 0, 1]:
for dy in [-1, 0, 1]:
for dz in [-1, 0, 1]:
offset = dx * strides[0] + dy * strides[1] + dz * strides[2]
pos = result + offset
pos = pos[np.logical_and(pos >= 0, pos < skeleton_f_len)]
skeleton_f[pos] = True
if verbose:
timer.printElapsedTime('Added paths with %d points to skeleton' %
(len(result), ))
#add dilated version to skeleton
if add_to_mask:
mask_f = np.reshape(mask, -1, order='A')
strides = mask.strides
for dx in [-1, 0, 1]:
for dy in [-1, 0, 1]:
for dz in [-1, 0, 1]:
offset = dx * strides[0] + dy * strides[1] + dz * strides[2]
pos = result + offset
pos = pos[np.logical_and(pos >= 0, pos < skeleton_f_len)]
mask_f[pos] = True
if verbose:
timer.printElapsedTime('Added paths with %d points to mask' %
(len(result), ))
timer_total.printElapsedTime(
'Connection post-processing added %d points' % (len(result), ))
return result