def _test():
#from importlib import reload
import ClearMap.ParallelProcessing.SharedMemoryManager as smm
reload(smm)
def propagate(t):
i, hdl = t
a = smm.get(hdl)
#if i % 100000 == 0:
# print('i=%d' % i)
for j in range(1):
a[i] = i
n = 5000000
hdl = smm.zeros(n, dtype=float)
print(hdl)
pool = smm.mp.Pool(processes=2)
smm.mp.Process()
pp = pool.map_async(propagate, zip(range(n), [hdl] * n))
#analysis:ignore
pool.close()
pool.join()
result = smm.get(hdl)
print(result)
smm.sma.is_shared(result)
smm.free(hdl)
smm.clean()
def _test():
#%%
import numpy as np
import scipy.ndimage as ndi
import ClearMap.DataProcessing.LargeData as ld
import ClearMap.Visualization.Plot3d as p3d
import ClearMap.DataProcessing.ConvolvePointList as cpl
import ClearMap.ImageProcessing.Skeletonization.Topology3d as t3d
import ClearMap.ImageProcessing.Skeletonization.SkeletonCleanUp as scu
import ClearMap.ImageProcessing.Tracing.Connect as con
reload(con)
data = np.load('/home/ckirst/Desktop/data.npy')
binary = np.load('/home/ckirst/Desktop/binarized.npy')
skel = np.load('/home/ckirst/Desktop/skel.npy')
#points = np.load('/home/ckirst/Desktop/pts.npy');
data = np.copy(data, order='F')
binary = np.copy(binary, order='F')
skel = np.copy(skel, order='F')
skel_copy = np.copy(skel, order='F')
points = np.ravel_multi_index(np.where(skel), skel.shape, order='F')
skel, points = scu.cleanOpenBranches(skel,
skel_copy,
points,
length=3,
clean=True)
deg = cpl.convolve3DIndex(skel, t3d.n26, points)
ends, isolated = con.findEndpoints(skel, points, border=25)
special = np.sort(np.hstack([ends, isolated]))
ends_xyz = np.array(np.unravel_index(ends, data.shape, order='F')).T
isolated_xyz = np.array(np.unravel_index(isolated, data.shape,
order='F')).T
special_xyz = np.vstack([ends_xyz, isolated_xyz])
#%%
import ClearMap.ParallelProcessing.SharedMemoryManager as smm
data_s = smm.asShared(data, order='F')
binary_s = smm.asShared(binary.view('uint8'), order='F')
skel_s = smm.asShared(skel.view('uint8'), order='F')
smm.clean()
res = con.addConnections(data_s,
binary_s,
skel_s,
points,
radius=20,
start_points=None,
add_to_skeleton=True,
add_to_mask=True,
verbose=True,
processes=4,
debug=False,
block_size=10)
skel_s = skel_s.view(bool)
binary_s = binary_s.view(bool)
#%%
mask_img = np.asarray(binary, dtype=int, order='A')
mask_img[:] = mask_img + binary_s
mask_img[:] = mask_img + skel
data_img = np.copy(data, order='A')
data_img[skel] = 120
mask_img_f = np.reshape(mask_img, -1, order='A')
data_img_f = np.reshape(data_img, -1, order='A')
mask_img_f[res] = 7
data_img_f[res] = 512
mask_img_f[special] = 8
data_img_f[special] = 150
for d in [3, 4, 5]:
mask_img_f[points[deg == d]] = d + 1
try:
con.viewer[0].setSource(mask_img)
con.viewer[1].setSource(data_img)
except:
con.viewer = p3d.plot([mask_img, data_img])
con.viewer[0].setMinMax([0, 8])
con.viewer[1].setMinMax([24, 160])
#%%
mask = binary
data_new = np.copy(data, order='A')
data_new[skel] = 120
skel_new = np.asarray(skel, dtype=int, order='A')
skel_new[:] = skel_new + binary
binary_new = np.copy(binary, order='A')
qs = []
for i, e in enumerate(special):
print('------')
print('%d / %d' % (i, len(special)))
path, quality = con.connectPoint(data,
mask,
special,
i,
radius=25,
skeleton=skel,
tubeness=None,
remove_local_mask=True,
min_quality=15.0,
verbose=True,
maxSteps=15000,
costPerDistance=1.0)
#print path, quality
if len(path) > 0:
qs.append(quality * 1.0 / len(path))
q = con.addPathToMask(skel_new, path, value=7)
q = con.addPathToMask(data_new, path, value=512)
binary_new = con.addDilatedPathToMask(binary_new,
path,
iterations=1)
skel_new[:] = skel_new + binary_new
q = con.addPathToMask(skel_new, special_xyz, value=6)
for d in [3, 4, 5]:
xyz = np.array(
np.unravel_index(points[deg == d], data.shape, order='F')).T
q = con.addPathToMask(skel_new, xyz, value=d)
q = con.addPathToMask(data_new, special_xyz, value=150)
try:
con.viewer[0].setSource(skel_new)
con.viewer[1].setSource(data_new)
except:
con.viewer = p3d.plot([skel_new, data_new])
con.viewer[0].setMinMax([0, 8])
con.viewer[1].setMinMax([24, 160])
#%%
import matplotlib.pyplot as plt
plt.figure(1)
plt.clf()
#plt.plot(qs);
plt.hist(qs)
#%%
i = 20
i = 21
i = 30
i = 40
r = 25
center = np.unravel_index(ends[i], data.shape)
print(center, data.shape)
mask = binary
path = con.tracePointToMask(data,
mask,
center,
radius=r,
points=special_xyz,
plot=True,
skel=skel,
binary=binary,
tubeness=None,
removeLocalMask=True,
maxSteps=None,
verbose=False,
costPerDistance=0.0)
#%%
nbs = ap.findNeighbours(ends, i, skel.shape, skel.strides, r)
center = np.unravel_index(ends[i], skel.shape)
nbs_xyz = np.array(np.unravel_index(nbs, skel.shape)).T
dists = nbs_xyz - center
dists = np.sum(dists * dists, axis=1)
nb = np.argmin(dists)
center = np.unravel_index(ends[i], data.shape)
print(center, data.shape)
mask = binary
path = con.tracePointToNeighbor(data,
mask,
center,
nbs_xyz[nb],
radius=r,
points=special_xyz,
plot=True,
skel=skel,
binary=binary,
tubeness=None,
removeLocalMask=True,
maxSteps=None,
verbose=False,
costPerDistance=0.0)
#%%
import ClearMap.ImageProcessing.Filter.FilterKernel as fkr
dog = fkr.filterKernel('DoG', size=(13, 13, 13))
dv.plot(dog)
data_filter = ndi.correlate(np.asarray(data, dtype=float), dog)
data_filter -= data_filter.min()
data_filter = data_filter / 3.0
#dv.dualPlot(data, data_filter);
#%%add all paths
reload(con)
r = 25
mask = binary
data_new = data.copy()
data_new[skel] = 120
skel_new = np.asarray(skel, dtype=int)
skel_new = skel_new + binary
binary_new = binary.copy()
for i, e in enumerate(special):
center = np.unravel_index(e, data.shape)
print(i, e, center)
path = con.tracePointToMask(data,
mask,
center,
radius=r,
points=special_xyz,
plot=False,
skel=skel,
binary=binary,
tubeness=None,
removeLocalMask=True,
maxSteps=15000,
costPerDistance=1.0)
q = con.addPathToMask(skel_new, path, value=7)
q = con.addPathToMask(data_new, path, value=512)
binary_new = con.addDilatedPathToMask(binary_new, path, iterations=1)
q = con.addPathToMask(skel_new, special_xyz, value=6)
for d in [3, 4, 5]:
xyz = np.array(np.unravel_index(points[deg == d], data.shape)).T
q = con.addPathToMask(skel_new, xyz, value=d)
q = con.addPathToMask(data_new, special_xyz, value=150)
skel_new = skel_new + binary_new
try:
con.viewer[0].setSource(skel_new)
con.viewer[1].setSource(data_new)
except:
con.viewer = dv.dualPlot(skel_new, data_new)
con.viewer[0].setMinMax([0, 8])
con.viewer[1].setMinMax([24, 160])
#%%
import ClearMap.ImageProcessing.Skeletonization.Skeletonize as skl
skel_2 = skl.skeletonize3D(binary_new.copy())
#%%
np.save('/home/ckirst/Desktop/binarized_con.npy', binary_new)
#%%
# write image
import ClearMap.IO.IO as io
#r = np.asarray(128 * binary_new, dtype = 'uint8');
#g = r.copy(); b = r.copy();
#r[:] = r + 127 * skel_2[0];
#g[:] = g - 128 * skel_2[0];
#b[:] = b - 128 * skel_2[0];
#img = np.stack((r,g,b), axis = 3)
img = np.asarray(128 * binary_new, dtype='uint8')
img[:] = img + 127 * skel_2[0]
io.writeData('/home/ckirst/Desktop/3d.tif', img)
def addConnections(data,
mask,
skeleton,
points,
radius=20,
start_points=None,
remove_local_mask=True,
min_quality=15.0,
add_to_skeleton=True,
add_to_mask=False,
verbose=True,
processes=mp.cpu_count(),
block_size=5000,
debug=False):
global temporary_folder
timer = tmr.Timer()
timer_total = tmr.Timer()
if start_points is None:
ends, isolated = findEndpoints(skeleton, points, border=20)
start_points = np.hstack([ends, isolated])
start_points = smm.asShared(start_points)
npts = len(start_points)
if verbose:
timer.printElapsedTime('Found %d endpoints' % (npts, ))
timer.reset()
assert smm.isShared(data)
assert smm.isShared(mask)
assert smm.isShared(skeleton)
#steps_total = npts / processes;
#steps_done = 0;
smm.clean()
data_hdl = smm.insert(data)
mask_hdl = smm.insert(mask)
skel_hdl = smm.insert(skeleton)
spts_hdl = smm.insert(start_points)
if not (data_hdl == 0 and mask_hdl == 1 and skel_hdl == 2
and spts_hdl == 3):
raise RuntimeError(
'The shared memory handles are invalid: %d, %d, %d, %d' %
(data_hdl, mask_hdl, skel_hdl, spts_hdl))
#generate temporary folder to write path too
temporary_folder = tmpf.mkdtemp()
if verbose:
timer.printElapsedTime('Preparation of %d connections' % (npts, ))
timer.reset()
# process in parallel / block processing is to clean up memory leaks for now
nblocks = max(1, int(np.ceil(1.0 * npts / processes / block_size)))
ranges = np.asarray(np.linspace(0, npts, nblocks + 1), dtype=int)
#nblocks = 1;
#ranges = [0, 100];
for b in range(nblocks):
argdata = np.arange(ranges[b], ranges[b + 1])
if debug:
result = [processSingleConnection(a) for a in argdata]
else:
#mp.process.current_process()._counter = mp.process.itertools.count(1); #reset worker counter
pool = mp.Pool(processes=processes)
#for i,_ in enumerate(pool.imap_unordered(processSingleConnection, argdata)):
# if i % 100 == 0:
# timer.printElapsedTime('Iteration %d / %d' % (i + ranges[b], npts));
pool.map(processSingleConnection, argdata)
pool.close()
pool.join()
gc.collect()
smm.free(data_hdl)
smm.free(mask_hdl)
smm.free(skel_hdl)
smm.free(spts_hdl)
if verbose:
timer.printElapsedTime('Processing of %d connections' % (npts, ))
timer.reset()
#return result;
#get paths from temporay files
result = []
for f in os.listdir(temporary_folder):
result.append(np.fromfile(os.path.join(temporary_folder, f),
dtype=int))
result = np.hstack(result)
#clean up
shutil.rmtree(temporary_folder)
temporary_folder = None
if verbose:
timer.printElapsedTime('Loading paths with %d points' %
(len(result), ))
timer.reset()
#add dilated version to skeleton
if add_to_skeleton:
skeleton_f = np.reshape(skeleton, -1, order='A')
strides = skeleton.strides
skeleton_f_len = len(skeleton_f)
for dx in [-1, 0, 1]:
for dy in [-1, 0, 1]:
for dz in [-1, 0, 1]:
offset = dx * strides[0] + dy * strides[1] + dz * strides[2]
pos = result + offset
pos = pos[np.logical_and(pos >= 0, pos < skeleton_f_len)]
skeleton_f[pos] = True
if verbose:
timer.printElapsedTime('Added paths with %d points to skeleton' %
(len(result), ))
#add dilated version to skeleton
if add_to_mask:
mask_f = np.reshape(mask, -1, order='A')
strides = mask.strides
for dx in [-1, 0, 1]:
for dy in [-1, 0, 1]:
for dz in [-1, 0, 1]:
offset = dx * strides[0] + dy * strides[1] + dz * strides[2]
pos = result + offset
pos = pos[np.logical_and(pos >= 0, pos < skeleton_f_len)]
mask_f[pos] = True
if verbose:
timer.printElapsedTime('Added paths with %d points to mask' %
(len(result), ))
timer_total.printElapsedTime(
'Connection post-processing added %d points' % (len(result), ))
return result