def processSingleConnection(args): global temporary_folder i = args #steps_done += 1; #if steps_done % 100 == 0: # print('Processing step %d / %d...' % (steps_done, steps_total)); if i % 100 == 0: print('Processing step %d...' % i) try: fid = mp.current_process()._identity[0] except: # mostlikely sequential mode fid = 0 data = smm.get(0) mask = smm.get(1) skel = smm.get(2) spts = smm.get(3) res = connectPoint(data, mask, spts, i, skeleton=skel, radius=20, tubeness=None, remove_local_mask=True, min_quality=15.0, verbose=False, maxSteps=12000, costPerDistance=1.0) #return res; path, score = res if len(path) > 0: #convert path to indices path_flat = np.ravel_multi_index(path.T, data.shape, order=order(data)) #append to file try: fid = mp.current_process()._identity[0] except: fid = 0 # sequential mode fn = '%s/path_%03d.tmp' % (temporary_folder, fid) #print fn; fb = open(fn, "ab") path_flat.tofile(fb) fb.close() return
def mesh_tube_from_coordinates_and_radii(coordinates, radii, indices, n_tube_points=None, edge_colors=None, processes=None, verbose=False): """Construct a mesh from the edge geometry of a graph.""" coordinates_hdl = smm.insert(coordinates) radii_hdl = smm.insert(radii) indices_hdl = smm.insert(indices) if n_tube_points is None: n_tube_points = 8 func = ft.partial(_parallel_mesh, coordinates_hdl=coordinates_hdl, radii_hdl=radii_hdl, indices_hdl=indices_hdl, n_tube_points=n_tube_points, verbose=verbose) argdata = np.arange(len(indices)) # process in parallel pool = smm.mp.Pool(processes=processes) results = pool.map(func, argdata) pool.close() pool.join() smm.free(coordinates_hdl) smm.free(radii_hdl) smm.free(indices_hdl) n_results = len(results) vertices = [np.reshape(r[0], (-1, 3)) for r in results] n_indices = [len(v) for v in vertices] n_indices = np.cumsum(n_indices) n_indices = np.hstack([[0], n_indices]) if edge_colors is not None: colors = [len(v) * [c] for v, c in zip(vertices, edge_colors)] colors = np.concatenate(colors) #print(colors.shape) else: colors = None vertices = np.concatenate(vertices) #print(grid.shape) faces = np.concatenate( [results[i][1] + n_indices[i] for i in range(n_results)]) return vertices, faces, colors
def propagate(t): i, hdl = t a = smm.get(hdl) #if i % 100000 == 0: # print('i=%d' % i) for j in range(1): a[i] = i
def _parallel_mesh(i, coordinates_hdl, radii_hdl, indices_hdl, n_tube_points=15, verbose=False): coordinates = smm.get(coordinates_hdl) radii = smm.get(radii_hdl) start, end = smm.get(indices_hdl)[i] coordinates = coordinates[start:end] radii = radii[start:end] if verbose: if i % 1000 == 0: print('Mesh calculation %d / %d.' % (i, len(smm.get(indices_hdl)))) return _mesh(coordinates, radii, n_tube_points)
def _test(): #from importlib import reload import ClearMap.ParallelProcessing.SharedMemoryManager as smm reload(smm) def propagate(t): i, hdl = t a = smm.get(hdl) #if i % 100000 == 0: # print('i=%d' % i) for j in range(1): a[i] = i n = 5000000 hdl = smm.zeros(n, dtype=float) print(hdl) pool = smm.mp.Pool(processes=2) smm.mp.Process() pp = pool.map_async(propagate, zip(range(n), [hdl] * n)) #analysis:ignore pool.close() pool.join() result = smm.get(hdl) print(result) smm.sma.is_shared(result) smm.free(hdl) smm.clean()
def _parallel_interpolate( i, coordinates_hdl, radii_hdl, indices_hdl, # coordinates_interp_hdl, radii_interp_hdl, indices_interp_hdl, smooth=5, order=2, points_per_pixel=0.5, verbose=False): coordinates = smm.get(coordinates_hdl) radii = smm.get(radii_hdl) start, end = smm.get(indices_hdl)[i] # coordinates_interp = smm.get(coordinates_interp_hdl); # radii_interp = smm.get(radii_interp_hdl); # start_interp,end_interp = smm.get(indices_interp_hdl)[i]; coordinates = coordinates[start:end] radii = radii[start:end] if verbose: if i % 1000 == 0: print('Mesh interpolation %d / %d.' % (i, len(smm.get(indices_hdl)))) #coordinates_interp[start:end], radii_interp[start:end]= n_points = _n_points_per_edge(end - start, points_per_pixel=points_per_pixel) #n_points = end_interp - start_interp; #coordinates_interp[start_interp:end_interp], radii_interp[start_interp:end_interp] = return _interpolate_edge(coordinates, radii, n_points=n_points, smooth=smooth, order=order)
def _shared(shape=None, dtype=None, order=None, array=None, handle=None): if handle is not None: array = smm.get(handle) if array is None: return sma.array(shape=shape, dtype=dtype, order=order) elif is_shared(array): if shape is None and dtype is None and order is None: return array shape = shape if shape is not None else array.shape dtype = dtype if dtype is not None else array.dtype order = order if order is not None else npy.order(array) if shape != array.shape: raise ValueError('Shapes do not match!') if np.dtype(dtype) == array.dtype and order == npy.order(array): return array else: new = sma.array(shape=shape, dtype=dtype, order=order) new[:] = array return new elif isinstance(array, (np.ndarray, list, tuple)): array = np.asarray(array) shape = shape if shape is not None else array.shape dtype = dtype if dtype is not None else array.dtype order = order if order is not None else npy.order(array) if shape != array.shape: raise ValueError('Shapes do not match!') new = sma.array(shape=shape, dtype=dtype, order=order) new[:] = array return new else: raise ValueError('Cannot create shared array from array %r!' % array)
def free(self): if self._handle is not None: smm.free(self._handle) self._handle = None
def handle(self): if self._handle is None: self._handle = smm.insert(self.array) return self._handle
def interpolate_edge_geometry(graph, smooth=5, order=2, points_per_pixel=0.5, processes=None, verbose=False): """Smooth center lines and radii of the edge geometry.""" if not graph.has_edge_geometry(): raise ValueError('Graph has no edge geometry!') coordinates, indices = graph.edge_geometry('coordinates', return_indices=True, as_list=False) radii = graph.edge_geometry('radii', as_list=False) # prepare result arrays #indices_interp = np.array([_n_points_per_edge(i[1]-i[0], points_per_pixel=points_per_pixel) for i in indices]); #indices_interp = np.cumsum(indices_interp); #indices_interp = np.hstack([[0], indices_interp]) #indices_interp = np.array([indices_interp[:-1], indices_interp[1:]]).T; #coordinates_interp = np.zeros((indices_interp[-1,1], coordinates.shape[1]), dtype=float); #radii_interp = np.zeros(indices_interp[-1,1], dtype=float); # process in parallel coordinates_hdl = smm.insert(coordinates) radii_hdl = smm.insert(radii) indices_hdl = smm.insert(indices) # coordinates_interp_hdl = smm.insert(coordinates_interp); # radii_interp_hdl = smm.insert(radii_interp); # indices_interp_hdl = smm.insert(indices_interp); func = ft.partial( _parallel_interpolate, coordinates_hdl=coordinates_hdl, radii_hdl=radii_hdl, indices_hdl=indices_hdl, # coordinates_interp_hdl=coordinates_interp_hdl, radii_interp_hdl=radii_interp_hdl, indices_interp_hdl=indices_interp_hdl, smooth=smooth, order=order, points_per_pixel=points_per_pixel, verbose=verbose) argdata = np.arange(len(indices)) pool = smm.mp.Pool(processes=processes) results = pool.map(func, argdata) pool.close() pool.join() smm.free(coordinates_hdl) smm.free(radii_hdl) smm.free(indices_hdl) # smm.free(coordinates_interp_hdl); # smm.free(radii_interp_hdl); # smm.free(indices_interp_hdl); indices_interp = np.array([len(r[1]) for r in results]) indices_interp = np.cumsum(indices_interp) indices_interp = np.hstack([[0], indices_interp]) indices_interp = np.array([indices_interp[:-1], indices_interp[1:]]).T coordinates_interp = np.vstack([r[0] for r in results]) radii_interp = np.hstack([r[1] for r in results]) return coordinates_interp, radii_interp, indices_interp
def _test(): #%% import numpy as np import scipy.ndimage as ndi import ClearMap.DataProcessing.LargeData as ld import ClearMap.Visualization.Plot3d as p3d import ClearMap.DataProcessing.ConvolvePointList as cpl import ClearMap.ImageProcessing.Skeletonization.Topology3d as t3d import ClearMap.ImageProcessing.Skeletonization.SkeletonCleanUp as scu import ClearMap.ImageProcessing.Tracing.Connect as con reload(con) data = np.load('/home/ckirst/Desktop/data.npy') binary = np.load('/home/ckirst/Desktop/binarized.npy') skel = np.load('/home/ckirst/Desktop/skel.npy') #points = np.load('/home/ckirst/Desktop/pts.npy'); data = np.copy(data, order='F') binary = np.copy(binary, order='F') skel = np.copy(skel, order='F') skel_copy = np.copy(skel, order='F') points = np.ravel_multi_index(np.where(skel), skel.shape, order='F') skel, points = scu.cleanOpenBranches(skel, skel_copy, points, length=3, clean=True) deg = cpl.convolve3DIndex(skel, t3d.n26, points) ends, isolated = con.findEndpoints(skel, points, border=25) special = np.sort(np.hstack([ends, isolated])) ends_xyz = np.array(np.unravel_index(ends, data.shape, order='F')).T isolated_xyz = np.array(np.unravel_index(isolated, data.shape, order='F')).T special_xyz = np.vstack([ends_xyz, isolated_xyz]) #%% import ClearMap.ParallelProcessing.SharedMemoryManager as smm data_s = smm.asShared(data, order='F') binary_s = smm.asShared(binary.view('uint8'), order='F') skel_s = smm.asShared(skel.view('uint8'), order='F') smm.clean() res = con.addConnections(data_s, binary_s, skel_s, points, radius=20, start_points=None, add_to_skeleton=True, add_to_mask=True, verbose=True, processes=4, debug=False, block_size=10) skel_s = skel_s.view(bool) binary_s = binary_s.view(bool) #%% mask_img = np.asarray(binary, dtype=int, order='A') mask_img[:] = mask_img + binary_s mask_img[:] = mask_img + skel data_img = np.copy(data, order='A') data_img[skel] = 120 mask_img_f = np.reshape(mask_img, -1, order='A') data_img_f = np.reshape(data_img, -1, order='A') mask_img_f[res] = 7 data_img_f[res] = 512 mask_img_f[special] = 8 data_img_f[special] = 150 for d in [3, 4, 5]: mask_img_f[points[deg == d]] = d + 1 try: con.viewer[0].setSource(mask_img) con.viewer[1].setSource(data_img) except: con.viewer = p3d.plot([mask_img, data_img]) con.viewer[0].setMinMax([0, 8]) con.viewer[1].setMinMax([24, 160]) #%% mask = binary data_new = np.copy(data, order='A') data_new[skel] = 120 skel_new = np.asarray(skel, dtype=int, order='A') skel_new[:] = skel_new + binary binary_new = np.copy(binary, order='A') qs = [] for i, e in enumerate(special): print('------') print('%d / %d' % (i, len(special))) path, quality = con.connectPoint(data, mask, special, i, radius=25, skeleton=skel, tubeness=None, remove_local_mask=True, min_quality=15.0, verbose=True, maxSteps=15000, costPerDistance=1.0) #print path, quality if len(path) > 0: qs.append(quality * 1.0 / len(path)) q = con.addPathToMask(skel_new, path, value=7) q = con.addPathToMask(data_new, path, value=512) binary_new = con.addDilatedPathToMask(binary_new, path, iterations=1) skel_new[:] = skel_new + binary_new q = con.addPathToMask(skel_new, special_xyz, value=6) for d in [3, 4, 5]: xyz = np.array( np.unravel_index(points[deg == d], data.shape, order='F')).T q = con.addPathToMask(skel_new, xyz, value=d) q = con.addPathToMask(data_new, special_xyz, value=150) try: con.viewer[0].setSource(skel_new) con.viewer[1].setSource(data_new) except: con.viewer = p3d.plot([skel_new, data_new]) con.viewer[0].setMinMax([0, 8]) con.viewer[1].setMinMax([24, 160]) #%% import matplotlib.pyplot as plt plt.figure(1) plt.clf() #plt.plot(qs); plt.hist(qs) #%% i = 20 i = 21 i = 30 i = 40 r = 25 center = np.unravel_index(ends[i], data.shape) print(center, data.shape) mask = binary path = con.tracePointToMask(data, mask, center, radius=r, points=special_xyz, plot=True, skel=skel, binary=binary, tubeness=None, removeLocalMask=True, maxSteps=None, verbose=False, costPerDistance=0.0) #%% nbs = ap.findNeighbours(ends, i, skel.shape, skel.strides, r) center = np.unravel_index(ends[i], skel.shape) nbs_xyz = np.array(np.unravel_index(nbs, skel.shape)).T dists = nbs_xyz - center dists = np.sum(dists * dists, axis=1) nb = np.argmin(dists) center = np.unravel_index(ends[i], data.shape) print(center, data.shape) mask = binary path = con.tracePointToNeighbor(data, mask, center, nbs_xyz[nb], radius=r, points=special_xyz, plot=True, skel=skel, binary=binary, tubeness=None, removeLocalMask=True, maxSteps=None, verbose=False, costPerDistance=0.0) #%% import ClearMap.ImageProcessing.Filter.FilterKernel as fkr dog = fkr.filterKernel('DoG', size=(13, 13, 13)) dv.plot(dog) data_filter = ndi.correlate(np.asarray(data, dtype=float), dog) data_filter -= data_filter.min() data_filter = data_filter / 3.0 #dv.dualPlot(data, data_filter); #%%add all paths reload(con) r = 25 mask = binary data_new = data.copy() data_new[skel] = 120 skel_new = np.asarray(skel, dtype=int) skel_new = skel_new + binary binary_new = binary.copy() for i, e in enumerate(special): center = np.unravel_index(e, data.shape) print(i, e, center) path = con.tracePointToMask(data, mask, center, radius=r, points=special_xyz, plot=False, skel=skel, binary=binary, tubeness=None, removeLocalMask=True, maxSteps=15000, costPerDistance=1.0) q = con.addPathToMask(skel_new, path, value=7) q = con.addPathToMask(data_new, path, value=512) binary_new = con.addDilatedPathToMask(binary_new, path, iterations=1) q = con.addPathToMask(skel_new, special_xyz, value=6) for d in [3, 4, 5]: xyz = np.array(np.unravel_index(points[deg == d], data.shape)).T q = con.addPathToMask(skel_new, xyz, value=d) q = con.addPathToMask(data_new, special_xyz, value=150) skel_new = skel_new + binary_new try: con.viewer[0].setSource(skel_new) con.viewer[1].setSource(data_new) except: con.viewer = dv.dualPlot(skel_new, data_new) con.viewer[0].setMinMax([0, 8]) con.viewer[1].setMinMax([24, 160]) #%% import ClearMap.ImageProcessing.Skeletonization.Skeletonize as skl skel_2 = skl.skeletonize3D(binary_new.copy()) #%% np.save('/home/ckirst/Desktop/binarized_con.npy', binary_new) #%% # write image import ClearMap.IO.IO as io #r = np.asarray(128 * binary_new, dtype = 'uint8'); #g = r.copy(); b = r.copy(); #r[:] = r + 127 * skel_2[0]; #g[:] = g - 128 * skel_2[0]; #b[:] = b - 128 * skel_2[0]; #img = np.stack((r,g,b), axis = 3) img = np.asarray(128 * binary_new, dtype='uint8') img[:] = img + 127 * skel_2[0] io.writeData('/home/ckirst/Desktop/3d.tif', img)
def addConnections(data, mask, skeleton, points, radius=20, start_points=None, remove_local_mask=True, min_quality=15.0, add_to_skeleton=True, add_to_mask=False, verbose=True, processes=mp.cpu_count(), block_size=5000, debug=False): global temporary_folder timer = tmr.Timer() timer_total = tmr.Timer() if start_points is None: ends, isolated = findEndpoints(skeleton, points, border=20) start_points = np.hstack([ends, isolated]) start_points = smm.asShared(start_points) npts = len(start_points) if verbose: timer.printElapsedTime('Found %d endpoints' % (npts, )) timer.reset() assert smm.isShared(data) assert smm.isShared(mask) assert smm.isShared(skeleton) #steps_total = npts / processes; #steps_done = 0; smm.clean() data_hdl = smm.insert(data) mask_hdl = smm.insert(mask) skel_hdl = smm.insert(skeleton) spts_hdl = smm.insert(start_points) if not (data_hdl == 0 and mask_hdl == 1 and skel_hdl == 2 and spts_hdl == 3): raise RuntimeError( 'The shared memory handles are invalid: %d, %d, %d, %d' % (data_hdl, mask_hdl, skel_hdl, spts_hdl)) #generate temporary folder to write path too temporary_folder = tmpf.mkdtemp() if verbose: timer.printElapsedTime('Preparation of %d connections' % (npts, )) timer.reset() # process in parallel / block processing is to clean up memory leaks for now nblocks = max(1, int(np.ceil(1.0 * npts / processes / block_size))) ranges = np.asarray(np.linspace(0, npts, nblocks + 1), dtype=int) #nblocks = 1; #ranges = [0, 100]; for b in range(nblocks): argdata = np.arange(ranges[b], ranges[b + 1]) if debug: result = [processSingleConnection(a) for a in argdata] else: #mp.process.current_process()._counter = mp.process.itertools.count(1); #reset worker counter pool = mp.Pool(processes=processes) #for i,_ in enumerate(pool.imap_unordered(processSingleConnection, argdata)): # if i % 100 == 0: # timer.printElapsedTime('Iteration %d / %d' % (i + ranges[b], npts)); pool.map(processSingleConnection, argdata) pool.close() pool.join() gc.collect() smm.free(data_hdl) smm.free(mask_hdl) smm.free(skel_hdl) smm.free(spts_hdl) if verbose: timer.printElapsedTime('Processing of %d connections' % (npts, )) timer.reset() #return result; #get paths from temporay files result = [] for f in os.listdir(temporary_folder): result.append(np.fromfile(os.path.join(temporary_folder, f), dtype=int)) result = np.hstack(result) #clean up shutil.rmtree(temporary_folder) temporary_folder = None if verbose: timer.printElapsedTime('Loading paths with %d points' % (len(result), )) timer.reset() #add dilated version to skeleton if add_to_skeleton: skeleton_f = np.reshape(skeleton, -1, order='A') strides = skeleton.strides skeleton_f_len = len(skeleton_f) for dx in [-1, 0, 1]: for dy in [-1, 0, 1]: for dz in [-1, 0, 1]: offset = dx * strides[0] + dy * strides[1] + dz * strides[2] pos = result + offset pos = pos[np.logical_and(pos >= 0, pos < skeleton_f_len)] skeleton_f[pos] = True if verbose: timer.printElapsedTime('Added paths with %d points to skeleton' % (len(result), )) #add dilated version to skeleton if add_to_mask: mask_f = np.reshape(mask, -1, order='A') strides = mask.strides for dx in [-1, 0, 1]: for dy in [-1, 0, 1]: for dz in [-1, 0, 1]: offset = dx * strides[0] + dy * strides[1] + dz * strides[2] pos = result + offset pos = pos[np.logical_and(pos >= 0, pos < skeleton_f_len)] mask_f[pos] = True if verbose: timer.printElapsedTime('Added paths with %d points to mask' % (len(result), )) timer_total.printElapsedTime( 'Connection post-processing added %d points' % (len(result), )) return result