def segment(pth, fname, npz_name, exp_re, immsk): print "segment" data = {} npz_cache = utils.file_cache(os.path.join(pth, results_path, npz_name), '/tmp/drmaize/') with np.load(npz_cache, 'r') as old_data: crn = old_data['crn'] lne = old_data['lne'] m = re.match(exp_re, fname) print m.group(3) # if m.group(3) == '01': # # 12 HPI # cmsk = 2.25, 7.5 # lmsk = 3.75 # elif m.group(3) == '02': # # 24 HPI # cmsk = 2., 5.25 # lmsk = 2.75 # elif m.group(3) == '03': # # 48 HPI # cmsk = 1.5, 5. # lmsk = 2.25 # else: # raise Exception # TODO these thresholds don't work for all images cmsk = np.log(crn[crn > 1e-6]).mean() + 4.0 * np.log(crn[crn > 1e-6]).std(), # 1.5, 4.5 lmsk = np.log(lne[lne > 1e-6]).mean() + 3.5 * np.log(lne[lne > 1e-6]).std() # 2.25 crn = (crn > cmsk[0]) # | (lne > cmsk[1]) lne = (lne > lmsk) seg = hysteresis(lne, crn, np.ones((3, 3), bool)) seg[~immsk] = 0 data['seg'] = seg npz_cache = utils.file_cache(os.path.join(pth, results_path, npz_name), '/tmp/drmaize/') with np.load(npz_cache, 'r') as old_data: old_data = dict(old_data) old_data.update(data) data = old_data np.savez_compressed(os.path.join(pth, results_path, npz_name), **data) print npz_name print npz_cache
def scale_analysis(pth, npz_name, im, immsk, sizes, nstds, orthstep, res): print 'scale_analysis' data = {} # compute scale factors as the L1 norm of large second order kernels nrm = list(np.abs(n).sum() for n in get_hessian_kernels(res, 50., nstds, orthstep)) scspace = [] for sz in sizes: print sz d2dx2 = hessian_eigenvalues(im, res, sz, nstds, orthstep, nrm) d2dx2.sort(d2dx2.ndim - 1) # sort eigenvalues into min/max or corner/line order scspace.append(d2dx2.astype(np.float32)) scspace = np.concatenate([sc[None, ...] for sc in scspace], 0) scspace *= -1 scspace[scspace < 1e-6] = 0 scspace[..., ~immsk, :] = 0 # combine scale space results using geometric mean scspace = (scipy.stats.gmean(1 + scspace, 0) - 1) ** .5 lne, crn = scspace[..., 0], scspace[..., 1] data['crn'] = crn data['lne'] = lne if os.path.isfile(os.path.join(pth, results_path, npz_name)): npz_cache = utils.file_cache(os.path.join(pth, results_path, npz_name), '/tmp/drmaize/') else: npz_cache = os.path.join(pth, results_path, npz_name) np.savez_compressed(os.path.join(pth, results_path, npz_name), **data) with np.load(npz_cache, 'r') as old_data: old_data = dict(old_data) old_data.update(data) data = old_data np.savez_compressed(os.path.join(pth, results_path, npz_name), **data) print npz_name print npz_cache
def pipeline(experiment): # TODO extract function for selecting filenames '/home/rhein/mnt/drmaize/image_data/e013SLB/microimages/reconstructed/HS' 'e013SLBp01wA1x20_1506111930rc001.ome.tif' '/mnt/data27/wisser/drmaize/image_data/' exp_re = 'e(\d{3})(SLB|NLB)p(\d{2})w([A-D])([1-6])x20_(\d*)rf001\.ome\.tif' # data_dir = '/home/rhein/mnt/drmaize/image_data/' data_dir = '/mnt/data27/wisser/drmaize/image_data' sub_dir = 'microimages/reconstructed/HS' exp = experiment if not os.path.isdir(os.path.join(data_dir, exp, sub_dir)): print "Experiment doesn't exist; exiting..." return sizes = .5 + 2 ** np.arange(4) nstds = 4. orthstep = 0.5 fnames = [] for f in sorted(os.listdir(os.path.join(data_dir, exp, sub_dir))): m = re.match(exp_re, f) if m: print m.group(6) fnames.append(os.path.join(data_dir, exp, sub_dir, f)) print f # if int(m.group(3)) == 1: # if 1 <= int(m.group(5)) <= 3: # if m.group(6) == '1506111930': # fnames.append(os.path.join(data_dir, exp, sub_dir, f)) # print f # if 4 <= int(m.group(5)) <= 6: # if m.group(6) == '1506121515': # fnames.append(os.path.join(data_dir, exp, sub_dir, f)) # print f # elif int(m.group(3)) == 2: # if 1 <= int(m.group(5)) <= 3: # if m.group(6) == '1506121700': # fnames.append(os.path.join(data_dir, exp, sub_dir, f)) # print f # if 4 <= int(m.group(5)) <= 6: # if m.group(6) == '1506221400': # fnames.append(os.path.join(data_dir, exp, sub_dir, f)) # print f # elif int(m.group(3)) == 3: # if 1 <= int(m.group(5)) <= 3: # if m.group(6) == '1505041720': # fnames.append(os.path.join(data_dir, exp, sub_dir, f)) # print f # if 4 <= int(m.group(5)) <= 6: # if m.group(6) == '1508062130': # fnames.append(os.path.join(data_dir, exp, sub_dir, f)) # print f seed = time.time() print 'seed', seed metrics = [] for fname in utils.shuffle(fnames, 0xDeadBeef): print 'filename', fname pth, fname = os.path.split(fname) if os.path.isfile(os.path.join(pth, 'MIP', fname)): cache_fname = utils.file_cache(os.path.join(pth, 'MIP', fname), '/tmp/drmaize') im = ndimage.imread(cache_fname) else: cache_fname = utils.file_cache(os.path.join(pth, fname), '/tmp/drmaize') im = utils.get_tif(cache_fname) im = np.max(im, 0) scipy.misc.imsave(os.path.join(pth, 'MIP', fname), im) print 'cache filename', cache_fname # cache_fname = drmaize.utils.file_cache(os.path.join(pth, fname), '/tmp/drmaize/') # res = utils.get_tif_res(cache_fname) res = np.array((1.,) * 3) print 'physical resolution', res res = res[1:] res = res / np.min(res) npz_name = '{}.npz'.format(os.path.splitext(os.path.splitext(fname)[0])[0]) print npz_name im = im.astype(float) # mask generation # TODO insert mask into cache file immsk = im > 4 scale_analysis(pth, npz_name, im, immsk, sizes, nstds, orthstep, res) segment(pth, fname, npz_name, exp_re, immsk) skeletonize(pth, npz_name, immsk) npz_cache = utils.file_cache(os.path.join(pth, results_path, npz_name), '/tmp/drmaize/') with np.load(npz_cache, 'r') as data: data = dict(data) seg = data['seg'] skel, dist = morphology.medial_axis(seg, return_distance=True) node, edge, leaf = (ndimage.label(g, np.ones((3, 3), bool))[0] for g in utils.skel2graph(skel)) # scale according to physical xy resolution dist = dist * 2.6240291219148313 # exp_re = 'exp(\d{3})(SLB|NLB)p(\d{2})w([A-D])([1-6])(\d*)rf002\.ome\.tif' m = re.match(exp_re, fname) met_row = OrderedDict() met_row['experiment'] = m.group(1) met_row['disease'] = m.group(2) met_row['plate'] = m.group(3) met_row['well_row'] = m.group(4) met_row['well_col'] = m.group(5) met_row['timestamp'] = m.group(6) met_row['host'] = np.count_nonzero(immsk) met_row['segmentation'] = np.count_nonzero(seg) met_row['width_mean'] = np.mean(dist[seg > 0].flat) met_row['width_median'] = np.median(dist[seg > 0].flat) met_row['width_variance'] = np.var(dist[seg > 0].flat) met_row['width_skewness'] = stats.skew(dist[seg > 0].flat) met_row['width_kurtosis'] = stats.kurtosis(dist[seg > 0].flat) met_row['leaf'] = np.count_nonzero(leaf) met_row['edge'] = np.count_nonzero(edge) met_row['node'] = np.count_nonzero(node) fname = os.path.join(pth, fname) head, tail = os.path.split(fname) tail = tail.replace('rf001.ome.tif', '_topsurface_optimized1.txt') surf = os.path.join(head, 'surfacemap', tail) cache_fname = utils.file_cache(surf, '/tmp/drmaize') surf = np.loadtxt(cache_fname, np.float32, delimiter=',') cache_fname = utils.file_cache(fname, '/tmp/drmaize') fung = utils.get_tif(cache_fname) fung = np.argmax(fung, 0) fung = fung.astype(np.float32) dpth = (fung - surf) dpth *= 1.2 # met_row['depth_mean'] = np.mean(dpth[skel > 0].flat) met_row['depth_median'] = np.median(dpth[skel > 0].flat) met_row['depth_variance'] = np.var(dpth[skel > 0].flat) met_row['depth_skewness'] = stats.skew(dpth[skel > 0].flat) met_row['depth_kurtosis'] = stats.kurtosis(dpth[skel > 0].flat) metrics.append(met_row) print (metrics) with open('metrics.csv', 'w') as csvfile: fieldnames = ['experiment', 'disease', 'plate', 'well_row', 'well_col', 'timestamp', \ 'host', 'segmentation', \ 'width_mean', 'width_median', 'width_variance', 'width_skewness', 'width_kurtosis', \ 'depth_mean', 'depth_median', 'depth_variance', 'depth_skewness', 'depth_kurtosis', \ 'leaf', 'edge', 'node'] writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() writer.writerows(metrics)
def skeletonize(pth, npz_name, immsk): print 'skeletonize' data = {} npz_cache = utils.file_cache(os.path.join(pth, results_path, npz_name), '/tmp/drmaize/') with np.load(npz_cache, 'r') as old_data: seg = old_data['seg'] # base skeleton comes from medial axis of segmented hyphae but it requires pruning skel, dist = morphology.medial_axis(seg, return_distance=True) # label skeletal pixels as one of leaf, edge or node node, edge, leaf = (ndimage.label(g, np.ones((3, 3), bool))[0] for g in utils.skel2graph(skel)) # determine which edges are candidates for pruning by finding edges adjacent to nodes trim_edge = (edge != 0) & ~(morphology.binary_dilation(node != 0, np.ones((3, 3), bool)) != 0) trim_edge = ndimage.label(trim_edge, np.ones((3, 3), bool))[0] # determine edges which are candidates from pruning by finding those adjacent to leaves leaf_edge_vals = morphology.binary_dilation(leaf != 0, np.ones((3, 3), bool)) != 0 leaf_edge_vals = np.unique(trim_edge[leaf_edge_vals]) leaf_edge_vals = leaf_edge_vals[leaf_edge_vals > 0] leaf_edge = leaf != 0 # TODO is this correct when we are setting the appropriate pixels below? # determine which edges are adjacent to leaves and nodes with fromarray(leaf_edge) as leaf_edge, fromarray(trim_edge) as trim_edge: joblib.Parallel(n_jobs=-1)( joblib.delayed(utils.set_msk)(leaf_edge, trim_edge, l) for l in leaf_edge_vals) trim_edge = np.copy(trim_edge) leaf_edge = np.copy(leaf_edge) # TODO what the heck is going on here leaf_edge[(morphology.binary_dilation(leaf_edge, np.ones((3, 3), bool)) != 0) & (edge != 0)] = True leaf_edge = ndimage.label(leaf_edge, np.ones((3, 3), bool))[0] leaf_edge_node = morphology.binary_dilation(leaf_edge != 0, np.ones((3, 3), bool)) != 0 leaf_edge_node = ((node != 0) & leaf_edge_node) | leaf_edge leaf_edge_node = ndimage.label(leaf_edge_node, np.ones((3, 3), bool))[0] cand_node = leaf_edge_node * (node != 0) cand_node = cand_node.nonzero() cand_node = np.transpose((leaf_edge_node[cand_node],) + cand_node + (2 * dist[cand_node],)) cand_leaf = leaf_edge_node * (leaf != 0) cand_leaf = cand_leaf.nonzero() cand_leaf = np.transpose((leaf_edge_node[cand_leaf],) + cand_leaf) # prune leaves and edges based on their distance from their adjacent node if len(cand_node) > 0 and len(cand_leaf) > 0: cand_leaf = np.array(cand_leaf) cand_node = np.array(cand_node) pruned = joblib.Parallel(n_jobs=-1)( joblib.delayed(prune_leaves)(cand_leaf, cand_node, j) for j in np.unique(cand_node[:, 0])) pruned_ind = [] for p in pruned: pruned_ind.extend(p) pruned_ind = tuple(np.transpose(pruned_ind)) pruned = ~skel pruned = np.array(pruned) leaf_edge = np.array(leaf_edge) joblib.Parallel(n_jobs=1, max_nbytes=None)( joblib.delayed(utils.set_msk)(pruned, leaf_edge, l) for l in np.unique(leaf_edge[pruned_ind])) pruned = ~pruned pruned[~immsk] = False else: pruned = np.zeros_like(skel) data['pruned'] = skel & ~pruned data['skel'] = pruned npz_cache = utils.file_cache(os.path.join(pth, results_path, npz_name), '/tmp/drmaize/') with np.load(npz_cache, 'r') as old_data: old_data = dict(old_data) old_data.update(data) data = old_data np.savez_compressed(os.path.join(pth, results_path, npz_name), **data) print npz_name print npz_cache