def example():
path = '/home/pape/Work/data/ilastik/mulastik/data/data.h5'
with h5py.File(path) as f:
raw = f['raw']
pred = f['prediction']
pred = to_source(pred, multichannel=True)
view(raw, pred)
def view_segmentations(version, raw_scale, seg_names=[], seg_scales=[], bb=np.s_[:]):
folder = os.path.join(ROOT, version, 'images', 'local')
raw_file = os.path.join(folder, 'sbem-6dpf-1-whole-raw.xml')
raw_file = get_data_path(raw_file, return_absolute_path=True)
raw_key = get_key(False, time_point=0, setup_id=0, scale=raw_scale)
with z5py.File(raw_file, 'r') as f:
ds = f[raw_key]
ds.n_threads = 16
raw = ds[bb]
ref_shape = raw.shape
data = [to_source(raw, name='raw')]
for seg_name, seg_scale in zip(seg_names, seg_scales):
seg_file = os.path.join(folder, seg_name + '.xml')
seg_file = get_data_path(seg_file, return_absolute_path=True)
seg_key = get_key(False, time_point=0, setup_id=0, scale=seg_scale)
with z5py.File(seg_file, 'r') as f:
ds = f[seg_key]
ds.n_threads = 16
seg = ds[bb].astype('uint32')
if seg.shape != ref_shape:
# FIXME this will fail with bounding box
print("Resize", ref_shape)
seg = ResizeWrapper(to_source(seg, name=seg_name), ref_shape)
data.append(to_source(seg, name=seg_name))
view(*data)
def check_objects(block_id, n_objects=5):
labels = labels_to_blocks[block_id]
if len(labels) == 0:
return
sampled_ids = np.random.choice(labels, n_objects, False)
for label_id in sampled_ids:
assignment_mask = assignments[:, 1] == label_id
assert assignment_mask.sum() > 0
this_assignments = assignments[assignment_mask]
this_ws_labels = this_assignments[:, 0]
roi_start = morpho[label_id, 5:8].astype('uint64')
roi_stop = morpho[label_id, 8:11].astype('uint64') + 1
bb = tuple(
slice(sta, sto)
for sta, sto in zip(roi_start.tolist(), roi_stop.tolist()))
ws = ds_seg[bb]
ws_mask = np.isin(ws, this_ws_labels)
print("Loading object", label_id)
print("N-foreground:", ws_mask.sum())
ws[~ws_mask] = 0
raw = ds_raw[bb]
view(raw, ws)
def example(split_channels=False):
""" Split channel toggles between old (False) and new (True) napari channel behaviour.
"""
path = '/home/pape/Work/data/ilastik/mulastik/data/data.h5'
with h5py.File(path) as f:
raw = f['raw']
pred = f['prediction']
pred = to_source(pred, channel_axis=0, split_channels=split_channels)
view(raw, pred)
def example():
path = '/home/pape/Work/data/cremi/example/sampleA.n5'
with z5py.File(path) as f:
raw = to_source(f['volumes/raw/s1'], scale=(1, 2, 2))
seg = f['volumes/segmentation/groundtruth']
seg.n_threads = 8
view(raw, seg)
def example():
path = '/home/pape/Work/data/cremi/example/sampleA.n5'
with z5py.File(path) as f:
raw = f['volumes/raw/s0']
raw.n_threads = 8
seg = f['volumes/segmentation/groundtruth']
seg.n_threads = 8
view(raw, seg)
def _show_inputs(inputs, scale_factors):
from heimdall import view, to_source
from vigra.sampling import resize
raw0 = inputs[0]
shape = raw0.shape
data = [to_source(raw0, name='scale0')]
for scale, (inp, scale_factor) in enumerate(zip(inputs[1:], scale_factors[1:]), 1):
exp_shape = [int(sh / sf) for sh, sf in zip(shape, scale_factor)]
act_shape = inp.shape
halo = [(ash - esh) // 2 for ash, esh in zip(act_shape, exp_shape)]
bb = tuple(slice(ha, sh - ha) for ha, sh in zip(halo, act_shape))
d = inp[bb]
d = resize(d.astype('float32'), shape=shape)
data.append(to_source(d, name='scale%i' % scale))
view(*data)
def debug_preprocessing_blocks():
p_blocks = './data.n5'
k_blocks = 'labels_for_subdivision'
halo = [50, 512, 512]
with z5py.File(p_blocks, 'r') as f:
ds = f[k_blocks]
ds.n_threads = 8
shape = ds.shape
center = [sh // 2 for sh in shape]
bb = tuple(slice(ce - ha, ce + ha) for ce, ha in zip(center, halo))
blocks = ds[bb]
p_raw = '../../../data/rawdata/sbem-6dpf-1-whole-raw.n5'
k_raw = 'setup0/timepoint0/s3'
with z5py.File(p_raw, 'r') as f:
ds = f[k_raw]
ds.n_threads = 8
raw = ds[bb]
view(raw, blocks)
def example():
path = '/home/pape/Work/data/cremi/example/sampleA.n5'
f = z5py.File(path)
# roi and factory
roi_start = [25, 256, 256]
roi_stop = [100, 768, 768]
factory = partial(roi_wrapper_pyramid_factory,
roi_start=roi_start,
roi_stop=roi_stop)
# raw pyramid source
raw = f['volumes/raw']
raw_source = to_source(raw, wrapper_factory=factory, name='raw')
# label source
labels = f['volumes/segmentation/groundtruth']
label_source = RoiWrapper(to_source(labels, name='labels'), roi_start,
roi_stop)
view(raw_source, label_source)
def check_exported(with_seg=False, with_boundaries=False, scale=3):
from heimdall import view, to_source
from elf.wrapper.resized_volume import ResizedVolume
path = '/g/arendt/EM_6dpf_segmentation/platy-browser-data/data/rawdata/sbem-6dpf-1-whole-raw.n5'
key = 'setup0/timepoint0/s%i' % (scale + 1, )
f = z5py.File(path, 'r')
ds = f[key]
ds.n_threads = 8
raw = ds[:]
shape = raw.shape
data = [to_source(raw, name='raw')]
path = './data.n5'
key = 'volumes/segmentation2/s%i' % scale
f = z5py.File(path, 'r')
if with_seg:
ds = f[key]
ds.n_threads = 8
seg = ds[:].astype('uint32')
data.append(to_source(seg, name='segmentation'))
key = 'volumes/clustering'
ds = f[key]
ds.n_threads = 8
clustered = ResizedVolume(ds[:], shape=shape)[:]
data.append(to_source(clustered, name='clustered'))
path = '/g/arendt/EM_6dpf_segmentation/corrections_and_proofreading/data.n5'
key = 'boundaries/s%i' % scale
f = z5py.File(path, 'r')
if with_boundaries:
ds = f[key]
ds.n_threads = 8
bd = ds[:]
data = to_source(bd, name='boundaries')
view(*data)
def check_cell_evaluation():
from mmpb.segmentation.validation.eval_cells import (eval_slice,
get_bounding_box)
praw = os.path.join(ROOT_FOLDER, 'rawdata/sbem-6dpf-1-whole-raw.h5')
pseg = os.path.join(
ROOT_FOLDER,
'0.5.5/segmentations/sbem-6dpf-1-whole-segmented-cells-labels.h5')
pann = os.path.join(ROOT_FOLDER,
'rawdata/evaluation/validation_annotations.h5')
table_path = '../../data/0.5.5/tables/sbem-6dpf-1-whole-segmented-cells-labels/regions.csv'
ignore_seg_ids = get_ignore_seg_ids(table_path)
with h5py.File(pseg, 'r') as fseg, h5py.File(pann, 'r') as fann:
ds_seg = fseg['t00000/s00/0/cells']
ds_ann = fann['xy/1000']
print("Run evaluation ...")
res, masks = eval_slice(ds_seg,
ds_ann,
ignore_seg_ids,
min_radius=16,
return_masks=True)
fm, fs = masks['merges'], masks['splits']
print()
print("Eval result")
print(res)
print()
print("Load raw data ...")
bb = get_bounding_box(ds_ann)
with h5py.File(praw, 'r') as f:
raw = f['t00000/s00/1/cells'][bb].squeeze()
print("Load seg data ...")
seg = ds_seg[bb].squeeze().astype('uint32')
view(to_source(raw, name='raw'), to_source(seg, name='seg'),
to_source(fm, name='merges'), to_source(fs, name='splits'))
def find_my_block(block_id):
from heimdall import view, to_source
scale = 5
rpath = '../../../data/rawdata/sbem-6dpf-1-whole-raw.n5'
k = 'setup0/timepoint0/s%i' % scale
f = z5py.File(rpath)
ds = f[k]
ds.n_thread = 8
raw = ds[:]
path = './data.n5'
k = 'volumes/clustering'
f = z5py.File(path)
ds = f[k]
ds.n_threads = 8
block = ds[:].astype('uint32')
if block_id is not None:
block = (block == block_id).astype('uint32')
view(to_source(raw, name='raw'), to_source(block, name='block-volume'))
def debug_preprocessing_seg():
p_seg = './data.n5'
k_seg = 'volumes/segmentation'
halo = [50, 512, 512]
with z5py.File(p_seg, 'r') as f:
ds = f[k_seg]
ds.n_threads = 8
shape = ds.shape
center = [sh // 2 for sh in shape]
bb = tuple(slice(ce - ha, ce + ha) for ce, ha in zip(center, halo))
seg = ds[bb]
p_raw = '../../../data/rawdata/sbem-6dpf-1-whole-raw.n5'
k_raw = 'setup0/timepoint0/s1'
with z5py.File(p_raw, 'r') as f:
ds = f[k_raw]
ds.n_threads = 8
print(ds.shape)
raw = ds[bb]
view(to_source(raw), to_source(seg))
def mask_gut_nuclei(view_mask=False):
p1 = '../data/0.0.0/images/prospr-6dpf-1-whole-segmented-Stomodeum.h5'
p2 = '../data/rawdata/sbem-6dpf-1-whole-segmented-tissue-labels.h5'
with h5py.File(p1, 'r') as f:
ds = f['t00000/s00/0/cells']
stomodeum = (ds[:] > 0).astype('float32')
with h5py.File(p2, 'r') as f:
ds = f['t00000/s00/2/cells']
gut = ds[:] == 23
stomodeum = vigra.sampling.resize(stomodeum, gut.shape,
order=0).astype('bool')
mask = np.logical_and(stomodeum == 0, gut)
if view_mask:
from heimdall import view, to_source
p_nuc = '../data/0.0.0/segmentations/sbem-6dpf-1-whole-segmented-nuclei-labels.h5'
with h5py.File(p_nuc, 'r') as f:
nuc = f['t00000/s00/2/cells'][:] > 0
nuc = vigra.sampling.resize(nuc.astype('float32'), gut.shape,
order=0).astype('bool')
view(to_source(gut.astype('uint32'), name='gut-em'),
to_source(stomodeum.astype('uint32'), name='stomodeum-prospr'),
to_source(mask.astype('uint32'), name='mask'),
to_source(nuc.astype('uint32'), name='nuclei'))
else:
resolution = [0.4, 0.32, 0.32]
mask = mask.astype('uint8') * 255
make_bdv(mask,
'nucleus_exclusion_mask',
unit='micrometer',
resolution=resolution)
def check_nucleus_evaluation():
from mmpb.segmentation.validation.eval_nuclei import (eval_slice,
get_bounding_box)
praw = os.path.join(ROOT_FOLDER, 'rawdata/sbem-6dpf-1-whole-raw.h5')
pseg = os.path.join(
ROOT_FOLDER,
'0.0.0/segmentations/sbem-6dpf-1-whole-segmented-nuclei-labels.h5')
pann = os.path.join(ROOT_FOLDER,
'rawdata/evaluation/validation_annotations.h5')
with h5py.File(pseg, 'r') as fseg, h5py.File(pann, 'r') as fann:
ds_seg = fseg['t00000/s00/0/cells']
ds_ann = fann['xy/1000']
print("Run evaluation ...")
res, masks = eval_slice(ds_seg,
ds_ann,
min_radius=6,
return_masks=True)
fm, fs = masks['merges'], masks['splits']
print()
print("Eval result")
print(res)
print()
print("Load raw data ...")
bb = get_bounding_box(ds_ann, scale_factor=4.)
with h5py.File(praw, 'r') as f:
raw = f['t00000/s00/3/cells'][bb].squeeze()
print("Load seg data ...")
seg = ds_seg[bb].squeeze().astype('uint32')
view(to_source(raw, name='raw'), to_source(seg, name='seg'),
to_source(fm, name='merges'), to_source(fs, name='splits'))
def example():
path = '/home/pape/Work/data/cremi/example/sampleA.n5'
with z5py.File(path) as f:
raw = f['volumes/raw']
view(raw)
def make_resin_mask_2d(z=None, scale=3):
path = '../data.n5'
raw_key = 'volumes/raw-samplexy/s%i' % scale
mask_path = '../../data/rawdata/sbem-6dpf-1-whole-segmented-inside.n5'
mask_key = 'setup0/timepoint0/s0'
n_threads = 8
f = z5py.File(path)
ds_raw = f[raw_key]
f_mask = z5py.File(mask_path)
ds = f_mask[mask_key]
ds.n_threads = n_threads
mask = ds[:]
mask = ResizedVolume(mask, ds_raw.shape, order=0)
size_thresh = 5000
def mask_2d(z):
print(z, "/", ds_raw.shape[0])
raw = ds_raw[z]
maskz = np.logical_not(mask[z])
maskz = np.logical_or(maskz, raw == 0)
maskz = np.logical_or(maskz, raw == 255)
# run otsu on the remaining data to get rid of the embedded silver
masked = raw[maskz]
thresh = threshold_otsu(masked)
maskz = np.logical_and(maskz, raw > thresh)
# get rid of upper quantile
masked = raw[maskz]
# thresh = threshold_otsu(masked)
thresh = np.quantile(masked, .9)
maskz = np.logical_and(maskz, raw < thresh)
# only keep the biggest component
ccs = vigra.analysis.labelImageWithBackground(maskz.astype('uint32'))
ids, sizes = np.unique(ccs, return_counts=True)
ids, sizes = ids[1:], sizes[1:]
keep_ids = ids[sizes > size_thresh]
maskz = np.isin(ccs, keep_ids)
maskz = maskz.astype('uint8') * 255
return maskz
if z is not None:
resin_mask = mask_2d(z)
raw = ds_raw[z]
mask = mask[z]
view(raw, mask, resin_mask)
else:
print("Compute mask")
with futures.ThreadPoolExecutor(n_threads) as tp:
tasks = [tp.submit(mask_2d, z) for z in range(ds_raw.shape[0])]
res = [t.result() for t in tasks]
resin_mask = np.concatenate([re[None] for re in res], axis=0)
print(resin_mask.shape)
# save as bdv, why not
# s0: .025, .02, .02
# s1: .025, .08, .08
# s2: .025, .16, .16
# s3: .025, .32, .32
res = [.025, .32, .32]
make_bdv(resin_mask, 'sbem-6dpf-1-whole-segmented-resin.n5',
downscale_factors=[[1, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2]],
resolution=res, unit='micrometer', downscale_mode='min')
def refine(seg_path, seg_key, ignore_seg_ids,
orientation, slice_id,
project_folder,
annotation_path=DEFAULT_ANNOTATION_PATH,
raw_path=DEFAULT_RAW_PATH,
raw_key='t00000/s00/1/cells'):
label_path = os.path.join(project_folder, 'labels.npy')
fm_path = os.path.join(project_folder, 'fm.npy')
fs_path = os.path.join(project_folder, 'fs.npy')
bb_path = os.path.join(project_folder, 'bounding_box.json')
if os.path.exists(project_folder):
print("Load from existing project")
labels = np.load(label_path) if os.path.exists(label_path) else None
fm = np.load(fm_path) if os.path.exists(fm_path) else None
fs = np.load(fs_path) if os.path.exists(fs_path) else None
else:
print("Start new project")
labels, fm, fs = None, None, None
with open_file(annotation_path, 'r') as fval:
ds = fval[orientation][str(slice_id)]
bb = get_bounding_box(ds)
ds.n_threads = 8
if labels is None:
labels = ds[:]
starts = [int(b.start) for b in bb]
stops = [int(b.stop) for b in bb]
with open_file(seg_path, 'r') as f:
ds = f[seg_key]
ds.n_threads = 8
seg = ds[bb].squeeze().astype('uint32')
with open_file(raw_path, 'r') as f:
ds = f[raw_key]
ds.n_threads = 8
raw = ds[bb].squeeze()
assert labels.shape == seg.shape == raw.shape
if fm is None:
assert fs is None
fm, fs = compute_masks(seg, labels, ignore_seg_ids)
else:
assert fs is not None
with napari.gui_qt():
viewer = view(to_source(raw, name='raw'), to_source(labels, name='labels'),
to_source(seg, name='seg'), to_source(fm, name='merges'),
to_source(fs, name='splits'), return_viewer=True)
@viewer.bind_key('s')
def save_labels(viewer):
print("Saving state ...")
layers = viewer.layers
os.makedirs(project_folder, exist_ok=True)
labels = layers['labels'].data
np.save(label_path, labels)
fm = layers['merges'].data
np.save(fm_path, fm)
fs = layers['splits'].data
np.save(fs_path, fs)
with open(bb_path, 'w') as f:
json.dump({'starts': starts, 'stops': stops}, f)
print("... done")
def debug(path):
import numpy as np
import z5py
from heimdall import view
p = os.path.join(path, 'data.n5')
p_raw = '/g/arendt/EM_6dpf_segmentation/platy-browser-data/data/rawdata/sbem-6dpf-1-whole-raw.n5'
k_raw = 'setup0/timepoint0/s1'
f_raw = z5py.File(p_raw, 'r')
ds_raw = f_raw[k_raw]
ds_raw.n_threads = 8
f_seg = z5py.File(p)
k_seg = 'volumes/segmentation_before_splitting'
ds_seg = f_seg[k_seg]
ds_seg.n_threads = 8
assert ds_raw.shape == ds_seg.shape, "%s, %s" % (ds_raw.shape,
ds_seg.shape)
proj_id = int(path[-2:])
this_path = os.path.abspath(__file__)
this_path = os.path.split(__file__)[0]
roi_file = os.path.join(this_path, 'configs/rois.json')
with open(roi_file) as f:
rois = json.load(f)
roi_begin, roi_end = rois[str(proj_id)]
halo = [50, 512, 512]
center = [rb + (re - rb) // 2 for re, rb in zip(roi_begin, roi_end)]
print(roi_begin, roi_end)
print(center)
bb = tuple(slice(ce - ha, ce + ha) for ce, ha in zip(center, halo))
print(bb)
raw = ds_raw[bb]
seg = ds_seg[bb]
view(raw, seg)
return
k = 'morphology'
with z5py.File(p, 'r') as f:
ds = f[k]
m = ds[:]
starts = m[:, 5:8]
stops = m[:, 8:11]
print(starts.min(axis=0))
print(starts.max(axis=0))
print()
print(stops.min(axis=0))
print(stops.max(axis=0))
return
seg_root_key = 'volumes/paintera'
ass_key = os.path.join(seg_root_key, 'fragment-segment-assignment')
with z5py.File(p, 'r') as f:
assignments = f[ass_key][:].T
seg_ids = assignments[:, 1]
unique_ids = np.unique(seg_ids)
if unique_ids[0] == 0:
unique_ids = unique_ids[1:]
unique_ids = unique_ids
# print(len(starts))
# print(unique_ids.max())
# return
attrs_p = os.path.join(path, 'attributes.json')
with open(attrs_p) as f:
attrs = json.load(f)
seg_state = attrs['paintera']['sourceInfo']['sources'][1]['state']
locked_ids = seg_state['lockedSegments']
flagged_ids = np.array(list(set(unique_ids.tolist()) - set(locked_ids)))
flagged_ids = flagged_ids[np.isin(flagged_ids, unique_ids)].tolist()
for flag_id in flagged_ids:
print(flag_id)
if flag_id >= len(starts):
print("%i is out of bounds %i" % (flag_id, len(starts)))
continue
print(starts[flag_id])
print(stops[flag_id])
print()