def prepare_table(make_full_table=False):
path = '/g/rompani/lgn-em-datasets/data/0.0.0/images/local/sbem-adult-1-lgn-boutons.n5'
bb = get_bounding_box(scale=1)
# this might be an issue if we get too large halos
f = z5py.File(path, 'r')
ds = f['setup0/timepoint0/s0']
ds.n_threads = 8
seg = ds[bb]
bouton_ids = np.unique(seg)[1:]
# make the annotation table
bouton_table_dir = '/g/rompani/lgn-em-datasets/data/0.0.0/tables/sbem-adult-1-lgn-boutons'
if make_full_table:
default_table = os.path.join(bouton_table_dir, 'default.csv')
default_table = pd.read_csv(default_table, sep='\t')
label_ids = default_table['label_id'].values
n_labels = len(label_ids)
annotations = np.array(n_labels * ["None"])
annotations[bouton_ids] = "unlabeled"
else:
label_ids = bouton_ids
annotations = np.array(len(label_ids) * ["unlabeled"])
out_table = np.concatenate([label_ids[:, None], annotations[:, None]],
axis=1)
out_table = pd.DataFrame(out_table, columns=['label_id', 'annotation'])
out_table_path = os.path.join(bouton_table_dir,
'bouton_annotations_v1.csv')
out_table.to_csv(out_table_path, sep='\t', index=False)
def label_vesicles():
# TODO bounding box is too aggressive, but predictions is
# also not good enough yet, need better training data
threshold = 250
path = './data.n5'
with z5py.File(path, 'a') as f:
ds = f['vesicles']
ds = ThresholdWrapper(ds, threshold)
out_key = 'segmentation/vesicles'
ds_out = f.require_dataset(out_key,
shape=ds.shape,
compression='gzip',
dtype='uint64',
chunks=ds.chunks)
bb = get_bounding_box(intersect_with_blocking=True)
label(ds,
ds_out,
with_background=True,
roi=bb,
verbose=True,
n_threads=32)
def write_proofread_boutons(annotation_table, keep_annotations):
out_key = 'boutons_proofread'
f_out = z5py.File(PATH, 'a')
if out_key in f_out:
return
bouton_ids, annotations = load_bouton_annotations(annotation_table)
bouton_ids = bouton_ids[np.isin(annotations,
keep_annotations)].astype('uint64')
bb = get_bounding_box(scale=1, intersect_with_blocking=True)
path = '/g/rompani/lgn-em-datasets/data/0.0.0/images/local/sbem-adult-1-lgn-boutons.n5'
f = z5py.File(path, 'r')
ds = f['setup0/timepoint0/s0']
ds.n_threads = 16
# this might not work for larger cututouts and we need to use cluster tools
seg = ds[bb]
seg[~np.isin(seg, bouton_ids)] = 0
ds_out = f_out.create_dataset(out_key,
shape=ds.shape,
chunks=ds.chunks,
compression='gzip',
dtype=seg.dtype)
ds_out[bb] = seg
def segment_boutons():
path = './data.n5'
with z5py.File(path, 'a') as f:
seg_neurons = f['segmentation/multicut']
seg_ves = f['segmentation/vesicles']
ds_out = f.require_dataset('segmentation/boutons',
shape=seg_ves.shape,
chunks=tuple(BLOCK_SHAPE),
dtype='uint64',
compression='gzip')
roi_begin, roi_end = get_bounding_box(intersect_with_blocking=True,
return_as_lists=True)
blocking = nt.blocking(roi_begin, roi_end, BLOCK_SHAPE)
n_blocks = blocking.numberOfBlocks
n_threads = 32
func = partial(intersect_segmentation,
seg_a=seg_neurons,
seg_b=seg_ves,
seg_out=ds_out,
blocking=blocking)
with futures.ThreadPoolExecutor(n_threads) as tp:
list(tqdm(tp.map(func, range(n_blocks)), total=n_blocks))
def check_mc():
import napari
bb = get_bounding_box(scale=0)
path = '/g/rompani/lgn-em-datasets/data/0.0.0/images/local/sbem-adult-1-lgn-raw.n5'
f = z5py.File(path, 'r')
ds = f['setup0/timepoint0/s0']
ds.n_threads = 8
raw = ds[bb]
path = './data.n5'
f = z5py.File(path, 'r')
ds = f['segmentation/multicut']
ds.n_threads = 8
seg = ds[bb]
# f = z5py.File(path, 'r')
# ds = f['boutons_proofread']
# ds.n_threads = 8
# bb_boutons = get_bounding_box(scale=1)
# boutons = ds[bb_boutons]
with napari.gui_qt():
viewer = napari.Viewer()
viewer.add_image(raw)
viewer.add_labels(seg)
def predict_boundaries(target, gpus, threads_per_job=6):
task = InferenceLocal if target == 'local' else InferenceSlurm
halo = [8, 64, 64]
output_key = {'affinities': [0, 3]}
tmp_folder = './tmp_prediction'
config_dir = os.path.join(tmp_folder, 'configs')
os.makedirs(config_dir, exist_ok=True)
roi_begin, roi_end = get_bounding_box(return_as_lists=True)
conf = task.default_global_config()
conf.update({
'block_shape': BLOCK_SHAPE,
'roi_begin': roi_begin,
'roi_end': roi_end
})
with open(os.path.join(config_dir, 'global.config'), 'w') as f:
json.dump(conf, f)
if target == 'local':
device_mapping = {ii: gpu for ii, gpu in enumerate(gpus)}
else:
device_mapping = None
conf = task.default_task_config()
conf.update({
'dtype': 'uint8',
'device_mapping': device_mapping,
'threads_per_job': threads_per_job
})
with open(os.path.join(config_dir, 'inference.config'), 'w') as f:
json.dump(conf, f)
input_path = '/g/rompani/lgn-em-datasets/data/0.0.0/images/local/sbem-adult-1-lgn-raw.n5'
input_key = 'setup0/timepoint0/s0'
# TODO for larger outputs we should put this on scratch
output_path = './data.n5'
t = task(tmp_folder=tmp_folder,
config_dir=config_dir,
max_jobs=len(gpus),
input_path=input_path,
input_key=input_key,
output_path=output_path,
output_key=output_key,
checkpoint_path=MODEL_PATH,
halo=halo,
framework='inferno')
assert luigi.build([t], local_scheduler=True)
def _make_global_config(configs):
os.makedirs(CONFIG_FOLDER, exist_ok=True)
roi_begin, roi_end = get_bounding_box(return_as_lists=True)
conf = configs['global']
conf.update({
'block_shape': BLOCK_SHAPE,
'roi_begin': roi_begin,
'roi_end': roi_end
})
with open(os.path.join(CONFIG_FOLDER, 'global.config'), 'w') as f:
json.dump(conf, f)
def check_boutons():
import napari
bb = get_bounding_box(scale=1)
path = '/g/rompani/lgn-em-datasets/data/0.0.0/images/local/sbem-adult-1-lgn-boutons.n5'
f = z5py.File(path, 'r')
ds = f['setup0/timepoint0/s0']
ds.n_threads = 8
seg = ds[bb]
path = '/g/rompani/lgn-em-datasets/data/0.0.0/images/local/sbem-adult-1-lgn-raw.n5'
f = z5py.File(path, 'r')
ds = f['setup0/timepoint0/s1']
ds.n_threads = 8
raw = ds[bb]
with napari.gui_qt():
viewer = napari.Viewer()
viewer.add_image(raw)
viewer.add_labels(seg)
def check_predictions():
import napari
bb = get_bounding_box(scale=0)
path = '/g/rompani/lgn-em-datasets/data/0.0.0/images/local/sbem-adult-1-lgn-raw.n5'
f = z5py.File(path, 'r')
ds = f['setup0/timepoint0/s0']
ds.n_threads = 8
raw = ds[bb]
path = './data.n5'
f = z5py.File(path, 'r')
ds = f['vesicles']
ds.n_threads = 8
pred = ds[bb]
with napari.gui_qt():
viewer = napari.Viewer()
viewer.add_image(raw)
viewer.add_image(pred)