def add_to_mobie(ds_name):
root = '/g/emcf/pape/sponge-fibsem-project/data'
seg_path = f'/g/emcf/pape/sponge-meshes/{ds_name}.n5'
if ds_name == 'cell1':
resolution = 3 * [0.008]
else:
resolution = [0.008, 0.01, 0.01]
scale_factors = 4 * [[2, 2, 2]]
chunks = 3 * (96, )
mobie_ds_name = 'cell2' if ds_name == 'cell3' else ds_name
names = list(open_file(seg_path, 'r')['pp'].keys())
for name in names:
in_key = 'pp/' + name
seg_name = f'fibsem-{name}'
tmp_folder = f'./tmp_{ds_name}_{name}'
add_segmentation(seg_path,
in_key,
root,
mobie_ds_name,
seg_name,
resolution=resolution,
scale_factors=scale_factors,
chunks=chunks,
target='local',
max_jobs=12,
tmp_folder=tmp_folder)
def seg_to_mobie(checkpoint, sample, seg_name, target='local', max_jobs=16):
checkpoint_name = os.path.split(checkpoint)[1]
input_path = f'data/{sample}.n5'
input_key = f'segmentation/{checkpoint_name}/{seg_name}_postprocessed'
with z5py.File(input_path, 'r') as f:
chunks = f[input_key].chunks
seg_name = f'{checkpoint_name}_{seg_name}'
tmp_folder = f'./tmp_folders/mobie_{sample}_{seg_name}'
im_dict = load_image_dict(
os.path.join(ROOT, sample, 'images', 'images.json'))
if seg_name in im_dict:
return
add_segmentation(input_path,
input_key,
ROOT,
sample,
seg_name,
resolution=RESOLUTION,
scale_factors=SCALE_FACTORS,
chunks=chunks,
max_jobs=max_jobs,
target=target,
tmp_folder=tmp_folder)
def add_segmentations(seg_paths):
path0 = seg_paths[0]
seg_name = 'lm-cells'
out_path = f'./data/{DS_NAME}/images/local/{seg_name}.n5'
key = 'data'
if not os.path.exists(out_path.replace('.n5', '.xml')):
add_segmentation(
path0, key,
'./data', DS_NAME,
segmentation_name=seg_name,
resolution=RESOLUTION,
chunks=CHUNKS,
scale_factors=SCALE_FACTORS,
max_jobs=8,
add_default_table=False
)
assert os.path.exists(out_path)
for tp, path in enumerate(seg_paths[1:], 1):
add_timepoint(path, out_path, tp, channel=0, mode='nearest',
in_key=key)
tp_key = get_key(False, tp, 0, 0)
add_max_id(out_path, tp_key, out_path, tp_key,
tmp_folder=f'tmp_max_ids/tp{tp}',
target='local',
max_jobs=8)
def test_add_segmentation_with_initial_table(self):
from mobie import add_segmentation
from mobie.tables import compute_default_table
dataset_folder = os.path.join(self.root, self.dataset_name)
seg_name = "seg"
tmp_folder = os.path.join(self.test_folder, "tmp-seg")
table_path = os.path.join(tmp_folder, "table.tsv")
compute_default_table(self.seg_path,
self.seg_key,
table_path,
resolution=(1, 1, 1),
tmp_folder=os.path.join(self.test_folder,
"tmp-table"),
target="local",
max_jobs=1)
scales = [[2, 2, 2]]
add_segmentation(self.seg_path,
self.seg_key,
self.root,
self.dataset_name,
seg_name,
resolution=(1, 1, 1),
scale_factors=scales,
chunks=(64, 64, 64),
tmp_folder=tmp_folder,
add_default_table=table_path)
self.check_segmentation(dataset_folder, seg_name)
def add_seg_to_dataset(dataset,
path,
seg_name,
key,
resolution,
target='local',
max_jobs=32):
scale_factors = 6 * [[2, 2, 2]]
seg_name = f'fibsem-{seg_name}'
tmp_folder = f'tmp_{dataset}_seg_name'
add_segmentation(path,
key,
root=ROOT,
tmp_folder=tmp_folder,
dataset_name=dataset,
segmentation_name=seg_name,
resolution=resolution,
scale_factors=scale_factors,
chunks=DEFAULT_CHUNKS,
target=target,
max_jobs=max_jobs,
add_default_table=True)
# convert tif stack to bdv.n5 format
dataset_folder = os.path.join(ROOT, dataset)
out_path = os.path.join(dataset_folder, 'images', 'local',
f'{seg_name}.n5')
xml_path = os.path.splitext(out_path)[0] + '.xml'
add_xml_for_s3(xml_path, out_path)
print("You also need to add the files in", dataset_folder, "to git")
def init_dataset(self):
data_path = os.path.join(self.test_folder, "data.h5")
data_key = "data"
with open_file(data_path, "a") as f:
f.create_dataset(data_key, data=np.random.rand(*self.shape))
seg_path = os.path.join(self.test_folder, "seg.h5")
with open_file(seg_path, "a") as f:
f.create_dataset(data_key,
data=np.random.randint(0, 100, size=self.shape))
scales = [[2, 2, 2]]
max_jobs = min(4, mp.cpu_count())
tmp_folder = os.path.join(self.test_folder, "tmp-init-raw")
mobie.add_image(data_path,
data_key,
self.root,
self.dataset_name,
self.raw_name,
resolution=(1, 1, 1),
chunks=self.chunks,
scale_factors=scales,
tmp_folder=tmp_folder,
max_jobs=max_jobs)
tmp_folder = os.path.join(self.test_folder, "tmp-init-seg")
mobie.add_segmentation(seg_path,
data_key,
self.root,
self.dataset_name,
self.seg_name,
resolution=(1, 1, 1),
chunks=self.chunks,
scale_factors=scales,
tmp_folder=tmp_folder,
max_jobs=max_jobs)
display_settings = [
mobie.metadata.get_image_display("image-group-0", [self.raw_name]),
mobie.metadata.get_segmentation_display("segmentation-group-1",
[self.seg_name]),
]
source_transforms = [
mobie.metadata.get_affine_source_transform(
[self.raw_name, self.seg_name], np.random.rand(12))
]
mobie.create_view(os.path.join(self.root, self.dataset_name),
"my-view", [[self.raw_name], [self.seg_name]],
display_settings=display_settings,
source_transforms=source_transforms)
def add_to_mobie(path, key_seg, ds_name, resolution):
scale_factors = 5 * [[2, 2, 2]]
chunks = [96, 96, 96]
root = '/g/emcf/pape/sponge-fibsem-project/data'
name = 'fibsem-segmentation'
add_segmentation(path,
key_seg,
root=root,
dataset_name=ds_name,
segmentation_name=name,
resolution=resolution,
chunks=chunks,
scale_factors=scale_factors,
target='local',
max_jobs=16)
def test_add_segmentation(self):
from mobie import add_segmentation
dataset_folder = os.path.join(self.root, self.dataset_name)
seg_name = "seg"
tmp_folder = os.path.join(self.test_folder, "tmp-seg")
scales = [[2, 2, 2]]
add_segmentation(self.seg_path,
self.seg_key,
self.root,
self.dataset_name,
seg_name,
resolution=(1, 1, 1),
scale_factors=scales,
chunks=(64, 64, 64),
tmp_folder=tmp_folder)
self.check_segmentation(dataset_folder, seg_name)
def add_seg_to_dataset(input_path, input_key, dataset_name, segmentation_name,
resolution, chunks, target, max_jobs):
scale_factors = 6 * [[2, 2, 2]]
add_segmentation(input_path,
input_key,
ROOT,
dataset_name,
segmentation_name,
resolution,
scale_factors,
chunks,
target=target,
max_jobs=max_jobs,
add_default_table=True)
# convert tif stack to bdv.n5 format
dataset_folder = os.path.join(ROOT, dataset_name)
out_path = os.path.join(dataset_folder, 'images', 'local',
f'{segmentation_name}.n5')
xml_path = os.path.splitext(out_path)[0] + '.xml'
add_xml_for_s3(xml_path, out_path)
print("You also need to add the files in", dataset_folder, "to git")
def test_add_segmentation_with_wrong_initial_table(self):
from mobie import add_segmentation
seg_name = "seg"
tmp_folder = os.path.join(self.test_folder, "tmp-seg")
os.makedirs(tmp_folder, exist_ok=True)
table_path = os.path.join(tmp_folder, "table.tsv")
table = pd.DataFrame(np.random.rand(128, 4),
columns=["a", "b", "c", "d"])
table.to_csv(table_path, index=False, sep="\t")
scales = [[2, 2, 2]]
with self.assertRaises(ValueError):
add_segmentation(self.seg_path,
self.seg_key,
self.root,
self.dataset_name,
seg_name,
resolution=(1, 1, 1),
scale_factors=scales,
chunks=(64, 64, 64),
tmp_folder=tmp_folder,
add_default_table=table_path)
# we add a segmentation for several objects visible in the em-overview image
input_path = os.path.join(example_input_data, 'em_segmentation.tif')
segmentation_name = "em-segmentation"
unit = "nanometer"
resolution = [1., 30., 30.]
chunks = [1, 256, 256]
scale_factors = [[1, 2, 2], [1, 2, 2], [1, 2, 2], [1, 2, 2]]
mobie.add_segmentation(
input_path=input_path,
input_key="",
root=mobie_project_folder,
dataset_name=dataset_name,
segmentation_name=segmentation_name,
resolution=resolution,
chunks=chunks,
scale_factors=scale_factors,
add_default_table=True
)
# finally, we update the default bookmark so that both the raw data and the segmentation is
# loaded upon opening the dataset
source_list = [[raw_name], [segmentation_name]]
settings = [
{"color": "white", "contrastLimits": [0., 255.]},
{"lut": "glasbey", "opacity": 0.75}
]
viewer_transform = {"affine": [2., 0., 0., 0.,
def init_dataset(self):
data_path = os.path.join(self.test_folder, "data.h5")
data_key = "data"
with open_file(data_path, "a") as f:
f.create_dataset(data_key, data=np.random.rand(*self.shape))
seg_path = os.path.join(self.test_folder, "seg.h5")
with open_file(seg_path, "a") as f:
f.create_dataset(data_key,
data=np.random.randint(0, 100, size=self.shape))
scales = [[2, 2, 2]]
max_jobs = min(4, mp.cpu_count())
tmp_folder = os.path.join(self.test_folder, "tmp-init-raw")
mobie.add_image(data_path,
data_key,
self.root,
self.dataset_name,
self.raw_name,
resolution=(1, 1, 1),
chunks=self.chunks,
scale_factors=scales,
tmp_folder=tmp_folder,
max_jobs=max_jobs)
tmp_folder = os.path.join(self.test_folder, "tmp-init-extra")
mobie.add_image(data_path,
data_key,
self.root,
self.dataset_name,
self.extra_name,
resolution=(1, 1, 1),
chunks=self.chunks,
scale_factors=scales,
tmp_folder=tmp_folder,
max_jobs=max_jobs)
tmp_folder = os.path.join(self.test_folder, "tmp-init-seg")
mobie.add_segmentation(seg_path,
data_key,
self.root,
self.dataset_name,
self.seg_name,
resolution=(1, 1, 1),
chunks=self.chunks,
scale_factors=scales,
tmp_folder=tmp_folder,
max_jobs=max_jobs)
tmp_folder = os.path.join(self.test_folder, "tmp-init-extra_seg")
mobie.add_segmentation(seg_path,
data_key,
self.root,
self.dataset_name,
self.extra_seg_name,
resolution=(1, 1, 1),
chunks=self.chunks,
scale_factors=scales,
tmp_folder=tmp_folder,
max_jobs=max_jobs)
def add_sample(experiment, sample, sample_dir,
is_default=False, dry_run=True, first_sample=False):
pattern = re.compile(f"({sample}_8bit_)(\w+)")
if experiment == 'nuclei':
file_pattern = '*linUnmix.tif'
else:
file_pattern = '*lynEGFP.tif'
unit = "micrometer"
chunks = (64, 64, 64)
scale_factors = 4 * [[2, 2, 2]]
#
# initialize the dataset with '...lynEGFP.tif'
#
im_path = glob(os.path.join(sample_dir, file_pattern))
assert len(im_path) == 1
im_path = im_path[0]
im_name = os.path.split(im_path)[1]
mobie_name = f'membrane-{sample}_' + pattern.match(im_name).group(2)
txt_file = im_path.replace('.tif', '.txt')
with open(txt_file) as f:
res = f.readline().rstrip()
res = list(map(float, res.split(',')))[::-1]
if first_sample:
if dry_run:
print("Initialize dataset:")
print(im_path)
print(mobie_name, res)
else:
initialize_dataset(
input_path=im_path,
input_key='',
root='./data',
dataset_name=experiment,
raw_name=mobie_name,
resolution=res,
scale_factors=scale_factors,
chunks=chunks,
unit=unit,
target='local',
max_jobs=16,
is_default=is_default
)
else:
if dry_run:
print("Add image:")
print(im_path)
print(mobie_name, res)
else:
add_image_data(
input_path=im_path,
input_key='',
root='./data',
dataset_name=experiment,
image_name=mobie_name,
resolution=res,
scale_factors=scale_factors,
chunks=chunks,
unit=unit,
target='local',
max_jobs=16
)
#
# add the segmentation with '...lynEGFP_seg.tif'
#
seg_path = glob(os.path.join(sample_dir, '*seg.tif'))
assert len(seg_path) == 1, f"{seg_path}"
seg_path = seg_path[0]
seg_name = os.path.split(seg_path)[1]
mobie_name = f'membrane-{sample}_' + pattern.match(seg_name).group(2)
txt_file = seg_path.replace('.tif', '.txt')
with open(txt_file) as f:
res = f.readline().rstrip()
res = list(map(float, res.split(',')))[::-1]
if dry_run:
print("Add seg:")
print(seg_path)
print(mobie_name, res)
else:
add_segmentation(
input_path=seg_path,
input_key='',
root='./data',
dataset_name=experiment,
segmentation_name=mobie_name,
scale_factors=scale_factors,
resolution=res,
chunks=chunks,
unit=unit,
target='local',
max_jobs=16
)
#
# add all other images
#
additional_paths = glob(os.path.join(sample_dir, '*.tif'))
for path in additional_paths:
if path == im_path or path == seg_path:
continue
if experiment == 'nuclei' and 'lynEGFP' in path:
continue
image_name = os.path.split(path)[1]
mobie_name = experiment + f'-{sample}_' + pattern.match(image_name).group(2)
txt_file = seg_path.replace('.tif', '.txt')
with open(txt_file) as f:
res = f.readline().rstrip()
res = list(map(float, res.split(',')))[::-1]
if dry_run:
print("Add additional image:")
print(path)
print(mobie_name, res)
else:
add_image_data(
input_path=path,
input_key='',
root='./data',
dataset_name=experiment,
image_name=mobie_name,
scale_factors=scale_factors,
resolution=res,
chunks=chunks,
unit=unit,
target='local',
max_jobs=16
)