def test_multi_setup(self): from pybdv import make_bdv from pybdv.metadata import get_affine shape = (64,) * 3 n_views = 2 data_dict = {} affine_dict = {} for vid in range(n_views): data = np.random.rand(*shape).astype('float32') affine = {'trafo1': [round(aff, 4) for aff in np.random.rand(12)], 'trafo2': [round(aff, 4) for aff in np.random.rand(12)]} make_bdv(data, self.out_path, setup_id=vid, affine=affine) data_dict[vid] = data affine_dict[vid] = affine # check implicit setup id data = np.random.rand(*shape).astype('float32') make_bdv(data, self.out_path) data_dict[n_views] = data for vid in range(n_views + 1): expected_key = get_key(self.is_h5, timepoint=0, setup_id=vid, scale=0) with open_file(self.out_path, 'r') as f: self.assertTrue(expected_key in f) data = f[expected_key][:] exp_data = data_dict[vid] self.assertTrue(np.allclose(data, exp_data)) # check affine trafos (only for explicit setup-ids) for vid in range(n_views): affine = affine_dict[vid] affine_out = get_affine(self.xml_path, vid) self.assertEqual(affine, affine_out)
def make_yolk_mask(): p = '../data/rawdata/sbem-6dpf-1-whole-segmented-tissue-labels.h5' with h5py.File(p, 'r') as f: names = f['semantic_names'][:] sem_ids = f['semantic_mapping'][:] for n, ids in zip(names, sem_ids): if n == 'yolk': yolk_ids = ids ds = f['t00000/s00/2/cells'] seg = ds[:].astype('uint32') mask = np.isin(seg, yolk_ids).astype('uint32') p = '../data/rawdata/sbem-6dpf-1-whole-raw.h5' with h5py.File(p, 'r') as f: ds = f['t00000/s00/5/cells'] rshape = ds.shape # need to resize due to mis aligned scales mask = vigra.sampling.resize(mask.astype('float32'), rshape, order=0).astype('uint8') print(mask.shape) mask *= 255 res = [.4, .32, .32] n_scales = 3 scales = n_scales * [[2, 2, 2]] make_bdv(mask, './em_yolk_mask', convert_dtype=False, unit='micrometer', resolution=res, downscale_factors=scales)
def test_multi_timepoint(self): from pybdv import make_bdv from pybdv.metadata import get_time_range n_timepoints = 6 shape = (64,) * 3 tp_data = [] tp_setups = [] for tp in range(n_timepoints): data = np.random.rand(*shape) # make sure that we at least have 2 setup ids that agree setup_id = np.random.randint(0, 20) if tp > 1 else 0 make_bdv(data, self.out_path, setup_id=setup_id, timepoint=tp) tp_data.append(data) tp_setups.append(setup_id) tstart, tstop = get_time_range(self.xml_path) self.assertEqual(tstart, 0) self.assertEqual(tstop, n_timepoints - 1) for tp in range(n_timepoints): setup_id = tp_setups[tp] tp_key = get_key(self.is_h5, timepoint=tp, setup_id=setup_id, scale=0) with open_file(self.out_path, 'r') as f: data = f[tp_key][:] data_exp = tp_data[tp] self.assertTrue(np.allclose(data, data_exp))
def combine_mask(): tmp_folder = './tmp_intensity_correction' os.makedirs(tmp_folder, exist_ok=True) mask_path1 = '../data/rawdata/sbem-6dpf-1-whole-mask-inside.h5' mask_path2 = '../data/rawdata/sbem-6dpf-1-whole-mask-resin.h5' print("Load inside mask ...") with h5py.File(mask_path1, 'r') as f: key = 't00000/s00/0/cells' mask1 = f[key][:].astype('bool') mask1 = binary_dilation(mask1, iterations=4) print("Load resin mask ..") with h5py.File(mask_path2, 'r') as f: key = 't00000/s00/1/cells' mask2 = f[key][:] print("Resize resin mask ...") mask2 = vigra.sampling.resize(mask2.astype('float32'), mask1.shape, order=0).astype('bool') mask = np.logical_or(mask1, mask2).astype('uint8') res = [.4, .32, .32] ds_factors = [[2, 2, 2], [2, 2, 2], [2, 2, 2]] make_bdv(mask, 'mask.h5', ds_factors, resolution=res, unit='micrometer')
def make_extrapolation_mask(): z0 = 800 # extrapolation for z < z0 z1 = 9800 # extraplation for z > z1 ref_path = '../data/rawdata/sbem-6dpf-1-whole-raw.h5' ref_scale = 4 ref_key = 't00000/s00/%i/cells' % ref_scale with h5py.File(ref_path, 'r') as f: shape = f[ref_key].shape mask = np.zeros(shape, dtype='uint8') # adapt to the resolution level z0 //= (2**(ref_scale - 1)) z1 //= (2**(ref_scale - 1)) print(z0, z1) mask[:z0] = 255 mask[z1:] = 255 print(mask.min(), mask.max()) scales = 3 * [[2, 2, 2]] res = [.2, .16, .16] out_path = './extrapolation_mask' make_bdv(mask, out_path, downscale_factors=scales, downscale_mode='nearest', resolution=res, unit='micrometer', convert_dtype=False)
def convert_to_bdv(correction_folder, name): raw = imread(os.path.join(correction_folder, name, 'corrected_raw.tiff')) make_bdv(raw, output_path=os.path.join(correction_folder, name, str(name) + '.h5'), resolution=[0.025, 0.32, 0.32], unit='micrometer')
def setUp(self): os.makedirs(self.test_folder, exist_ok=True) self.im_path = os.path.join(self.test_folder, "im.n5") self.xml_path = os.path.join(self.test_folder, "im.xml") self.data = np.random.rand(*self.shape) scale_factors = 2 * [[2, 2, 2]] pybdv.make_bdv(self.data, self.im_path, downscale_factors=scale_factors, resolution=[0.5, 0.5, 0.5], unit="micrometer", setup_name=self.image_name)
def _test_overwrite(self, mode): from pybdv import make_bdv from pybdv.util import get_scale_factors, absolute_to_relative_scale_factors from pybdv.metadata import get_attributes, get_affine def _check(exp_data, exp_sf, exp_attrs, exp_affine): key = get_key(self.is_h5, timepoint=0, setup_id=0, scale=0) with open_file(self.out_path, 'r') as f: data = f[key][:] self.assertTrue(np.allclose(data, exp_data)) sf = get_scale_factors(self.out_path, setup_id=0) sf = absolute_to_relative_scale_factors(sf) self.assertEqual(sf, [[1, 1, 1]] + exp_sf) attrs = get_attributes(self.xml_path, setup_id=0) self.assertEqual(attrs, exp_attrs) affine = get_affine(self.xml_path, setup_id=0, timepoint=0)['affine0'] self.assertTrue(np.allclose(np.array(affine), np.array(exp_affine), atol=1e-4)) shape1 = (64,) * 3 data1 = np.random.rand(*shape1) sf1 = [[2, 2, 2]] attrs1 = {'channel': {'id': 1}, 'angle': {'id': 2}} affine1 = np.random.rand(12).tolist() shape2 = (72,) * 3 data2 = np.random.rand(*shape2) sf2 = [[1, 2, 2], [2, 2, 2]] attrs2 = {'channel': {'id': 3}, 'angle': {'id': 6}} affine2 = np.random.rand(12).tolist() make_bdv(data1, self.out_path, setup_id=0, timepoint=0, downscale_factors=sf1, attributes=attrs1, affine=affine1) _check(data1, sf1, attrs1, affine1) make_bdv(data2, self.out_path, setup_id=0, timepoint=0, downscale_factors=sf2, attributes=attrs2, affine=affine2, overwrite=mode) if mode == 'skip': _check(data1, sf1, attrs1, affine1) elif mode == 'all': _check(data2, sf2, attrs2, affine2) elif mode == 'data': _check(data2, sf2, attrs1, affine1) elif mode == 'metadata': _check(data1, sf1, attrs2, affine2) else: raise ValueError("Invalid over-write mode")
def copy_file(in_path, out_path, resolution=[.55, .55, .55]): chunks = (96, ) * 3 if os.path.exists(out_path + '.xml'): return print("Copy", in_path, "to", out_path) vol = np.asarray(imageio.volread(in_path + '-ch0.tif')) downscale_factors = [[2, 2, 2], [2, 2, 2], [2, 2, 2]] make_bdv(vol, out_path + '.n5', downscale_factors, unit='micrometer', resolution=resolution, chunks=chunks)
def _test_simple(self, shape, affine=None): from pybdv import make_bdv data = np.random.rand(*shape).astype('float32') make_bdv(data, self.out_path, affine=affine) key = get_key(self.is_h5, timepoint=0, setup_id=0, scale=0) self.assertTrue(os.path.exists(self.out_path)) with open_file(self.out_path, 'r') as f: self.assertTrue(key in f) ds = f[key] self.assertEqual(ds.shape, shape) out_data = ds[:] self.assertTrue(np.allclose(data, out_data))
def test_multi_threaded(self): from pybdv import make_bdv shape = (128,) * 3 chunks = (64,) * 3 data = np.random.rand(*shape) scale_factors = 2 * [[2, 2, 2]] make_bdv(data, self.out_path, chunks=chunks, n_threads=4, downscale_factors=scale_factors) key = get_key(self.is_h5, timepoint=0, setup_id=0, scale=0) with open_file(self.out_path, 'r') as f: d = f[key][:] self.assertTrue(np.allclose(d, data))
def write_ganglion_segmentation(): seg_path = os.path.join( '/g/arendt/EM_6dpf_segmentation/platy-browser-data/data', '0.6.5/images/local/sbem-6dpf-1-whole-segmented-cells.n5') tab_path = os.path.join( '/g/arendt/EM_6dpf_segmentation/platy-browser-data/data', '0.6.6/tables/sbem-6dpf-1-whole-segmented-cells/ganglia_ids.csv') tab = pd.read_csv(tab_path, sep='\t') seg_key = 'setup0/timepoint0/s0' with z5py.File(seg_path) as f: ds = f[seg_key] max_id = int(ds.attrs['maxId']) + 1 label_ids = tab['label_id'].values.astype('uint32') ganglion_labels = tab['ganglion_id'].values.astype('uint32') label_mapping = np.zeros(max_id, 'uint32') label_mapping[label_ids] = ganglion_labels seg_key = 'setup0/timepoint0/s3' out_path = './sbem-6dpf-1-whole-segmented-ganglia.n5' print("Reading segmentation ...") with z5py.File(seg_path, 'r') as f: ds = f[seg_key] ds.n_threads = 16 seg = ds[:].astype('uint32') print("To ganglion segmentation ...") seg = nt.take(label_mapping, seg) seg = seg.astype('int16') print(seg.shape) print("Writing segmentation ...") n_scales = 4 res = [.2, .16, .16] chunks = (128, ) * 3 downscale_factors = n_scales * [[2, 2, 2]] make_bdv(seg, out_path, downscale_factors, resolution=res, unit='micrometer', chunks=chunks, n_threads=16) with z5py.File(out_path) as f: ds = f['setup0/timepoint0/s0'] ds.attrs['maxId'] = max_id
def make_nephridia_segmentation(): table_path = '../../data/0.6.5/tables/sbem-6dpf-1-whole-segmented-cilia/cell_mapping.csv' seg_path = '../../data/0.6.5/segmentations/sbem-6dpf-1-whole-segmented-cells.n5' out_path = '../../data/0.6.5/segmentations/sbem-6dpf-1-whole-segmented-nephridia.xml' table = pd.read_csv(table_path, sep='\t') cell_ids = table['cell_id'].values cell_ids = np.unique(cell_ids) if cell_ids[0] == 0: cell_ids = cell_ids[1:] print(cell_ids) scale = 4 is_h5 = is_h5_file(seg_path) key = get_key(is_h5, setup_id=0, time_point=0, scale=scale) with open_file(seg_path, 'r') as f: ds = f[key] seg = ds[:].astype('uint32') bshape = (32, 256, 256) tmp = np.zeros_like(seg) print("Isin ...") tmp = elf.parallel.isin(seg, cell_ids, out=tmp, n_threads=16, verbose=True, block_shape=bshape) print("Label ...") tmp = vigra.analysis.labelVolumeWithBackground(tmp) print("Size filter ...") ids, counts = elf.parallel.unique(tmp, return_counts=True, n_threads=16, verbose=True, block_shape=bshape) keep_ids = np.argsort(counts)[::-1] keep_ids = ids[keep_ids[:3]] assert keep_ids[0] == 0 out = np.zeros(tmp.shape, dtype='uint8') for new_id, keep_id in enumerate(keep_ids[1:], 1): out[tmp == keep_id] = new_id factors = 3 * [[2, 2, 2]] res = [.4, .32, .32] make_bdv(out, out_path, factors, resolution=res, unit='micrometer')
def test_custom_chunks(self): from pybdv import make_bdv shape = (128,) * 3 chunks = (64, 42, 59) data = np.random.rand(*shape) make_bdv(data, self.out_path, chunks=chunks) key = get_key(self.is_h5, timepoint=0, setup_id=0, scale=0) with open_file(self.out_path, 'r') as f: ds = f[key] chunks_out = ds.chunks d = ds[:] self.assertEqual(chunks, chunks_out) self.assertTrue(np.allclose(d, data))
def _register(p, key, name, setup_id, res, trafos): with open_file(p, 'r') as f: raw = f[key] raw.n_threads = 16 raw = raw[:] affine_trafo = { "AmiraTranslation": matrix_to_parameters(trafos[0]), "AmiraTransform": matrix_to_parameters(trafos[1]), "AmiraOrigin": matrix_to_parameters(trafos[2]) } make_bdv(raw, out_file, downscale_factors=ds_factors, resolution=res, setup_id=setup_id, affine=affine_trafo)
def test_external(self): from pybdv import make_bdv d = imageio.volread('../data/example.tif') res_path = os.path.join(self.tmp_folder, 'data.h5') exp_path = '../data/example.h5' make_bdv(d, res_path, convert_dtype=True) with h5py.File(exp_path, 'r') as f: ds = f['t00000/s00/0/cells'] exp = ds[:] with h5py.File(res_path, 'r') as f: ds = f['t00000/s00/0/cells'] res = ds[:] self.assertEqual(res.dtype, exp.dtype) self.assertEqual(res.shape, exp.shape) self.assertTrue(np.array_equal(res, exp))
def test_dtype(self): if not self.is_h5: return from pybdv import make_bdv shape = (128,) * 3 val = np.iinfo('uint16').max + 1 data = np.full(shape, val, dtype='uint32') make_bdv(data, self.out_path, convert_dtype=False) with open_file(self.out_path, 'r') as f: key = get_key(self.is_h5, timepoint=0, setup_id=0, scale=0) d = f[key][:] self.assertTrue(np.array_equal(d, data)) with self.assertRaises(RuntimeError): make_bdv(d, './tmp.test2.h5', convert_dtype=True)
def get_mapped_ids(ids, out_path, scale=4): resolution = [0.025, 0.02, 0.02] resolution = [res * 2**scale for res in resolution] # load the segmentation and map fp ids to it path = os.path.join('/g/arendt/EM_6dpf_segmentation/platy-browser-data/data/0.3.1/segmentations', 'sbem-6dpf-1-whole-segmented-cells-labels.h5') print("Load and project seg ...") with h5py.File(path, 'r') as f: ds = f['t00000/s00/%i/cells' % scale] seg = ds[:] seg = map_ids_to_seg(seg, ids) ds_factors = [[2, 2, 2], [2, 2, 2], [2, 2, 2]] make_bdv(seg, out_path, downscale_factors=ds_factors, resolution=resolution, unit='micrometer')
def _test_ds(self, shape, mode): from pybdv import make_bdv data = np.random.rand(*shape).astype('float32') n_scales = 4 ndim = len(shape) downscale_factors = n_scales * [[2] * ndim] make_bdv(data, self.out_path, downscale_factors, downscale_mode=mode) exp_shape = shape self.assertTrue(os.path.exists(self.out_path)) with open_file(self.out_path, 'r') as f: for scale in range(n_scales): key = get_key(self.is_h5, timepoint=0, setup_id=0, scale=scale) self.assertTrue(key in f) ds = f[key] self.assertEqual(ds.shape, exp_shape) exp_shape = tuple(sh // sf for sh, sf in zip(exp_shape, downscale_factors[scale]))
def get_iou(start_index=(0, 0, 0)): # open file z, y, x = start_index filename = 's4a2_mc_' + str(z) + '_' + str(y) + '_' + str( x) + '_' + '1024_1024_1024.h5' filename_bdv = 's4a2_mc_' + str(z) + '_' + str(y) + '_' + str( x) + '_bin_4.h5' if not os.path.exists(res_path + filename_bdv): with h5py.File(res_path + filename, 'r') as f: current_label = f['data'][:, :, :].astype('uint64') current_label, max_label, mapping = vigra.analysis.relabelConsecutive( current_label, start_label=1, keep_zeros=True) binary_current_label = (current_label != 1).astype('uint') make_bdv(binary_current_label, res_path + filename_bdv, downscale_factors=scale_factors, downscale_mode=mode) #convert_to_bdv(res_path + filename, 'data', res_path + filename_bdv, downscale_factors = scale_factors, downscale_mode = mode) with h5py.File(res_path + filename_bdv, 'r') as f: bin_down_label = f['/t00000/s00/2/cells'][:, :, :] shape = bin_down_label.shape bb = np.s_[z // 4:z // 4 + shape[0], y // 4:y // 4 + shape[1], x // 4:x // 4 + shape[2]] with h5py.File(mask_filename, 'r') as f: mask = f['data'][bb] bin_down_label = bin_down_label * mask with h5py.File(res_path_j + res_filename_j, 'r') as f: control_results = f['bin_4'][bb] * mask size1 = np.count_nonzero(bin_down_label) size2 = np.count_nonzero(control_results) interception = bin_down_label * control_results size_inter = np.count_nonzero(interception) if size1 == 0 and size2 == 0: iou = 1 else: iou = size_inter / (size2 + size1 - size_inter) print(start_index, size1, size2, iou) return (start_index, size1, size2, iou)
def update_shell(): p_tiff = '../../EM-Prospr/shell_seg.tiff' p_res_n5 = '../data/rawdata/sbem-6dpf-1-whole-segmented-resin.n5' p_res_xml = '../data/rawdata/sbem-6dpf-1-whole-segmented-resin.xml' scale_factors = get_scale_factors(p_res_n5, 0)[1:] resolution = get_resolution(p_res_xml, 0) scale_factors = [[2, 2, 1], [2, 2, 2], [2, 2, 2], [2, 2, 2]] print(scale_factors) print(resolution) print("Load tiff ...") shell = np.asarray(imageio.volread(p_tiff)) print(shell.shape) print("Write bdv") out_path = 'sbem-6dpf-1-whole-segmented-shell.n5' make_bdv(shell, out_path, downscale_factors=scale_factors, resolution=resolution, unit='micrometer', n_threads=8, chunks=(96,) * 3, convert_dtype=False)
def mask_gut_nuclei(view_mask=False): p1 = '../data/0.0.0/images/prospr-6dpf-1-whole-segmented-Stomodeum.h5' p2 = '../data/rawdata/sbem-6dpf-1-whole-segmented-tissue-labels.h5' with h5py.File(p1, 'r') as f: ds = f['t00000/s00/0/cells'] stomodeum = (ds[:] > 0).astype('float32') with h5py.File(p2, 'r') as f: ds = f['t00000/s00/2/cells'] gut = ds[:] == 23 stomodeum = vigra.sampling.resize(stomodeum, gut.shape, order=0).astype('bool') mask = np.logical_and(stomodeum == 0, gut) if view_mask: from heimdall import view, to_source p_nuc = '../data/0.0.0/segmentations/sbem-6dpf-1-whole-segmented-nuclei-labels.h5' with h5py.File(p_nuc, 'r') as f: nuc = f['t00000/s00/2/cells'][:] > 0 nuc = vigra.sampling.resize(nuc.astype('float32'), gut.shape, order=0).astype('bool') view(to_source(gut.astype('uint32'), name='gut-em'), to_source(stomodeum.astype('uint32'), name='stomodeum-prospr'), to_source(mask.astype('uint32'), name='mask'), to_source(nuc.astype('uint32'), name='nuclei')) else: resolution = [0.4, 0.32, 0.32] mask = mask.astype('uint8') * 255 make_bdv(mask, 'nucleus_exclusion_mask', unit='micrometer', resolution=resolution)
def make_midgut_volume(): table_path = 'midgut_table.csv' table = pd.read_csv(table_path, sep='\t') label_ids = table['label_id'].values midgut = table['midgut'].values assert len(label_ids) == len(midgut) assert np.array_equal(np.unique(midgut), np.array([0, 1])) midgut_ids = label_ids[midgut == 1] scale = 4 seg_path = '../../data/0.6.5/segmentations/sbem-6dpf-1-whole-segmented-cells.n5' res = [0.4, 0.32, 0.32] key = get_key(False, time_point=0, setup_id=0, scale=scale) with open_file(seg_path, 'r') as f: ds = f[key] seg = ds[:] chunks = ds.chunks midgut_seg = 255 * np.isin(seg, midgut_ids).astype('uint8') out_path = './sbem-6dpf-1-whole-segmented-midgut.n5' factors = 3 * [[2, 2, 2]] make_bdv(midgut_seg, out_path, factors, resolution=res, unit='micrometer', convert_dtype=False, chunks=chunks)
# use 'nearest' instead mode = 'interpolate' # resolution of the data, set appropriately resolution = [pixel * 1e6, pixel * 1e6, pixel * 1e6] #save big data format folder_h5 = folder_result + 'bdv/' if not os.path.exists(folder_h5): os.makedirs(folder_h5) make_bdv(data, folder_h5 + data_name, downscale_factors=scale_factors, downscale_mode=mode, resolution=resolution, unit='micrometer', setup_name=data_name) # ============================================================================= # save as tiff # ============================================================================= folder_tiff = folder_result + 'tiff/' if not os.path.exists(folder_tiff): os.makedirs(folder_tiff) dxchange.write_tiff_stack(data, fname=folder_tiff + data_name + '/tomo') # =============================================================================
def setUp(self): os.makedirs(self.tmp_folder, exist_ok=True) data = np.random.rand(*self.shape) make_bdv(data, self.out_path, downscale_factors=self.scale_factors)
def test_custom_attributes(self): from pybdv import make_bdv from pybdv.metadata import get_attributes shape = (64,) * 3 data = np.random.rand(*shape) chan_name = 'DAPI' tile_name = 'some-tile' angle_name = 'some-angle' # write setup 0 make_bdv(data, self.out_path, setup_id=0, attributes={'channel': {'id': None, 'name': chan_name}, 'tile': {'id': 2, 'name': tile_name}, 'angle': {'id': 0, 'name': angle_name}}) attrs_out = get_attributes(self.xml_path, 0) attrs_exp = {'channel': {'id': 0, 'name': chan_name}, 'tile': {'id': 2, 'name': tile_name}, 'angle': {'id': 0, 'name': angle_name}} self.assertEqual(attrs_out, attrs_exp) # write setup 1 make_bdv(data, self.out_path, setup_id=None, attributes={'channel': {'id': None}, 'tile': {'id': 2}, 'angle': {'id': 0}}) attrs_out = get_attributes(self.xml_path, 1) attrs_exp = {'channel': {'id': 1}, 'tile': {'id': 2, 'name': tile_name}, 'angle': {'id': 0, 'name': angle_name}} self.assertEqual(attrs_out, attrs_exp) # write to setup 0 again with different timepoint make_bdv(data, self.out_path, setup_id=0, timepoint=1, attributes={'channel': {'id': None}, 'tile': {'id': 2}, 'angle': {'id': 0}}) attrs_out = get_attributes(self.xml_path, 0) attrs_exp = {'channel': {'id': 0, 'name': chan_name}, 'tile': {'id': 2, 'name': tile_name}, 'angle': {'id': 0, 'name': angle_name}} self.assertEqual(attrs_out, attrs_exp) # write next setup id without specifying all attribute names # -> should fail with self.assertRaises(ValueError): make_bdv(data, self.out_path, setup_id=None, attributes={'channel': {'id': 5}, 'tile': {'id': 2}}) # write next setup id with a new attribute name # -> should fail with self.assertRaises(ValueError): make_bdv(data, self.out_path, setup_id=None, attributes={'channel': {'id': 5}, 'settings': {'id': 2}}) # write exisiting setup id with different attribute setup # -> should fail with self.assertRaises(ValueError): make_bdv(data, self.out_path, setup_id=0, timepoint=2, attributes={'channel': {'id': 5}, 'tile': {'id': 2}, 'angle': {'id': 0}})