def main():
dataset_name = 'jrc_UT21-1413-003'
source_path = ('/nrs/flyem/render/n5',
'/flyem/z_corr/NIH_J1/v1_acquire_align___20220321_170126')
dest_path = f'/groups/cellmap/cellmap/data/{dataset_name}/{dataset_name}.n5/volumes/raw'
source_group = access_n5(*source_path)
dest_group = access(dest_path, mode='a')
dest_group.attrs.update(**source_group.attrs)
dest_group.attrs['source'] = os.path.join(*source_path)
new_chunks = (128, ) * 3
dry = False
with get_cluster() as clust, Client(clust) as cl:
print(cl.cluster.dashboard_link)
cl.cluster.scale(120)
storage_ops = []
for key, value in source_group.arrays():
print(f'array {key}')
dest_array = access(dest_path + '/' + key,
shape=value.shape,
dtype=value.dtype,
chunks=new_chunks,
mode='a')
dest_array.attrs.update(**value.attrs)
dar = da.from_array(value, chunks=[6 * c for c in new_chunks])
if not dry:
storage_ops.append(
dar.store(dest_array, lock=False, compute=False))
cl.compute(storage_ops, sync=True)
def test_accessing_group_zarr_n5():
store = tempfile.mkdtemp(suffix='.n5')
atexit.register(shutil.rmtree, store)
data = np.zeros(100, dtype='uint8') + 42
zg = zarr.open(store, mode='a')
zg.attrs.update({'foo': 'bar'})
zg['foo'] = data
assert dict(access(store, mode='r').attrs) == {'foo': 'bar'}
assert np.array_equal(access(store, mode='r')['foo'][:], data)
def test_accessing_group_zarr():
store = tempfile.mkdtemp(suffix='.zarr')
atexit.register(shutil.rmtree, store)
data = np.zeros(100, dtype='uint8') + 42
zg = zarr.open(store, mode='w')
zg['foo'] = data
assert access(store, mode='a') == zg
zg = access(store, mode='w')
zg['foo'] = data
assert zarr.open(store, mode='a') == zg
def test_accessing_group_zarr_n5():
store = 'data/group.n5'
data = np.zeros(100, dtype='uint8') + 42
zg = zarr.open(store, mode='w')
zg['foo'] = data
assert access(store, mode='a') == zg
zg = access(store, mode='w')
zg['foo'] = data
assert zarr.open(store, mode='a') == zg
shutil.rmtree(store)
def main(h5_path: str, n5_path: str, skip_confirmation: bool = False):
crop_name = Path(split_by_suffix(h5_path, ('.h5', ))[0]).parent.name
num_n5_suffixes = len(
tuple(filter(lambda v: v.endswith('.n5'),
Path(n5_path).parts)))
if num_n5_suffixes != 1:
raise ValueError(
'The n5 path must have one and only one element ending with ".n5". The path given has {num_n5_suffixes}'
)
dataset_name = Path(split_by_suffix(n5_path, ('.n5', ))[0]).name
sheet_df = get_sheet_df(credfile, sheetname, sheetpage)
crop_attrs = generate_crop_attrs(dataset_name, crop_name, sheet_df,
classNameDict)
gt_data = read(h5_path)
if not skip_confirmation:
print(
f'The data created at {n5_path} will have the following attributes: '
)
print(crop_attrs)
click.confirm('Do you wish to continue?', default=True)
output_array = access(n5_path,
shape=gt_data.shape,
dtype=gt_data.dtype,
mode='a')
output_array[:] = gt_data
output_array.attrs.update(**crop_attrs)
log.info(f'Saved data and attributes to {n5_path}')
return 0
def reunit(url: str, new_units: Dict[str, str]):
validate_axis_keys(list(new_units.keys()))
if not len(set(new_units.values())) == 1:
raise ValueError(
"Due to legacy support for the PixelResolution attribute, units for all axes must be identical."
)
new_unit: str = list(new_units.values())[0]
new_meta: Dict[str, Any] = {}
arr = access(url, mode="a")
if not isinstance(arr, zarr.core.Array):
raise ValueError(
"This function only works on Zarr arrays or N5 arrays accessed via zarr"
)
if arr.attrs.get("pixelResolution", None) is not None:
old_pixr = arr.attrs["pixelResolution"]
# pixel resolution is x y z
new_meta.update({
"[pixelResolution":
neuroglancer.PixelResolution(dimensions=old_pixr["dimensions"],
unit=new_unit).asdict()
})
if arr.attrs.get("transform", None) is not None:
old_transform = cosem_ome.SpatialTransform(
**arr.attrs.get("transform"))
new_transform = cosem_ome.SpatialTransform(**old_transform.asdict())
for key, value in new_units.items():
idx = old_transform.axes.index(key)
new_transform.units[idx] = new_unit
new_meta.update({"transform": new_transform.asdict()})
return new_meta
def rescale(url: str, new_scale: Dict[str, float]) -> Dict[str, Any]:
validate_axis_keys(list(new_scale.keys()))
new_meta: Dict[str, Any] = {}
arr = access(url, mode="a")
if not isinstance(arr, zarr.core.Array):
raise ValueError(
"This function only works on Zarr arrays or N5 arrays accessed via zarr"
)
if arr.attrs.get("pixelResolution", None) is not None:
old_pixr = arr.attrs["pixelResolution"]
# pixel resolution is x y z
_new_scale = [new_scale[k] for k in ["x", "y", "z"]]
new_meta.update({
"[pixelResolution":
neuroglancer.PixelResolution(dimensions=_new_scale,
unit=old_pixr["unit"]).asdict()
})
if arr.attrs.get("transform", None) is not None:
old_transform = cosem_ome.SpatialTransform(
**arr.attrs.get("transform"))
new_transform = cosem_ome.SpatialTransform(**old_transform.asdict())
for key, value in new_scale.items():
idx = old_transform.axes.index(key)
new_transform.scale[idx] = value
new_meta.update({"transform": new_transform.asdict()})
return new_meta
def retranslate(url: str, new_translate: Dict[str, float]):
validate_axis_keys(list(new_translate.keys()))
new_meta: Dict[str, Any] = {}
arr = access(url, mode="a")
if not isinstance(arr, zarr.core.Array):
raise ValueError(
"This function only works on Zarr arrays or N5 arrays accessed via zarr"
)
if arr.attrs.get("transform", None) is not None:
old_transform = cosem_ome.SpatialTransform(**arr.attrs.get("transform"))
new_transform = cosem_ome.SpatialTransform(**old_transform.asdict())
for key, value in new_translate.items():
idx = old_transform.axes.index(key)
new_transform.translate[idx] = value
new_meta.update({"transform": new_transform.asdict()})
return new_meta
def main(group_path: str):
group = access(group_path, mode='a')
arrays = dict(sorted(group.arrays(), key = lambda kv: int(kv[0][1:])))
base_pixelRes = neuroglancer.PixelResolution(**group.attrs['pixelResolution'])
base_transform = cosem.SpatialTransform(axes=group.attrs['axes'][::-1],
units=(base_pixelRes.unit,) * 3,
translate=[0.0,] * 3,
scale=base_pixelRes.dimensions[::-1])
scale_factors = (2,2,2)
full_rank = {k: v for k,v in arrays.items() if np.all(np.array(v.shape) > 1)}
base_data = DataArray(da.from_array(group['s0']), coords=transform_to_coords(group['s0'], base_transform))
multi = multiscale(base_data, da.mean, scale_factors=scale_factors)[:len(full_rank)]
multi_meta = [cosem.SpatialTransform.fromDataArray(m).dict() for m in multi]
[a.attrs.update({'transform' : mmeta}) for a,mmeta in zip(full_rank.values(), multi_meta)]
group_meta = cosem.COSEMGroupMetadata.fromDataArrays(multi, name='raw', paths=tuple(full_rank.keys())).dict()
group.attrs.update(group_meta)
return 1
def copy_crop(gt_path: str, cosemSheet: pd.DataFrame,
classNameDict: Dict[str, Any]):
dataset_name = Path(gt_path).parts[3]
cropname_matches = cropname_matcher.findall(gt_path)
if len(cropname_matches) == 0:
raise ValueError(
f'Could not infer the crop name from {gt_path}. Crop names should take the form `crop130`.'
)
else:
crop_name = cropname_matches[0]
labelList = createLabelList(dataset_name, crop_name, cosemSheet,
classNameDict)
offset = getOffset(dataset_name, crop_name, cosemSheet)
base_resolution = getBaseResolution(dataset_name, crop_name, cosemSheet)
annotation_resolution = getAnnotationResolution(dataset_name, crop_name,
cosemSheet)
transform = SpatialTransform(
scale=annotation_resolution[::-1],
translate=[r * o for r, o in zip(base_resolution, offset)][::-1],
axes=('x', 'y', 'z')[::-1],
units=('nm', 'nm', 'nm'))
pixr = PixelResolution(unit='nm', dimensions=annotation_resolution)
new_path = update_path(gt_path)
old_array = read(gt_path)
new_attrs = {
'pixelResolution': pixr.dict(),
'transform': transform.dict(),
**labelList
}
new_array = access(new_path,
shape=old_array.shape,
dtype=old_array.dtype,
mode='a')
new_array[:] = old_array[:]
new_array.attrs.update(**new_attrs)
def rescale_and_invert(arr, min, max):
return arr.map_blocks(rescale_intensity,
in_range=(min, max),
out_range=(255, 0),
dtype='uint8')
if __name__ == '__main__':
source = '/groups/cellmap/cellmap/data/jrc_ctl-id8-1/jrc_ctl-id8-1.n5/volumes/raw'
dest = '/groups/cellmap/cellmap/data/jrc_ctl-id8-1/jrc_ctl-id8-1.n5/volumes/raw_normalized'
source_group = read(source)
target_group = access(dest, mode='a')
target_group.attrs.update(**source_group.attrs)
with get_cluster() as clust, Client(clust) as cl:
print(cl.cluster.dashboard_link)
cl.cluster.scale(num_workers)
_arr = da.from_array(source_group['s0'],
chunks=[x * 4 for x in source_group['s0'].chunks])
min, max = cl.compute((_arr.min(), _arr.max()), sync=True)
for array_path, arr in source_group.arrays():
arr_dest = access(dest + f'/{array_path}',
shape=arr.shape,
dtype='uint8',
chunks=arr.chunks,
mode='a')