def create_dataset(output_n5,
template_n5,
compression='same',
dtype='same',
overwrite=True):
data_set = '/s0'
template = zarr.open(store=zarr.N5Store(template_n5), mode='r')[data_set]
out = zarr.open(store=zarr.N5Store(output_n5), mode='a')
if compression == 'raw':
compressor = None
elif compression == 'same':
compressor = template.compressor
else:
compressor = codecs.get_codec(dict(id=compression))
if dtype == 'same':
dtype = template.dtype
print("Using compressor:", compressor or 'raw')
print("Creating n5 data set with:")
print(f" compressor: {compressor}")
print(f" shape: {template.shape}")
print(f" chunking: {template.chunks}")
print(f" dtype: {dtype}")
print(f" to path: {output_n5}{data_set}")
out.create_dataset(data_set,
shape=template.shape,
chunks=template.chunks,
dtype=dtype,
compressor=compressor,
overwrite=overwrite)
def autodetect_iteration(path, ds):
n5_ds = zarr.open(zarr.N5Store(path), 'r')[ds]
try:
return n5_ds.attrs['iteration']
except KeyError:
print(path,ds)
return None
def add_metadata(n5_path,
downsampling_factors=(2, 2, 2),
axes=("x", "y", "z"),
pixel_res=None,
pixel_res_units="nm"):
store = zarr.N5Store(n5_path)
scales = []
for idx in range(20):
try:
zarr.open(store, path=f'/s{idx}', mode='r')
except PathNotFoundError:
break
factors = tuple([int(math.pow(f, idx)) for f in downsampling_factors])
print(f"Setting downsampling factors for scale {idx} to {factors}")
scales.append(factors)
if idx > 0:
z = zarr.open(store, path=f'/s{idx}', mode='a')
z.attrs["downsamplingFactors"] = factors
# Add metadata at the root
z = zarr.open(store, path='/', mode='a')
z.attrs["scales"] = scales
if axes:
z.attrs["axes"] = axes
if pixel_res:
z.attrs["pixelResolution"] = {
"dimensions": pixel_res,
"unit": pixel_res_units
}
print("Added multiscale metadata to", n5_path)
def write_n5(path, shape, block_size, compressor):
store = zarr.N5Store(path)
data = np.arange(np.prod(shape), dtype=np.uint16)
data = data.reshape(shape)
data_transpose = data.transpose()
z = zarr.zeros(
data_transpose.shape,
chunks=block_size[::-1],
store=store,
dtype=data.dtype,
overwrite=True,
compressor=compressor)
z[...] = data_transpose
def read_n5_block(path, data_set, start, end):
"""
Reads and returns an image block from the specified n5 location.
path: path to the N5 directory
data_set: path to the data set inside the n5, e.g. "/s0"
start: tuple x,y,z indicating the starting corner of the data block
end: tuple (x,y,z) indicating the ending corner of the data block
"""
n5_path = path + data_set
img = zarr.open(store=zarr.N5Store(n5_path), mode='r')
img = img[start[2]:end[2], start[1]:end[1], start[0]:end[0]]
# zarr loads zyx order
return img[...].transpose(2, 1, 0)
def write_n5_block(path, data_set, start, end, data):
"""
Writes the given image block to the specified n5 location.
path: path to the N5 directory
data_set: path to the data set inside the n5, e.g. "/s0"
start: tuple x,y,z indicating the starting corner of the data block
end: tuple (x,y,z) indicating the ending corner of the data block
"""
n5_path = path + data_set
img = zarr.open(store=zarr.N5Store(n5_path), mode='a')
# zarr writes zyx order
img = img[...].transpose(2, 1, 0)
img[start[2]:end[2], start[1]:end[1], start[0]:end[0]] = data
def create_tile_directory(self, series, resolution, width, height):
tile_directory = os.path.join(self.slide_directory,
"data.%s" % self.file_type)
self.zarr_store = zarr.DirectoryStore(tile_directory)
if self.file_type == "n5":
self.zarr_store = zarr.N5Store(tile_directory)
self.zarr_group = zarr.group(store=self.zarr_store)
self.zarr_group.attrs['bioformats2raw.layout'] = LAYOUT_VERSION
# important to explicitly set the chunk size to 1 for non-XY dims
# setting to None may cause all planes to be chunked together
# ordering is TZCYX and hard-coded since Z and T are not present
self.zarr_group.create_dataset(
"%s/%s" % (str(series), str(resolution)),
shape=(1, 1, 3, height, width),
chunks=(1, 1, 1, self.tile_height, self.tile_width),
dtype='B')
def create_tile_directory(self, resolution, width, height):
tile_directory = os.path.join(self.slide_directory, str(resolution))
if self.file_type in ("n5", "zarr"):
tile_directory = os.path.join(self.slide_directory,
"pyramid.%s" % self.file_type)
self.zarr_store = zarr.DirectoryStore(tile_directory)
if self.file_type == "n5":
self.zarr_store = zarr.N5Store(tile_directory)
self.zarr_group = zarr.group(store=self.zarr_store)
self.zarr_group.create_dataset(str(resolution),
shape=(3, height, width),
chunks=(None, self.tile_height,
self.tile_width),
dtype='B')
else:
os.mkdir(tile_directory)
return tile_directory
def add_multiscale(n5_path, data_set, downsampling_factors=(2,2,2), \
downsampling_method=np.mean, thumbnail_size_yx=None):
'''
Given an n5 with "s0", generate downsampled versions s1, s2, etc., up to the point where
the smallest version is larger than thumbnail_size_yx (which defaults to the chunk size).
'''
print('Generating multiscale for', n5_path)
store = zarr.N5Store(n5_path)
# Find out what compression is used for s0, so we can use the same for the multiscale
fullscale = f'{data_set}/s0'
r = zarr.open(store=store, mode='r')
compressor = r[fullscale].compressor
volume = da.from_zarr(store, component=fullscale)
chunk_size = volume.chunksize
thumbnail_size_yx = thumbnail_size_yx or chunk_size
multi = multiscale(volume,
downsampling_method,
downsampling_factors,
chunks=chunk_size)
thumbnail_sized = [
np.less_equal(m.shape, thumbnail_size_yx).all() for m in multi
]
cutoff = thumbnail_sized.index(True)
multi_to_save = multi[0:cutoff + 1]
for idx, m in enumerate(multi_to_save):
if idx == 0: continue
print(f'Saving level {idx}')
component = f'{data_set}/s{idx}'
m.data.to_zarr(store,
component=component,
overwrite=True,
compressor=compressor)
z = zarr.open(store, path=component, mode='a')
z.attrs["downsamplingFactors"] = tuple(
[int(math.pow(f, idx)) for f in downsampling_factors])
print("Added multiscale imagery to", n5_path)
def read_image(path, dtype, n5_path=None):
"""Read an image and its voxel spacing"""
x = splitext(path)[1]
ext = x if x != '.gz' else splitext(splitext(path)[0])[1]
if ext == '.nii':
import nibabel
img = nibabel.load(abspath(path))
img_meta = img.header
img_data = img.get_data().squeeze()
img_data = dtype(img_data)
img_vox = np.array(img.header.get_zooms()[0:3])
return img_data, img_vox, img_meta
elif ext == '.nrrd':
import nrrd
img_data, img_meta = nrrd.read(abspath(path))
img_data = dtype(img_data)
img_vox = np.diag(img_meta['space directions'].astype(dtype))
return img_data, img_vox, img_meta
elif isdir(path) and n5_path is not None:
import zarr, json
img = zarr.open(store=zarr.N5Store(path), mode='r')
slices = parse_n5_slice(n5_path[1])
if len(slices) == 3:
img_data = img[n5_path[0]][slices[0], slices[1], slices[2]]
img_data = np.ascontiguousarray(np.moveaxis(img_data, (0, 2), (2, 0)))
elif len(slices) == 2:
img_data = img[n5_path[0]][slices[0], slices[1]]
img_data = np.ascontiguousarray(np.moveaxis(img_data, 0, -1))
img_data = img_data.astype(dtype)
with open(path + n5_path[0] + '/attributes.json') as atts:
atts = json.load(atts)
img_vox = np.absolute(np.array(atts['pixelResolution']) *
np.array(atts['downsamplingFactors']))
img_vox = img_vox.astype(dtype)
return img_data, img_vox, atts
def autodetect_setup(path, ds):
n5_ds = zarr.open(zarr.N5Store(path), 'r')[ds]
try:
return n5_ds.attrs['setup']
except KeyError:
return None
def autodetect_labelnames(path, crop):
n5 = zarr.open(zarr.N5Store(path), 'r')
labels_predicted = set(n5.array_keys())
labels_in_crop = set(crop_utils.get_all_present_labelnames(crop))
labels_eval = list(labels_predicted.intersection(labels_in_crop))
return labels_eval
def multifish_registration_pipeline(
fixed_file_path,
fixed_lowres_dataset,
fixed_highres_dataset,
moving_file_path,
moving_lowres_dataset,
moving_highres_dataset,
transform_write_path,
inv_transform_write_path,
scratch_directory,
global_affine_params={},
local_affine_params={},
deform_params={},
):
"""
"""
# merge params with defaults
global_affine_params = {
**(configuration.multifish_registration_global_affine_defaults),
**global_affine_params,
}
local_affine_params = {
**(configuration.multifish_registration_local_affine_defaults),
**local_affine_params,
}
deform_params = {
**(configuration.multifish_registration_deform_defaults),
**deform_params,
}
# wrap inputs as zarr objects
fix_zarr = zarr.open(store=zarr.N5Store(fixed_file_path), mode='r')
mov_zarr = zarr.open(store=zarr.N5Store(moving_file_path), mode='r')
# get lowres data and spacing
fix_lowres = fix_zarr[fixed_lowres_dataset]
mov_lowres = mov_zarr[moving_lowres_dataset]
fix_meta = fix_lowres.attrs.asdict()
mov_meta = mov_lowres.attrs.asdict()
fix_lowres_spacing = np.array(fix_meta['pixelResolution'])
fix_lowres_spacing *= fix_meta['downsamplingFactors']
mov_lowres_spacing = np.array(mov_meta['pixelResolution'])
mov_lowres_spacing *= mov_meta['downsamplingFactors']
# lowres data is assumed to fit into RAM
fix_lowres = fix_lowres[...].transpose(2, 1, 0) # zarr loads zyx order
mov_lowres = mov_lowres[...].transpose(2, 1, 0)
# global affine alignment
global_affine = affine.dog_ransac_affine(
fix_lowres,
mov_lowres,
fix_lowres_spacing,
mov_lowres_spacing,
nspots=global_affine_params['nspots'],
cc_radius=global_affine_params['cc_radius'],
match_threshold=global_affine_params['match_threshold'],
align_threshold=global_affine_params['align_threshold'],
)
# apply global affine
mov_lowres_aligned = transform.apply_global_affine(
fix_lowres,
mov_lowres,
fix_lowres_spacing,
mov_lowres_spacing,
global_affine,
)
# local affine alignment
local_affines = affine.dog_ransac_affine_distributed(
fix_lowres,
mov_lowres_aligned,
fix_lowres_spacing,
fix_lowres_spacing,
nspots=local_affine_params['nspots'],
cc_radius=local_affine_params['cc_radius'],
match_threshold=local_affine_params['match_threshold'],
align_threshold=local_affine_params['align_threshold'],
blocksize=local_affine_params['blocksize'],
cluster_extra=local_affine_params['cluster_extra'],
)
# get highres data and spacing
fix_highres = fix_zarr[fixed_highres_dataset]
mov_highres = mov_zarr[moving_highres_dataset]
fix_meta = fix_highres.attrs.asdict()
mov_meta = mov_highres.attrs.asdict()
fix_highres_spacing = np.array(fix_meta['pixelResolution'])
fix_highres_spacing *= fix_meta['downsamplingFactors']
mov_highres_spacing = np.array(mov_meta['pixelResolution'])
mov_highres_spacing *= mov_meta['downsamplingFactors']
# compose global and local affines to single position field
# highres objects assumed too big for RAM, backed by disk
dbs = configuration.multifish_registration_deform_defaults[
'deform_blocksize']
global_affine = transform.global_affine_to_position_field(
fix_highres.shape,
fix_highres_spacing,
global_affine,
scratch_directory + '/global_affine_position_field.zarr',
blocksize=dbs,
)
local_affines = transform.local_affine_to_position_field(
fix_highres.shape,
fix_highres_spacing,
local_affines,
scratch_directory + '/local_affines_position_field.zarr',
blocksize=dbs,
block_multiplier=[2, 2, 1],
)
return "WORKED!"
def access_n5(dir_path: Union[str, Path], container_path: Union[str, Path],
**kwargs) -> Any:
dir_path = zarr.N5Store(dir_path)
return access_zarr(dir_path, container_path, **kwargs)
def single_block_inference(net_name, input_shape, output_shape, ckpt, outputs, input_file,
input_ds_name="volumes/raw/s0", coordinate=(0, 0, 0), output_file='prediction.n5',
voxel_size_input=(4, 4, 4), voxel_size_output=(4, 4, 4), input="raw", factor=None):
logging.info("Preparing output file {0:}".format(output_file))
store = zarr.N5Store(output_file)
root = zarr.group(store=store)
compr = GZip(level=6)
input_shape = tuple(int(ins) for ins in input_shape)
output_shape = tuple(int(outs) for outs in output_shape)
for output_key in outputs:
root.require_dataset(output_key, shape=output_shape, chunks=output_shape, dtype='float32', compressor=compr)
root.require_dataset(input, shape=input_shape, chunks=input_shape, dtype='float32',
compressor=compr)
logging.info("Reading input data from {0:}".format(os.path.join(input_file, input_ds_name)))
sf = zarr.open(input_file, mode="r")
input_ds = sf[input_ds_name]
input_data = input_ds[tuple(slice(c, c+w) for c, w in zip(coordinate, input_shape))]
logging.info("Preprocessing input data")
if factor is None:
if input_ds.dtype == np.uint8:
factor = 255.
elif input_ds.dtype == np.uint16:
factor = 256.*256.-1
else:
raise ValueError("don't know which factor to assume for data of type {0:}".format(input_ds.dtype))
logging.debug("Normalization factor set to {0:} based on dtype {1:}".format(factor, input_ds.dtype))
try:
contr_adj = input_ds.attrs["contrastAdjustment"]
scale = float(factor) / (float(contr_adj["max"]) - float(contr_adj["min"]))
shift = - scale * float(contr_adj["min"])
input_data = input_data * scale + shift
except KeyError:
logging.debug("Attribute `contrastAdjustment` not found in {0:}, keeping contrast as is".format(os.path.join(
input_file, input_ds_name)))
logging.debug("Normalizing input data to range -1, 1")
input_data = input_data.astype(np.float32) / factor
input_data = input_data * 2 - 1
# prepare input and output definition for model
with open('net_io_names.json'.format(net_name))as f:
net_io_names = json.load(f)
network_input_key = net_io_names[input]
network_output_keys = []
for output_key in outputs:
network_output_keys.append(net_io_names[output_key])
logging.info("Running inference using ckpt {0:} with network {1:}".format(ckpt, net_name+'_inference.meta'))
graph = tf.Graph()
sess = tf.Session(graph=graph)
with graph.as_default():
saver = tf.train.import_meta_graph(net_name + '_inference.meta', clear_devices=True)
saver.restore(sess, ckpt)
output_data = sess.run(network_output_keys, feed_dict={network_input_key: input_data})
logging.info("Writing data to file {0:}".format(output_file))
# write input data to file
root[input][...] = input_data
root[input].attrs["offset"] = (0, 0, 0)
root[input].attrs["resolution"] = voxel_size_input
# write output data to file
offset = tuple(((np.array(input_shape) * np.array(voxel_size_input)) - (np.array(output_shape) * np.array(
voxel_size_output)) )/ 2.)
for output_key, data in zip(outputs, output_data):
root[output_key][...] = data
root[output_key].attrs["offset"] = offset
root[output_key].attrs["resolution"] = voxel_size_output
if len(sys.argv) > 7:
dapi_channel = sys.argv[7]
else:
dapi_channel = 'c2'
if len(sys.argv) > 8:
bleed_channel = sys.argv[8]
else:
bleed_channel = 'c3'
lb = imread(label_path)
roi = np.unique(lb)
# get n5 image data
ch_scale_path = '{}/{}'.format(channel, scale)
print('Image path:', puncta_path, ch_scale_path)
im = zarr.open(store=zarr.N5Store(puncta_path), mode='r')
img = im[ch_scale_path][...]
if channel == bleed_channel:
dapi_ch_scale_path = '{}/{}'.format(dapi_channel, scale) # c2/s2
dapi = im[dapi_ch_scale_path][...]
lo = np.percentile(np.ndarray.flatten(dapi), 99.5)
bg_dapi = np.percentile(np.ndarray.flatten(dapi[dapi != 0]), 1)
bg_img = np.percentile(np.ndarray.flatten(img[img != 0]), 1)
dapi_factor = np.median(
(img[dapi > lo] - bg_img) / (dapi[dapi > lo] - bg_dapi))
img = np.maximum(0, img - bg_img - dapi_factor *
(dapi - bg_dapi)).astype('float32')
print('bleed_through:', dapi_factor)
print('DAPI background:', bg_dapi)
print('bleed_through channel background:', bg_img)
