def downsample(ctx, src_layer_spec, dst_layer_spec, mip_start,
mip_end, chunk_xy, chunk_z, mips_per_task, start_coord,
end_coord, coord_mip):
scheduler = ctx.obj['scheduler']
corgie_logger.debug("Setting up Source and Destination layers...")
src_layer = create_layer_from_spec(src_layer_spec,
caller_name='src layer',
readonly=True)
if dst_layer_spec is None:
corgie_logger.info("Destination layer not specified. Using Source layer "
"as Destination.")
dst_layer = src_layer
dst_layer.readonly = False
else:
dst_layer = create_layer_from_spec(dst_layer_spec,
caller_name='dst_layer layer',
readonly=False,
reference=src_layer, chunk_z=chunk_z, overwrite=True)
bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip)
downsample_job = DownsampleJob(src_layer=src_layer,
dst_layer=dst_layer,
mip_start=mip_start,
mip_end=mip_end,
bcube=bcube,
chunk_xy=chunk_xy,
chunk_z=chunk_z,
mips_per_task=mips_per_task)
# create scheduler and execute the job
scheduler.register_job(downsample_job, job_name="downsample")
scheduler.execute_until_completion()
result_report = f"Downsampled {src_layer} from {mip_start} to {mip_end}. Result in {dst_layer}"
corgie_logger.info(result_report)
def compute_field(ctx, src_layer_spec, tgt_layer_spec, dst_layer_spec, suffix,
processor_spec, pad, crop, chunk_xy, start_coord,
processor_mip, end_coord, coord_mip, blend_xy, tgt_z_offset,
chunk_z, reference_key, clear_nontissue_field):
if suffix is None:
suffix = ''
else:
suffix = f"_{suffix}"
scheduler = ctx.obj['scheduler']
corgie_logger.debug("Setting up layers...")
src_stack = create_stack_from_spec(src_layer_spec,
name='src',
readonly=True)
tgt_stack = create_stack_from_spec(tgt_layer_spec,
name='tgt',
readonly=True,
reference=src_stack)
reference_layer = None
if reference_key in src_stack.layers:
reference_layer = src_stack.layers[reference_key]
dst_layer = create_layer_from_spec(dst_layer_spec,
allowed_types=['field'],
default_type='field',
readonly=False,
caller_name='dst_layer',
reference=reference_layer,
overwrite=True)
bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip)
if crop is None:
crop = pad
compute_field_job = ComputeFieldJob(
src_stack=src_stack,
tgt_stack=tgt_stack,
dst_layer=dst_layer,
chunk_xy=chunk_xy,
chunk_z=chunk_z,
blend_xy=blend_xy,
processor_spec=processor_spec,
pad=pad,
crop=crop,
bcube=bcube,
tgt_z_offset=tgt_z_offset,
suffix=suffix,
processor_mip=processor_mip,
clear_nontissue_field=clear_nontissue_field)
# create scheduler and execute the job
scheduler.register_job(compute_field_job,
job_name="Compute field {}, tgt z offset {}".format(
bcube, tgt_z_offset))
scheduler.execute_until_completion()
def invert_field(ctx, src_layer_spec, dst_layer_spec, pad, crop, chunk_xy,
start_coord, mip, end_coord, coord_mip, blend_xy, chunk_z,
force_chunk_xy):
scheduler = ctx.obj['scheduler']
if force_chunk_xy:
force_chunk_xy = chunk_xy
else:
force_chunk_xy = None
corgie_logger.debug("Setting up layers...")
src_layer = create_layer_from_spec(src_layer_spec,
allowed_types=['field'],
default_type='field',
readonly=True,
caller_name='src_layer')
dst_layer = create_layer_from_spec(dst_layer_spec,
allowed_types=['field'],
default_type='field',
readonly=False,
caller_name='dst_layer',
reference=src_layer,
overwrite=True,
force_chunk_xy=force_chunk_xy)
bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip)
if crop is None:
crop = pad
invert_field_job = InvertFieldJob(src_layer=src_layer,
dst_layer=dst_layer,
chunk_xy=chunk_xy,
chunk_z=chunk_z,
blend_xy=blend_xy,
pad=pad,
mip=mip,
crop=crop,
bcube=bcube)
# create scheduler and execute the job
scheduler.register_job(invert_field_job,
job_name="Invert Field {}".format(bcube))
scheduler.execute_until_completion()
def downsample_by_spec(ctx, src_layer_spec, spec_path, dst_layer_spec,
mip_start, mip_end, chunk_xy, chunk_z, mips_per_task,
start_coord, end_coord, coord_mip):
scheduler = ctx.obj['scheduler']
corgie_logger.debug("Setting up Source and Destination layers...")
src_layer = create_layer_from_spec(src_layer_spec,
caller_name='src layer',
readonly=True)
with open(spec_path, 'r') as f:
spec = set(json.load(f))
if dst_layer_spec is None:
corgie_logger.info(
"Destination layer not specified. Using Source layer "
"as Destination.")
dst_layer = src_layer
dst_layer.readonly = False
else:
dst_layer = create_layer_from_spec(dst_layer_spec,
caller_name='dst_layer layer',
readonly=False,
reference=src_layer,
chunk_z=chunk_z,
overwrite=True)
bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip)
for z in range(*bcube.z_range()):
if z in spec:
job_bcube = bcube.reset_coords(zs=z, ze=z + 1, in_place=False)
downsample_job = DownsampleJob(src_layer=src_layer,
dst_layer=dst_layer,
mip_start=mip_start,
mip_end=mip_end,
bcube=job_bcube,
chunk_xy=chunk_xy,
chunk_z=chunk_z,
mips_per_task=mips_per_task)
# create scheduler and execute the job
scheduler.register_job(downsample_job,
job_name=f"Downsample {job_bcube}")
scheduler.execute_until_completion()
result_report = f"Downsampled {src_layer} from {mip_start} to {mip_end}. Result in {dst_layer}"
corgie_logger.info(result_report)
def combine_masks(
ctx,
src_layer_spec,
dst_layer_spec,
exp,
chunk_xy,
chunk_z,
force_chunk_xy,
force_chunk_z,
start_coord,
end_coord,
coord_mip,
mip,
pad,
):
scheduler = ctx.obj["scheduler"]
if not force_chunk_xy:
force_chunk_xy = chunk_xy
corgie_logger.debug("Setting up layers...")
src_stack = create_stack_from_spec(src_layer_spec, name="src", readonly=True)
reference_layer = src_stack.reference_layer
dst_layer = create_layer_from_spec(
dst_layer_spec,
allowed_types=["mask"],
default_type="mask",
readonly=False,
caller_name="dst_layer",
reference=reference_layer,
force_chunk_xy=force_chunk_xy,
force_chunk_z=force_chunk_z,
overwrite=True,
)
bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip)
combine_masks_job = CombineMasksJob(
src_stack=src_stack,
exp=json.loads(exp),
dst_layer=dst_layer,
mip=mip,
bcube=bcube,
pad=pad,
chunk_xy=chunk_xy,
chunk_z=chunk_z,
)
# create scheduler and execute the job
scheduler.register_job(combine_masks_job, job_name="Combine Masks {}".format(bcube))
scheduler.execute_until_completion()
result_report = f"Results in {str(dst_layer)}"
corgie_logger.info(result_report)
def apply_processor_by_spec(ctx, src_layer_spec, spec_path, dst_layer_spec,
processor_spec, pad, crop, chunk_xy, start_coord,
processor_mip, end_coord, coord_mip, blend_xy,
chunk_z, reference_key):
scheduler = ctx.obj['scheduler']
corgie_logger.debug("Setting up layers...")
src_stack = create_stack_from_spec(src_layer_spec,
name='src',
readonly=True)
with open(spec_path, 'r') as f:
spec = set(json.load(f))
reference_layer = None
if reference_key in src_stack.layers:
reference_layer = src_stack.layers[reference_key]
dst_layer = create_layer_from_spec(dst_layer_spec,
allowed_types=['img', 'mask'],
default_type='img',
readonly=False,
caller_name='dst_layer',
reference=reference_layer,
overwrite=True)
bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip)
if crop is None:
crop = pad
for z in range(*bcube.z_range()):
if z in spec:
job_bcube = bcube.reset_coords(zs=z, ze=z + 1, in_place=False)
apply_processor_job = ApplyProcessorJob(
src_stack=src_stack,
dst_layer=dst_layer,
chunk_xy=chunk_xy,
chunk_z=chunk_z,
blend_xy=blend_xy,
processor_spec=processor_spec,
pad=pad,
crop=crop,
bcube=job_bcube,
processor_mip=processor_mip)
# create scheduler and execute the job
scheduler.register_job(
apply_processor_job,
job_name="Apply Processor {}".format(job_bcube))
scheduler.execute_until_completion()
def spec_to_layer_dict_overwrite(layer_specs, reference_layer, default_type):
"""Create dict of layers from a corgie spec indexed by unique id
These layers will be of type overwrite
Args:
layer_specs (dict): layer specs indexed by unique id
reference_layer (layer)
default_type (str): e.g. img, field
"""
layers = {}
for k, s in layer_specs.items():
layers[k] = create_layer_from_spec(json.dumps(s),
default_type=default_type,
readonly=False,
reference=reference_layer,
overwrite=True)
return layers
def apply_processor(ctx, src_layer_spec, dst_layer_spec, processor_spec, pad,
crop, chunk_xy, start_coord, processor_mip, end_coord,
coord_mip, blend_xy, chunk_z, reference_key):
scheduler = ctx.obj['scheduler']
corgie_logger.debug("Setting up layers...")
src_stack = create_stack_from_spec(src_layer_spec,
name='src',
readonly=True)
reference_layer = None
if reference_key in src_stack.layers:
reference_layer = src_stack.layers[reference_key]
dst_layer = create_layer_from_spec(dst_layer_spec,
allowed_types=['img'],
default_type='img',
readonly=False,
caller_name='dst_layer',
reference=reference_layer,
overwrite=True)
bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip)
if crop is None:
crop = pad
apply_processor_job = ApplyProcessorJob(src_stack=src_stack,
dst_layer=dst_layer,
chunk_xy=chunk_xy,
chunk_z=chunk_z,
blend_xy=blend_xy,
processor_spec=processor_spec,
pad=pad,
crop=crop,
bcube=bcube,
processor_mip=processor_mip)
# create scheduler and execute the job
scheduler.register_job(apply_processor_job,
job_name="Apply Processor {}".format(bcube))
scheduler.execute_until_completion()
def compare_sections(
ctx,
src_layer_spec,
tgt_layer_spec,
dst_layer_spec,
suffix,
processor_spec,
pad,
chunk_xy,
force_chunk_xy,
start_coord,
mip,
dst_mip,
end_coord,
coord_mip,
tgt_z_offset,
reference_key,
):
if suffix is None:
suffix = ""
else:
suffix = f"_{suffix}"
if not force_chunk_xy:
force_chunk_xy = chunk_xy
if not dst_mip:
dst_mip = mip
scheduler = ctx.obj["scheduler"]
corgie_logger.debug("Setting up layers...")
src_stack = create_stack_from_spec(src_layer_spec,
name="src",
readonly=True)
tgt_stack = create_stack_from_spec(tgt_layer_spec,
name="tgt",
readonly=True,
reference=src_stack)
reference_layer = None
if reference_key in src_stack.layers:
reference_layer = src_stack.layers[reference_key]
dst_layer = create_layer_from_spec(
dst_layer_spec,
allowed_types=["img", "mask"],
default_type="field",
readonly=False,
caller_name="dst_layer",
reference=reference_layer,
force_chunk_xy=force_chunk_xy,
overwrite=True,
)
bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip)
compare_job = CompareSectionsJob(
src_stack=src_stack,
tgt_stack=tgt_stack,
dst_layer=dst_layer,
chunk_xy=chunk_xy,
processor_spec=processor_spec,
pad=pad,
bcube=bcube,
tgt_z_offset=tgt_z_offset,
suffix=suffix,
mip=mip,
dst_mip=dst_mip,
)
# create scheduler and execute the job
scheduler.register_job(
compare_job,
job_name="Compare Job {}, tgt z offset {}".format(bcube, tgt_z_offset),
)
scheduler.execute_until_completion()
def compute_stats_fn(ctx, src_layer_spec, dst_folder, suffix, mip, chunk_xy,
chunk_z, start_coord, end_coord, coord_mip):
if chunk_z != 1:
raise NotImplemented("Compute Statistics command currently only \
supports per-section statistics.")
scheduler = ctx.obj['scheduler']
src_layer = create_layer_from_spec(src_layer_spec,
caller_name='src layer',
readonly=True)
mask_layers = [
create_layer_from_spec(mask_spec,
caller_name='src mask',
readonly=True,
allowed_types=['mask'],
default_type='mask')
for mask_spec in mask_layers_spec
]
if suffix is None:
suffix = ''
else:
suffix = '_' + suffix
src_stack = create_stack_from_spec(src_layer_spec,
name='src',
readonly=True)
bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip)
mask_layers = src_stack.get_layers_of_type(["mask"])
non_mask_layers = src_stack.get_layers_of_type(["img", "field"])
for l in non_mask_layers:
mean_layer = src_layer.get_sublayer(
name=f"mean{suffix}",
path=os.path.join(dst_dir, f"mean{suffix}"),
layer_type="section_value",
)
var_layer = src_layer.get_sublayer(path=os.path.join(
dst_dir, f"var{suffix}"),
layer_type="section_value")
compute_stats_job = ComputeStatsJob(src_layer=l,
mask_layers=mask_layers,
mean_layer=mean_layer,
var_layer=var_layer,
bcube=bcube,
mip=mip,
chunk_xy=chunk_xy,
chunk_z=chunk_z)
# create scheduler and execute the job
scheduler.register_job(
compute_stats_job,
job_name=f"Compute Stats. Layer: {l}, Bcube: {bcube}")
scheduler.execute_until_completion()
def compute_field(
ctx,
src_layer_spec,
tgt_layer_spec,
dst_layer_spec,
suffix,
processor_spec,
pad,
crop,
chunk_xy,
start_coord,
mip,
end_coord,
coord_mip,
tgt_z_offset,
chunk_z,
reference_key,
):
scheduler = ctx.obj["scheduler"]
corgie_logger.debug("Setting up layers...")
src_stack = create_stack_from_spec(src_layer_spec, name="src", readonly=True)
tgt_stack = create_stack_from_spec(
tgt_layer_spec, name="tgt", readonly=True, reference=src_stack
)
reference_layer = None
if reference_key in src_stack.layers:
reference_layer = src_stack.layers[reference_key]
dst_layer = create_layer_from_spec(
dst_layer_spec,
allowed_types=["field"],
default_type="field",
readonly=False,
caller_name="dst_layer",
reference=reference_layer,
)
bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip)
if crop is None:
crop = pad
compute_field_job = ComputeFieldJob(
src_stack=src_stack,
tgt_stack=tgt_stack,
dst_layer=dst_layer,
chunk_xy=chunk_xy,
chunk_z=chunk_z,
processor_spec=processor_spec,
pad=pad,
crop=crop,
bcube=bcube,
tgt_z_offset=tgt_z_offset,
suffix=suffix,
mip=mip,
)
# create scheduler and execute the job
scheduler.register_job(
align_block_job,
job_name="Compute field {}, tgt z offset {}".format(bcube, tgt_z_offset),
)
scheduler.execute_until_completion()
def apply_processor(
ctx,
src_layer_spec,
dst_layer_spec,
spec_path,
processor_spec,
pad,
crop,
chunk_xy,
start_coord,
force_chunk_xy,
processor_mip,
end_coord,
coord_mip,
blend_xy,
chunk_z,
reference_key,
):
scheduler = ctx.obj["scheduler"]
corgie_logger.debug("Setting up layers...")
src_stack = create_stack_from_spec(src_layer_spec, name="src", readonly=True)
reference_layer = None
if reference_key in src_stack.layers:
reference_layer = src_stack.layers[reference_key]
if force_chunk_xy is None:
force_chunk_xy = chunk_xy
dst_layer = create_layer_from_spec(
dst_layer_spec,
allowed_types=["img", "mask", "section_value", "field"],
default_type="img",
readonly=False,
caller_name="dst_layer",
force_chunk_xy=force_chunk_xy,
reference=reference_layer,
overwrite=True,
)
bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip)
if crop is None:
crop = pad
if spec_path:
with open(spec_path, "r") as f:
spec = json.load(f)
for z in spec:
job_bcube = bcube.reset_coords(zs=z, ze=z + 1, in_place=False)
apply_processor_job = ApplyProcessorJob(
src_stack=src_stack,
dst_layer=dst_layer,
chunk_xy=chunk_xy,
chunk_z=chunk_z,
blend_xy=blend_xy,
processor_spec=processor_spec,
pad=pad,
crop=crop,
bcube=job_bcube,
processor_mip=processor_mip,
)
# create scheduler and execute the job
scheduler.register_job(
apply_processor_job, job_name="Apply Processor {}".format(job_bcube)
)
else:
apply_processor_job = ApplyProcessorJob(
src_stack=src_stack,
dst_layer=dst_layer,
chunk_xy=chunk_xy,
chunk_z=chunk_z,
blend_xy=blend_xy,
processor_spec=processor_spec,
pad=pad,
crop=crop,
bcube=bcube,
processor_mip=processor_mip,
)
# create scheduler and execute the job
scheduler.register_job(
apply_processor_job, job_name="Apply Processor {}".format(bcube)
)
scheduler.execute_until_completion()
