def compute_field(ctx, src_layer_spec, tgt_layer_spec, dst_layer_spec, suffix, processor_spec, pad, crop, chunk_xy, start_coord, processor_mip, end_coord, coord_mip, blend_xy, tgt_z_offset, chunk_z, reference_key, clear_nontissue_field): if suffix is None: suffix = '' else: suffix = f"_{suffix}" scheduler = ctx.obj['scheduler'] corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name='src', readonly=True) tgt_stack = create_stack_from_spec(tgt_layer_spec, name='tgt', readonly=True, reference=src_stack) reference_layer = None if reference_key in src_stack.layers: reference_layer = src_stack.layers[reference_key] dst_layer = create_layer_from_spec(dst_layer_spec, allowed_types=['field'], default_type='field', readonly=False, caller_name='dst_layer', reference=reference_layer, overwrite=True) bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) if crop is None: crop = pad compute_field_job = ComputeFieldJob( src_stack=src_stack, tgt_stack=tgt_stack, dst_layer=dst_layer, chunk_xy=chunk_xy, chunk_z=chunk_z, blend_xy=blend_xy, processor_spec=processor_spec, pad=pad, crop=crop, bcube=bcube, tgt_z_offset=tgt_z_offset, suffix=suffix, processor_mip=processor_mip, clear_nontissue_field=clear_nontissue_field) # create scheduler and execute the job scheduler.register_job(compute_field_job, job_name="Compute field {}, tgt z offset {}".format( bcube, tgt_z_offset)) scheduler.execute_until_completion()
def create_skeletons( ctx, seg_layer_spec, dst_folder, timestamp, mip, teasar_scale, teasar_const, ids, ids_filepath, tick_threshold, chunk_xy, chunk_z, single_merge_mode, start_coord, end_coord, coord_mip, ): scheduler = ctx.obj["scheduler"] corgie_logger.debug("Setting up layers...") seg_stack = create_stack_from_spec(seg_layer_spec, name="src", readonly=True) bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) object_ids = ids if ids_filepath is not None: object_ids = [] with open(ids_filepath, "r") as f: line = f.readline() while line: object_ids.append(int(line)) line = f.readline() if object_ids is None or len(object_ids) == 0: raise ValueError("Must specify ids to skeletonize") object_ids = list(object_ids) teasar_params = {"scale": teasar_scale, "const": teasar_const} seg_layer = seg_stack.get_layers_of_type("segmentation")[0] skeleton_job = SkeletonJob( seg_layer=seg_layer, dst_path=dst_folder, timestamp=timestamp, bcube=bcube, chunk_xy=chunk_xy, chunk_z=chunk_z, mip=mip, teasar_params=teasar_params, object_ids=object_ids, tick_threshold=tick_threshold, single_merge_mode=single_merge_mode, ) scheduler.register_job(skeleton_job, job_name="Skeletonize {}".format(bcube)) scheduler.execute_until_completion() result_report = f"Skeletonized {str(seg_layer)}. " corgie_logger.info(result_report)
def transform_skeletons( ctx, vector_field_spec, src_folder, dst_folder, field_mip, ids, ids_filepath, task_vertex_size, calculate_skeleton_lengths, mip0_field, ): scheduler = ctx.obj["scheduler"] corgie_logger.debug("Setting up layers...") vf_stack = create_stack_from_spec(vector_field_spec, name="src", readonly=True) skeleton_ids = ids if ids_filepath is not None: skeleton_ids = [] with open(ids_filepath, "r") as f: line = f.readline() while line: skeleton_ids.append(int(line)) line = f.readline() if len(skeleton_ids) == 0: skeleton_ids = None else: skeleton_ids = list(skeleton_ids) skeleton_length_file = None if calculate_skeleton_lengths: import time if not os.path.exists("skeleton_lengths"): os.makedirs("skeleton_lengths") skeleton_length_file = f"skeleton_lengths/skeleton_lengths_{int(time.time())}" vf_layer = vf_stack.get_layers_of_type("field")[0] transform_skeletons_job = TransformSkeletonsJob( vector_field_layer=vf_layer, src_path=src_folder, dst_path=dst_folder, field_mip=field_mip, skeleton_ids=skeleton_ids, task_vertex_size=task_vertex_size, skeleton_length_file=skeleton_length_file, mip0_field=mip0_field, ) scheduler.register_job( transform_skeletons_job, job_name="Transforming skeletons in {}".format(src_folder), ) scheduler.execute_until_completion() result_report = f"Transformed skeletons stored at {dst_folder}. " corgie_logger.info(result_report)
def multi_section_compare( ctx, src_layer_spec, dst_folder, chunk_xy, pad, crop, force_chunk_xy, z_offsets, processor_spec, processor_mip, start_coord, end_coord, coord_mip, ): scheduler = ctx.obj["scheduler"] if crop is None: crop = pad bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name="src", readonly=True) if force_chunk_xy is None: force_chunk_xy = chunk_xy dst_stack = stack.Stack(name="dst", folder=dst_folder) dst_stack.reference_layer = src_stack.get_layers()[0] for z_offset in z_offsets: dst_stack.create_sublayer( name=z_offset, layer_type="img", dtype="float32", force_chunk_xy=force_chunk_xy, overwrite=True, ) multi_section_compare_job = MultiSectionCompareJob( src_stack=src_stack, dst_stack=dst_stack, chunk_xy=chunk_xy, bcube=bcube, pad=pad, processor_spec=processor_spec, mip=processor_mip[0], dst_mip=processor_mip[0], ) scheduler.register_job( multi_section_compare_job, job_name=f"MultiSectionCompare {bcube}", ) scheduler.execute_until_completion() corgie_logger.debug("Done!")
def fill_nearest( ctx, src_layer_spec, dst_folder, chunk_xy, start_coord, end_coord, coord_mip, suffix, mip, radius, force_chunk_z=1, ): scheduler = ctx.obj["scheduler"] if suffix is None: suffix = "_seethrough" else: suffix = f"_{suffix}" crop, pad = 0, 0 corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name="src", readonly=True) src_stack.folder = dst_folder dst_stack = stack.create_stack_from_reference( reference_stack=src_stack, folder=dst_folder, name="dst", types=["img", "mask"], readonly=False, suffix=suffix, overwrite=True, force_chunk_z=force_chunk_z, ) bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) fill_nearest_job = FillNearestJob( src_stack=src_stack, dst_stack=dst_stack, bcube=bcube, radius=radius, mip=mip, chunk_xy=chunk_xy, ) # create scheduler and execute the job scheduler.register_job(fill_nearest_job, job_name="Fill Nearest Block {}".format(bcube)) scheduler.execute_until_completion() result_report = ( f"Rendered layers {[str(l) for l in src_stack.get_layers_of_type('img')]}. " f"Results in {[str(l) for l in dst_stack.get_layers_of_type('img')]}") corgie_logger.info(result_report)
def combine_masks( ctx, src_layer_spec, dst_layer_spec, exp, chunk_xy, chunk_z, force_chunk_xy, force_chunk_z, start_coord, end_coord, coord_mip, mip, pad, ): scheduler = ctx.obj["scheduler"] if not force_chunk_xy: force_chunk_xy = chunk_xy corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name="src", readonly=True) reference_layer = src_stack.reference_layer dst_layer = create_layer_from_spec( dst_layer_spec, allowed_types=["mask"], default_type="mask", readonly=False, caller_name="dst_layer", reference=reference_layer, force_chunk_xy=force_chunk_xy, force_chunk_z=force_chunk_z, overwrite=True, ) bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) combine_masks_job = CombineMasksJob( src_stack=src_stack, exp=json.loads(exp), dst_layer=dst_layer, mip=mip, bcube=bcube, pad=pad, chunk_xy=chunk_xy, chunk_z=chunk_z, ) # create scheduler and execute the job scheduler.register_job(combine_masks_job, job_name="Combine Masks {}".format(bcube)) scheduler.execute_until_completion() result_report = f"Results in {str(dst_layer)}" corgie_logger.info(result_report)
def render(ctx, src_layer_spec, dst_folder, pad, render_masks, blackout_masks, seethrough, chunk_xy, chunk_z, start_coord, end_coord, mips, coord_mip, force_chunk_xy, force_chunk_z, suffix): scheduler = ctx.obj['scheduler'] if suffix is None: suffix = '_rendered' else: suffix = f"_{suffix}" corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name='src', readonly=True) if force_chunk_xy: force_chunk_xy = chunk_xy else: force_chunk_xy = None if force_chunk_z: force_chunk_z = chunk_z else: force_chunk_z = None dst_stack = stack.create_stack_from_reference( reference_stack=src_stack, folder=dst_folder, name="dst", types=["img", "mask"], force_chunk_xy=force_chunk_xy, force_chunk_z=force_chunk_z, suffix=suffix, overwrite=True) bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) render_job = RenderJob(src_stack=src_stack, dst_stack=dst_stack, mips=mips, pad=pad, bcube=bcube, chunk_xy=chunk_xy, chunk_z=chunk_z, render_masks=render_masks, blackout_masks=blackout_masks, seethrough=seethrough) # create scheduler and execute the job scheduler.register_job(render_job, job_name="Render {}".format(bcube)) scheduler.execute_until_completion()
def apply_processor_by_spec(ctx, src_layer_spec, spec_path, dst_layer_spec, processor_spec, pad, crop, chunk_xy, start_coord, processor_mip, end_coord, coord_mip, blend_xy, chunk_z, reference_key): scheduler = ctx.obj['scheduler'] corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name='src', readonly=True) with open(spec_path, 'r') as f: spec = set(json.load(f)) reference_layer = None if reference_key in src_stack.layers: reference_layer = src_stack.layers[reference_key] dst_layer = create_layer_from_spec(dst_layer_spec, allowed_types=['img', 'mask'], default_type='img', readonly=False, caller_name='dst_layer', reference=reference_layer, overwrite=True) bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) if crop is None: crop = pad for z in range(*bcube.z_range()): if z in spec: job_bcube = bcube.reset_coords(zs=z, ze=z + 1, in_place=False) apply_processor_job = ApplyProcessorJob( src_stack=src_stack, dst_layer=dst_layer, chunk_xy=chunk_xy, chunk_z=chunk_z, blend_xy=blend_xy, processor_spec=processor_spec, pad=pad, crop=crop, bcube=job_bcube, processor_mip=processor_mip) # create scheduler and execute the job scheduler.register_job( apply_processor_job, job_name="Apply Processor {}".format(job_bcube)) scheduler.execute_until_completion()
def copy(ctx, src_layer_spec, dst_folder, copy_masks, blackout_masks, chunk_xy, chunk_z, start_coord, end_coord, coord_mip, mip, suffix, force_chunk_xy, force_chunk_z): scheduler = ctx.obj['scheduler'] if suffix is None: suffix = '' else: suffix = f"_{suffix}" corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name='src', readonly=True) if force_chunk_xy: force_chunk_xy = chunk_xy else: force_chunk_xy = None if force_chunk_z: force_chunk_z = chunk_z else: force_chunk_z = None dst_stack = stack.create_stack_from_reference( reference_stack=src_stack, folder=dst_folder, name="dst", types=["img", "mask"], readonly=False, suffix=suffix, force_chunk_xy=force_chunk_xy, force_chunk_z=force_chunk_z, overwrite=True) bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) copy_job = CopyJob(src_stack=src_stack, dst_stack=dst_stack, mip=mip, bcube=bcube, chunk_xy=chunk_xy, chunk_z=chunk_z, copy_masks=copy_masks, blackout_masks=blackout_masks) # create scheduler and execute the job scheduler.register_job(copy_job, job_name="Copy {}".format(bcube)) scheduler.execute_until_completion()
def apply_processor(ctx, src_layer_spec, dst_layer_spec, processor_spec, pad, crop, chunk_xy, start_coord, processor_mip, end_coord, coord_mip, blend_xy, chunk_z, reference_key): scheduler = ctx.obj['scheduler'] corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name='src', readonly=True) reference_layer = None if reference_key in src_stack.layers: reference_layer = src_stack.layers[reference_key] dst_layer = create_layer_from_spec(dst_layer_spec, allowed_types=['img'], default_type='img', readonly=False, caller_name='dst_layer', reference=reference_layer, overwrite=True) bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) if crop is None: crop = pad apply_processor_job = ApplyProcessorJob(src_stack=src_stack, dst_layer=dst_layer, chunk_xy=chunk_xy, chunk_z=chunk_z, blend_xy=blend_xy, processor_spec=processor_spec, pad=pad, crop=crop, bcube=bcube, processor_mip=processor_mip) # create scheduler and execute the job scheduler.register_job(apply_processor_job, job_name="Apply Processor {}".format(bcube)) scheduler.execute_until_completion()
def normalize_by_spec(ctx, src_layer_spec, spec_path, dst_folder, stats_mip, mip_start, mip_end, chunk_xy, chunk_z, start_coord, end_coord, coord_mip, suffix, recompute_stats, mask_value): if chunk_z != 1: raise NotImplemented("Compute Statistics command currently only \ supports per-section statistics.") result_report = "" scheduler = ctx.obj['scheduler'] if suffix is None: suffix = '_norm' else: suffix = f"_{suffix}" if stats_mip is None: stats_mip = mip_end with open(spec_path, 'r') as f: spec = set(json.load(f)) corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name='src', readonly=True) # dst_stack = stack.create_stack_from_reference(reference_stack=src_stack, # folder=dst_folder, name="dst", types=["img"], readonly=False, # suffix=suffix, overwrite=True) bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) img_layers = src_stack.get_layers_of_type("img") mask_layers = src_stack.get_layers_of_type("mask") field_layers = src_stack.get_layers_of_type("field") assert len(field_layers) == 0 for l in img_layers: mean_layer = l.get_sublayer( name=f"mean{suffix}", path=os.path.join(dst_folder, f"mean{suffix}"), layer_type="section_value", ) var_layer = l.get_sublayer(name=f"var{suffix}", path=os.path.join(dst_folder, f"var{suffix}"), layer_type="section_value") if recompute_stats: for z in range(*bcube.z_range()): if z in spec: job_bcube = bcube.reset_coords(zs=z, ze=z + 1, in_place=False) compute_stats_job = ComputeStatsJob( src_layer=l, mask_layers=mask_layers, mean_layer=mean_layer, var_layer=var_layer, bcube=job_bcube, mip=stats_mip, chunk_xy=chunk_xy, chunk_z=chunk_z) # create scheduler and execute the job scheduler.register_job( compute_stats_job, job_name= f"Compute Stats. Layer: {l}, Bcube: {job_bcube}") scheduler.execute_until_completion() dst_layer = l.get_sublayer(name=f"{l.name}{suffix}", path=os.path.join(dst_folder, "img", f"{l.name}{suffix}"), layer_type=l.get_layer_type(), dtype='float32', overwrite=True) for z in range(*bcube.z_range()): if z in spec: job_bcube = bcube.reset_coords(zs=z, ze=z + 1, in_place=False) result_report += f"Normalized {l} -> {dst_layer}\n" for mip in range(mip_start, mip_end + 1): normalize_job = NormalizeJob(src_layer=l, mask_layers=mask_layers, dst_layer=deepcopy(dst_layer), mean_layer=mean_layer, var_layer=var_layer, stats_mip=stats_mip, mip=mip, bcube=job_bcube, chunk_xy=chunk_xy, chunk_z=chunk_z, mask_value=mask_value) # create scheduler and execute the job scheduler.register_job( normalize_job, job_name=f"Normalize {job_bcube}, MIP {mip}") scheduler.execute_until_completion() corgie_logger.info(result_report)
def merge_copy(ctx, src_layer_spec, dst_folder, spec_path, chunk_xy, start_coord, end_coord, coord_mip, mip, suffix): scheduler = ctx.obj['scheduler'] if suffix is None: suffix = '' else: suffix = f"_{suffix}" corgie_logger.debug("Setting up layers...") assert(len(src_layer_spec) % 2 == 0) src_stacks = {} for k in range(len(src_layer_spec) // 2): src_stack = create_stack_from_spec(src_layer_spec[2*k:2*k+2], name='src', readonly=True) name = src_stack.get_layers_of_type('img')[0].path src_stacks[name] = src_stack with open(spec_path, 'r') as f: spec = json.load(f) # if force_chunk_xy: # force_chunk_xy = chunk_xy # else: # force_chunk_xy = None # if force_chunk_z: # force_chunk_z = chunk_z # else: # force_chunk_z = None dst_stack = stack.create_stack_from_reference(reference_stack=list(src_stacks.values())[0], folder=dst_folder, name="dst", types=["img"], readonly=False, suffix=suffix, force_chunk_xy=None, force_chunk_z =None, overwrite=True) bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) for z in range(*bcube.z_range()): spec_z = str(z) if spec_z in spec.keys(): src_dict = spec[str(z)] job_bcube = bcube.reset_coords(zs=z, ze=z+1, in_place=False) src_stack = src_stacks[src_dict['cv_path']] z_list = src_dict['z_list'] copy_job = MergeCopyJob(src_stack=src_stack, dst_stack=dst_stack, mip=mip, bcube=job_bcube, chunk_xy=chunk_xy, z_list=z_list) # create scheduler and execute the job scheduler.register_job(copy_job, job_name="MergeCopy {}".format(job_bcube)) scheduler.execute_until_completion()
def align( ctx, src_layer_spec, dst_folder, render_pad, render_chunk_xy, processor_spec, pad, crop, processor_mip, chunk_xy, start_coord, end_coord, coord_mip, bad_starter_path, block_size, stitch_size, vote_dist, consensus_threshold, blur_sigma, kernel_size, blend_xy, force_chunk_xy, suffix, seethrough_spec, seethrough_limit, seethrough_spec_mip, decay_dist, blur_rate, restart_stage, restart_suffix, ): scheduler = ctx.obj["scheduler"] if suffix is None: suffix = "_aligned" else: suffix = f"_{suffix}" if (restart_suffix is None) or (restart_stage == 0): restart_suffix = suffix if crop is None: crop = pad corgie_logger.debug("Setting up layers...") # TODO: store stitching images in layer other than even & odd if vote_dist + stitch_size - 2 >= block_size: raise exceptions.CorgieException( "block_size too small for stitching + voting requirements (stitch_size + vote_dist)" ) corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name="src", readonly=True) src_stack.folder = dst_folder if force_chunk_xy is None: force_chunk_xy = chunk_xy dst_stack = stack.create_stack_from_reference( reference_stack=src_stack, folder=dst_folder, name="dst", types=["img", "mask"], readonly=False, suffix=restart_suffix, force_chunk_xy=force_chunk_xy, overwrite=True, ) even_stack = stack.create_stack_from_reference( reference_stack=src_stack, folder=os.path.join(dst_folder, "even"), name="even", types=["img", "mask"], readonly=False, suffix=suffix, force_chunk_xy=force_chunk_xy, overwrite=True, ) odd_stack = stack.create_stack_from_reference( reference_stack=src_stack, folder=os.path.join(dst_folder, "odd"), name="odd", types=["img", "mask"], readonly=False, suffix=suffix, force_chunk_xy=force_chunk_xy, overwrite=True, ) corgie_logger.debug("Done!") bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) corgie_logger.debug("Calculating blocks...") skip_list = [] if bad_starter_path is not None: with open(bad_starter_path) as f: line = f.readline() while line: skip_list.append(int(line)) line = f.readline() blocks = get_blocks( start=bcube.z_range()[0], stop=bcube.z_range()[1], block_size=block_size, block_overlap=1, skip_list=skip_list, src_stack=src_stack, even_stack=even_stack, odd_stack=odd_stack, ) stitch_blocks = [b.overlap(stitch_size) for b in blocks[1:]] corgie_logger.debug("All Blocks") for block, stitch_block in zip(blocks, [None] + stitch_blocks): corgie_logger.debug(block) corgie_logger.debug(f"Stitch {stitch_block}") corgie_logger.debug("\n") max_blur_mip = (math.ceil(math.log(decay_dist * blur_rate + 1, 2)) + processor_mip[-1]) corgie_logger.debug(f"Max blur mip for stitching field: {max_blur_mip}") # Set all field names, adjusting for restart suffix block_field_name = f"field{suffix}" stitch_estimated_suffix = f"_stitch_estimated{suffix}" stitch_estimated_name = f"field{stitch_estimated_suffix}" stitch_corrected_name = f"stitch_corrected{suffix}" stitch_corrected_field = None composed_name = f"composed{suffix}" if restart_stage <= 2: stitch_estimated_suffix = f"_stitch_estimated{restart_suffix}" stitch_estimated_name = f"field{stitch_estimated_suffix}" stitch_corrected_name = f"stitch_corrected{restart_suffix}" if restart_stage <= 3: composed_name = f"composed{restart_suffix}" render_method = helpers.PartialSpecification( f=RenderJob, pad=render_pad, chunk_xy=render_chunk_xy, chunk_z=1, render_masks=False, ) cf_method = helpers.PartialSpecification( f=ComputeFieldJob, pad=pad, crop=crop, processor_mip=processor_mip, processor_spec=processor_spec, chunk_xy=chunk_xy, blend_xy=blend_xy, chunk_z=1, ) if seethrough_spec != tuple(): assert seethrough_spec_mip is not None seethrough_method = helpers.PartialSpecification( f=SeethroughCompareJob, mip=seethrough_spec_mip, processor_spec=seethrough_spec, chunk_xy=chunk_xy, pad=pad, crop=pad, seethrough_limit=seethrough_limit, ) else: seethrough_method = None #restart_stage = 4 #import pdb; pdb.set_trace() if restart_stage == 0: corgie_logger.debug("Aligning blocks...") for block in blocks: block_bcube = block.get_bcube(bcube) # Use copies of src & dst so that aligning the stitching blocks # is not affected by these block fields. # Copying also allows local compute to not modify objects for other tasks align_block_job_forv = AlignBlockJob( src_stack=deepcopy(block.src_stack), dst_stack=deepcopy(block.dst_stack), bcube=block_bcube, render_method=render_method, cf_method=cf_method, vote_dist=vote_dist, seethrough_method=seethrough_method, suffix=suffix, copy_start=True, use_starters=True, backward=False, consensus_threshold=consensus_threshold, blur_sigma=blur_sigma, kernel_size=kernel_size, ) scheduler.register_job( align_block_job_forv, job_name=f"Forward Align {block} {block_bcube}", ) scheduler.execute_until_completion() corgie_logger.debug("Done!") if restart_stage <= 1: corgie_logger.debug("Aligning stitching blocks...") for stitch_block in stitch_blocks: block_bcube = stitch_block.get_bcube(bcube) # These blocks will have block-aligned images, but not # the block_fields that warped them. align_block_job_forv = AlignBlockJob( src_stack=deepcopy(stitch_block.src_stack), dst_stack=deepcopy(stitch_block.dst_stack), bcube=block_bcube, render_method=render_method, cf_method=cf_method, vote_dist=vote_dist, seethrough_method=seethrough_method, suffix=stitch_estimated_suffix, copy_start=False, use_starters=False, backward=False, consensus_threshold=consensus_threshold, blur_sigma=blur_sigma, kernel_size=kernel_size, ) scheduler.register_job( align_block_job_forv, job_name=f"Stitch Align {stitch_block} {block_bcube}", ) scheduler.execute_until_completion() corgie_logger.debug("Done!") # Add in the stitch_estimated fields that were just created above even_stack.create_sublayer( stitch_estimated_name, layer_type="field", overwrite=False, ) odd_stack.create_sublayer( stitch_estimated_name, layer_type="field", overwrite=False, ) if restart_stage <= 2: if stitch_size > 1: corgie_logger.debug("Voting over stitching blocks") stitch_corrected_field = dst_stack.create_sublayer( stitch_corrected_name, layer_type="field", overwrite=True) for stitch_block in stitch_blocks: stitch_estimated_field = stitch_block.dst_stack[ stitch_estimated_name] block_bcube = bcube.reset_coords( zs=stitch_block.start, ze=stitch_block.start + 1, in_place=False, ) z_offsets = [ z - block_bcube.z_range()[0] for z in range(stitch_block.start, stitch_block.stop) ] vote_stitch_job = VoteJob( input_fields=[stitch_estimated_field], output_field=stitch_corrected_field, chunk_xy=chunk_xy, bcube=block_bcube, z_offsets=z_offsets, mip=processor_mip[-1], consensus_threshold=consensus_threshold, blur_sigma=blur_sigma, kernel_size=kernel_size, ) scheduler.register_job( vote_stitch_job, job_name=f"Stitching Vote {stitch_block} {block_bcube}", ) scheduler.execute_until_completion() corgie_logger.debug("Done!") for stitch_block in stitch_blocks: block_bcube = bcube.reset_coords(zs=stitch_block.start, ze=stitch_block.start + 1, in_place=False) field_to_downsample = stitch_block.dst_stack[stitch_estimated_name] if stitch_corrected_field is not None: field_to_downsample = stitch_corrected_field # Hack for fafb field_info = field_to_downsample.get_info() for scale in field_info['scales']: scale['chunk_sizes'][-1][-1] = 1 scale['encoding'] = 'raw' field_to_downsample.cv.store_info(field_info) field_to_downsample.cv.fetch_info() downsample_field_job = DownsampleJob( src_layer=field_to_downsample, mip_start=processor_mip[-1], mip_end=max_blur_mip, bcube=block_bcube, chunk_xy= chunk_xy, # TODO: This probably needs to be modified at highest mips chunk_z=1, mips_per_task=2, ) scheduler.register_job( downsample_field_job, job_name=f"Downsample stitching field {block_bcube}", ) scheduler.execute_until_completion() corgie_logger.debug("Done!") # Add in the block-align fields even_stack.create_sublayer( block_field_name, layer_type="field", overwrite=False, ) odd_stack.create_sublayer( block_field_name, layer_type="field", overwrite=False, ) composed_field = dst_stack.create_sublayer(composed_name, layer_type="field", overwrite=True) if (restart_stage > 2) and (stitch_size > 1): stitch_corrected_field = dst_stack.create_sublayer( stitch_corrected_name, layer_type="field", overwrite=False) if restart_stage <= 3: corgie_logger.debug("Stitching blocks...") for block, stitch_block in zip(blocks[1:], stitch_blocks): block_bcube = block.broadcastable().get_bcube(bcube) block_list = block.get_neighbors(dist=decay_dist) corgie_logger.debug(f"src_block: {block}") corgie_logger.debug(f"influencing blocks: {block_list}") z_list = [b.stop for b in block_list] # stitch_corrected_field used if there is multi-section block overlap, # which requires voting to produce a corrected field. # If there is only single-section block overlap, then use # stitch_estimated_fields from each stitch_block if stitch_corrected_field is not None: stitching_fields = [stitch_corrected_field] else: # Order with furthest block first (convention of FieldSet). stitching_fields = [ stitch_block.dst_stack[stitch_estimated_name], stitch_block.src_stack[stitch_estimated_name], ] broadcast_job = BroadcastJob( block_field=block.dst_stack[block_field_name], stitching_fields=stitching_fields, output_field=composed_field, chunk_xy=chunk_xy, bcube=block_bcube, pad=pad, z_list=z_list, mip=processor_mip[-1], decay_dist=decay_dist, blur_rate=blur_rate, ) scheduler.register_job(broadcast_job, job_name=f"Broadcast {block} {block_bcube}") scheduler.execute_until_completion() corgie_logger.debug("Done!") if len(blocks) > 1: block_bcube = blocks[0].get_bcube(bcube) copy_job = CopyLayerJob( src_layer=even_stack[block_field_name], dst_layer=composed_field, mip=processor_mip[-1], bcube=block_bcube, chunk_xy=chunk_xy, chunk_z=1, ) scheduler.register_job( copy_job, job_name=f"Copy first block_field to composed_field location") scheduler.execute_until_completion() corgie_logger.debug("Done!") if restart_stage <= 4: if len(blocks) == 1: block_bcube = blocks[0].get_bcube(bcube) render_job = RenderJob( src_stack=src_stack, dst_stack=dst_stack, mips=processor_mip[-1], pad=pad, bcube=block_bcube, chunk_xy=chunk_xy, chunk_z=1, render_masks=True, blackout_masks=False, additional_fields=[even_stack[block_field_name]], ) scheduler.register_job( render_job, job_name=f"Render first block {block_bcube}") else: block_bcube = bcube.reset_coords(zs=blocks[0].start, ze=blocks[-1].stop, in_place=False) render_job = RenderJob( src_stack=src_stack, dst_stack=dst_stack, mips=processor_mip[-1], pad=pad, bcube=block_bcube, chunk_xy=chunk_xy, chunk_z=1, render_masks=True, blackout_masks=True, additional_fields=[composed_field], ) scheduler.register_job(render_job, job_name=f"Render all blocks {block_bcube}") scheduler.execute_until_completion() corgie_logger.debug("Done!") result_report = ( f"Aligned layers {[str(l) for l in src_stack.get_layers_of_type('img')]}. " f"Results in {[str(l) for l in dst_stack.get_layers_of_type('img')]}") corgie_logger.info(result_report)
def align_block( ctx, src_layer_spec, dst_folder, vote_dist, render_pad, render_chunk_xy, processor_spec, pad, crop, processor_mip, chunk_xy, start_coord, end_coord, coord_mip, blend_xy, force_chunk_xy, suffix, copy_start, use_starters, seethrough_spec, seethrough_limit, seethrough_spec_mip, mode, chunk_z=1, ): scheduler = ctx.obj["scheduler"] if suffix is None: suffix = "_aligned" else: suffix = f"_{suffix}" if crop is None: crop = pad corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name="src", readonly=True) src_stack.folder = dst_folder force_chunk_xy = chunk_xy if force_chunk_xy else None dst_stack = stack.create_stack_from_reference( reference_stack=src_stack, folder=dst_folder, name="dst", types=["img", "mask"], readonly=False, suffix=suffix, force_chunk_xy=force_chunk_xy, overwrite=True, ) render_method = helpers.PartialSpecification( f=RenderJob, pad=render_pad, chunk_xy=render_chunk_xy, chunk_z=1, render_masks=False, ) if seethrough_spec != tuple(): assert seethrough_spec_mip is not None seethrough_method = helpers.PartialSpecification( f=SeethroughCompareJob, mip=seethrough_spec_mip, processor_spec=seethrough_spec, chunk_xy=chunk_xy, pad=pad, crop=pad, seethrough_limit=seethrough_limit, ) else: seethrough_method = None cf_method = helpers.PartialSpecification( f=ComputeFieldJob, pad=pad, crop=crop, processor_mip=processor_mip, processor_spec=processor_spec, chunk_xy=chunk_xy, blend_xy=blend_xy, chunk_z=1, ) bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) if mode == "bidirectional": z_mid = (bcube.z_range()[1] + bcube.z_range()[0]) // 2 bcube_back = bcube.reset_coords(ze=z_mid, in_place=False) bcube_forv = bcube.reset_coords(zs=z_mid, in_place=False) align_block_job_back = AlignBlockJob( src_stack=src_stack, dst_stack=dst_stack, bcube=bcube_back, render_method=render_method, cf_method=cf_method, seethrough_method=seethrough_method, suffix=suffix, copy_start=copy_start, backward=True, vote_dist=vote_dist, use_starters=use_starters, ) scheduler.register_job( align_block_job_back, job_name="Backward Align Block {}".format(bcube), ) align_block_job_forv = AlignBlockJob( src_stack=src_stack, dst_stack=deepcopy(dst_stack), bcube=bcube_forv, render_method=render_method, cf_method=cf_method, seethrough_method=seethrough_method, suffix=suffix, copy_start=True, backward=False, vote_dist=vote_dist, use_starters=use_starters, ) scheduler.register_job( align_block_job_forv, job_name="Forward Align Block {}".format(bcube), ) else: align_block_job = AlignBlockJob( src_stack=src_stack, dst_stack=dst_stack, bcube=bcube, render_method=render_method, cf_method=cf_method, seethrough_method=seethrough_method, suffix=suffix, copy_start=copy_start, backward=mode == "backward", vote_dist=vote_dist, use_starters=use_starters, ) # create scheduler and execute the job scheduler.register_job(align_block_job, job_name="Align Block {}".format(bcube)) scheduler.execute_until_completion() result_report = ( f"Aligned layers {[str(l) for l in src_stack.get_layers_of_type('img')]}. " f"Results in {[str(l) for l in dst_stack.get_layers_of_type('img')]}" ) corgie_logger.info(result_report)
def vote( ctx, src_layer_spec, dst_folder, chunk_xy, mip, z_offsets, force_chunk_xy, consensus_threshold, blur_sigma, kernel_size, start_coord, end_coord, coord_mip, suffix, ): scheduler = ctx.obj["scheduler"] if suffix is None: suffix = "_voted" else: suffix = f"_{suffix}" bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name="src", readonly=True) if force_chunk_xy is None: force_chunk_xy = chunk_xy dst_stack = stack.create_stack_from_reference( reference_stack=src_stack, folder=dst_folder, name="dst", types=["field", "float_tensor"], readonly=False, suffix=suffix, force_chunk_xy=force_chunk_xy, overwrite=True, ) vote_weights = dst_stack.create_sublayer( name="vote_weights", layer_type="float_tensor", overwrite=True, num_channels=max(len(src_stack), len(z_offsets)), ) voted_field = dst_stack.create_sublayer( name="voted_field", layer_type="field", overwrite=True ) vote_stitch_job = VoteJob( input_fields=src_stack.get_layers(), output_field=voted_field, chunk_xy=chunk_xy, bcube=bcube, z_offsets=z_offsets, mip=mip, consensus_threshold=consensus_threshold, blur_sigma=blur_sigma, kernel_size=kernel_size, weights_layer=vote_weights, ) scheduler.register_job( vote_stitch_job, job_name=f"Vote {bcube}", ) scheduler.execute_until_completion() corgie_logger.debug("Done!")
def seethrough_block( ctx, src_layer_spec, dst_folder, chunk_xy, start_coord, end_coord, coord_mip, suffix, seethrough_spec, seethrough_limit, seethrough_spec_mip, force_chunk_z=1, ): scheduler = ctx.obj["scheduler"] if suffix is None: suffix = "_seethrough" else: suffix = f"_{suffix}" crop, pad = 0, 0 corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name="src", readonly=True) src_stack.folder = dst_folder dst_stack = stack.create_stack_from_reference( reference_stack=src_stack, folder=dst_folder, name="dst", types=["img", "mask"], readonly=False, suffix=suffix, overwrite=True, force_chunk_z=force_chunk_z, ) render_method = helpers.PartialSpecification( f=RenderJob, pad=pad, chunk_xy=chunk_xy, chunk_z=1, render_masks=False, ) seethrough_method = helpers.PartialSpecification( f=SeethroughCompareJob, mip=seethrough_spec_mip, processor_spec=seethrough_spec, chunk_xy=chunk_xy, pad=pad, crop=pad, seethrough_limit=seethrough_limit, ) bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) seethrough_block_job = SeethroughBlockJob( src_stack=src_stack, dst_stack=dst_stack, bcube=bcube, render_method=render_method, seethrough_method=seethrough_method, suffix=suffix, ) # create scheduler and execute the job scheduler.register_job(seethrough_block_job, job_name="Seethrough Block {}".format(bcube)) scheduler.execute_until_completion() result_report = ( f"Rendered layers {[str(l) for l in src_stack.get_layers_of_type('img')]}. " f"Results in {[str(l) for l in dst_stack.get_layers_of_type('img')]}") corgie_logger.info(result_report)
def align(ctx, src_layer_spec, tgt_layer_spec, dst_folder, render_pad, render_chunk_xy, processor_spec, pad, crop, processor_mip, chunk_xy, start_coord, end_coord, coord_mip, bad_starter_path, block_size, block_overlap, blend_xy, force_chunk_xy, suffix, copy_start, seethrough_spec, seethrough_spec_mip): scheduler = ctx.obj['scheduler'] if suffix is None: suffix = '_aligned' else: suffix = f"_{suffix}" if crop is None: crop = pad corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name='src', readonly=True) src_stack.folder = dst_folder tgt_stack = create_stack_from_spec(tgt_layer_spec, name='tgt', readonly=True, reference=src_stack) if force_chunk_xy is None: force_chunk_xy = chunk_xy dst_stack = stack.create_stack_from_reference( reference_stack=src_stack, folder=dst_folder, name="dst", types=["img", "mask"], readonly=False, suffix=suffix, force_chunk_xy=force_chunk_xy, overwrite=True) even_stack = stack.create_stack_from_reference( reference_stack=src_stack, folder=os.path.join(dst_folder, 'even'), name="even", types=["img", "mask"], readonly=False, suffix=suffix, force_chunk_xy=force_chunk_xy, overwrite=True) odd_stack = stack.create_stack_from_reference( reference_stack=src_stack, folder=os.path.join(dst_folder, 'odd'), name="odd", types=["img", "mask"], readonly=False, suffix=suffix, force_chunk_xy=force_chunk_xy, overwrite=True) corgie_logger.debug("Done!") bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) corgie_logger.debug("Calculating blocks...") # TODO: read in bad starter sections bad_starter_sections = [] blocks = [] z = bcube.z_range()[0] while z < bcube.z_range()[-1]: block_start = z block_end = z + block_size while block_end + block_overlap in bad_starter_sections and \ block_end + block_overlap < bcube.z_range()[-1]: block_end += 1 block = Block(block_start, block_end + block_overlap) blocks.append(block) z = block_end corgie_logger.debug("Done!") render_method = helpers.PartialSpecification( f=RenderJob, pad=render_pad, chunk_xy=render_chunk_xy, chunk_z=1, render_masks=False, ) cf_method = helpers.PartialSpecification(f=ComputeFieldJob, pad=pad, crop=crop, processor_mip=processor_mip, processor_spec=processor_spec, chunk_xy=chunk_xy, blend_xy=blend_xy, chunk_z=1) if seethrough_spec is not None: assert seethrough_spec_mip is not None seethrough_method = helpers.PartialSpecification( f=CompareSectionsJob, mip=seethrough_spec_mip, processor_spec=seethrough_spec, chunk_xy=chunk_xy, pad=pad, crop=pad, ) else: seethrough_method = None corgie_logger.debug("Aligning blocks...") for i in range(len(blocks)): block = blocks[i] block_bcube = bcube.copy() block_bcube.reset_coords(zs=block.z_start, ze=block.z_end) if i % 2 == 0: block_dst_stack = even_stack else: block_dst_stack = odd_stack align_block_job_forv = AlignBlockJob( src_stack=src_stack, tgt_stack=tgt_stack, dst_stack=block_dst_stack, bcube=block_bcube, render_method=render_method, cf_method=cf_method, seethrough_method=seethrough_method, suffix=suffix, copy_start=copy_start, backward=False) scheduler.register_job(align_block_job_forv, job_name=f"Forward Align {block} {block_bcube}") scheduler.execute_until_completion() corgie_logger.debug("Done!") corgie_logger.debug("Stitching blocks...") #TODO #stitch_blocks_job = StitchBlockJob( # blocks=blocks, # src_stack=src_stack, # dst_stack=dst_stack, # bcube=bcube, # suffix=suffix, # render_method=render_method, # cf_method=cf_method #) #scheduler.register_job(stitch_blocks_job, job_name=f"Stitch blocks {bcube}") #scheduler.execute_until_completion() corgie_logger.debug("Done!") result_report = f"Aligned layers {[str(l) for l in src_stack.get_layers_of_type('img')]}. " \ f"Results in {[str(l) for l in dst_stack.get_layers_of_type('img')]}" corgie_logger.info(result_report)
def apply_processor( ctx, src_layer_spec, dst_layer_spec, spec_path, processor_spec, pad, crop, chunk_xy, start_coord, force_chunk_xy, processor_mip, end_coord, coord_mip, blend_xy, chunk_z, reference_key, ): scheduler = ctx.obj["scheduler"] corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name="src", readonly=True) reference_layer = None if reference_key in src_stack.layers: reference_layer = src_stack.layers[reference_key] if force_chunk_xy is None: force_chunk_xy = chunk_xy dst_layer = create_layer_from_spec( dst_layer_spec, allowed_types=["img", "mask", "section_value", "field"], default_type="img", readonly=False, caller_name="dst_layer", force_chunk_xy=force_chunk_xy, reference=reference_layer, overwrite=True, ) bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) if crop is None: crop = pad if spec_path: with open(spec_path, "r") as f: spec = json.load(f) for z in spec: job_bcube = bcube.reset_coords(zs=z, ze=z + 1, in_place=False) apply_processor_job = ApplyProcessorJob( src_stack=src_stack, dst_layer=dst_layer, chunk_xy=chunk_xy, chunk_z=chunk_z, blend_xy=blend_xy, processor_spec=processor_spec, pad=pad, crop=crop, bcube=job_bcube, processor_mip=processor_mip, ) # create scheduler and execute the job scheduler.register_job( apply_processor_job, job_name="Apply Processor {}".format(job_bcube) ) else: apply_processor_job = ApplyProcessorJob( src_stack=src_stack, dst_layer=dst_layer, chunk_xy=chunk_xy, chunk_z=chunk_z, blend_xy=blend_xy, processor_spec=processor_spec, pad=pad, crop=crop, bcube=bcube, processor_mip=processor_mip, ) # create scheduler and execute the job scheduler.register_job( apply_processor_job, job_name="Apply Processor {}".format(bcube) ) scheduler.execute_until_completion()
def compute_field( ctx, src_layer_spec, tgt_layer_spec, dst_layer_spec, suffix, processor_spec, pad, crop, chunk_xy, start_coord, mip, end_coord, coord_mip, tgt_z_offset, chunk_z, reference_key, ): scheduler = ctx.obj["scheduler"] corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name="src", readonly=True) tgt_stack = create_stack_from_spec( tgt_layer_spec, name="tgt", readonly=True, reference=src_stack ) reference_layer = None if reference_key in src_stack.layers: reference_layer = src_stack.layers[reference_key] dst_layer = create_layer_from_spec( dst_layer_spec, allowed_types=["field"], default_type="field", readonly=False, caller_name="dst_layer", reference=reference_layer, ) bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) if crop is None: crop = pad compute_field_job = ComputeFieldJob( src_stack=src_stack, tgt_stack=tgt_stack, dst_layer=dst_layer, chunk_xy=chunk_xy, chunk_z=chunk_z, processor_spec=processor_spec, pad=pad, crop=crop, bcube=bcube, tgt_z_offset=tgt_z_offset, suffix=suffix, mip=mip, ) # create scheduler and execute the job scheduler.register_job( align_block_job, job_name="Compute field {}, tgt z offset {}".format(bcube, tgt_z_offset), ) scheduler.execute_until_completion()
def normalize( ctx, src_layer_spec, dst_folder, stats_mip, mip_start, mip_end, chunk_xy, chunk_z, force_chunk_xy, start_coord, end_coord, coord_mip, suffix, recompute_stats, mask_value, ): if chunk_z != 1: raise NotImplemented("Compute Statistics command currently only \ supports per-section statistics.") result_report = "" scheduler = ctx.obj["scheduler"] if suffix is None: suffix = "_norm" else: suffix = f"_{suffix}" if stats_mip is None: stats_mip = mip_end if not force_chunk_xy: force_chunk_xy = chunk_xy corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name="src", readonly=True) dst_stack = stack.create_stack_from_reference( reference_stack=src_stack, folder=dst_folder, name="dst", types=["img"], readonly=False, suffix=suffix, overwrite=True, ) bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) img_layers = src_stack.get_layers_of_type("img") mask_layers = src_stack.get_layers_of_type("mask") field_layers = src_stack.get_layers_of_type("field") assert len(field_layers) == 0 for l in img_layers: mean_layer = l.get_sublayer( name=f"mean_{l.name}{suffix}", path=os.path.join(dst_folder, f"mean_{l.name}{suffix}"), layer_type="section_value", ) var_layer = l.get_sublayer( name=f"var_{l.name}{suffix}", path=os.path.join(dst_folder, f"var_{l.name}{suffix}"), layer_type="section_value", ) if recompute_stats: compute_stats_job = ComputeStatsJob( src_layer=l, mask_layers=mask_layers, mean_layer=mean_layer, var_layer=var_layer, bcube=bcube, mip=stats_mip, chunk_xy=chunk_xy, chunk_z=chunk_z, ) # create scheduler and execute the job scheduler.register_job( compute_stats_job, job_name=f"Compute Stats. Layer: {l}, Bcube: {bcube}") scheduler.execute_until_completion() dst_layer = l.get_sublayer( name=f"{l.name}{suffix}", path=os.path.join(dst_folder, "img", f"{l.name}{suffix}"), layer_type=l.get_layer_type(), dtype="float32", force_chunk_xy=force_chunk_xy, overwrite=True, ) result_report += f"Normalized {l} -> {dst_layer}\n" for mip in range(mip_start, mip_end + 1): normalize_job = NormalizeJob( src_layer=l, mask_layers=mask_layers, dst_layer=dst_layer, mean_layer=mean_layer, var_layer=var_layer, stats_mip=stats_mip, mip=mip, bcube=bcube, chunk_xy=chunk_xy, chunk_z=chunk_z, mask_value=mask_value, ) # create scheduler and execute the job scheduler.register_job(normalize_job, job_name=f"Normalize {bcube}, MIP {mip}") scheduler.execute_until_completion() corgie_logger.info(result_report)
def compare_sections( ctx, src_layer_spec, tgt_layer_spec, dst_layer_spec, suffix, processor_spec, pad, chunk_xy, force_chunk_xy, start_coord, mip, dst_mip, end_coord, coord_mip, tgt_z_offset, reference_key, ): if suffix is None: suffix = "" else: suffix = f"_{suffix}" if not force_chunk_xy: force_chunk_xy = chunk_xy if not dst_mip: dst_mip = mip scheduler = ctx.obj["scheduler"] corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name="src", readonly=True) tgt_stack = create_stack_from_spec(tgt_layer_spec, name="tgt", readonly=True, reference=src_stack) reference_layer = None if reference_key in src_stack.layers: reference_layer = src_stack.layers[reference_key] dst_layer = create_layer_from_spec( dst_layer_spec, allowed_types=["img", "mask"], default_type="field", readonly=False, caller_name="dst_layer", reference=reference_layer, force_chunk_xy=force_chunk_xy, overwrite=True, ) bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) compare_job = CompareSectionsJob( src_stack=src_stack, tgt_stack=tgt_stack, dst_layer=dst_layer, chunk_xy=chunk_xy, processor_spec=processor_spec, pad=pad, bcube=bcube, tgt_z_offset=tgt_z_offset, suffix=suffix, mip=mip, dst_mip=dst_mip, ) # create scheduler and execute the job scheduler.register_job( compare_job, job_name="Compare Job {}, tgt z offset {}".format(bcube, tgt_z_offset), ) scheduler.execute_until_completion()
def align_block(ctx, src_layer_spec, tgt_layer_spec, dst_folder, render_pad, render_chunk_xy, processor_spec, pad, crop, processor_mip, chunk_xy, start_coord, end_coord, coord_mip, suffix, copy_start, mode, chunk_z=1): scheduler = ctx.obj['scheduler'] if suffix is None: suffix = '_aligned' else: suffix = f"_{suffix}" if crop is None: crop = pad corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name='src', readonly=True) tgt_stack = create_stack_from_spec(tgt_layer_spec, name='tgt', readonly=True, reference=src_stack) dst_stack = stack.create_stack_from_reference(reference_stack=src_stack, folder=dst_folder, name="dst", types=["img", "mask"], readonly=False, suffix=suffix) render_method = helpers.PartialSpecification( f=RenderJob, pad=render_pad, chunk_xy=render_chunk_xy, chunk_z=1, blackout_masks=False, render_masks=True, mip=min(processor_mip) ) cf_method = helpers.PartialSpecification( f=ComputeFieldJob, pad=pad, crop=crop, processor_mip=processor_mip, processor_spec=processor_spec, chunk_xy=chunk_xy, chunk_z=1 ) bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) if mode == 'bidirectional': z_mid = (bcube.z_range()[1] + bcube.z_range()[0]) // 2 bcube_back = bcube.reset_coords(ze=z_mid, in_place=False) bcube_forv = bcube.reset_coords(zs=z_mid, in_place=False) align_block_job_back = AlignBlockJob(src_stack=src_stack, tgt_stack=tgt_stack, dst_stack=dst_stack, bcube=bcube_back, render_method=render_method, cf_method=cf_method, suffix=suffix, copy_start=copy_start, backward=True) scheduler.register_job(align_block_job_back, job_name="Backward Align Block {}".format(bcube)) align_block_job_forv = AlignBlockJob(src_stack=src_stack, tgt_stack=tgt_stack, dst_stack=deepcopy(dst_stack), bcube=bcube_forv, render_method=render_method, cf_method=cf_method, suffix=suffix, copy_start=True, backward=False) scheduler.register_job(align_block_job_forv, job_name="Forward Align Block {}".format(bcube)) else: align_block_job = AlignBlockJob(src_stack=src_stack, tgt_stack=tgt_stack, dst_stack=dst_stack, bcube=bcube, render_method=render_method, cf_method=cf_method, suffix=suffix, copy_start=copy_start, backward=mode=='backward') # create scheduler and execute the job scheduler.register_job(align_block_job, job_name="Align Block {}".format(bcube)) scheduler.execute_until_completion() result_report = f"Aligned layers {[str(l) for l in src_stack.get_layers_of_type('img')]}. " \ f"Results in {[str(l) for l in dst_stack.get_layers_of_type('img')]}" corgie_logger.info(result_report)
def copy( ctx, src_layer_spec, dst_folder, copy_masks, blackout_masks, chunk_xy, chunk_z, start_coord, end_coord, coord_mip, mip, suffix, force_chunk_xy, force_chunk_z, ): scheduler = ctx.obj["scheduler"] if suffix is None: suffix = "" else: suffix = f"_{suffix}" corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name="src", readonly=True) if not force_chunk_xy: force_chunk_xy = chunk_xy if not force_chunk_z: force_chunk_z = chunk_z dst_stack = stack.create_stack_from_reference( reference_stack=src_stack, folder=dst_folder, name="dst", types=["img", "mask"], readonly=False, suffix=suffix, force_chunk_xy=force_chunk_xy, force_chunk_z=force_chunk_z, overwrite=True, ) bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) copy_job = CopyJob( src_stack=src_stack, dst_stack=dst_stack, mip=mip, bcube=bcube, chunk_xy=chunk_xy, chunk_z=chunk_z, copy_masks=copy_masks, blackout_masks=blackout_masks, ) # create scheduler and execute the job scheduler.register_job(copy_job, job_name="Copy {}".format(bcube)) scheduler.execute_until_completion() result_report = ( f"Copied layers {[str(l) for l in src_stack.get_layers_of_type('img')]}. " f"to {[str(l) for l in dst_stack.get_layers_of_type('img')]}") corgie_logger.info(result_report)
def create_segmentation_masks( ctx, src_layer_spec, dst_folder, processor_spec, pad, crop, processor_mip, dst_mip, chunk_xy, chunk_z, force_chunk_xy, start_coord, end_coord, coord_mip, suffix, similarity_threshold, compute_similarities, compute_slip_mask, compute_step_mask, compute_affinity_mask, ): scheduler = ctx.obj["scheduler"] if suffix is None: suffix = "_aligned" else: suffix = f"_{suffix}" if crop is None: crop = pad corgie_logger.debug("Setting up layers...") src_stack = create_stack_from_spec(src_layer_spec, name="src", readonly=True) src_stack.folder = dst_folder if force_chunk_xy is None: force_chunk_xy = chunk_xy dst_stack = stack.create_stack_from_reference( reference_stack=src_stack, folder=dst_folder, name="dst", types=["img", "mask"], readonly=False, suffix=suffix, force_chunk_xy=force_chunk_xy, overwrite=True, ) corgie_logger.debug("Done!") bcube = get_bcube_from_coords(start_coord, end_coord, coord_mip) if compute_similarities: z_offsets = range(-1, -4, -1) for tgt_z_offset in z_offsets: if tgt_z_offset not in src_stack: dst_layer = dst_stack.create_sublayer( name=tgt_z_offset, layer_type="img", overwrite=True, layer_args={"dtype": "uint8"}, ) proc_spec = json.loads(processor_spec) if isinstance(proc_spec, dict): assert str(tgt_z_offset) in proc_spec proc_spec = json.dumps(proc_spec[str(tgt_z_offset)]) else: proc_spec = processor_spec compare_job = CompareSectionsJob( src_stack=src_stack, tgt_stack=src_stack, dst_layer=dst_layer, chunk_xy=chunk_xy, processor_spec=proc_spec, pad=pad, bcube=bcube, tgt_z_offset=tgt_z_offset, suffix=suffix, mip=processor_mip, dst_mip=dst_mip, ) scheduler.register_job( compare_job, job_name="Compare Sections Job {}, tgt z offset {}".format( bcube, tgt_z_offset ), ) scheduler.execute_until_completion() result_report = f"Similarity results in {[str(l) for l in dst_stack.get_layers_of_type('img')]}" corgie_logger.info(result_report) # If similarities were just computed, add them to the src_stack as masks # Otherwise, they need to be included with src_stack_specs as masks # See arg help above for similarity mask and misalignment mask definitions. for layer_name in z_offsets: img_layer = dst_stack[layer_name] binarizer = { "binarization": ["gt", similarity_threshold], # "cv_params": {"cache": True}, } layer_dict = { "path": img_layer.path, "name": img_layer.name, "type": "mask", "args": binarizer, } mask_layer = create_layer_from_dict(layer_dict, reference=dst_stack) src_stack.add_layer(mask_layer) corgie_logger.info("Computing slip & step masks") if compute_slip_mask: slip_layer = dst_stack.create_sublayer( name="slip", layer_type="mask", overwrite=True ) slip_bcube = bcube.reset_coords( zs=bcube.z[0] + 1, ze=bcube.z[1] - 1, in_place=False ) slip_misalignments_job = DetectSlipMisalignmentsJob( src_stack=src_stack, dst_layer=slip_layer, mip=dst_mip, bcube=slip_bcube, pad=pad, chunk_xy=chunk_xy, chunk_z=chunk_z, ) scheduler.register_job( slip_misalignments_job, job_name="Detect Slip Misalignments {}".format(bcube), ) if compute_step_mask: step_layer = dst_stack.create_sublayer( name="step", layer_type="mask", overwrite=True ) step_bcube = bcube.reset_coords( zs=bcube.z[0] + 2, ze=bcube.z[1] - 2, in_place=False ) step_misalignments_job = DetectStepMisalignmentsJob( src_stack=src_stack, dst_layer=step_layer, mip=dst_mip, bcube=step_bcube, pad=pad, chunk_xy=chunk_xy, chunk_z=chunk_z, ) scheduler.register_job( step_misalignments_job, job_name="Detect Step Misalignments {}".format(bcube), ) if compute_slip_mask or compute_step_mask: # Execute slip & step masks at the same time scheduler.execute_until_completion() result_report = f"Slip & step masks in in {str(slip_layer), str(step_layer)}" corgie_logger.info(result_report) if compute_affinity_mask: corgie_logger.info("Creating affinity masks") affinity_layer = dst_stack.create_sublayer( name="affinity", layer_type="mask", overwrite=True ) three_consecutive_exp = DetectConsecutiveMasksJob.get_exp(n=3, key="slip") exp = { "inputs": [ three_consecutive_exp, {"weight": 1, "key": "step", "offset": 0}, ], "threshold": 0, } affinity_masks_job = CombineMasksJob( src_stack=dst_stack, exp=exp, dst_layer=affinity_layer, mip=dst_mip, bcube=bcube, pad=pad, chunk_xy=chunk_xy, chunk_z=chunk_z, ) scheduler.register_job( affinity_masks_job, job_name="Affinity Masks {}".format(bcube) ) scheduler.execute_until_completion() result_report = f"Results in {str(affinity_layer)}" corgie_logger.info(result_report)