def bl_links_worker(block_idx, block_rows, digits_row, digits_col, all_blocks_dir, cell_dim, link_cells_d, overlap): links_disp_d = {} lnk_coord = "{:0{dig_row}d}_{:0{dig_col}d}".format(*block_idx, dig_row=digits_row, dig_col=digits_col) lnk_file = "lk-" + lnk_coord lnk_dir = setup_directories(os.path.join(all_blocks_dir, lnk_coord)) lnk_path = os.path.join(all_blocks_dir, lnk_file) lnk_cells = [cell_n for cell_n, cell_idx in link_cells_d.values()] (success, tiles_file, body_matrix, block, pre_crop_file) = fiji_grid_stitch(lnk_dir, lnk_cells, lnk_path, False, pre_copy_files=False, tile_overlap=overlap) if not success: debug_log("Could not compute links to block", block_idx) else: inverse_d = { os.path.basename(fpath): coord for coord, (fpath, cidx) in link_cells_d.items() } displs = TileConfigurator().parse(tiles_file) links_disp_d[block_idx] = { coord: displs[fname] for fname, coord in inverse_d.items() } return {'success': success, 'links': links_disp_d}
def __call__(self, identifier, partner_identifier, row_target, col_target, scale, reduction_axis): scale_dir = ensure_dir(os.path.join(self.work_dir, str(scale))) next_scale_dir = ensure_dir(os.path.join(self.work_dir, str(scale + 1))) temp_dir = ensure_dir(os.path.join(scale_dir, "work")) input_img = [identifier, partner_identifier] axis_name = ['rows', 'cols'] print("Partners over {}: {} and {}".format( axis_name[reduction_axis], os.path.basename(identifier), os.path.basename(partner_identifier))) target_name = "{}_{}".format(row_target, col_target) target_file = "{}.{}".format(target_name, img_data_fmt) dir_stitch = setup_directories(os.path.join(temp_dir, target_name)) stitched_img = os.path.join(next_scale_dir, target_file) if reduction_axis == 0: success = assemble_row(dir_stitch, input_img, stitched_img) else: success = assemble_column(dir_stitch, input_img, stitched_img) return success, stitched_img
def bl_intrablock_worker(block_idx, block_rows, digits_row, digits_col, all_blocks_dir, cell_dim): blocks_dim_d = {} blocks_disp_d = {} block_coord = "{:0{dig_row}d}_{:0{dig_col}d}".format(*block_idx, dig_row=digits_row, dig_col=digits_col) block_file = "bl-" + block_coord block_dir = setup_directories(os.path.join(all_blocks_dir, block_coord)) block_path = os.path.join(all_blocks_dir, block_file) block_cells = sum(block_rows, []) success, tiles_file = fiji_grid_stitch(block_dir, block_cells, block_path, False, pre_copy_files=False)[:2] if not success: debug_log("Could not assemble", block_path) else: displs = TileConfigurator().parse(tiles_file) block_stitch_size, block_origind = compute_stitch_size( cell_dim, displs) blocks_dim_d[block_idx] = (block_stitch_size, block_origind) cell_disp = [[ displs[os.path.basename(cpath)] - block_origind for cpath in row ] for row in block_rows] blocks_disp_d[block_idx] = cell_disp return {'success': success, 'dim': blocks_dim_d, 'disp': blocks_disp_d}
def multistep_worker(block_i, block_j, block_rows, img_type, blocks_dir, digits_row, digits_col, keep_files, overlap): block_coord = "{:0{dig_row}d}_{:0{dig_col}d}".format(block_i, block_j, dig_row=digits_row, dig_col=digits_col) block_file = "{}.{}".format(block_coord, img_type) block_dir = setup_directories(os.path.join(blocks_dir, block_coord)) block_path = os.path.join(blocks_dir, block_file) block_cells = sum(block_rows, []) success, _, _, _, _ = fiji_grid_stitch(block_dir, block_cells, block_path, True, keep_uncropped=False, pre_copy_files=False, tile_overlap=overlap) if not success: debug_log("Could not assemble", block_path) if success and not keep_files: debug_log("Removing block cells:") for cell in block_cells: print cell, os.remove(cell) print return {'success': success, 'block-path': block_path}
def assemble_image_by_rows(directories, files, out_file_name, max_threads): n_rows, n_cols, row_cells, img_type, rows, _, _ = parse_image_grid_list( files) pool = mp.Pool(processes=max_threads) rows_dir = ensure_dir(os.path.join(directories['out'], "rows")) row_files = [ os.path.join(rows_dir, "row_{}.{}".format(row, img_data_fmt)) for row in rows ] temp_dirs = [ setup_directories(os.path.join(rows_dir, "row_{}".format(i))) for i in rows ] row_jobs = [ pool.apply_async(assemble_row, (temp_dirs[i], row_cells[i], row_files[i])) for i in range(n_rows) ] pool.close() pool.join() row_successes = [job.get() for job in row_jobs] if all(row_successes): debug_log("Assembling of rows into", rows_dir, "is complete.", "Assembling image...") return assemble_column(directories, row_files, out_file_name) else: return False
def main(): opt = process_command_line() print opt work_dir = tempfile.mkdtemp(dir=opt.work_dir) work_dirs = setup_directories(work_dir) debug_log("JOB", "Staging files in", work_dir) output_file = os.path.abspath(opt.output_file) img_path, img_fname_ext = os.path.split(output_file) img_fname, img_ext = os.path.splitext(img_fname_ext) if opt.job_type == 'row': success = assemble_row(work_dirs, opt.files, output_file) elif opt.job_type == 'col': success = assemble_column(work_dirs, opt.files, output_file) elif opt.job_type == 'image-by-rows': success = assemble_image_by_rows(work_dirs, opt.files, output_file, opt.threads) elif opt.job_type == 'image-by-blocks': success = assemble_image_by_blocks(work_dirs, opt.files, output_file, opt.threads) elif opt.job_type == 'two-step': success = assemble_twostep(work_dirs, opt.files, output_file) elif opt.job_type == 'multi-step': success = assemble_multistep(work_dirs, opt.files, output_file, keep_files=opt.keep_work_dir, threads=opt.threads, overlap=opt.tile_overlap) elif opt.job_type == 'block-links': success, img, _ = assemble_blocklink(work_dirs, opt.files, output_file, threads=opt.threads, keep_files=opt.keep_work_dir, overlap=opt.tile_overlap, do_contrast_stretch=opt.contrast) if success and opt.also_save_reduced < 100: img_path, img_fname = os.path.split(output_file) reduced_fname = os.path.splitext(img_fname)[0] + ".small.jpg" reduced_fpath = os.path.join(img_path, reduced_fname) reduced_img = image_resize(img, opt.also_save_reduced) cv2_to_file(reduced_img, reduced_fpath) if success and not opt.keep_work_dir: shutil.rmtree(work_dir) return shell_retcode(success)
def apply_displacements(target_dir, angle, displacements, work_dirs, kind, do_register, blur, files_in): file_out = os.path.join(target_dir, "full-{}.{}".format(angle, img_data_fmt)) if not pth.isfile(file_out): kind_dir = ensure_dir(pth.join(work_dirs['temp'], kind)) my_work_dirs = setup_directories(pth.join(kind_dir, str(angle))) if blur is not None: new_dir = blankfield_guard(files_in, kind, my_work_dirs['in'], blur) else: new_dir = pth.dirname(files_in[0]) coords_fi = pth.join(new_dir, "tiles.txt") TileConfigurator().generate(displacements, coords_fi) success, img, _ = fiji_grid_fuse(my_work_dirs, coords_fi, file_out, do_crop=False) shutil.rmtree(my_work_dirs['in']) else: debug_log("Ensemble", file_out, "already exists.") img = None cropped_fpath = os.path.join(target_dir, "center-crop-{}.jpg".format(angle)) if not pth.isfile(cropped_fpath): load_img = img if img is not None else file_to_cv2(file_out) cropped_img = crop_center_chunk(load_img, 1024) cv2_to_file(cropped_img, cropped_fpath) else: debug_log("Center-cropped chunk", cropped_fpath, "already exists.") reduced_fpath = os.path.join(target_dir, "full-{}.small.jpg".format(angle)) if not pth.isfile(reduced_fpath): load_img = img if img is not None else file_to_cv2(file_out) reduced_img = image_resize(load_img, 30) cv2_to_file(reduced_img, reduced_fpath) else: debug_log("Reduced size image", reduced_fpath, "already exists.") if not do_register: pyramid_archive = pth.join(target_dir, "pyramid-{}.tar".format(angle)) if not pth.isfile(pyramid_archive): make_pyramid(img, pyramid_archive, "{}-{}.dzi".format(kind, angle)) else: debug_log("Pyramid archive", pyramid_archive, "already exists.") return file_out
def assemble_multigrid(directories, files, out_file_name, pre_resize=100, base_cell_size=4, max_threads=cpu_count(), keep_uncropped=False, blankfield_file=None): n_rows, n_cols, row_cells, img_type, _, _, _ = parse_image_grid_list(files) pool = Pool(processes=max_threads) cs = float(base_cell_size) base_cells_dims = np.ceil(np.r_[n_rows, n_cols] / cs).astype(int) cells_dir = ensure_dir(os.path.join(directories['out'], "cells")) cell_jobs = [] cell_files = [] blankfield = (None if blankfield_file is None else image_resize( file_to_cv2(blankfield_file), pre_resize)) digits_row, digits_col = map(get_digits, base_cells_dims) for b_row in range(base_cells_dims[0]): for b_col in range(base_cells_dims[1]): base_name = "{:0{w_r}d}_{:0{w_c}d}".format(b_row, b_col, w_r=digits_row, w_c=digits_col) temp_dirs = setup_directories(os.path.join(cells_dir, base_name)) files = [ file_image_preprocess( row_cells[i][j], os.path.join(temp_dirs['in'], os.path.basename(row_cells[i][j])), pre_resize, blankfield) for i in range(base_cell_size * b_row, min(base_cell_size * (b_row + 1), n_rows)) for j in range(base_cell_size * b_col, min(base_cell_size * (b_col + 1), n_cols)) ] out_file = os.path.join(cells_dir, "{}.{}".format(base_name, img_data_fmt)) cell_jobs.append( pool.apply_async(fiji_grid_stitch, (temp_dirs, files, out_file), { 'pre_copy_files': False, 'keep_uncropped': keep_uncropped })) cell_files.append(out_file) pool.close() pool.join() cell_successes = [job.get() for job in cell_jobs] if all(cell_successes): debug_log("Assembling of base cells into", cells_dir, "is complete.", "Assembling image...") return fiji_grid_stitch(directories, cell_files, out_file_name, keep_uncropped=keep_uncropped) else: return False
parser.add_argument( '-r', "--remove-work-tree", action='store_true', help=("Remove intermediate files if the stitch is successful")) parser.add_argument('-k', "--keep-uncropped", action='store_true', help=("Keep uncropped intermediate images")) opt = parser.parse_args() print opt #TODO receive arguments for Fiji's stitching thresholds work_dir = tempfile.mkdtemp(dir=opt.work_dir) work_dirs = setup_directories(work_dir) debug_log("JOB", "Staging files in", work_dir) output_file = os.path.abspath(opt.output_file) if opt.multi_grid: success = assemble_multigrid(work_dirs, opt.files, output_file, opt.pre_resize, opt.mg_cell_size, opt.threads, opt.keep_uncropped, opt.blankfield_file) else: success = fiji_grid_stitch(work_dirs, opt.files, output_file, keep_uncropped=opt.keep_uncropped) if success and opt.remove_work_tree:
def main(): opt = process_command_line() print opt output_dir = pth.abspath(opt.output_dir) ensure_dir(output_dir) ensure_dir(opt.work_dir) stack_name = pth.basename(opt.stack_base) work_dir = tempfile.mkdtemp(prefix=stack_name + '-', dir=opt.work_dir) disk_work_dir = ensure_dir(opt.disk_work_dir) # Reference cell grid for registration and stitching computation ref_kind, ref_angle = 'ppl', 0 src_dirs, tgt_dirs, tgt_base_dir = set_stack_structure( opt.stack_base, output_dir, opt.preregister) stack_config, s_rows, s_cols = determine_stack_config(src_dirs) # 0. Register cells across all angles of all image kinds if opt.preregister: success = align_cells(stack_config, src_dirs, tgt_dirs, ref_kind, ref_angle, work_dir, max_threads, opt.crop_size, opt.ppl_blankfield_file) else: print "Cells pre-registration skipped" success = skip_align_cells(opt.stack_base, stack_config, tgt_base_dir) result = "done" if success else "failed" debug_log("Registration job", result) if success: work_pending = {kind: cfg.keys() for kind, cfg in stack_config.items()} # 1. Apply blankfield correction to PPL images for ppl_angle in work_pending['ppl']: ppl_angle_dir = pth.join(tgt_base_dir, "ppl", str(ppl_angle)) blankfield = pth.join(ppl_angle_dir, "blankfield-stat.png") if opt.ppl_blankfield_file is None: images = list(iglob(pth.join(ppl_angle_dir, "??_??.png"))) if not pth.isfile(blankfield): print "Computing blankfield for PPL image angle", ppl_angle generate_write_blankfield(images, ppl_angle_dir, threads=opt.threads) else: print "Blankfield for PPL image angle", ppl_angle, "exists" else: if not opt.preregister: ppl_bfield = pth.expanduser(opt.ppl_blankfield_file) shutil.copy(ppl_bfield, blankfield) print "Copied blankfield for PPL image angle", ppl_angle, "from", ppl_bfield, "into", blankfield # 2. Find stitch displacements from the reference cell grid displacements_savefile = pth.join(tgt_base_dir, "stack-displacements.txt") tc = TileConfigurator() if pth.isfile(displacements_savefile): displacements = tc.parse(displacements_savefile) success = True else: files_in = [ pth.join(tgt_dirs[ref_kind], str(ref_angle), fname) for row in stack_config[ref_kind][ref_angle]['row_cells'] for fname in row ] work_dirs = setup_directories(work_dir) success, _, displacements = assemble_blocklink( work_dirs, files_in, None, keep_files=opt.keep_work_dir, threads=opt.threads, do_crop=False, do_fuse=False) tc.generate(displacements, displacements_savefile) # 3. Apply stitch displacements to all angles of all image kinds work_order = imerge(*[[(kind, ang) for ang in angles] for kind, angles in work_pending.items()]) disk_work_dirs = setup_directories(disk_work_dir) if opt.ppl_blankfield_file is not None: blur = None if opt.preregister else -1 else: blur = 0.6 job_args = [(tgt_dirs[kind], angle, displacements, disk_work_dirs, kind, opt.post_register, blur, [ pth.join(tgt_dirs[kind], str(angle), fname) for row in stack_config[kind][angle]['row_cells'] for fname in row ]) for kind, angle in work_order] if opt.threads == 1: files = [apply_displacements(*args) for args in job_args] else: pool = mp.Pool(processes=opt.threads) jobs = [ pool.apply_async(apply_displacements, args) for args in job_args ] pool.close() pool.join() files = [job.get() for job in jobs] if not opt.keep_work_dir: shutil.rmtree(work_dir) # 4. Register the images for removing mismatch due to birrefringence target_dir = pth.join(tgt_base_dir, "registered") if opt.post_register: files_map = { fi: re.sub(r"([xp]pl)/full", r"registered/\1", fi) for fi in files } if not all(pth.isfile(fo) for fo in files_map.values()): register_images(files, 4096, opt.threads, ensure_dir(target_dir), True, False, target_files_map=files_map, make_jpeg=opt.write_also_jpeg) job_args_gen = ((fi, pth.splitext(pth.basename(fo))[0]) for fi, fo in files_map.items()) job_args = [(fi, pth.join(target_dir, "pyr-{}.tar".format(fo)), "{}.dzi".format(fo)) for fi, fo in job_args_gen] if opt.threads == 1: success = [make_pyramid(*args) for args in job_args] else: pool = mp.Pool(processes=opt.threads) jobs = [ pool.apply_async(make_pyramid, args) for args in job_args ] pool.close() pool.join() success = [job.get() for job in jobs] origs = (arc + ".orig" for _, arc, _ in job_args) [os.remove(orig) for orig in origs if pth.exists(orig)] # A flag file is a dirty way of notifying other processes that this # job is complete flag_file = pth.join(tgt_base_dir, "done") with open(flag_file, 'a'): pass # Touch file result = "done" if success else "failed" debug_log("Stack job", result) return shell_retcode(success)
def assemble_twostep(directories, files, out_file_name, corner_dim_min=[2, 3]): n_rows, n_cols, row_cells, img_type, _, _, _ = parse_image_grid_list(files) body_dir = ensure_dir(os.path.join(directories['out'], "body")) debug_log("Assembling of body into", body_dir, "started.") body_work_dir = setup_directories(body_dir) body_simple_file = os.path.join(body_dir, "body-straight.png") (success, tiles_file, body_matrix, body_simple, pre_crop_file) = fiji_grid_stitch(body_work_dir, files, body_simple_file, True, keep_uncropped=True, pre_copy_files=False, threshold_reg=0.5, threshold_maxavg=10, threshold_abs=50) if not success: debug_log("Could not assemble body") return False c_h, c_w = body_simple.shape[:2] grey_chunk = color_to_gray(body_simple[:c_h / 2, :c_w / 2, :]) left_black = find_left_border(grey_chunk, c_w / 2) top_black = find_top_border(grey_chunk, c_h / 2) pre_corner = body_simple[:top_black, :left_black] corner_is_black = (pre_corner.max() == 0) debug_log("Corner void size:", left_black, "x", top_black, corner_is_black) if (left_black == 0 and top_black == 0) or (not corner_is_black): debug_log("Apparently no corner stitching is necessary!") shutil.move(body_simple_file, out_file_name) return True dim_h, dim_w = file_to_cv2(files[0]).shape[:2] corner_dim = [ max(corner_dim_min[0], 1 + 2 * int(ceil(float(top_black) / dim_h))), max(corner_dim_min[1], 1 + 2 * int(ceil(float(left_black) / dim_w))) ] debug_log("Corner dimensions: {}".format(corner_dim)) corner_dir = ensure_dir(os.path.join(directories['out'], "corner")) corner_work_dir = setup_directories(corner_dir) corner_file = os.path.join(corner_dir, "corner.png") corner_input_files = [ cell for row in row_cells[:corner_dim[0]] for cell in row[:corner_dim[1]] ] debug_log("Assembling of corner into", corner_dir, "started.") success, _, corner_matrix, corner, _ = fiji_grid_stitch( corner_work_dir, corner_input_files, corner_file, True, pre_copy_files=False) if not success: debug_log("Could not assemble corner") return False debug_log("Generating Laplacian blending weights for corner") body = patch_body_corner_inmem(body_simple, body_matrix, corner, corner_matrix) debug_log("Saving blended image to", out_file_name) success = cv2_to_file(body, out_file_name) result_name = "done" if success else "failed" debug_log("Saving blended image", out_file_name, result_name) return success