def bl_links_worker(block_idx, block_rows, digits_row, digits_col,
all_blocks_dir, cell_dim, link_cells_d, overlap):
links_disp_d = {}
lnk_coord = "{:0{dig_row}d}_{:0{dig_col}d}".format(*block_idx,
dig_row=digits_row,
dig_col=digits_col)
lnk_file = "lk-" + lnk_coord
lnk_dir = setup_directories(os.path.join(all_blocks_dir, lnk_coord))
lnk_path = os.path.join(all_blocks_dir, lnk_file)
lnk_cells = [cell_n for cell_n, cell_idx in link_cells_d.values()]
(success, tiles_file, body_matrix, block,
pre_crop_file) = fiji_grid_stitch(lnk_dir,
lnk_cells,
lnk_path,
False,
pre_copy_files=False,
tile_overlap=overlap)
if not success:
debug_log("Could not compute links to block", block_idx)
else:
inverse_d = {
os.path.basename(fpath): coord
for coord, (fpath, cidx) in link_cells_d.items()
}
displs = TileConfigurator().parse(tiles_file)
links_disp_d[block_idx] = {
coord: displs[fname]
for fname, coord in inverse_d.items()
}
return {'success': success, 'links': links_disp_d}
def __call__(self, identifier, partner_identifier, row_target, col_target,
scale, reduction_axis):
scale_dir = ensure_dir(os.path.join(self.work_dir, str(scale)))
next_scale_dir = ensure_dir(os.path.join(self.work_dir,
str(scale + 1)))
temp_dir = ensure_dir(os.path.join(scale_dir, "work"))
input_img = [identifier, partner_identifier]
axis_name = ['rows', 'cols']
print("Partners over {}: {} and {}".format(
axis_name[reduction_axis], os.path.basename(identifier),
os.path.basename(partner_identifier)))
target_name = "{}_{}".format(row_target, col_target)
target_file = "{}.{}".format(target_name, img_data_fmt)
dir_stitch = setup_directories(os.path.join(temp_dir, target_name))
stitched_img = os.path.join(next_scale_dir, target_file)
if reduction_axis == 0:
success = assemble_row(dir_stitch, input_img, stitched_img)
else:
success = assemble_column(dir_stitch, input_img, stitched_img)
return success, stitched_img
def bl_intrablock_worker(block_idx, block_rows, digits_row, digits_col,
all_blocks_dir, cell_dim):
blocks_dim_d = {}
blocks_disp_d = {}
block_coord = "{:0{dig_row}d}_{:0{dig_col}d}".format(*block_idx,
dig_row=digits_row,
dig_col=digits_col)
block_file = "bl-" + block_coord
block_dir = setup_directories(os.path.join(all_blocks_dir, block_coord))
block_path = os.path.join(all_blocks_dir, block_file)
block_cells = sum(block_rows, [])
success, tiles_file = fiji_grid_stitch(block_dir,
block_cells,
block_path,
False,
pre_copy_files=False)[:2]
if not success:
debug_log("Could not assemble", block_path)
else:
displs = TileConfigurator().parse(tiles_file)
block_stitch_size, block_origind = compute_stitch_size(
cell_dim, displs)
blocks_dim_d[block_idx] = (block_stitch_size, block_origind)
cell_disp = [[
displs[os.path.basename(cpath)] - block_origind for cpath in row
] for row in block_rows]
blocks_disp_d[block_idx] = cell_disp
return {'success': success, 'dim': blocks_dim_d, 'disp': blocks_disp_d}
def multistep_worker(block_i, block_j, block_rows, img_type, blocks_dir,
digits_row, digits_col, keep_files, overlap):
block_coord = "{:0{dig_row}d}_{:0{dig_col}d}".format(block_i,
block_j,
dig_row=digits_row,
dig_col=digits_col)
block_file = "{}.{}".format(block_coord, img_type)
block_dir = setup_directories(os.path.join(blocks_dir, block_coord))
block_path = os.path.join(blocks_dir, block_file)
block_cells = sum(block_rows, [])
success, _, _, _, _ = fiji_grid_stitch(block_dir,
block_cells,
block_path,
True,
keep_uncropped=False,
pre_copy_files=False,
tile_overlap=overlap)
if not success:
debug_log("Could not assemble", block_path)
if success and not keep_files:
debug_log("Removing block cells:")
for cell in block_cells:
print cell,
os.remove(cell)
print
return {'success': success, 'block-path': block_path}
def assemble_image_by_rows(directories, files, out_file_name, max_threads):
n_rows, n_cols, row_cells, img_type, rows, _, _ = parse_image_grid_list(
files)
pool = mp.Pool(processes=max_threads)
rows_dir = ensure_dir(os.path.join(directories['out'], "rows"))
row_files = [
os.path.join(rows_dir, "row_{}.{}".format(row, img_data_fmt))
for row in rows
]
temp_dirs = [
setup_directories(os.path.join(rows_dir, "row_{}".format(i)))
for i in rows
]
row_jobs = [
pool.apply_async(assemble_row,
(temp_dirs[i], row_cells[i], row_files[i]))
for i in range(n_rows)
]
pool.close()
pool.join()
row_successes = [job.get() for job in row_jobs]
if all(row_successes):
debug_log("Assembling of rows into", rows_dir, "is complete.",
"Assembling image...")
return assemble_column(directories, row_files, out_file_name)
else:
return False
def main():
opt = process_command_line()
print opt
work_dir = tempfile.mkdtemp(dir=opt.work_dir)
work_dirs = setup_directories(work_dir)
debug_log("JOB", "Staging files in", work_dir)
output_file = os.path.abspath(opt.output_file)
img_path, img_fname_ext = os.path.split(output_file)
img_fname, img_ext = os.path.splitext(img_fname_ext)
if opt.job_type == 'row':
success = assemble_row(work_dirs, opt.files, output_file)
elif opt.job_type == 'col':
success = assemble_column(work_dirs, opt.files, output_file)
elif opt.job_type == 'image-by-rows':
success = assemble_image_by_rows(work_dirs, opt.files, output_file,
opt.threads)
elif opt.job_type == 'image-by-blocks':
success = assemble_image_by_blocks(work_dirs, opt.files, output_file,
opt.threads)
elif opt.job_type == 'two-step':
success = assemble_twostep(work_dirs, opt.files, output_file)
elif opt.job_type == 'multi-step':
success = assemble_multistep(work_dirs,
opt.files,
output_file,
keep_files=opt.keep_work_dir,
threads=opt.threads,
overlap=opt.tile_overlap)
elif opt.job_type == 'block-links':
success, img, _ = assemble_blocklink(work_dirs,
opt.files,
output_file,
threads=opt.threads,
keep_files=opt.keep_work_dir,
overlap=opt.tile_overlap,
do_contrast_stretch=opt.contrast)
if success and opt.also_save_reduced < 100:
img_path, img_fname = os.path.split(output_file)
reduced_fname = os.path.splitext(img_fname)[0] + ".small.jpg"
reduced_fpath = os.path.join(img_path, reduced_fname)
reduced_img = image_resize(img, opt.also_save_reduced)
cv2_to_file(reduced_img, reduced_fpath)
if success and not opt.keep_work_dir:
shutil.rmtree(work_dir)
return shell_retcode(success)
def apply_displacements(target_dir, angle, displacements, work_dirs, kind,
do_register, blur, files_in):
file_out = os.path.join(target_dir,
"full-{}.{}".format(angle, img_data_fmt))
if not pth.isfile(file_out):
kind_dir = ensure_dir(pth.join(work_dirs['temp'], kind))
my_work_dirs = setup_directories(pth.join(kind_dir, str(angle)))
if blur is not None:
new_dir = blankfield_guard(files_in, kind, my_work_dirs['in'],
blur)
else:
new_dir = pth.dirname(files_in[0])
coords_fi = pth.join(new_dir, "tiles.txt")
TileConfigurator().generate(displacements, coords_fi)
success, img, _ = fiji_grid_fuse(my_work_dirs,
coords_fi,
file_out,
do_crop=False)
shutil.rmtree(my_work_dirs['in'])
else:
debug_log("Ensemble", file_out, "already exists.")
img = None
cropped_fpath = os.path.join(target_dir,
"center-crop-{}.jpg".format(angle))
if not pth.isfile(cropped_fpath):
load_img = img if img is not None else file_to_cv2(file_out)
cropped_img = crop_center_chunk(load_img, 1024)
cv2_to_file(cropped_img, cropped_fpath)
else:
debug_log("Center-cropped chunk", cropped_fpath, "already exists.")
reduced_fpath = os.path.join(target_dir, "full-{}.small.jpg".format(angle))
if not pth.isfile(reduced_fpath):
load_img = img if img is not None else file_to_cv2(file_out)
reduced_img = image_resize(load_img, 30)
cv2_to_file(reduced_img, reduced_fpath)
else:
debug_log("Reduced size image", reduced_fpath, "already exists.")
if not do_register:
pyramid_archive = pth.join(target_dir, "pyramid-{}.tar".format(angle))
if not pth.isfile(pyramid_archive):
make_pyramid(img, pyramid_archive, "{}-{}.dzi".format(kind, angle))
else:
debug_log("Pyramid archive", pyramid_archive, "already exists.")
return file_out
def assemble_multigrid(directories,
files,
out_file_name,
pre_resize=100,
base_cell_size=4,
max_threads=cpu_count(),
keep_uncropped=False,
blankfield_file=None):
n_rows, n_cols, row_cells, img_type, _, _, _ = parse_image_grid_list(files)
pool = Pool(processes=max_threads)
cs = float(base_cell_size)
base_cells_dims = np.ceil(np.r_[n_rows, n_cols] / cs).astype(int)
cells_dir = ensure_dir(os.path.join(directories['out'], "cells"))
cell_jobs = []
cell_files = []
blankfield = (None if blankfield_file is None else image_resize(
file_to_cv2(blankfield_file), pre_resize))
digits_row, digits_col = map(get_digits, base_cells_dims)
for b_row in range(base_cells_dims[0]):
for b_col in range(base_cells_dims[1]):
base_name = "{:0{w_r}d}_{:0{w_c}d}".format(b_row,
b_col,
w_r=digits_row,
w_c=digits_col)
temp_dirs = setup_directories(os.path.join(cells_dir, base_name))
files = [
file_image_preprocess(
row_cells[i][j],
os.path.join(temp_dirs['in'],
os.path.basename(row_cells[i][j])),
pre_resize, blankfield)
for i in range(base_cell_size *
b_row, min(base_cell_size *
(b_row + 1), n_rows))
for j in range(base_cell_size *
b_col, min(base_cell_size *
(b_col + 1), n_cols))
]
out_file = os.path.join(cells_dir,
"{}.{}".format(base_name, img_data_fmt))
cell_jobs.append(
pool.apply_async(fiji_grid_stitch,
(temp_dirs, files, out_file), {
'pre_copy_files': False,
'keep_uncropped': keep_uncropped
}))
cell_files.append(out_file)
pool.close()
pool.join()
cell_successes = [job.get() for job in cell_jobs]
if all(cell_successes):
debug_log("Assembling of base cells into", cells_dir, "is complete.",
"Assembling image...")
return fiji_grid_stitch(directories,
cell_files,
out_file_name,
keep_uncropped=keep_uncropped)
else:
return False
parser.add_argument(
'-r',
"--remove-work-tree",
action='store_true',
help=("Remove intermediate files if the stitch is successful"))
parser.add_argument('-k',
"--keep-uncropped",
action='store_true',
help=("Keep uncropped intermediate images"))
opt = parser.parse_args()
print opt
#TODO receive arguments for Fiji's stitching thresholds
work_dir = tempfile.mkdtemp(dir=opt.work_dir)
work_dirs = setup_directories(work_dir)
debug_log("JOB", "Staging files in", work_dir)
output_file = os.path.abspath(opt.output_file)
if opt.multi_grid:
success = assemble_multigrid(work_dirs, opt.files, output_file,
opt.pre_resize, opt.mg_cell_size,
opt.threads, opt.keep_uncropped,
opt.blankfield_file)
else:
success = fiji_grid_stitch(work_dirs,
opt.files,
output_file,
keep_uncropped=opt.keep_uncropped)
if success and opt.remove_work_tree:
def main():
opt = process_command_line()
print opt
output_dir = pth.abspath(opt.output_dir)
ensure_dir(output_dir)
ensure_dir(opt.work_dir)
stack_name = pth.basename(opt.stack_base)
work_dir = tempfile.mkdtemp(prefix=stack_name + '-', dir=opt.work_dir)
disk_work_dir = ensure_dir(opt.disk_work_dir)
# Reference cell grid for registration and stitching computation
ref_kind, ref_angle = 'ppl', 0
src_dirs, tgt_dirs, tgt_base_dir = set_stack_structure(
opt.stack_base, output_dir, opt.preregister)
stack_config, s_rows, s_cols = determine_stack_config(src_dirs)
# 0. Register cells across all angles of all image kinds
if opt.preregister:
success = align_cells(stack_config, src_dirs, tgt_dirs, ref_kind,
ref_angle, work_dir, max_threads, opt.crop_size,
opt.ppl_blankfield_file)
else:
print "Cells pre-registration skipped"
success = skip_align_cells(opt.stack_base, stack_config, tgt_base_dir)
result = "done" if success else "failed"
debug_log("Registration job", result)
if success:
work_pending = {kind: cfg.keys() for kind, cfg in stack_config.items()}
# 1. Apply blankfield correction to PPL images
for ppl_angle in work_pending['ppl']:
ppl_angle_dir = pth.join(tgt_base_dir, "ppl", str(ppl_angle))
blankfield = pth.join(ppl_angle_dir, "blankfield-stat.png")
if opt.ppl_blankfield_file is None:
images = list(iglob(pth.join(ppl_angle_dir, "??_??.png")))
if not pth.isfile(blankfield):
print "Computing blankfield for PPL image angle", ppl_angle
generate_write_blankfield(images,
ppl_angle_dir,
threads=opt.threads)
else:
print "Blankfield for PPL image angle", ppl_angle, "exists"
else:
if not opt.preregister:
ppl_bfield = pth.expanduser(opt.ppl_blankfield_file)
shutil.copy(ppl_bfield, blankfield)
print "Copied blankfield for PPL image angle", ppl_angle, "from", ppl_bfield, "into", blankfield
# 2. Find stitch displacements from the reference cell grid
displacements_savefile = pth.join(tgt_base_dir,
"stack-displacements.txt")
tc = TileConfigurator()
if pth.isfile(displacements_savefile):
displacements = tc.parse(displacements_savefile)
success = True
else:
files_in = [
pth.join(tgt_dirs[ref_kind], str(ref_angle), fname)
for row in stack_config[ref_kind][ref_angle]['row_cells']
for fname in row
]
work_dirs = setup_directories(work_dir)
success, _, displacements = assemble_blocklink(
work_dirs,
files_in,
None,
keep_files=opt.keep_work_dir,
threads=opt.threads,
do_crop=False,
do_fuse=False)
tc.generate(displacements, displacements_savefile)
# 3. Apply stitch displacements to all angles of all image kinds
work_order = imerge(*[[(kind, ang) for ang in angles]
for kind, angles in work_pending.items()])
disk_work_dirs = setup_directories(disk_work_dir)
if opt.ppl_blankfield_file is not None:
blur = None if opt.preregister else -1
else:
blur = 0.6
job_args = [(tgt_dirs[kind], angle, displacements, disk_work_dirs,
kind, opt.post_register, blur, [
pth.join(tgt_dirs[kind], str(angle), fname)
for row in stack_config[kind][angle]['row_cells']
for fname in row
]) for kind, angle in work_order]
if opt.threads == 1:
files = [apply_displacements(*args) for args in job_args]
else:
pool = mp.Pool(processes=opt.threads)
jobs = [
pool.apply_async(apply_displacements, args)
for args in job_args
]
pool.close()
pool.join()
files = [job.get() for job in jobs]
if not opt.keep_work_dir:
shutil.rmtree(work_dir)
# 4. Register the images for removing mismatch due to birrefringence
target_dir = pth.join(tgt_base_dir, "registered")
if opt.post_register:
files_map = {
fi: re.sub(r"([xp]pl)/full", r"registered/\1", fi)
for fi in files
}
if not all(pth.isfile(fo) for fo in files_map.values()):
register_images(files,
4096,
opt.threads,
ensure_dir(target_dir),
True,
False,
target_files_map=files_map,
make_jpeg=opt.write_also_jpeg)
job_args_gen = ((fi, pth.splitext(pth.basename(fo))[0])
for fi, fo in files_map.items())
job_args = [(fi, pth.join(target_dir, "pyr-{}.tar".format(fo)),
"{}.dzi".format(fo)) for fi, fo in job_args_gen]
if opt.threads == 1:
success = [make_pyramid(*args) for args in job_args]
else:
pool = mp.Pool(processes=opt.threads)
jobs = [
pool.apply_async(make_pyramid, args) for args in job_args
]
pool.close()
pool.join()
success = [job.get() for job in jobs]
origs = (arc + ".orig" for _, arc, _ in job_args)
[os.remove(orig) for orig in origs if pth.exists(orig)]
# A flag file is a dirty way of notifying other processes that this
# job is complete
flag_file = pth.join(tgt_base_dir, "done")
with open(flag_file, 'a'):
pass # Touch file
result = "done" if success else "failed"
debug_log("Stack job", result)
return shell_retcode(success)
def assemble_twostep(directories, files, out_file_name, corner_dim_min=[2, 3]):
n_rows, n_cols, row_cells, img_type, _, _, _ = parse_image_grid_list(files)
body_dir = ensure_dir(os.path.join(directories['out'], "body"))
debug_log("Assembling of body into", body_dir, "started.")
body_work_dir = setup_directories(body_dir)
body_simple_file = os.path.join(body_dir, "body-straight.png")
(success, tiles_file, body_matrix, body_simple,
pre_crop_file) = fiji_grid_stitch(body_work_dir,
files,
body_simple_file,
True,
keep_uncropped=True,
pre_copy_files=False,
threshold_reg=0.5,
threshold_maxavg=10,
threshold_abs=50)
if not success:
debug_log("Could not assemble body")
return False
c_h, c_w = body_simple.shape[:2]
grey_chunk = color_to_gray(body_simple[:c_h / 2, :c_w / 2, :])
left_black = find_left_border(grey_chunk, c_w / 2)
top_black = find_top_border(grey_chunk, c_h / 2)
pre_corner = body_simple[:top_black, :left_black]
corner_is_black = (pre_corner.max() == 0)
debug_log("Corner void size:", left_black, "x", top_black, corner_is_black)
if (left_black == 0 and top_black == 0) or (not corner_is_black):
debug_log("Apparently no corner stitching is necessary!")
shutil.move(body_simple_file, out_file_name)
return True
dim_h, dim_w = file_to_cv2(files[0]).shape[:2]
corner_dim = [
max(corner_dim_min[0], 1 + 2 * int(ceil(float(top_black) / dim_h))),
max(corner_dim_min[1], 1 + 2 * int(ceil(float(left_black) / dim_w)))
]
debug_log("Corner dimensions: {}".format(corner_dim))
corner_dir = ensure_dir(os.path.join(directories['out'], "corner"))
corner_work_dir = setup_directories(corner_dir)
corner_file = os.path.join(corner_dir, "corner.png")
corner_input_files = [
cell for row in row_cells[:corner_dim[0]]
for cell in row[:corner_dim[1]]
]
debug_log("Assembling of corner into", corner_dir, "started.")
success, _, corner_matrix, corner, _ = fiji_grid_stitch(
corner_work_dir,
corner_input_files,
corner_file,
True,
pre_copy_files=False)
if not success:
debug_log("Could not assemble corner")
return False
debug_log("Generating Laplacian blending weights for corner")
body = patch_body_corner_inmem(body_simple, body_matrix, corner,
corner_matrix)
debug_log("Saving blended image to", out_file_name)
success = cv2_to_file(body, out_file_name)
result_name = "done" if success else "failed"
debug_log("Saving blended image", out_file_name, result_name)
return success