def test_no_im(image_dir):
with pytest.raises(IOError):
io_utils.read_gray_im(os.path.join(image_dir, 'no_im.png'))
def test_read_gray_im(image_dir):
im = io_utils.read_gray_im(os.path.join(image_dir, 'A1.png'))
im_shape = im.shape
assert im_shape[0] == 5
assert im_shape[1] == 10
def interp(input_dir, output_dir):
MetaData(input_dir, output_dir)
# Initialize background estimator
bg_estimator = background_estimator.BackgroundEstimator2D(
block_size=128,
order=2,
normalize=False,
)
reporter = report.ReportWriter()
well_xlsx_path = os.path.join(
constants.RUN_PATH,
'stats_per_well.xlsx',
)
well_xlsx_writer = pd.ExcelWriter(well_xlsx_path)
antigen_df = reporter.get_antigen_df()
antigen_df.to_excel(well_xlsx_writer, sheet_name='antigens')
# ================
# loop over images => good place for multiproc? careful with columns in report
# ================
well_images = io_utils.get_image_paths(input_dir)
for well_name, im_path in well_images.items():
start = time.time()
image = io_utils.read_gray_im(im_path)
spot_props_array = txt_parser.create_array(
constants.params['rows'],
constants.params['columns'],
dtype=object,
)
bgprops_array = txt_parser.create_array(
constants.params['rows'],
constants.params['columns'],
dtype=object,
)
# finding center of well and cropping
well_center, well_radi, well_mask = image_parser.find_well_border(
image, detmethod='region', segmethod='otsu')
im_crop, _ = img_processing.crop_image_at_center(
image, well_center, 2 * well_radi, 2 * well_radi)
# find center of spots from crop
spot_mask = img_processing.thresh_and_binarize(im_crop,
method='bright_spots')
spot_props = image_parser.generate_props(spot_mask,
intensity_image=im_crop)
# if debug:
crop_coords = image_parser.grid_from_centroids(
spot_props, constants.params['rows'], constants.params['columns'])
# convert to float64
im_crop = im_crop / np.iinfo(im_crop.dtype).max
background = bg_estimator.get_background(im_crop)
spots_df, spot_props = array_gen.get_spot_intensity(
coords=crop_coords,
im=im_crop,
background=background,
params=constants.params,
)
# Write metrics for each spot in grid in current well
spots_df.to_excel(well_xlsx_writer, sheet_name=well_name)
# Assign well OD, intensity, and background stats to plate
reporter.assign_well_to_plate(well_name, spots_df)
stop = time.time()
print(f"\ttime to process={stop-start}")
# SAVE FOR DEBUGGING
if constants.DEBUG:
# Save spot and background intensities.
output_name = os.path.join(constants.RUN_PATH, well_name)
# # Save mask of the well, cropped grayscale image, cropped spot segmentation.
io.imsave(output_name + "_well_mask.png",
(255 * well_mask).astype('uint8'))
io.imsave(output_name + "_crop.png",
(255 * im_crop).astype('uint8'))
io.imsave(output_name + "_crop_binary.png",
(255 * spot_mask).astype('uint8'))
# Evaluate accuracy of background estimation with green (image), magenta (background) overlay.
im_bg_overlay = np.stack([background, im_crop, background], axis=2)
io.imsave(output_name + "_crop_bg_overlay.png",
(255 * im_bg_overlay).astype('uint8'))
# This plot shows which spots have been assigned what index.
debug_plots.plot_centroid_overlay(
im_crop,
constants.params,
spot_props,
output_name,
)
debug_plots.plot_od(
spots_df=spots_df,
nbr_grid_rows=constants.params['rows'],
nbr_grid_cols=constants.params['columns'],
output_name=output_name,
)
# save a composite of all spots, where spots are from source or from region prop
debug_plots.save_composite_spots(
spot_props,
output_name,
image=im_crop,
)
debug_plots.save_composite_spots(
spot_props,
output_name,
image=im_crop,
from_source=True,
)
stop2 = time.time()
print(f"\ttime to save debug={stop2-stop}")
# After running all wells, write plate reports
well_xlsx_writer.close()
reporter.write_reports()
def point_registration(input_dir, output_dir):
"""
For each image in input directory, detect spots using particle filtering
to register fiducial spots to blobs detected in the image.
:param str input_dir: Input directory containing images and an xml file
with parameters
:param str output_dir: Directory where output is written to
"""
logger = logging.getLogger(constants.LOG_NAME)
metadata.MetaData(input_dir, output_dir)
nbr_outliers = constants.params['nbr_outliers']
# Create reports instance for whole plate
reporter = report.ReportWriter()
# Create writer for stats per well
well_xlsx_path = os.path.join(
constants.RUN_PATH,
'stats_per_well.xlsx',
)
well_xlsx_writer = pd.ExcelWriter(well_xlsx_path)
antigen_df = reporter.get_antigen_df()
antigen_df.to_excel(well_xlsx_writer, sheet_name='antigens')
# Initialize background estimator
bg_estimator = background_estimator.BackgroundEstimator2D(
block_size=128,
order=2,
normalize=False,
)
# Get grid rows and columns from params
nbr_grid_rows = constants.params['rows']
nbr_grid_cols = constants.params['columns']
fiducials_idx = constants.FIDUCIALS_IDX
# Create spot detector instance
spot_detector = img_processing.SpotDetector(
imaging_params=constants.params, )
well_images = io_utils.get_image_paths(input_dir)
well_names = list(well_images)
# If rerunning only a subset of wells
if constants.RERUN:
logger.info("Rerunning wells: {}".format(constants.RERUN_WELLS))
txt_parser.rerun_xl_od(
well_names=well_names,
well_xlsx_path=well_xlsx_path,
rerun_names=constants.RERUN_WELLS,
xlsx_writer=well_xlsx_writer,
)
reporter.load_existing_reports()
well_names = constants.RERUN_WELLS
# remove debug images from old runs
for f in os.listdir(constants.RUN_PATH):
if f.split('_')[0] in well_names:
os.remove(os.path.join(constants.RUN_PATH, f))
else:
reporter.create_new_reports()
# ================
# loop over well images
# ================
for well_name in well_names:
start_time = time.time()
im_path = well_images[well_name]
image = io_utils.read_gray_im(im_path)
logger.info("Extracting well: {}".format(well_name))
# Get max intensity
max_intensity = io_utils.get_max_intensity(image)
logger.debug("Image max intensity: {}".format(max_intensity))
# Crop image to well only
"""""
try:
well_center, well_radi, _ = image_parser.find_well_border(
image,
detmethod='region',
segmethod='otsu',
)
im_well, _ = img_processing.crop_image_at_center(
im=image,
center=well_center,
height=2 * well_radi,
width=2 * well_radi,
)
except IndexError:
logging.warning("Couldn't find well in {}".format(well_name))
im_well = image
""" ""
im_well = image
# Find spot center coordinates
spot_coords = spot_detector.get_spot_coords(
im=im_well,
max_intensity=max_intensity,
)
if spot_coords.shape[0] < constants.MIN_NBR_SPOTS:
logging.warning("Not enough spots detected in {},"
"continuing.".format(well_name))
continue
# Create particle filter registration instance
register_inst = registration.ParticleFilter(
spot_coords=spot_coords,
im_shape=im_well.shape,
fiducials_idx=fiducials_idx,
)
register_inst.particle_filter()
if not register_inst.registration_ok:
logger.warning("Registration failed for {}, "
"repeat with outlier removal".format(well_name))
register_inst.particle_filter(nbr_outliers=nbr_outliers)
# Transform grid coordinates
registered_coords = register_inst.compute_registered_coords()
# Check that registered coordinates are inside well
registration_ok = register_inst.check_reg_coords()
if not registration_ok:
logger.warning("Final registration failed,"
"will not write OD for {}".format(well_name))
if constants.DEBUG:
debug_plots.plot_registration(
im_well,
spot_coords,
register_inst.fiducial_coords,
registered_coords,
os.path.join(constants.RUN_PATH, well_name + '_failed'),
max_intensity=max_intensity,
)
continue
# Crop image
im_crop, crop_coords = img_processing.crop_image_from_coords(
im=im_well,
coords=registered_coords,
)
im_crop = im_crop / max_intensity
# Estimate background
background = bg_estimator.get_background(im_crop)
# Find spots near grid locations and compute properties
spots_df, spot_props = array_gen.get_spot_intensity(
coords=crop_coords,
im=im_crop,
background=background,
)
# Write metrics for each spot in grid in current well
spots_df.to_excel(well_xlsx_writer, sheet_name=well_name)
# Assign well OD, intensity, and background stats to plate
reporter.assign_well_to_plate(well_name, spots_df)
time_msg = "Time to extract OD in {}: {:.3f} s".format(
well_name,
time.time() - start_time,
)
print(time_msg)
logger.info(time_msg)
# ==================================
# SAVE FOR DEBUGGING
if constants.DEBUG:
start_time = time.time()
# Save spot and background intensities
output_name = os.path.join(constants.RUN_PATH, well_name)
# Save OD plots, composite spots and registration
debug_plots.plot_od(
spots_df=spots_df,
nbr_grid_rows=nbr_grid_rows,
nbr_grid_cols=nbr_grid_cols,
output_name=output_name,
)
debug_plots.save_composite_spots(
spot_props=spot_props,
output_name=output_name,
image=im_crop,
)
debug_plots.plot_background_overlay(
im_crop,
background,
output_name,
)
debug_plots.plot_registration(
image=im_well,
spot_coords=spot_coords,
grid_coords=register_inst.fiducial_coords,
reg_coords=registered_coords,
output_name=output_name,
max_intensity=max_intensity,
)
logger.debug(
"Time to save debug images: {:.3f} s".format(time.time() -
start_time), )
# After running all wells, write plate reports
well_xlsx_writer.close()
reporter.write_reports()
def well_analysis(input_dir, output_dir, method='segmentation'):
"""
Workflow that pulls all images scanned on a multi-well plate in a standard ELISA format (one antigen per well)
It loops over the images in the input_folder (for images acquired using Micro-Manager ONLY).
Extracts the center of the well, calculates the median intensity of that spot and background, then computes OD.
Finally, it writes a summary report (.xlsx) containing plate info and the computed values.
:param input_dir: str path to experiment directory
:param output_dir: str output path to write report and diagnostic images
:param method: str 'segmentation' or 'crop'. Methods to estimate the boundaries of the well
:return:
"""
start = time.time()
# metadata isn't used for the well format
MetaData(input_dir, output_dir)
# Read plate info
plate_info = pd.read_excel(
os.path.join(input_dir, 'Plate_Info.xlsx'),
usecols='A:M',
sheet_name=None,
index_col=0,
)
# Write an excel file that can be read into jupyter notebook with minimal parsing.
xlwriter_int = pd.ExcelWriter(
os.path.join(constants.RUN_PATH, 'intensities.xlsx'))
# get well directories
well_images = io_utils.get_image_paths(input_dir)
int_well = []
for well_name, im_path in well_images.items():
# read image
image = io_utils.read_gray_im(im_path)
print(well_name)
# measure intensity
if method == 'segmentation':
# segment well using otsu thresholding
well_mask = image_parser.get_well_mask(image, segmethod='otsu')
int_well_ = image_parser.get_well_intensity(image, well_mask)
elif method == 'crop':
# get intensity at square crop in the middle of the image
img_size = image.shape
radius = np.floor(0.1 * np.min(img_size)).astype('int')
cx = np.floor(img_size[1] / 2).astype('int')
cy = np.floor(img_size[0] / 2).astype('int')
im_crop = processing.crop_image(image, cx, cy, radius, border_=0)
well_mask = np.ones_like(im_crop, dtype='bool')
int_well_ = image_parser.get_well_intensity(im_crop, well_mask)
int_well.append(int_well_)
# SAVE FOR DEBUGGING
if constants.DEBUG:
output_name = os.path.join(constants.RUN_PATH, well_name)
# Save mask of the well, cropped grayscale image, cropped spot segmentation.
io.imsave(output_name + "_well_mask.png",
(255 * well_mask).astype('uint8'))
# Save masked image
if method == 'segmentation':
img_ = image.copy()
img_[~well_mask] = 0
elif method == 'crop':
img_ = im_crop.copy()
img_[~well_mask] = 0
else:
raise NotImplementedError(
f'method of type {method} not supported')
io.imsave(output_name + "_masked_image.png",
(img_ / 256).astype('uint8'))
df_int = pd.DataFrame(
np.reshape(int_well, (8, 12)),
index=list(string.ascii_uppercase[:8]),
columns=range(1, 13),
)
plate_info.update({'intensity': df_int})
# compute optical density
sample_info = plate_info['sample']
blanks = np.any(np.dstack((sample_info == 'Blank', sample_info == 'blank',
sample_info == 'BLANK')),
axis=2)
if blanks.any():
# blank intensity is averaged over all blank wells
int_blank = np.mean(df_int.to_numpy()[blanks])
df_od = np.log10(int_blank / df_int)
plate_info.update({'od': df_od})
# save analysis results
for k, v in plate_info.items():
v.to_excel(xlwriter_int, sheet_name=k)
xlwriter_int.close()
stop = time.time()
print(f"\ttime to process={stop - start}")
def interp(input_dir, output_dir):
MetaData(input_dir, output_dir)
# Write an excel file that can be read into jupyter notebook with minimal parsing.
xlwriter_od_well = pd.ExcelWriter(
os.path.join(constants.RUN_PATH, 'median_ODs_per_well.xlsx'), )
# Initialize background estimator
bg_estimator = background_estimator.BackgroundEstimator2D(
block_size=128,
order=2,
normalize=False,
)
# ================
# loop over images => good place for multiproc? careful with columns in report
# ================
well_images = io_utils.get_image_paths(input_dir)
for well_name, im_path in well_images.items():
start = time.time()
image = io_utils.read_gray_im(im_path)
spot_props_array = txt_parser.create_array(
constants.params['rows'],
constants.params['columns'],
dtype=object,
)
bgprops_array = txt_parser.create_array(
constants.params['rows'],
constants.params['columns'],
dtype=object,
)
# finding center of well and cropping
well_center, well_radi, well_mask = image_parser.find_well_border(
image, detmethod='region', segmethod='otsu')
im_crop, _ = img_processing.crop_image_at_center(
image, well_center, 2 * well_radi, 2 * well_radi)
# find center of spots from crop
spot_mask = img_processing.thresh_and_binarize(im_crop,
method='bright_spots')
spot_props = image_parser.generate_props(spot_mask,
intensity_image_=im_crop)
# if debug:
crop_coords = image_parser.grid_from_centroids(
spot_props, constants.params['rows'], constants.params['columns'])
# convert to float64
im_crop = im_crop / np.iinfo(im_crop.dtype).max
background = bg_estimator.get_background(im_crop)
props_by_loc, bgprops_by_loc = array_gen.get_spot_intensity(
coords=crop_coords,
im_int=im_crop,
background=background,
params=constants.params)
props_array_placed = image_parser.assign_props_to_array(
spot_props_array, props_by_loc)
bgprops_array = image_parser.assign_props_to_array(
bgprops_array, bgprops_by_loc)
od_well, int_well, bg_well = image_parser.compute_od(
props_array_placed, bgprops_array)
pd_OD = pd.DataFrame(od_well)
pd_OD.to_excel(xlwriter_od_well, sheet_name=well_name)
# populate 96-well plate constants with OD, INT, BG arrays
report.write_od_to_plate(od_well, well_name, 'od')
report.write_od_to_plate(int_well, well_name, 'int')
report.write_od_to_plate(bg_well, well_name, 'bg')
stop = time.time()
print(f"\ttime to process={stop-start}")
# SAVE FOR DEBUGGING
if constants.DEBUG:
# Save spot and background intensities.
output_name = os.path.join(constants.RUN_PATH, well_name)
# # Save mask of the well, cropped grayscale image, cropped spot segmentation.
io.imsave(output_name + "_well_mask.png",
(255 * well_mask).astype('uint8'))
io.imsave(output_name + "_crop.png",
(255 * im_crop).astype('uint8'))
io.imsave(output_name + "_crop_binary.png",
(255 * spot_mask).astype('uint8'))
# Evaluate accuracy of background estimation with green (image), magenta (background) overlay.
im_bg_overlay = np.stack([background, im_crop, background], axis=2)
io.imsave(output_name + "_crop_bg_overlay.png",
(255 * im_bg_overlay).astype('uint8'))
# This plot shows which spots have been assigned what index.
debug_plots.plot_centroid_overlay(
im_crop,
constants.params,
props_by_loc,
bgprops_by_loc,
output_name,
)
debug_plots.plot_od(
od_well,
int_well,
bg_well,
output_name,
)
# save a composite of all spots, where spots are from source or from region prop
debug_plots.save_composite_spots(im_crop,
props_array_placed,
output_name,
from_source=True)
debug_plots.save_composite_spots(im_crop,
props_array_placed,
output_name,
from_source=False)
stop2 = time.time()
print(f"\ttime to save debug={stop2-stop}")
xlwriter_od_well.close()
# create excel writers to write reports
xlwriter_od = pd.ExcelWriter(
os.path.join(constants.RUN_PATH, 'python_median_ODs.xlsx'))
xlwriter_int = pd.ExcelWriter(
os.path.join(constants.RUN_PATH, 'python_median_intensities.xlsx'))
xlwriter_bg = pd.ExcelWriter(
os.path.join(constants.RUN_PATH, 'python_median_backgrounds.xlsx'))
report.write_antigen_report(xlwriter_od, 'od')
report.write_antigen_report(xlwriter_int, 'int')
report.write_antigen_report(xlwriter_bg, 'bg')
xlwriter_od.close()
xlwriter_int.close()
xlwriter_bg.close()
def point_registration(input_dir, output_dir):
"""
For each image in input directory, detect spots using particle filtering
to register fiducial spots to blobs detected in the image.
:param str input_dir: Input directory containing images and an xml file
with parameters
:param str output_dir: Directory where output is written to
"""
MetaData(input_dir, output_dir)
xlwriter_od_well = pd.ExcelWriter(
os.path.join(constants.RUN_PATH, 'median_ODs_per_well.xlsx'), )
pdantigen = pd.DataFrame(constants.ANTIGEN_ARRAY)
pdantigen.to_excel(xlwriter_od_well, sheet_name='antigens')
# Initialize background estimator
bg_estimator = background_estimator.BackgroundEstimator2D(
block_size=128,
order=2,
normalize=False,
)
# Get grid rows and columns from params
nbr_grid_rows = constants.params['rows']
nbr_grid_cols = constants.params['columns']
fiducials_idx = constants.FIDUCIALS_IDX
# Create spot detector instance
spot_detector = img_processing.SpotDetector(
imaging_params=constants.params, )
# ================
# loop over well images
# ================
well_images = io_utils.get_image_paths(input_dir)
for well_name, im_path in well_images.items():
start_time = time.time()
image = io_utils.read_gray_im(im_path)
# Get max intensity
max_intensity = np.iinfo(image.dtype).max
# Crop image to well only
try:
well_center, well_radi, well_mask = image_parser.find_well_border(
image,
detmethod='region',
segmethod='otsu',
)
im_well, _ = img_processing.crop_image_at_center(
image,
well_center,
2 * well_radi,
2 * well_radi,
)
except IndexError:
warnings.warn("Couldn't find well in {}".format(well_name))
im_well = image
# Find spot center coordinates
spot_coords = spot_detector.get_spot_coords(im_well)
# Initial estimate of spot center
center_point = tuple((im_well.shape[0] / 2, im_well.shape[1] / 2))
grid_coords = registration.create_reference_grid(
center_point=center_point,
nbr_grid_rows=nbr_grid_rows,
nbr_grid_cols=nbr_grid_cols,
spot_dist=constants.SPOT_DIST_PIX,
)
fiducial_coords = grid_coords[fiducials_idx, :]
# Use particle filter to register fiducials to detected spots
reg_ok = True
particles = registration.create_gaussian_particles(
stds=np.array(constants.STDS),
nbr_particles=4000,
)
t_matrix, min_dist = registration.particle_filter(
fiducial_coords=fiducial_coords,
spot_coords=spot_coords,
particles=particles,
stds=np.array(constants.STDS),
debug=constants.DEBUG,
)
if min_dist > constants.REG_DIST_THRESH:
warnings.warn("Registration failed, repeat with outlier removal")
t_matrix, min_dist = registration.particle_filter(
fiducial_coords=fiducial_coords,
spot_coords=spot_coords,
particles=particles,
stds=np.array(constants.STDS),
remove_outlier=True,
debug=constants.DEBUG,
)
# Warn if fit is still bad
if min_dist > constants.REG_DIST_THRESH:
reg_ok = False
if not reg_ok:
warnings.warn("Final registration failed,"
"will not write OD for {}".format(well_name))
if constants.DEBUG:
reg_coords = np.squeeze(
cv.transform(np.array([grid_coords]), t_matrix))
debug_plots.plot_registration(
im_well,
spot_coords,
grid_coords[fiducials_idx, :],
reg_coords,
os.path.join(constants.RUN_PATH, well_name + '_failed'),
max_intensity=max_intensity,
)
continue
# Transform grid coordinates
reg_coords = np.squeeze(cv.transform(np.array([grid_coords]),
t_matrix))
# Crop image
im_crop, crop_coords = img_processing.crop_image_from_coords(
im=im_well,
grid_coords=reg_coords,
)
im_crop = im_crop / max_intensity
# Estimate background
background = bg_estimator.get_background(im_crop)
# Find spots near grid locations
props_placed_by_loc, bgprops_by_loc = array_gen.get_spot_intensity(
coords=crop_coords,
im_int=im_crop,
background=background,
params=constants.params,
)
# Create arrays and assign properties
props_array = txt_parser.create_array(
constants.params['rows'],
constants.params['columns'],
dtype=object,
)
bgprops_array = txt_parser.create_array(
constants.params['rows'],
constants.params['columns'],
dtype=object,
)
props_array_placed = image_parser.assign_props_to_array(
props_array,
props_placed_by_loc,
)
bgprops_array = image_parser.assign_props_to_array(
bgprops_array,
bgprops_by_loc,
)
od_well, int_well, bg_well = image_parser.compute_od(
props_array_placed,
bgprops_array,
)
# populate 96-well plate constants with OD, INT, BG arrays
report.write_od_to_plate(od_well, well_name, 'od')
report.write_od_to_plate(int_well, well_name, 'int')
report.write_od_to_plate(bg_well, well_name, 'bg')
# Write ODs per well
pd_od = pd.DataFrame(od_well)
pd_od.to_excel(xlwriter_od_well, sheet_name=well_name)
print(
"Time to register grid to {}: {:.3f} s".format(
well_name,
time.time() - start_time), )
# ==================================
# SAVE FOR DEBUGGING
if constants.DEBUG:
start_time = time.time()
# Save spot and background intensities
output_name = os.path.join(constants.RUN_PATH, well_name)
# Save OD plots, composite spots and registration
debug_plots.plot_od(
od_well,
int_well,
bg_well,
output_name,
)
debug_plots.save_composite_spots(
im_crop,
props_array_placed,
output_name,
from_source=True,
)
debug_plots.plot_background_overlay(
im_crop,
background,
output_name,
)
debug_plots.plot_registration(
im_well,
spot_coords,
grid_coords[fiducials_idx, :],
reg_coords,
output_name,
max_intensity=max_intensity,
)
print(
"Time to save debug images: {:.3f} s".format(time.time() -
start_time), )
xlwriter_od = pd.ExcelWriter(
os.path.join(constants.RUN_PATH, 'median_ODs.xlsx'), )
xlwriter_int = pd.ExcelWriter(
os.path.join(constants.RUN_PATH, 'median_intensities.xlsx'), )
xlwriter_bg = pd.ExcelWriter(
os.path.join(constants.RUN_PATH, 'median_backgrounds.xlsx'), )
report.write_antigen_report(xlwriter_od, 'od')
report.write_antigen_report(xlwriter_int, 'int')
report.write_antigen_report(xlwriter_bg, 'bg')
xlwriter_od.close()
xlwriter_int.close()
xlwriter_bg.close()
xlwriter_od_well.close()