def calibrate(ims_import_res,
n_best_fields=6,
divs=5,
metadata=None,
progress=None):
if metadata is None:
metadata = {}
calib = Calibration({f"metadata.instrument": metadata})
qdf = ims_import_res.qualities()
quality = qdf.sort_values(["field_i", "channel_i",
"cycle_i"])["quality"].values.reshape(
(ims_import_res.n_fields,
ims_import_res.n_channels,
ims_import_res.n_cycles))
best_field_iz = np.argsort(np.sum(
quality, axis=(1, 2)))[::-1][0:n_best_fields].tolist()
n_cycles = ims_import_res.n_cycles
zstack_depths = [
0
] * n_cycles # TASK: This will need to come from ims_import_res metadata
calib.add({f"zstack_depths.instrument": zstack_depths})
_calibrate(ims_import_res.ims[best_field_iz, :, :],
calib,
divs=divs,
progress=progress)
return calib
def validate(self):
# Note: does not call super because the override_nones is set to false here
self.schema.apply_defaults(self.defaults,
apply_to=self,
override_nones=False)
self.schema.validate(self, context=self.__class__.__name__)
self.calibration = Calibration(self.calibration)
if self.instrument_subject_id is not None:
self.calibration.filter_subject_ids(self.instrument_subject_id)
if len(self.calibration.keys()) == 0:
raise ValueError(
f"All calibration records removed after filter_subject_ids on subject_id '{self.instrument_subject_id}'"
)
assert not self.calibration.has_subject_ids()
if self.radiometry_channels is not None:
pat = re.compile(r"[0-9a-z_]+")
for name, channel_i in self.radiometry_channels.items():
self._validate(
pat.fullmatch(name),
"radiometry_channels name must be lower-case alphanumeric (including underscore)",
)
self._validate(isinstance(channel_i, int),
"channel_i must be an integer")
def it_filters():
c = Calibration({
"p_failure_to_bind_amino_acid.label[C].sn1234": 0.5,
"p_failure_to_bind_amino_acid.label[C].sn0": 1.0,
})
c.filter_subject_ids({"sn1234"})
assert len(c) == 1
assert c["p_failure_to_bind_amino_acid.label[C]"] == 0.5
def it_sets_subject_id():
c = Calibration({
"p_failure_to_bind_amino_acid.label[C]": 1.0,
"p_failure_to_attach_to_dye.label[C]": 2.0,
})
c.set_subject_id("test")
assert c["p_failure_to_bind_amino_acid.label[C].test"] == 1.0
assert c["p_failure_to_attach_to_dye.label[C].test"] == 2.0
def it_balances_regionally():
sigproc_params = SigprocV2Params(
radiometry_channels=dict(aaa=0, bbb=1),
calibration=Calibration(
{
"regional_illumination_balance.instrument_channel[0].test": [
[1.0, 5.0],
[1.0, 1.0],
],
"regional_illumination_balance.instrument_channel[1].test": [
[7.0, 1.0],
[1.0, 1.0],
],
"regional_bg_mean.instrument_channel[0].test": [
[100.0, 100.0],
[100.0, 100.0],
],
"regional_bg_mean.instrument_channel[1].test": [
[200.0, 200.0],
[200.0, 200.0],
],
}
),
instrument_subject_id="test",
)
chcy_ims = np.ones((2, 1, 128, 128))
chcy_ims[0] *= 1000.0
chcy_ims[1] *= 2000.0
balanced_ims = worker._import_balanced_images(chcy_ims, sigproc_params)
assert np.all(np.isclose(balanced_ims[0, 0, 0, 0], (1000 - 100) * 2))
assert np.all(np.isclose(balanced_ims[0, 0, 0, 127], (1000 - 100) * 2 * 5))
assert np.all(np.isclose(balanced_ims[1, 0, 0, 0], (2000 - 200) * 1 * 7))
assert np.all(np.isclose(balanced_ims[1, 0, 127, 0], (2000 - 200) * 1))
def generate(self):
run_descs = []
calibration = Calibration.from_yaml(self.calibration_file)
sigproc_tasks = self.sigprocs_v2(
calibration=calibration, instrument_subject_id=self.instrument_subject_id,
)
if len(sigproc_tasks) == 0:
raise ValueError(
"No sigprocv2 tasks were found. This might be due to an empty block of another switch."
)
for sigproc_i, sigproc_task in enumerate(sigproc_tasks):
lnfit_tasks = self.lnfits()
sigproc_source = ""
for k, v in sigproc_task.items():
if "ims_import" in k:
sigproc_source = local.path(v.inputs.src_dir).name
break
run_name = f"sigproc_v2_{sigproc_i}_{sigproc_source}"
if self.force_run_name is not None:
run_name = self.force_run_name
run_desc = Munch(run_name=run_name, **sigproc_task, **lnfit_tasks,)
sigproc_template = "sigproc_v2_template.ipynb"
if self.movie:
sigproc_template = "sigproc_v2_movie_template.ipynb"
self.report_section_markdown(f"# RUN {run_desc.run_name}\n")
self.report_section_run_object(run_desc)
self.report_section_from_template(sigproc_template)
if lnfit_tasks:
self.report_section_from_template("lnfit_template.ipynb")
run_descs += [run_desc]
n_run_descs = len(run_descs)
self.report_preamble(
utils.smart_wrap(
f"""
# Signal Processing Overview
## {n_run_descs} run(s) processed.
"""
)
)
return run_descs
def sigproc(sigproc_params, ims_import_result, progress=None):
"""
Analyze all fields
"""
calib = Calibration(sigproc_params.calibration)
assert not calib.is_empty()
channel_weights = _compute_channel_weights(sigproc_params)
sigproc_result = SigprocV2Result(
params=sigproc_params,
n_input_channels=ims_import_result.n_channels,
n_channels=sigproc_params.n_output_channels,
n_cycles=ims_import_result.n_cycles,
channel_weights=channel_weights,
)
n_fields = ims_import_result.n_fields
n_fields_limit = sigproc_params.n_fields_limit
if n_fields_limit is not None and n_fields_limit < n_fields:
n_fields = n_fields_limit
zap.work_orders(
[
Munch(
fn=_do_sigproc_field,
ims_import_result=ims_import_result,
sigproc_params=sigproc_params,
field_i=field_i,
sigproc_result=sigproc_result,
) for field_i in range(n_fields)
],
_trap_exceptions=False,
_progress=progress,
)
return sigproc_result
def _setup(corner_bal):
divs = 5
bg = 100 * np.ones((divs, divs))
bal = np.ones((divs, divs))
bal[0, 0] = corner_bal
chcy_ims = 101 * np.ones((2, 4, 512, 512))
chcy_ims[1, :, :, :] += 1
calib = Calibration(
{
"regional_bg_mean.instrument_channel[0]": bg.tolist(),
"regional_illumination_balance.instrument_channel[0]": bal.tolist(),
"regional_bg_mean.instrument_channel[1]": bg.tolist(),
"regional_illumination_balance.instrument_channel[1]": bal.tolist(),
}
)
return chcy_ims, calib
def it_returns_balanced_channels():
sigproc_params = SigprocV2Params(
radiometry_channels=dict(aaa=0, bbb=1),
calibration=Calibration(
{
"regional_bg_mean.instrument_channel[0].test": [
[100.0, 100.0],
[100.0, 100.0],
],
"regional_bg_mean.instrument_channel[1].test": [
[200.0, 200.0],
[200.0, 200.0],
],
}
),
instrument_subject_id="test",
)
balance = worker._compute_channel_weights(sigproc_params)
assert np.all(balance == [2.0, 1.0])
def it_checks_variable_subtype():
with zest.raises(TypeError):
Calibration(
{"p_failure_to_bind_amino_acid.label[foo].sn1234": 1.0})
def _calibrate(flchcy_ims, divs=5, progress=None, overload_psf=None):
"""
Accumulate calibration data using a set of fields.
Arguments:
flchcy_ims: frame, channel, cycles ims to be analyzed
These are typically only a small subset of high quality fields.
NOTE: "Cycles" here are considered to be z-stack slices, NOT chem-cycles.
divs: The regional sub-divisions.
"""
n_fields, n_channels, n_cycles = flchcy_ims.shape[0:3]
n_z_slices = n_cycles # This is just an alias to remind me that cycle=z-slice here.
peak_mea = 11
peak_dim = (peak_mea, peak_mea)
if overload_psf is not None:
# This is used for testing
peak_dim = overload_psf.shape
calib = Calibration()
for ch_i in range(n_channels):
z_and_region_to_psf = np.zeros((n_z_slices, divs, divs, *peak_dim))
# BACKGROUND
# Masks out the foreground and uses remaining pixels to estimate
# regional background mean and std.
# --------------------------------------------------------------
flcy_calibs = [
_calibrate_bg_and_psf_im(flchcy_ims[fl_i, ch_i, cy_i])
for fl_i in range(n_fields) for cy_i in range(n_cycles)
]
calib.add({
f"regional_bg_mean.instrument_channel[{ch_i}]":
np.mean([
np.array(
flcy_calibs[f"regional_bg_mean.instrument_channel[{ch_i}]"]
) for c in flcy_calibs
])
})
# reg_psfs = np.sum([
# np.array(c[f"regional_psf_zstack.instrument_channel[{ch_i}]"])
# for c in flcy_calibs
# ], axis=(2, 3))
#
# denominator = np.sum(z_and_region_to_psf, axis=(2, 3))[:, :, None, None]
# calib.add({
# f"regional_psf_zstack.instrument_channel[{ch_i}]": reg_psfs /
# })
#
# z_and_region_to_psf = utils.np_safe_divide(z_and_region_to_psf, denominator)
#
# calib.add({
# f"regional_bg_std.instrument_channel[{ch_i}]": np.mean([
# np.array(c[f"regional_bg_std.instrument_channel[{ch_i}]"])
# for c in flcy_calibs
# ])
# })
# if overload_psf is not None:
# # This is used for testing
# z_and_region_to_psf = np.broadcast_to(
# overload_psf, (n_z_slices, divs, divs, *peak_dim)
# ).tolist()
#
# else:
# # PSF
# # Accumulate the PSF regionally over every field
# # Then divide each PSF though by it's own mass so that the
# # AUC under each PSF is 1.
# # --------------------------------------------------------------
# [
# _calibrate_bg_im(flchcy_ims[fl_i, ch_i, cy_i], regional_bg_mean, regional_bg_std)
# for fl_i in range(n_fields)
# for cy_i in range(n_cycles)
# ]
#
# for fl_i in range(n_fields):
#
# for cy_i in range(n_cycles):
# # Remember: cy_i is a pseudo-cycle: it is really a z-slice
# # with z_depths[cy_i] holding the actual depth
#
# regional_bg_mean = np.array(
# calib[f"regional_bg_mean.instrument_channel[{ch_i}]"]
# )
# _calibrate_psf_im(flchcy_ims[fl_i, ch_i, cy_i], regional_bg_mean)
#
# # ACCUMULATE each field, will normalize at the end
# z_and_region_to_psf[cy_i] += reg_psfs
# # NORMALIZE all psfs
# denominator = np.sum(z_and_region_to_psf, axis=(3, 4))[:, :, :, None, None]
# z_and_region_to_psf = utils.np_safe_divide(z_and_region_to_psf, denominator)
#
# calib.add(
# {
# f"regional_psf_zstack.instrument_channel[{ch_i}]": z_and_region_to_psf.tolist()
# }
# )
# FOREGROUND
# Runs the standard sigproc_field analysis (without balancing)
# to get the regional radmats for regional histogram balancing.
# This requires that the PSF already be estimated so that the
# radiometry can run.
# --------------------------------------------------------------
# Spoof the sigproc_v2 worker into bypassing illumination balance by giving it all zeros
calib.add({
f"regional_illumination_balance.instrument_channel[{ch_i}]":
np.ones((divs, divs)).tolist()
})
sigproc_params = SigprocV2Params(
calibration=calib,
instrument_subject_id=None,
radiometry_channels=dict(ch=ch_i),
)
fl_radmats = []
fl_locs = []
for fl_i in range(n_fields):
if progress is not None:
progress(fl_i, n_fields)
chcy_ims = flchcy_ims[fl_i, ch_i:(ch_i + 1), :]
(
chcy_ims,
locs,
radmat,
aln_offsets,
aln_scores,
) = sigproc_field(chcy_ims, sigproc_params)
fl_radmats += [radmat]
fl_locs += [locs]
fl_radmat = np.concatenate(fl_radmats)
fl_loc = np.concatenate(fl_locs)
# BALANCE
sig = np.nan_to_num(fl_radmat[:, ch_i, :, 0].flatten())
noi = fl_radmat[:, ch_i, :, 1].flatten()
snr = np.nan_to_num(sig / noi)
locs = np.tile(fl_loc, (1, n_cycles)).reshape((-1, 2))
snr_mask = snr > 10
sig = sig[snr_mask]
locs = locs[snr_mask]
top = np.max((locs[:, 0], locs[:, 1]))
y = utils.ispace(0, top, divs + 1)
x = utils.ispace(0, top, divs + 1)
def regional_locs_mask(yi, xi):
"""Create a mask for locs inside of a region"""
mask = (y[yi] <= locs[:, 0]) & (locs[:, 0] < y[yi + 1])
mask &= (x[xi] <= locs[:, 1]) & (locs[:, 1] < x[xi + 1])
return mask
medians = np.zeros((divs, divs))
for yi in range(len(y) - 1):
for xi in range(len(x) - 1):
loc_mask = regional_locs_mask(yi, xi)
bright_mask = sig > 2.0
_sig = sig[loc_mask & bright_mask]
medians[yi, xi] = np.median(_sig)
center = np.max(medians)
balance = np.zeros((divs, divs))
for yi in range(len(y) - 1):
for xi in range(len(x) - 1):
loc_mask = regional_locs_mask(yi, xi)
bright_mask = sig > 2.0
_sig = sig[loc_mask & bright_mask]
for yi in range(len(y) - 1):
for xi in range(len(x) - 1):
loc_mask = regional_locs_mask(yi, xi)
bright_mask = sig > 2.0
_sig = sig[loc_mask & bright_mask]
median = np.median(_sig)
balance[yi, xi] = center / median
_sig *= balance[yi, xi]
calib.add({
f"regional_illumination_balance.instrument_channel[{ch_i}]":
balance.tolist()
})
return calib
def it_adds():
c = Calibration(
{"p_failure_to_bind_amino_acid.label[C].batch_2020_03_01": 1.0})
c.add({"p_failure_to_attach_to_dye.label[C].batch_2020_03_01": 2.0})
assert len(c) == 2
def it_checks_prop():
with zest.raises(TypeError):
Calibration({"not_a_property.instrument.sn1234": 1})
def it_checks_subject_type():
with zest.raises(TypeError):
Calibration(
{"p_failure_to_bind_amino_acid.not_a_subject.sn1234": 1})
def calib(self):
return Calibration(self.params.calibration)
def it_checks_value():
with zest.raises(TypeError):
Calibration({
"p_failure_to_bind_amino_acid.label[C].sn1234":
"not_a_float"
})
def it_checks_propsub():
with zest.raises(TypeError):
Calibration(
{"p_failure_to_bind_amino_acid.instrument.sn1234": 1})
def it_checks_subject_id():
with zest.raises(TypeError):
Calibration({"p_failure_to_bind_amino_acid.label[C].1234": 1})
def it_checks_metadata():
with zest.raises(TypeError):
Calibration({"metadata.instrument.sn1234": "not a dict"})
def it_allows_metadata():
c = Calibration({"metadata.instrument.sn1234": dict(a=1)})
assert c["metadata.instrument.sn1234"]["a"] == 1
class SigprocV2Params(Params):
defaults = dict(
radiometry_channels=None,
n_fields_limit=None,
save_full_signal_radmat_npy=False,
# use_cycle_zero_psfs_only=False,
)
schema = s(
s.is_kws_r(
radiometry_channels=s.is_dict(noneable=True),
n_fields_limit=s.is_int(noneable=True),
save_full_signal_radmat_npy=s.is_bool(),
calibration=s.is_dict(),
instrument_subject_id=s.is_str(noneable=True),
# use_cycle_zero_psfs_only=s.is_bool(),
))
def validate(self):
# Note: does not call super because the override_nones is set to false here
self.schema.apply_defaults(self.defaults,
apply_to=self,
override_nones=False)
self.schema.validate(self, context=self.__class__.__name__)
self.calibration = Calibration(self.calibration)
if self.instrument_subject_id is not None:
self.calibration.filter_subject_ids(self.instrument_subject_id)
if len(self.calibration.keys()) == 0:
raise ValueError(
f"All calibration records removed after filter_subject_ids on subject_id '{self.instrument_subject_id}'"
)
assert not self.calibration.has_subject_ids()
if self.radiometry_channels is not None:
pat = re.compile(r"[0-9a-z_]+")
for name, channel_i in self.radiometry_channels.items():
self._validate(
pat.fullmatch(name),
"radiometry_channels name must be lower-case alphanumeric (including underscore)",
)
self._validate(isinstance(channel_i, int),
"channel_i must be an integer")
def set_radiometry_channels_from_input_channels_if_needed(
self, n_channels):
if self.radiometry_channels is None:
# Assume channels from nd2 manifest
channels = list(range(n_channels))
self.radiometry_channels = {f"ch_{ch}": ch for ch in channels}
@property
def n_output_channels(self):
return len(self.radiometry_channels.keys())
@property
def n_input_channels(self):
return len(self.radiometry_channels.keys())
# @property
# def channels_cycles_dim(self):
# # This is a cache set in sigproc_v1.
# # It is a helper for the repetitive call:
# # n_outchannels, n_inchannels, n_cycles, dim =
# return self._outchannels_inchannels_cycles_dim
def _input_channels(self):
"""
Return a list that converts channel number of the output to the channel of the input
Example:
input might have channels ["foo", "bar"]
the radiometry_channels has: {"bar": 0}]
Thus this function returns [1] because the 0th output channel is mapped
to the "1" input channel
"""
return [
self.radiometry_channels[name]
for name in sorted(self.radiometry_channels.keys())
]
# def input_names(self):
# return sorted(self.radiometry_channels.keys())
def output_channel_to_input_channel(self, out_ch):
return self._input_channels()[out_ch]
def input_channel_to_output_channel(self, in_ch):
"""Not every input channel necessarily has an output; can return None"""
return utils.filt_first_arg(self._input_channels(),
lambda x: x == in_ch)
def sigproc_field(chcy_ims, sigproc_params, snr_thresh=None):
"""
Analyze one field and return values (do not save)
Arguments:
chcy_ims: In input order (from ims_import_result)
sigproc_params: The SigprocParams
snr_thresh: if non-None keeps only locs with S/R > snr_thresh
This is useful for debugging.
"""
calib = Calibration(sigproc_params.calibration)
# Step 1: Load the images in output channel order, balance, equalize
chcy_ims = _import_balanced_images(chcy_ims, sigproc_params)
# At this point, chcy_ims has its background subtracted and it is
# regionally balanced and channel equalized. It may contain negative
# values
#
# NOTE: at this point, chcy_ims are in OUTPUT CHANNEL order!
n_out_channels, n_cycles = chcy_ims.shape[0:2]
assert n_out_channels == sigproc_params.n_output_channels
# Step 2: Remove anomalies
for ch_i, cy_ims in enumerate(chcy_ims):
chcy_ims[ch_i] = imops.stack_map(cy_ims, _mask_anomalies_im)
# Step 3: Find alignment offsets
aln_offsets, aln_scores = _align(np.mean(chcy_ims, axis=0))
# Step 4: Composite with alignment
chcy_ims = _composite_with_alignment_offsets_chcy_ims(
chcy_ims, aln_offsets)
# chcy_ims is now only the intersection region so it may be smaller than the original
# Step 5: Peak find on combined channels
# The goal of previous channel equalization and regional balancing is that
# all pixels are now on an equal footing so we can now use
# a single values for fg_thresh and bg_thresh.
ch_mean_of_cy0_im = np.mean(chcy_ims[:, 0, :, :], axis=0)
locs = _peak_find(ch_mean_of_cy0_im)
# Step 6: Radiometry over each channel, cycle
# TASK: Eventually this will examine each cycle and decide
# which z-depth of the PSFs is best fit to that cycle.
# The result will be a per-cycle index into the chcy_regional_psfs
# Until then the index is hard-coded to the zero-th index of regional_psf_zstack
ch_z_reg_psfs = np.stack(
[
np.array(calib[
f"regional_psf_zstack.instrument_channel[{sigproc_params.output_channel_to_input_channel(out_ch_i)}]"]
) for out_ch_i in range(n_out_channels)
],
axis=0,
)
assert ch_z_reg_psfs.shape[0] == n_out_channels
cycle_to_z_index = np.zeros((n_cycles, )).astype(int)
radmat = _radiometry(chcy_ims, locs, ch_z_reg_psfs, cycle_to_z_index)
# Step 7: Remove empties
# Keep any loc that has a signal > 20 times the minimum bg std in any channel
# The 20 was found somewhat empirically and may need to be adjusted
keep_mask = np.zeros((radmat.shape[0], )) > 0
for out_ch_i in range(n_out_channels):
in_ch_i = sigproc_params.output_channel_to_input_channel(out_ch_i)
bg_std = np.min(
calib[f"regional_bg_std.instrument_channel[{in_ch_i}]"])
keep_mask = keep_mask | np.any(radmat[:, out_ch_i, :, 0] > 20 * bg_std,
axis=1)
if snr_thresh is not None:
snr = radmat[:, :, :, 0] / radmat[:, :, :, 1]
keep_mask = keep_mask & np.any(np.nan_to_num(snr) > snr_thresh,
axis=(1, 2))
return chcy_ims, locs[keep_mask], radmat[
keep_mask], aln_offsets, aln_scores