def it_balances_regionally(): sigproc_params = SigprocV2Params( radiometry_channels=dict(aaa=0, bbb=1), calibration=Calibration( { "regional_illumination_balance.instrument_channel[0].test": [ [1.0, 5.0], [1.0, 1.0], ], "regional_illumination_balance.instrument_channel[1].test": [ [7.0, 1.0], [1.0, 1.0], ], "regional_bg_mean.instrument_channel[0].test": [ [100.0, 100.0], [100.0, 100.0], ], "regional_bg_mean.instrument_channel[1].test": [ [200.0, 200.0], [200.0, 200.0], ], } ), instrument_subject_id="test", ) chcy_ims = np.ones((2, 1, 128, 128)) chcy_ims[0] *= 1000.0 chcy_ims[1] *= 2000.0 balanced_ims = worker._import_balanced_images(chcy_ims, sigproc_params) assert np.all(np.isclose(balanced_ims[0, 0, 0, 0], (1000 - 100) * 2)) assert np.all(np.isclose(balanced_ims[0, 0, 0, 127], (1000 - 100) * 2 * 5)) assert np.all(np.isclose(balanced_ims[1, 0, 0, 0], (2000 - 200) * 1 * 7)) assert np.all(np.isclose(balanced_ims[1, 0, 127, 0], (2000 - 200) * 1))
def validate(self): # Note: does not call super because the override_nones is set to false here self.schema.apply_defaults(self.defaults, apply_to=self, override_nones=False) self.schema.validate(self, context=self.__class__.__name__) self.calibration = Calibration(self.calibration) if self.instrument_subject_id is not None: self.calibration.filter_subject_ids(self.instrument_subject_id) if len(self.calibration.keys()) == 0: raise ValueError( f"All calibration records removed after filter_subject_ids on subject_id '{self.instrument_subject_id}'" ) assert not self.calibration.has_subject_ids() if self.radiometry_channels is not None: pat = re.compile(r"[0-9a-z_]+") for name, channel_i in self.radiometry_channels.items(): self._validate( pat.fullmatch(name), "radiometry_channels name must be lower-case alphanumeric (including underscore)", ) self._validate(isinstance(channel_i, int), "channel_i must be an integer")
def calibrate(ims_import_res, n_best_fields=6, divs=5, metadata=None, progress=None): if metadata is None: metadata = {} calib = Calibration({f"metadata.instrument": metadata}) qdf = ims_import_res.qualities() quality = qdf.sort_values(["field_i", "channel_i", "cycle_i"])["quality"].values.reshape( (ims_import_res.n_fields, ims_import_res.n_channels, ims_import_res.n_cycles)) best_field_iz = np.argsort(np.sum( quality, axis=(1, 2)))[::-1][0:n_best_fields].tolist() n_cycles = ims_import_res.n_cycles zstack_depths = [ 0 ] * n_cycles # TASK: This will need to come from ims_import_res metadata calib.add({f"zstack_depths.instrument": zstack_depths}) _calibrate(ims_import_res.ims[best_field_iz, :, :], calib, divs=divs, progress=progress) return calib
def it_sets_subject_id(): c = Calibration({ "p_failure_to_bind_amino_acid.label[C]": 1.0, "p_failure_to_attach_to_dye.label[C]": 2.0, }) c.set_subject_id("test") assert c["p_failure_to_bind_amino_acid.label[C].test"] == 1.0 assert c["p_failure_to_attach_to_dye.label[C].test"] == 2.0
def it_filters(): c = Calibration({ "p_failure_to_bind_amino_acid.label[C].sn1234": 0.5, "p_failure_to_bind_amino_acid.label[C].sn0": 1.0, }) c.filter_subject_ids({"sn1234"}) assert len(c) == 1 assert c["p_failure_to_bind_amino_acid.label[C]"] == 0.5
def _setup(corner_bal): divs = 5 bg = 100 * np.ones((divs, divs)) bal = np.ones((divs, divs)) bal[0, 0] = corner_bal chcy_ims = 101 * np.ones((2, 4, 512, 512)) chcy_ims[1, :, :, :] += 1 calib = Calibration( { "regional_bg_mean.instrument_channel[0]": bg.tolist(), "regional_illumination_balance.instrument_channel[0]": bal.tolist(), "regional_bg_mean.instrument_channel[1]": bg.tolist(), "regional_illumination_balance.instrument_channel[1]": bal.tolist(), } ) return chcy_ims, calib
def it_returns_balanced_channels(): sigproc_params = SigprocV2Params( radiometry_channels=dict(aaa=0, bbb=1), calibration=Calibration( { "regional_bg_mean.instrument_channel[0].test": [ [100.0, 100.0], [100.0, 100.0], ], "regional_bg_mean.instrument_channel[1].test": [ [200.0, 200.0], [200.0, 200.0], ], } ), instrument_subject_id="test", ) balance = worker._compute_channel_weights(sigproc_params) assert np.all(balance == [2.0, 1.0])
def sigproc(sigproc_params, ims_import_result, progress=None): """ Analyze all fields """ calib = Calibration(sigproc_params.calibration) assert not calib.is_empty() channel_weights = _compute_channel_weights(sigproc_params) sigproc_result = SigprocV2Result( params=sigproc_params, n_input_channels=ims_import_result.n_channels, n_channels=sigproc_params.n_output_channels, n_cycles=ims_import_result.n_cycles, channel_weights=channel_weights, ) n_fields = ims_import_result.n_fields n_fields_limit = sigproc_params.n_fields_limit if n_fields_limit is not None and n_fields_limit < n_fields: n_fields = n_fields_limit zap.work_orders( [ Munch( fn=_do_sigproc_field, ims_import_result=ims_import_result, sigproc_params=sigproc_params, field_i=field_i, sigproc_result=sigproc_result, ) for field_i in range(n_fields) ], _trap_exceptions=False, _progress=progress, ) return sigproc_result
def it_checks_variable_subtype(): with zest.raises(TypeError): Calibration( {"p_failure_to_bind_amino_acid.label[foo].sn1234": 1.0})
def it_adds(): c = Calibration( {"p_failure_to_bind_amino_acid.label[C].batch_2020_03_01": 1.0}) c.add({"p_failure_to_attach_to_dye.label[C].batch_2020_03_01": 2.0}) assert len(c) == 2
def it_allows_metadata(): c = Calibration({"metadata.instrument.sn1234": dict(a=1)}) assert c["metadata.instrument.sn1234"]["a"] == 1
def calib(self): return Calibration(self.params.calibration)
def it_checks_value(): with zest.raises(TypeError): Calibration({ "p_failure_to_bind_amino_acid.label[C].sn1234": "not_a_float" })
def it_checks_propsub(): with zest.raises(TypeError): Calibration( {"p_failure_to_bind_amino_acid.instrument.sn1234": 1})
def it_checks_subject_id(): with zest.raises(TypeError): Calibration({"p_failure_to_bind_amino_acid.label[C].1234": 1})
def it_checks_subject_type(): with zest.raises(TypeError): Calibration( {"p_failure_to_bind_amino_acid.not_a_subject.sn1234": 1})
def it_checks_prop(): with zest.raises(TypeError): Calibration({"not_a_property.instrument.sn1234": 1})
def _calibrate(flchcy_ims, divs=5, progress=None, overload_psf=None): """ Accumulate calibration data using a set of fields. Arguments: flchcy_ims: frame, channel, cycles ims to be analyzed These are typically only a small subset of high quality fields. NOTE: "Cycles" here are considered to be z-stack slices, NOT chem-cycles. divs: The regional sub-divisions. """ n_fields, n_channels, n_cycles = flchcy_ims.shape[0:3] n_z_slices = n_cycles # This is just an alias to remind me that cycle=z-slice here. peak_mea = 11 peak_dim = (peak_mea, peak_mea) if overload_psf is not None: # This is used for testing peak_dim = overload_psf.shape calib = Calibration() for ch_i in range(n_channels): z_and_region_to_psf = np.zeros((n_z_slices, divs, divs, *peak_dim)) # BACKGROUND # Masks out the foreground and uses remaining pixels to estimate # regional background mean and std. # -------------------------------------------------------------- flcy_calibs = [ _calibrate_bg_and_psf_im(flchcy_ims[fl_i, ch_i, cy_i]) for fl_i in range(n_fields) for cy_i in range(n_cycles) ] calib.add({ f"regional_bg_mean.instrument_channel[{ch_i}]": np.mean([ np.array( flcy_calibs[f"regional_bg_mean.instrument_channel[{ch_i}]"] ) for c in flcy_calibs ]) }) # reg_psfs = np.sum([ # np.array(c[f"regional_psf_zstack.instrument_channel[{ch_i}]"]) # for c in flcy_calibs # ], axis=(2, 3)) # # denominator = np.sum(z_and_region_to_psf, axis=(2, 3))[:, :, None, None] # calib.add({ # f"regional_psf_zstack.instrument_channel[{ch_i}]": reg_psfs / # }) # # z_and_region_to_psf = utils.np_safe_divide(z_and_region_to_psf, denominator) # # calib.add({ # f"regional_bg_std.instrument_channel[{ch_i}]": np.mean([ # np.array(c[f"regional_bg_std.instrument_channel[{ch_i}]"]) # for c in flcy_calibs # ]) # }) # if overload_psf is not None: # # This is used for testing # z_and_region_to_psf = np.broadcast_to( # overload_psf, (n_z_slices, divs, divs, *peak_dim) # ).tolist() # # else: # # PSF # # Accumulate the PSF regionally over every field # # Then divide each PSF though by it's own mass so that the # # AUC under each PSF is 1. # # -------------------------------------------------------------- # [ # _calibrate_bg_im(flchcy_ims[fl_i, ch_i, cy_i], regional_bg_mean, regional_bg_std) # for fl_i in range(n_fields) # for cy_i in range(n_cycles) # ] # # for fl_i in range(n_fields): # # for cy_i in range(n_cycles): # # Remember: cy_i is a pseudo-cycle: it is really a z-slice # # with z_depths[cy_i] holding the actual depth # # regional_bg_mean = np.array( # calib[f"regional_bg_mean.instrument_channel[{ch_i}]"] # ) # _calibrate_psf_im(flchcy_ims[fl_i, ch_i, cy_i], regional_bg_mean) # # # ACCUMULATE each field, will normalize at the end # z_and_region_to_psf[cy_i] += reg_psfs # # NORMALIZE all psfs # denominator = np.sum(z_and_region_to_psf, axis=(3, 4))[:, :, :, None, None] # z_and_region_to_psf = utils.np_safe_divide(z_and_region_to_psf, denominator) # # calib.add( # { # f"regional_psf_zstack.instrument_channel[{ch_i}]": z_and_region_to_psf.tolist() # } # ) # FOREGROUND # Runs the standard sigproc_field analysis (without balancing) # to get the regional radmats for regional histogram balancing. # This requires that the PSF already be estimated so that the # radiometry can run. # -------------------------------------------------------------- # Spoof the sigproc_v2 worker into bypassing illumination balance by giving it all zeros calib.add({ f"regional_illumination_balance.instrument_channel[{ch_i}]": np.ones((divs, divs)).tolist() }) sigproc_params = SigprocV2Params( calibration=calib, instrument_subject_id=None, radiometry_channels=dict(ch=ch_i), ) fl_radmats = [] fl_locs = [] for fl_i in range(n_fields): if progress is not None: progress(fl_i, n_fields) chcy_ims = flchcy_ims[fl_i, ch_i:(ch_i + 1), :] ( chcy_ims, locs, radmat, aln_offsets, aln_scores, ) = sigproc_field(chcy_ims, sigproc_params) fl_radmats += [radmat] fl_locs += [locs] fl_radmat = np.concatenate(fl_radmats) fl_loc = np.concatenate(fl_locs) # BALANCE sig = np.nan_to_num(fl_radmat[:, ch_i, :, 0].flatten()) noi = fl_radmat[:, ch_i, :, 1].flatten() snr = np.nan_to_num(sig / noi) locs = np.tile(fl_loc, (1, n_cycles)).reshape((-1, 2)) snr_mask = snr > 10 sig = sig[snr_mask] locs = locs[snr_mask] top = np.max((locs[:, 0], locs[:, 1])) y = utils.ispace(0, top, divs + 1) x = utils.ispace(0, top, divs + 1) def regional_locs_mask(yi, xi): """Create a mask for locs inside of a region""" mask = (y[yi] <= locs[:, 0]) & (locs[:, 0] < y[yi + 1]) mask &= (x[xi] <= locs[:, 1]) & (locs[:, 1] < x[xi + 1]) return mask medians = np.zeros((divs, divs)) for yi in range(len(y) - 1): for xi in range(len(x) - 1): loc_mask = regional_locs_mask(yi, xi) bright_mask = sig > 2.0 _sig = sig[loc_mask & bright_mask] medians[yi, xi] = np.median(_sig) center = np.max(medians) balance = np.zeros((divs, divs)) for yi in range(len(y) - 1): for xi in range(len(x) - 1): loc_mask = regional_locs_mask(yi, xi) bright_mask = sig > 2.0 _sig = sig[loc_mask & bright_mask] for yi in range(len(y) - 1): for xi in range(len(x) - 1): loc_mask = regional_locs_mask(yi, xi) bright_mask = sig > 2.0 _sig = sig[loc_mask & bright_mask] median = np.median(_sig) balance[yi, xi] = center / median _sig *= balance[yi, xi] calib.add({ f"regional_illumination_balance.instrument_channel[{ch_i}]": balance.tolist() }) return calib
def it_checks_metadata(): with zest.raises(TypeError): Calibration({"metadata.instrument.sn1234": "not a dict"})
def sigproc_field(chcy_ims, sigproc_params, snr_thresh=None): """ Analyze one field and return values (do not save) Arguments: chcy_ims: In input order (from ims_import_result) sigproc_params: The SigprocParams snr_thresh: if non-None keeps only locs with S/R > snr_thresh This is useful for debugging. """ calib = Calibration(sigproc_params.calibration) # Step 1: Load the images in output channel order, balance, equalize chcy_ims = _import_balanced_images(chcy_ims, sigproc_params) # At this point, chcy_ims has its background subtracted and it is # regionally balanced and channel equalized. It may contain negative # values # # NOTE: at this point, chcy_ims are in OUTPUT CHANNEL order! n_out_channels, n_cycles = chcy_ims.shape[0:2] assert n_out_channels == sigproc_params.n_output_channels # Step 2: Remove anomalies for ch_i, cy_ims in enumerate(chcy_ims): chcy_ims[ch_i] = imops.stack_map(cy_ims, _mask_anomalies_im) # Step 3: Find alignment offsets aln_offsets, aln_scores = _align(np.mean(chcy_ims, axis=0)) # Step 4: Composite with alignment chcy_ims = _composite_with_alignment_offsets_chcy_ims( chcy_ims, aln_offsets) # chcy_ims is now only the intersection region so it may be smaller than the original # Step 5: Peak find on combined channels # The goal of previous channel equalization and regional balancing is that # all pixels are now on an equal footing so we can now use # a single values for fg_thresh and bg_thresh. ch_mean_of_cy0_im = np.mean(chcy_ims[:, 0, :, :], axis=0) locs = _peak_find(ch_mean_of_cy0_im) # Step 6: Radiometry over each channel, cycle # TASK: Eventually this will examine each cycle and decide # which z-depth of the PSFs is best fit to that cycle. # The result will be a per-cycle index into the chcy_regional_psfs # Until then the index is hard-coded to the zero-th index of regional_psf_zstack ch_z_reg_psfs = np.stack( [ np.array(calib[ f"regional_psf_zstack.instrument_channel[{sigproc_params.output_channel_to_input_channel(out_ch_i)}]"] ) for out_ch_i in range(n_out_channels) ], axis=0, ) assert ch_z_reg_psfs.shape[0] == n_out_channels cycle_to_z_index = np.zeros((n_cycles, )).astype(int) radmat = _radiometry(chcy_ims, locs, ch_z_reg_psfs, cycle_to_z_index) # Step 7: Remove empties # Keep any loc that has a signal > 20 times the minimum bg std in any channel # The 20 was found somewhat empirically and may need to be adjusted keep_mask = np.zeros((radmat.shape[0], )) > 0 for out_ch_i in range(n_out_channels): in_ch_i = sigproc_params.output_channel_to_input_channel(out_ch_i) bg_std = np.min( calib[f"regional_bg_std.instrument_channel[{in_ch_i}]"]) keep_mask = keep_mask | np.any(radmat[:, out_ch_i, :, 0] > 20 * bg_std, axis=1) if snr_thresh is not None: snr = radmat[:, :, :, 0] / radmat[:, :, :, 1] keep_mask = keep_mask & np.any(np.nan_to_num(snr) > snr_thresh, axis=(1, 2)) return chcy_ims, locs[keep_mask], radmat[ keep_mask], aln_offsets, aln_scores