def it_runs():
reg_illum = PriorsMLEFixtures.reg_illum_uniform(im_mea)
reg_psf = PriorsMLEFixtures.reg_psf_uniform(hyper_im_mea=im_mea)
priors = Priors(hyper_n_channels=n_channels)
priors.add("reg_illum", reg_illum, "test")
priors.add("reg_psf", reg_psf, "test")
sigproc_v2_params = SigprocV2Params(mode="analyze", priors=priors)
raw_chcy_ims, peaks = _synth()
_analyze_field(raw_chcy_ims, sigproc_v2_params)
def from_prep_fixture(cls, prep_result, labels: str, n_edmans=5, priors=None):
"""
Run a (likely small) simulation to make a SimResult fixture for testing
labels: a CSV list of aas. Eg: "DE,C"
Common labels: "DE", "C", "Y", "K", "H"
"""
from plaster.run.sim_v2.sim_v2_worker import sim_v2
from plaster.tools.schema import check
from plaster.run.priors import PriorsMLEFixtures
check.t(labels, str)
labels = labels.split(",")
if priors is None:
priors = PriorsMLEFixtures.val_defaults()
sim_v2_params = SimV2Params.from_aa_list_fixture(
labels,
priors=priors,
n_pres=1,
n_mocks=0,
n_edmans=n_edmans,
use_lognormal_model=False,
n_samples_train=100,
n_samples_test=100,
train_includes_radmat=True,
)
return sim_v2(sim_v2_params, prep_result)
def it_compares_to_all_dyetracks_with_row_fit():
_priors = PriorsMLEFixtures.illumination(row_k_sigma=0.2)
radmat, true_dyt_iz, true_ks = synthetic_radmat_from_dyemat(
dyemat, _priors, n_channels=1, n_samples=500)
mask = true_dyt_iz > 0
radmat = radmat[mask]
true_ks = true_ks[mask]
nn_v2_context = _test(
radmat,
n_neighbors=0,
run_against_all_dyetracks=True,
run_row_k_fit=True,
)
# Uncomment this to compare to random
# true_ks = np.random.normal(1.0, 0.5, true_ks.shape[0])
# In this mode I expect to get back outputs for every radrow vs every dytrow
assert nn_v2_context.against_all_dyetrack_pvals.shape == (
radmat.shape[0],
dyemat.shape[0],
)
assert nn_v2_context.against_all_dyetrack_pred_ks.shape == (
radmat.shape[0],
dyemat.shape[0],
)
pks = nn_v2_context.pred_ks
mask = ~np.isnan(pks)
pear_r, _ = stats.pearsonr(true_ks[mask], pks[mask])
assert pear_r > 0.5
def it_makes_cycles_array():
priors = PriorsMLEFixtures.nbt_defaults()
src_params = SimV2Params.from_aa_list_fixture(["DE", "Y"],
priors=priors,
n_pres=1,
n_mocks=2,
n_edmans=3)
assert src_params.cycles_array().tolist() == [0, 1, 1, 2, 2, 2]
def _synth(n_peaks=500):
with synth.Synth(n_channels=1, n_cycles=3, dim=(512, 512)) as s:
reg_psf = PriorsMLEFixtures.reg_psf_uniform()
(
synth.PeaksModelPSF(reg_psf, n_peaks=n_peaks)
.amps_constant(5000)
.locs_randomize()
)
synth.CameraModel(100, 30)
synth.HaloModel()
return s.aln_offsets, s.render_chcy()
def _sim(priors=None, _prep_result=None, sim_kwargs=None):
if _prep_result is None:
_prep_result = prep_result
priors = PriorsMLEFixtures.fixture_no_errors(**(priors or {}))
if sim_kwargs is None:
sim_kwargs = {}
sim_kwargs["use_lognormal_model"] = True
sim_v2_params = SimV2Params.from_aa_list_fixture(
["A", "B"], priors=priors, n_edmans=4, **(sim_kwargs or {})
)
return sim_v2_worker.sim_v2(sim_v2_params, _prep_result), sim_v2_params
def it_generates_multichannel_radmat_different_betas():
_priors = PriorsMLEFixtures.illumination_multi_channel(
gain_mus_ch=[5000, 15000],
gain_sigmas_ch=[0, 0],
bg_sigmas_ch=[0, 0],
row_k_sigma=0.0,
)
radmat, true_dyt_iz, true_ks = synthetic_radmat_from_dyemat(
dyemat_2ch, _priors, n_channels=2, n_samples=1)
# fmt off
expected = np.array([
[
0,
0,
0,
0,
0,
0,
],
[
5000,
0,
0,
15000,
0,
0,
],
[
10000,
5000,
5000,
30000,
15000,
15000,
],
[
15000,
10000,
5000,
45000,
30000,
15000,
],
], )
# fmt on
assert np.allclose(radmat, expected)
def it_classifies_multichannel():
_priors = PriorsMLEFixtures.illumination_multi_channel(
gain_mus_ch=[5000, 15000],
gain_sigmas_ch=[0, 0],
bg_sigmas_ch=[0, 0],
)
# gain_model = GainModel.multi_channel(
# beta=[5000, 15000], sigma=[0, 0], zero_sigma=[0, 0]
# )
radmat, true_dyt_iz, true_ks = synthetic_radmat_from_dyemat(
dyemat_2ch, _priors, n_channels=2, n_samples=5)
nn_v2_context = _test(radmat, _dyemat=dyemat_2ch)
n_same = (true_dyt_iz == nn_v2_context.pred_dyt_iz).sum()
# This is almost always higher than 0.5, but how to prevent the
# occasional hiccup? Classify n times? You don't want to just
# retry and allow a terrible result to slip through.
assert n_same >= int(0.5 * true_dyt_iz.shape[0])
def it_fits_k():
# Does it make a copy of
_priors = PriorsMLEFixtures.illumination(row_k_sigma=0.2)
radmat, true_dyt_iz, true_ks = synthetic_radmat_from_dyemat(
dyemat, _priors, n_channels=1, n_samples=500)
mask = true_dyt_iz > 0
radmat = radmat[mask]
true_ks = true_ks[mask]
nn_v2_context = _test(radmat)
# Uncomment this to compare to random
# true_ks = np.random.normal(1.0, 0.5, true_ks.shape[0])
# Check that there's a reasonable correlation between true and pred k
# I ran this several times and found with random true_ks
# ie: true_ks = np.random.normal(1.0, 0.5, true_ks.shape[0])
# was like 0.02 where real was > 0.4
pks = nn_v2_context.pred_ks
mask = ~np.isnan(pks)
pear_r, _ = stats.pearsonr(true_ks[mask], pks[mask])
assert pear_r > 0.4
def zest_radiometry():
reg_psf = PriorsMLEFixtures.reg_psf_uniform()
reg_psf_samples = reg_psf.sample_params_grid(16)
def _peak(into_im, x, y, rho=0.0, mea=15, amp=1000.0, peak_sigma=1.8):
corner_y = int(y - mea / 2.0 + 0.5) # int
corner_x = int(x - mea / 2.0 + 0.5)
off_y = y - corner_y # float
off_x = x - corner_x
peak_im = imops.gauss2_rho_form(amp, peak_sigma, peak_sigma, off_x,
off_y, rho, 0.0, mea)
imops.accum_inplace(into_im,
peak_im,
loc=XY(corner_x, corner_y),
center=False)
def it_finds_one_peak_centered():
peak_mea = reg_psf.hyper_peak_mea
w, h = peak_mea * 2, peak_mea * 2
n_cycles = 1
y = h / 2.0
x = w / 2.0
cy_ims = np.zeros((n_cycles, h, w))
_peak(cy_ims[0], x, y)
radrow = radiometry_cy_ims(
cy_ims,
locs=np.array([[y, x]]),
reg_psf_samples=reg_psf_samples,
peak_mea=reg_psf.hyper_peak_mea,
)
sig = radrow[0, 0, 0]
noi = radrow[0, 0, 1]
bg_med = radrow[0, 0, 2]
bg_std = radrow[0, 0, 3]
assert np.abs(sig - 1000.0) < 2.00
assert np.abs(noi) < 0.1
assert np.abs(bg_med) < 0.05
assert np.abs(bg_std - 3.0) < 1.0
def it_finds_off_center():
n_cycles = 1
w, h = 21, 21
y = h / 2.0 - 2.0
x = w / 2.0 + 1.0
cy_ims = np.zeros((n_cycles, h, w))
_peak(cy_ims[0], x, y)
radrow = radiometry_cy_ims(
cy_ims,
locs=np.array([[y, x]]),
reg_psf_samples=reg_psf_samples,
peak_mea=reg_psf.hyper_peak_mea,
)
sig = radrow[0, 0, 0]
noi = radrow[0, 0, 1]
bg_med = radrow[0, 0, 2]
bg_std = radrow[0, 0, 3]
assert np.abs(sig - 1000.0) < 2.00
assert np.abs(noi) < 0.1
assert np.abs(bg_med) < 0.05
assert np.abs(bg_std - 3.0) < 1.0
def it_finds_way_off_center():
n_cycles = 1
w, h = 21, 21
y = h / 2.0 - 3.4
x = w / 2.0 + 2.1
cy_ims = np.zeros((n_cycles, h, w))
_peak(cy_ims[0], x, y)
radrow = radiometry_cy_ims(
cy_ims,
locs=np.array([[y, x]]),
reg_psf_samples=reg_psf_samples,
peak_mea=reg_psf.hyper_peak_mea,
)
sig = radrow[0, 0, 0]
noi = radrow[0, 0, 1]
bg_med = radrow[0, 0, 2]
bg_std = radrow[0, 0, 3]
assert np.abs(sig - 1000.0) < 2.00
assert np.abs(noi) < 0.1
assert np.abs(bg_med) < 0.05
assert np.abs(bg_std - 3.0) < 1.0
def it_finds_fractional_loc():
n_cycles = 1
w, h = 21, 21
for y in np.linspace(h / 2.0 - 2.0, h / 2.0 + 2.0, 7):
for x in np.linspace(w / 2.0 - 2.0, w / 2.0 + 2.0, 7):
cy_ims = np.zeros((n_cycles, h, w))
_peak(cy_ims[0], x, y)
radrow = radiometry_cy_ims(
cy_ims,
locs=np.array([[y, x]]),
reg_psf_samples=reg_psf_samples,
peak_mea=reg_psf.hyper_peak_mea,
)
sig = radrow[0, 0, 0]
noi = radrow[0, 0, 1]
bg_med = radrow[0, 0, 2]
bg_std = radrow[0, 0, 3]
assert np.abs(sig - 1000.0) < 2.00
assert np.abs(noi) < 0.1
assert np.abs(bg_med) < 0.05
assert np.abs(bg_std - 3.0) < 1.0
def it_finds_many_locs_with_jitter():
n_cycles = 1
w, h = 128, 128
cy_ims = np.zeros((n_cycles, h, w))
locs = [(y, x) for y in np.linspace(10, 110, 8)
for x in np.linspace(10, 110, 8)]
locs = np.array(locs)
locs = locs + np.random.uniform(-1, 1, size=locs.shape)
for loc in locs:
_peak(cy_ims[0], loc[1], loc[0])
radmat = radiometry_cy_ims(
cy_ims,
locs=locs,
reg_psf_samples=reg_psf_samples,
peak_mea=reg_psf.hyper_peak_mea,
)
# assert np.all(np.abs(radmat[:, :, 0] - 1000.0) < 0.01)
# # Why is this suddenly larger? Originally it was alwasy less than 0.2
# # but now it varys up to 1. But I don't see where the difference is
# assert np.all(np.abs(radmat[:, :, 1]) < 1.0)
sig = radmat[:, :, 0]
noi = radmat[:, :, 1]
bg_med = radmat[:, :, 2]
bg_std = radmat[:, :, 3]
assert np.all(np.abs(sig - 1000.0) < 3.20)
assert np.all(np.abs(noi) < 1.0)
assert np.all(np.abs(bg_med) < 0.10)
assert np.all(np.abs(bg_std - 3.0) < 1.0)
def it_finds_changes_over_cycles():
n_cycles = 3
w, h = 21, 21
cy_ims = np.zeros((n_cycles, h, w))
x, y = w / 2.0, h / 2.0
amp_per_cycle = [1000.0, 900.0, 100.0]
for cy_i in range(n_cycles):
_peak(cy_ims[cy_i], x, y, amp=amp_per_cycle[cy_i])
radrow = radiometry_cy_ims(
cy_ims,
locs=np.array([[y, x]]),
reg_psf_samples=reg_psf_samples,
peak_mea=reg_psf.hyper_peak_mea,
)
for cy_i in range(n_cycles):
sig = radrow[0, cy_i, 0]
noi = radrow[0, cy_i, 1]
bg_med = radrow[0, cy_i, 2]
bg_std = radrow[0, cy_i, 3]
assert np.abs(sig - amp_per_cycle[cy_i]) < 2.00
assert np.abs(noi) < 0.1
assert np.abs(bg_med) < 0.05
def corrects_for_background_shifts():
n_cycles = 1
w, h = 128, 128
x, y = 16.5, 20.5
def it_bg_corrects_with_no_collisions():
cy_ims = np.zeros((n_cycles, h, w))
_peak(cy_ims[0], x, y)
# A uniform drop is a common thing that happens as a result of a filter
# so simulate that here with a constant
bg_shift = -2.0
_cy_ims = cy_ims + bg_shift
radrow = radiometry_cy_ims(
_cy_ims,
locs=np.array([[y, x]]),
reg_psf_samples=reg_psf_samples,
peak_mea=reg_psf.hyper_peak_mea,
)
sig = radrow[0, 0, 0]
bg_med = radrow[0, 0, 2]
bg_std = radrow[0, 0, 3]
assert np.abs(sig - 999.0) < 1.0
assert np.abs(bg_med - bg_shift) < 0.5
assert np.abs(bg_std - 3.0) < 0.5
def it_catches_collisions_in_bg():
cy_ims = np.zeros((n_cycles, h, w))
_peak(cy_ims[0], x, y)
# Add a second peak nearby
offset_x = 5.0
_peak(cy_ims[0], x + offset_x, y)
# A uniform drop is a common thing that happens as a result of a filter
# so simulate that here with a constant
bg_shift = -2.0
_cy_ims = cy_ims + bg_shift
radrow = radiometry_cy_ims(
_cy_ims,
locs=np.array([[y, x], [y, x + offset_x]]),
reg_psf_samples=reg_psf_samples,
peak_mea=reg_psf.hyper_peak_mea,
)
sig = radrow[0, 0, 0]
noi = radrow[0, 0, 1]
bg_med = radrow[0, 0, 2]
bg_std = radrow[0, 0, 3]
# It will be brighter because of the contention
assert np.abs(sig - 1134.0) < 20.0 # 1134.0 is empirical
assert np.abs(bg_med - bg_shift) < 0.5
assert np.abs(bg_std) > 50 # empirical
zest()
zest()
def zest_c_nn_v2():
# fmt: off
dyemat = np.array([
[0, 0, 0],
[1, 0, 0],
[2, 1, 1],
[3, 2, 1],
],
dtype=DyeType)
# fmt: on
# fmt: off
dyemat_2ch = np.array([
[0, 0, 0, 0, 0, 0],
[1, 0, 0, 1, 0, 0],
[2, 1, 1, 2, 1, 1],
[3, 2, 1, 3, 2, 1],
],
dtype=DyeType)
# fmt: on
dyepeps = np.array(
[
# (dyt_i, pep_i, count)
[1, 2, 30],
[1, 1, 10],
[2, 1, 10],
[3, 1, 10],
],
dtype=np.uint64,
)
n_channels = 1
priors = PriorsMLEFixtures.illumination()
def _test(
radmat,
n_neighbors=4,
run_against_all_dyetracks=False,
run_row_k_fit=True,
_dyepeps=dyepeps,
_dyemat=dyemat,
):
with c_nn_v2.context(
_dyemat,
_dyepeps,
radmat.astype(RadType),
None,
priors,
n_channels=n_channels,
n_neighbors=n_neighbors,
run_row_k_fit=run_row_k_fit,
run_against_all_dyetracks=run_against_all_dyetracks,
) as nn_v2_context:
c_nn_v2.do_classify_radrows(nn_v2_context, 0, len(radmat))
return nn_v2_context
def it_catches_non_sequential_dyt_iz_in_dyepeps():
_dyepeps = np.array(
[
[1, 1, 10],
[2, 1, 10],
[1, 2, 30],
],
dtype=np.uint64,
)
radmat, true_dyt_iz, true_ks = synthetic_radmat_from_dyemat(
dyemat, priors, n_channels=1, n_samples=5)
with zest.raises(CException, in_args="Non sequential dyt_i"):
_test(radmat, _dyepeps=_dyepeps)
def it_enforces_reverse_sort_on_count_per_dyt():
_dyepeps = np.array(
[
[1, 1, 10],
[1, 2, 30],
],
dtype=np.uint64,
)
radmat, true_dyt_iz, true_ks = synthetic_radmat_from_dyemat(
dyemat, priors, n_channels=1, n_samples=5)
with zest.raises(CException,
in_args="must be reverse sorted by count per dyt"):
_test(radmat, _dyepeps=_dyepeps)
def it_generates_multichannel_radmat_different_betas():
_priors = PriorsMLEFixtures.illumination_multi_channel(
gain_mus_ch=[5000, 15000],
gain_sigmas_ch=[0, 0],
bg_sigmas_ch=[0, 0],
row_k_sigma=0.0,
)
radmat, true_dyt_iz, true_ks = synthetic_radmat_from_dyemat(
dyemat_2ch, _priors, n_channels=2, n_samples=1)
# fmt off
expected = np.array([
[
0,
0,
0,
0,
0,
0,
],
[
5000,
0,
0,
15000,
0,
0,
],
[
10000,
5000,
5000,
30000,
15000,
15000,
],
[
15000,
10000,
5000,
45000,
30000,
15000,
],
], )
# fmt on
assert np.allclose(radmat, expected)
def it_classifies():
radmat, true_dyt_iz, true_ks = synthetic_radmat_from_dyemat(
dyemat, priors, n_channels=1, n_samples=5)
nn_v2_context = _test(radmat)
n_same = (true_dyt_iz == nn_v2_context.pred_dyt_iz).sum()
assert n_same >= int(0.65 * true_dyt_iz.shape[0])
def it_classifies_multichannel():
_priors = PriorsMLEFixtures.illumination_multi_channel(
gain_mus_ch=[5000, 15000],
gain_sigmas_ch=[0, 0],
bg_sigmas_ch=[0, 0],
)
# gain_model = GainModel.multi_channel(
# beta=[5000, 15000], sigma=[0, 0], zero_sigma=[0, 0]
# )
radmat, true_dyt_iz, true_ks = synthetic_radmat_from_dyemat(
dyemat_2ch, _priors, n_channels=2, n_samples=5)
nn_v2_context = _test(radmat, _dyemat=dyemat_2ch)
n_same = (true_dyt_iz == nn_v2_context.pred_dyt_iz).sum()
# This is almost always higher than 0.5, but how to prevent the
# occasional hiccup? Classify n times? You don't want to just
# retry and allow a terrible result to slip through.
assert n_same >= int(0.5 * true_dyt_iz.shape[0])
def it_fits_k():
# Does it make a copy of
_priors = PriorsMLEFixtures.illumination(row_k_sigma=0.2)
radmat, true_dyt_iz, true_ks = synthetic_radmat_from_dyemat(
dyemat, _priors, n_channels=1, n_samples=500)
mask = true_dyt_iz > 0
radmat = radmat[mask]
true_ks = true_ks[mask]
nn_v2_context = _test(radmat)
# Uncomment this to compare to random
# true_ks = np.random.normal(1.0, 0.5, true_ks.shape[0])
# Check that there's a reasonable correlation between true and pred k
# I ran this several times and found with random true_ks
# ie: true_ks = np.random.normal(1.0, 0.5, true_ks.shape[0])
# was like 0.02 where real was > 0.4
pks = nn_v2_context.pred_ks
mask = ~np.isnan(pks)
pear_r, _ = stats.pearsonr(true_ks[mask], pks[mask])
assert pear_r > 0.4
def it_compares_to_all_dyetracks_without_row_fit():
radmat, true_dyt_iz, true_ks = synthetic_radmat_from_dyemat(
dyemat, priors, n_channels=1, n_samples=5)
nn_v2_context = _test(radmat,
n_neighbors=0,
run_against_all_dyetracks=True,
run_row_k_fit=False)
# In this mode I expect to get back outputs for every radrow vs every dytrow
# But the radrows generated from empties can be ignore
mask = true_dyt_iz > 0
n_good_calls = (
true_dyt_iz[mask] == nn_v2_context.pred_dyt_iz[mask]).sum()
assert n_good_calls >= int(0.75 * mask.sum())
assert np.all(nn_v2_context.against_all_dyetrack_pred_ks[:, 1:] == 1.0)
assert nn_v2_context.against_all_dyetrack_pvals.shape == (
radmat.shape[0],
dyemat.shape[0],
)
assert nn_v2_context.against_all_dyetrack_pred_ks.shape == (
radmat.shape[0],
dyemat.shape[0],
)
def it_compares_to_all_dyetracks_with_row_fit():
_priors = PriorsMLEFixtures.illumination(row_k_sigma=0.2)
radmat, true_dyt_iz, true_ks = synthetic_radmat_from_dyemat(
dyemat, _priors, n_channels=1, n_samples=500)
mask = true_dyt_iz > 0
radmat = radmat[mask]
true_ks = true_ks[mask]
nn_v2_context = _test(
radmat,
n_neighbors=0,
run_against_all_dyetracks=True,
run_row_k_fit=True,
)
# Uncomment this to compare to random
# true_ks = np.random.normal(1.0, 0.5, true_ks.shape[0])
# In this mode I expect to get back outputs for every radrow vs every dytrow
assert nn_v2_context.against_all_dyetrack_pvals.shape == (
radmat.shape[0],
dyemat.shape[0],
)
assert nn_v2_context.against_all_dyetrack_pred_ks.shape == (
radmat.shape[0],
dyemat.shape[0],
)
pks = nn_v2_context.pred_ks
mask = ~np.isnan(pks)
pear_r, _ = stats.pearsonr(true_ks[mask], pks[mask])
assert pear_r > 0.5
zest()