def start(self):
params = NNV2Params(source="NNV2Task", **self.config.parameters)
sigproc_v2_result = None
rad_filter_result = None
if params.include_sigproc:
sigproc_v2_result = SigprocV2Result.load_from_folder(
self.inputs.sigproc, prop_list=["n_cycles", "n_channels"])
rad_filter_result = RadFilterResult.load_from_folder(
self.inputs.rad_filter)
prep_result = None
if self.inputs.get("prep") is not None:
prep_result = PrepResult.load_from_folder(self.inputs.prep)
sim_v2_result = None
if self.inputs.get("sim_v2") is not None:
sim_v2_result = SimV2Result.load_from_folder(self.inputs.sim_v2)
nn_v2(
params,
prep_result,
sim_v2_result,
sigproc_v2_result,
rad_filter_result,
progress=self.progress,
pipeline=self,
).save()
def start(self):
rf_v2_params = RFV2Params(**self.config.parameters)
radmat = None
if self.inputs.get("sim_v2"):
sim_result = SimV2Result.load_from_folder(self.inputs.sim_v2)
radmat = sim_result.flat_test_radmat()
elif self.inputs.get("sigproc_v2"):
sigproc_v2_result = SigprocV2Result.load_from_folder(self.inputs.sigproc_v2)
radmat = sigproc_v2_result.sig(flat_chcy=True)
check.array_t(radmat, ndim=2)
rf_train_v2_result = RFTrainV2Result.load_from_folder(self.inputs.rf_train_v2)
rf_v2_result = rf_classify(
rf_v2_params, rf_train_v2_result, radmat, progress=self.progress,
)
if self.inputs.get("sim_v2"):
# Stuff the true value into the results to simplify downstream processing
sim_result = SimV2Result.load_from_folder(self.inputs.sim_v2)
rf_v2_result.true_pep_iz = sim_result.test_true_pep_iz
rf_v2_result.save()
def start(self):
rad_filter_params = RadFilterParams(**self.config.parameters)
ims_import_result = ImsImportResult.load_from_folder(
self.inputs.ims_import)
sigproc_v2_result = SigprocV2Result.load_from_folder(
self.inputs.sigproc)
rad_filter_result = rad_filter(
rad_filter_params,
ims_import_result,
sigproc_v2_result,
progress=self.progress,
pipeline=self,
)
rad_filter_result.save()
def start(self):
lnfit_params = LNFitParams(**self.config.parameters)
# We don't need all of sigproc loaded, but do need a couple attrs. The rest
# will be loaded on demand.
load_props = ["n_channels", "n_cycles"]
try:
sigproc_result = SigprocV1Result.load_from_folder(
self.inputs.sigproc_dir, prop_list=load_props
)
except FileNotFoundError:
sigproc_result = SigprocV2Result.load_from_folder(
self.inputs.sigproc_dir, prop_list=load_props
)
lnfit_result = lnfit(lnfit_params, sigproc_result)
lnfit_result.save()
def sigproc(sigproc_params, ims_import_result, progress=None):
"""
Analyze all fields
"""
calib = Calibration(sigproc_params.calibration)
assert not calib.is_empty()
channel_weights = _compute_channel_weights(sigproc_params)
sigproc_result = SigprocV2Result(
params=sigproc_params,
n_input_channels=ims_import_result.n_channels,
n_channels=sigproc_params.n_output_channels,
n_cycles=ims_import_result.n_cycles,
channel_weights=channel_weights,
)
n_fields = ims_import_result.n_fields
n_fields_limit = sigproc_params.n_fields_limit
if n_fields_limit is not None and n_fields_limit < n_fields:
n_fields = n_fields_limit
zap.work_orders(
[
Munch(
fn=_do_sigproc_field,
ims_import_result=ims_import_result,
sigproc_params=sigproc_params,
field_i=field_i,
sigproc_result=sigproc_result,
) for field_i in range(n_fields)
],
_trap_exceptions=False,
_progress=progress,
)
return sigproc_result
def zest_v2_all_df():
props = Munch(
radmat=npf(
[
[
[[3.0, 2.0, 1.0], [1.0, 0.0, 1.0]],
[[4.0, 2.0, 1.0], [1.0, 1.0, 0.0]],
[[5.0, 2.0, 1.0], [1.0, 1.0, 1.0]],
[[6.0, 3.0, 0.9], [1.0, 0.0, 1.0]],
],
[
[[0.3, 0.2, 0.1], [0.3, 0.2, 0.1]],
[[0.4, 0.2, 0.1], [0.3, 0.2, 0.1]],
[[0.5, 0.2, 0.1], [0.3, 0.2, 0.1]],
[[0.6, 0.3, 2.9], [0.3, 0.2, 0.1]],
],
[
[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]],
[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]],
[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]],
[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]],
],
[
[[1.1, 1.2, 1.3], [1.4, 1.5, 1.6]],
[[1.1, 1.2, 1.3], [1.4, 1.5, 1.6]],
[[1.1, 1.2, 1.3], [1.4, 1.5, 1.6]],
[[1.1, 1.2, 1.3], [1.4, 1.5, 1.6]],
],
],
),
peak_df=pd.DataFrame(
[
(0, 0, 0, 1.0, 11.0),
(0, 0, 1, 2.0, 12.0),
(0, 1, 0, 3.0, 13.0),
(0, 1, 1, 4.0, 14.0),
],
columns=["peak_i", "field_i", "field_peak_i", "aln_x", "aln_y"],
),
field_df=pd.DataFrame(
[
(0, 0, 0, 1, 2, 0.1, 100.0, 2, 10, 5, 10, 500, 20, 490),
(0, 0, 1, 2, 3, 0.2, 110.0, 2, 10, 5, 10, 500, 20, 490),
(0, 0, 2, 3, 4, 0.3, 120.0, 2, 10, 5, 10, 500, 20, 490),
(0, 1, 0, 4, 5, 0.4, 130.0, 2, 10, 5, 10, 500, 20, 490),
(0, 1, 1, 5, 6, 0.5, 140.0, 2, 10, 5, 10, 500, 20, 490),
(0, 1, 2, 6, 7, 0.6, 150.0, 2, 10, 5, 10, 500, 20, 490),
(1, 0, 0, 7, 8, 0.7, 160.0, 2, 10, 6, 20, 510, 30, 480),
(1, 0, 1, 8, 9, 0.8, 170.0, 2, 10, 6, 20, 510, 30, 480),
(1, 0, 2, 9, 0, 0.9, 180.0, 2, 10, 6, 20, 510, 30, 480),
(1, 1, 0, 0, 1, 0.0, 190.0, 2, 10, 6, 20, 510, 30, 480),
(1, 1, 1, 1, 2, 0.1, 200.0, 2, 10, 6, 20, 510, 30, 480),
(1, 1, 2, 2, 3, 0.2, 210.0, 2, 10, 6, 20, 510, 30, 480),
],
columns=[
"field_i",
"channel_i",
"cycle_i",
"shift_y",
"shift_x",
"aln_score",
"bg_median",
"n_mask_rects",
"mask_area",
"border_size",
"aligned_roi_l",
"aligned_roi_r",
"aligned_roi_b",
"aligned_roi_t",
],
),
mask_rects_df=pd.DataFrame(
[
(0, 0, 0, 1, 2, 3, 4),
(0, 0, 1, 2, 2, 3, 4),
(0, 0, 2, 3, 2, 3, 4),
(0, 1, 0, 4, 2, 3, 4),
(0, 1, 1, 5, 2, 3, 4),
(0, 1, 2, 6, 2, 3, 4),
(1, 0, 0, 7, 2, 3, 4),
(1, 0, 1, 8, 2, 3, 4),
(1, 0, 2, 9, 2, 3, 4),
(1, 1, 0, 0, 2, 3, 4),
(1, 1, 1, 1, 2, 3, 4),
(1, 1, 2, 2, 2, 3, 4),
],
columns=["field_i", "channel_i", "cycle_i", "l", "r", "w", "h",],
),
)
# ramar got refactored and I just reorder it here
props["radmat"] = np.moveaxis(props["radmat"], 0, 3)
def _mock_load_field_prop(inst, field_i, prop):
return props[prop]
zest.stack_mock(
SigprocV2Result._load_field_prop,
substitute_fn=_mock_load_field_prop,
reset_before_each=False,
)
res = SigprocV2Result(
is_loaded_result=True,
field_files=[""],
n_peaks=4,
n_channels=2,
n_cycles=3,
n_fields=1,
)
def it_np_signal_radmat():
assert np_array_same(res.sig(), props.radmat[:, :, :, 0])
def it_np_noise_radmat():
assert np_array_same(res.noi(), props.radmat[:, :, :, 1])
# All of the following are testing the DataFrames
def it_fields():
assert res.fields().equals(props.field_df)
def it_radmats():
rad_df = res.radmats()
check.df_t(rad_df, SigprocV2Result.radmat_df_schema)
assert len(rad_df) == 4 * 2 * 3
# Sanity check a few
assert (
rad_df[
(rad_df.peak_i == 1) & (rad_df.channel_i == 1) & (rad_df.cycle_i == 1)
].signal.values[0]
== 1.0
)
assert (
rad_df[
(rad_df.peak_i == 2) & (rad_df.channel_i == 0) & (rad_df.cycle_i == 0)
].signal.values[0]
== 5.0
)
zest()
return df, r_mask, (df_path, mask_path)
# ------------------------------------------------------------
# Main
# ------------------------------------------------------------
if __name__ == "__main__":
import sys
from plaster.run.sigproc_v2.sigproc_v2_result import SigprocV2Result
# Paths
JOBS_FOLDER = "/erisyon/internal/jobs_folder"
SIGPROC_PATH = "sigproc_v2/plaster_output/sigproc_v2"
args = sys.argv[1:]
if len(args) == 0:
print(f"Usage: python {sys.argv[0]} run_names...")
sys.exit(0)
for run_name in args:
base_run_path = os.path.join(JOBS_FOLDER, run_name)
sig_path = os.path.join(base_run_path, SIGPROC_PATH)
print("Processing ", run_name)
results = SigprocV2Result(sig_path)
df, r_mask, paths = region_report(results, run_name, base_run_path)
def tasks_for_sigproc_v2(self):
tasks = {}
if self.sigproc_source:
ims_import_task = task_templates.ims_import(
self.sigproc_source,
is_movie=self.movie,
n_cycles_limit=self.n_cycles_limit,
start_cycle=self.start_cycle,
dst_ch_i_to_src_ch_i=self.dst_ch_i_to_src_ch_i,
)
calib_priors = None
if self.calibration_job is not None:
calib_src_path = (local.path(self.calibration_job) /
"sigproc_v2_calib/plaster_output/sigproc_v2")
calib_result = SigprocV2Result.load_from_folder(
calib_src_path, prop_list=["calib_priors"])
calib_priors = calib_result.calib_priors
if self.calib_dst_ch_i_to_src_ch_i is not None:
# Convert a string like 2,1,0 and remap
check.t(self.calib_dst_ch_i_to_src_ch_i, str)
calib_dst_ch_i_to_src_ch_i = [
int(ch_i)
for ch_i in self.calib_dst_ch_i_to_src_ch_i.split(",")
]
ch_remapped_priors = Priors.copy(calib_priors)
ch_remapped_priors.delete_ch_specific_records()
ch_aln_prior = ch_remapped_priors.get_exact(f"ch_aln")
if ch_aln_prior is not None:
ch_aln_prior = ChannelAlignPrior.ch_remap(
ch_aln_prior.prior, calib_dst_ch_i_to_src_ch_i)
for dst_ch_i, src_ch_i in enumerate(
calib_dst_ch_i_to_src_ch_i):
def remap(src_key, dst_key):
prior = calib_priors.get_exact(src_key)
if prior is not None:
ch_remapped_priors.add(
dst_key, prior.prior,
"remapped channel in gen")
remap(f"reg_illum.ch_{src_ch_i}",
f"reg_illum.ch_{dst_ch_i}")
remap(f"reg_psf.ch_{src_ch_i}",
f"reg_psf.ch_{dst_ch_i}")
calib_priors = ch_remapped_priors
ch_aln = None
if self.ch_aln is not None:
ch_aln = np.array([float(i) for i in self.ch_aln.split(",")])
assert ch_aln.shape[0] % 2 == 0
ch_aln = ch_aln.reshape((-1, 2))
sigproc_v2_task = task_templates.sigproc_v2_analyze(
calib_priors=calib_priors,
self_calib=self.self_calib,
ch_aln=ch_aln,
ch_for_alignment=self.ch_for_alignment,
)
tasks = Munch(**ims_import_task, **sigproc_v2_task)
return tasks