def _run(radmat, radmat_filter_mask, dyemat, dyepeps, n_channels):
with c_nn_v2.context(
train_dyemat=dyemat,
train_dyepeps=dyepeps,
radmat=radmat,
radmat_filter_mask=radmat_filter_mask,
priors=params.priors,
n_channels=n_channels,
n_neighbors=params.n_neighbors,
run_row_k_fit=params.run_row_k_fit,
run_against_all_dyetracks=params.run_against_all_dyetracks,
scoring_verbose=params.scoring_verbose,
scoring_verbose_cc=params.scoring_verbose_cc,
row_k_score_factor=params.row_k_score_factor,
) as nn_v2_context:
# _nn_v2.c chokes if a batch is larger than 1024*16
batches = zap.make_batch_slices(n_rows=radmat.shape[0],
_batch_size=_batch_size)
with zap.Context(mode="thread",
trap_exceptions=False,
progress=progress):
# This must be thread mode because it operates on the context in shared memory.
zap.work_orders([
dict(
fn=c_nn_v2.do_classify_radrows,
radrow_start_i=batch[0],
n_radrows=batch[1] - batch[0],
nn_v2_context=nn_v2_context,
) for batch in batches
])
return nn_v2_context
def radiometry_cy_ims(cy_ims, locs, reg_psf_samples, peak_mea):
"""
Compute radiometry on the stack of cycle images for one field on one channel
Returns:
output_radmat: ndarray(n_peaks, n_cycles, (sig, noi, bg_med, bg_std))
"""
with context(cy_ims=cy_ims,
locs=locs,
reg_psf_samples=reg_psf_samples,
peak_mea=peak_mea) as ctx:
check.array_t(locs, ndim=2, dtype=np.float64)
n_peaks = locs.shape[0]
if n_peaks > 0:
batches = zap.make_batch_slices(n_rows=locs.shape[0],
_batch_size=100)
with zap.Context(trap_exceptions=False, mode="thread"):
zap.work_orders([
dict(
fn=_do_radiometry_field_stack_peak_batch,
ctx=ctx,
peak_start_i=batch[0],
peak_stop_i=batch[1],
) for batch in batches
])
return ctx._out_radiometry
def classify(self, X, progress=None):
check.array_t(X, ndim=2)
n_rows = X.shape[0]
if n_rows < 100:
winner_y, winner_scores, runnerup_y, runnerup_scores = _do_predict(
classifier=self.classifier, X=X)
else:
n_work_orders = n_rows // 100
with zap.Context(progress=progress, trap_exceptions=False):
results = zap.work_orders([
Munch(classifier=self.classifier, X=X, fn=_do_predict)
for X in np.array_split(X, n_work_orders, axis=0)
])
winner_y = utils.listi(results, 0)
winner_scores = utils.listi(results, 1)
runnerup_y = utils.listi(results, 2)
runnerup_scores = utils.listi(results, 3)
winner_y = np.concatenate(winner_y)
winner_scores = np.concatenate(winner_scores)
runnerup_y = np.concatenate(runnerup_y)
runnerup_scores = np.concatenate(runnerup_scores)
return winner_y, winner_scores, runnerup_y, runnerup_scores
def peps_above_thresholds(self, precision=0.0, recall=0.0):
with zap.Context(mode="thread"):
df = zap.df_groups(
_do_peps_above_thresholds,
self.pr_curve_by_pep().groupby("pep_i"),
precision=precision,
recall=recall,
)
df = df.reset_index().sort_index().rename(columns={0: "passes"})
return np.argwhere(df.passes.values).flatten()
def psf_fields_one_channel(ims_import_result,
sigproc_v2_params,
field_iz,
channel_i,
progress=None) -> priors.RegPSFPrior:
"""
Build up a regional PSF for one channel on the RAW images.
Implemented in a parallel zap over every field and then combine the
fields into a single RegPSF which stores: (divs, divs, peak_mea, peak_mea)
"""
if ims_import_result.n_fields == 0:
return None
with zap.Context(progress=progress):
region_to_psf_per_field = zap.arrays(
_do_psf_one_field_one_channel,
dict(field_i=field_iz),
_stack=True,
peak_mea=sigproc_v2_params.peak_mea,
divs=sigproc_v2_params.divs,
bandpass_kwargs=dict(
low_inflection=sigproc_v2_params.low_inflection,
low_sharpness=sigproc_v2_params.low_sharpness,
high_inflection=sigproc_v2_params.high_inflection,
high_sharpness=sigproc_v2_params.high_sharpness,
),
ims_import_result=ims_import_result,
channel_i=channel_i,
n_cycles_limit=sigproc_v2_params.n_cycles_limit,
)
# SUM over fields
psf_ims = np.sum(region_to_psf_per_field, axis=0)
psf_ims = psf_normalize(psf_ims)
# At this point psf_ims is a pixel image of the PSF at each reg div.
# ie, 4 dimensional: (divs_y, divs_x, n_pixels_h, n_pixels_w)
# Now we convert it to Gaussian Parameters by fitting so we don't have
# to store the pixels anymore: just the 3 critical shape parameters:
# sigma_x, sigma_y, and rho.
# Use one frame of ims_import_result to sort out dimensions
im = ims_import_result.ims[0, 0, 0]
check.array_t(im, is_square=True)
reg_psf = priors.RegPSFPrior.from_psf_ims(im.shape[-1], psf_ims)
return reg_psf
def sigproc(sigproc_params, ims_import_result, progress=None):
# CACHE n_channel, n_cycles, dim into sigproc_params by loading one field
ims = ims_import_result.field_chcy_ims(field_i=0)
n_inchannels, n_cycles, h, w = ims.shape
assert h == w
n_outchannels = sigproc_params.n_output_channels
sigproc_params._outchannels_inchannels_cycles_dim = (
n_outchannels,
n_inchannels,
n_cycles,
h,
)
if not sigproc_params.channel_indices_for_alignment:
sigproc_params.channel_indices_for_alignment = list(
range(n_inchannels))
sigproc_result = SigprocV1Result(
params=sigproc_params,
n_input_channels=n_inchannels,
n_channels=n_outchannels,
n_cycles=n_cycles,
)
n_fields = ims_import_result.n_fields
n_fields_limit = sigproc_params.n_fields_limit
if n_fields_limit is not None and n_fields_limit < n_fields:
n_fields = n_fields_limit
# TASK: I think this would be nicer with the parallel array map
with zap.Context(trap_exceptions=False, progress=progress):
results = zap.work_orders([
Munch(
fn=_do_field,
field_i=field_i,
sigproc_params=sigproc_params,
ims_import_result=ims_import_result,
sigproc_result=sigproc_result,
) for field_i in range(n_fields)
])
# SET the result n_channels (possibly different from input n_channels)
n_inchannels = np.array(results)
assert np.all(n_inchannels == n_inchannels[0])
sigproc_result.n_channels = int(n_inchannels[0])
return sigproc_result
def _run_sim(sim_params, pep_seqs_df, name, n_peps, n_samples, progress):
if sim_params.get("random_seed") is not None:
# Increment so that train and test will be different
sim_params.random_seed += 1
np.random.seed(sim_params.random_seed)
dyemat = ArrayResult(
f"{name}_dyemat",
shape=(n_peps, n_samples, sim_params.n_channels, sim_params.n_cycles),
dtype=DyeType,
mode="w+",
)
radmat = ArrayResult(
f"{name}_radmat",
shape=(n_peps, n_samples, sim_params.n_channels, sim_params.n_cycles),
dtype=RadType,
mode="w+",
)
recall = ArrayResult(
f"{name}_recall",
shape=(n_peps, ),
dtype=RecallType,
mode="w+",
)
with zap.Context(trap_exceptions=False, progress=progress):
flus__remainders = zap.df_groups(
_do_pep_sim,
pep_seqs_df.groupby("pep_i"),
sim_params=sim_params,
n_samples=n_samples,
output_dyemat=dyemat,
output_radmat=radmat,
output_recall=recall,
)
flus = np.array(utils.listi(flus__remainders, 0))
flu_remainders = np.array(utils.listi(flus__remainders, 1))
true_pep_iz = np.repeat(np.arange(n_peps), n_samples)
return dyemat, radmat, recall, flus, flu_remainders, true_pep_iz
def all_dfs(self, fn, parallel=False):
"""
Run fn on every run, assert that each returns af DataFrame
and then pd.concat all the results into one adding a run_i
column to that DataFrame.
Example:
df = job.all_dfs(lambda run: run.prep.pros())
"""
df_list = []
if parallel:
def wrap_fn(run, run_i):
res_df = fn(run)
assert isinstance(res_df, pd.DataFrame)
res_df["run_i"] = run_i
res_df["run_name"] = run.manifest.run_name
return res_df
work_orders = [
{"fn": wrap_fn, "args": [run, run_i]}
for run_i, run in enumerate(self._run_results.values())
]
# TODO: it would be nice to integrate this progress stuff into zap as an optional argument
progress = tqdm(total=len(work_orders))
def progress_callback(i, j, retry):
if not retry:
progress.update()
with zap.Context(trap_exceptions=False, progress=progress_callback):
df_list = zap.work_orders(work_orders)
progress.close()
else:
for run_i, run in enumerate(self._run_results.values()):
res_df = fn(run)
assert isinstance(res_df, pd.DataFrame)
res_df["run_i"] = run_i
res_df["run_name"] = run.manifest.run_name
df_list += [res_df]
return pd.concat(df_list).reset_index(drop=True)
def main(self, job_folder=None):
switches = utils.plumbum_switches(self)
if job_folder is None:
job_folder = self.job
job_folder = assets.validate_job_folder(job_folder)
# At this point job_folder is a plumbum path
# Add a new handler so we get PER-RUN log files into the job folder
per_run_log_path = job_folder / f"{int(time.time()):06x}.log"
formatter = zlog.ColorfulFormatter(
"%(name)s %(asctime)s %(levelname)s %(message)s %(filename)s %(lineno)d"
)
handler = logging.StreamHandler(open(per_run_log_path, "w"))
handler.setFormatter(formatter)
zlog.add_handler(handler)
tell(f"Trapping run logs into {per_run_log_path}")
if job_folder is None:
log.error(f"No job_folder was specified")
return 1
tell(
f"Plaster run {job_folder} limit={self.limit} started at {arrow.utcnow().format()}"
)
if not job_folder.exists():
log.error(f"Unable to find the path {job_folder}")
return 1
# Load the job_uuid if available, evntually this will be nice for logging
job_uuid = None
job_yaml = job_folder / "job_manifest.yaml"
if job_yaml.exists():
job_manifest = utils.yaml_load_munch(job_yaml)
job_uuid = job_manifest.uuid
# Find all the plaster_run.yaml files. They might be in run subfolders
found = list(
job_folder.walk(filter=lambda p: p.name == "plaster_run.yaml"))
run_dirs = [p.dirname for p in found]
if len(run_dirs) == 0:
log.error(
"Plaster: Nothing to do because no run_dirs have plaster_run.yaml files"
)
return 1
def run_reports():
report_paths = [job_folder / "report.ipynb"
] + (job_folder / "_reports" // "*.ipynb")
for report_src_path in report_paths:
report_dst_path = report_src_path.with_suffix(".html")
if report_src_path.exists() and (self.force or out_of_date(
report_src_path, report_dst_path)):
tell(f"Running report {report_src_path}")
self.run_ipynb(report_src_path)
if self.reports_only:
run_reports()
return 0
# A normal run where all happens in this process
failure_count = 0
for run_dir_i, run_dir in enumerate(sorted(run_dirs)):
zlog.metrics(
f"Starting run subdirectory {run_dir}. {run_dir_i + 1} of {len(run_dirs)}",
log=log,
_type="plaster_start",
run_dir=run_dir,
run_dir_i=run_dir_i,
run_dir_n=len(run_dirs),
**switches,
)
try:
with zap.Context(
cpu_limit=self.cpu_limit,
mode="debug" if self.debug_mode else None,
allow_inner_parallelism=True,
):
# allow_inner_parallelism=True needs to be true so that each task such as sigproc_v2
# can allocate parallel jobs to each field.
run = RunExecutor(run_dir).load()
if "_erisyon" in run.config:
zlog.metrics(
"run metrics",
log=log,
_type="erisyon_block",
**run.config._erisyon,
)
failure_count += run.execute(
force=self.force,
limit=self.limit.split(",") if self.limit else None,
clean=self.clean,
n_fields_limit=self.n_fields_limit,
no_progress=self.no_progress,
)
except Exception as e:
failure_count += 1
if not self.continue_on_error:
raise e
if failure_count == 0 and self.limit is None and not self.clean:
# WRITE job_info.yaml with metadata used by the indexer
n_runs = len(run_dirs)
job_info = Munch(n_runs=n_runs, job_uuid=job_uuid)
if n_runs == 1:
job = JobResult(job_folder=job_folder)
tsv_data = {}
try:
tsv_data = job.runs[0].ims_import.tsv_data
except:
pass
nd2_metadata = {}
try:
nd2_metadata = job.runs[0].ims_import._nd2_metadata
except:
pass
job_info.update(tsv_data=tsv_data, nd2_metadata=nd2_metadata)
utils.yaml_save(job_folder / "job_info.yaml", job_info)
# RUN reports if not skipped
if not self.skip_reports:
run_reports()
return failure_count
def pr_curve_by_pep(self,
return_auc=False,
pep_iz=None,
force_compute=False,
progress=None):
"""
Obtain pr_curves for every peptide.
If all params are default, may returned cached information computed
during the run.
Returns:
A (potentially HUGE) df of every P/R for every peptide
A smaller df with just the pep_i and the Area-Under-Curve
This uses the work_order system (as opposed to the
higher-level array_split_map()) because the _do_pep_pr_curve
returns 3 identical returns AND one scalar; array_split_map() doesn't
like that.
"""
# The PR for all peptides is computed during the run (no auc).
if not return_auc and not force_compute and self._cached_pr is not None:
df = self._cached_pr
if pep_iz is not None:
df = df[df.pep_i.isin(pep_iz)]
return df.copy()
if pep_iz is None:
pep_iz = self._prep_result.peps().pep_i.values
if isinstance(pep_iz, np.ndarray):
pep_iz = pep_iz.tolist()
check.list_t(pep_iz, int)
with zap.Context(mode="thread",
trap_exceptions=False,
progress=progress):
results = zap.work_orders([
Munch(
fn=_do_pep_pr_curve,
pep_i=pep_i,
bag=self,
) for pep_i in pep_iz
], )
df_per_pep = [
pd.DataFrame(
dict(
pep_i=np.repeat(np.array([pep_i]), prec.shape[0]),
prec=prec,
recall=recall,
score=score,
)) for pep_i, (prec, recall, score, _) in results
]
if len(df_per_pep) > 0:
pr_df = pd.concat(df_per_pep, axis=0)
else:
pr_df = None
auc_df = pd.DataFrame(
[(pep_i, auc) for pep_i, (_, _, _, auc) in results],
columns=["pep_i", "auc"],
)
if return_auc:
return pr_df, auc_df
else:
return pr_df
def ims_import(src_dir: Path,
ims_import_params: ImsImportParams,
progress=None,
pipeline=None):
reference_nd2_file_for_metadata = None
scan_result = _scan_files(src_dir)
if len(scan_result.nd2_paths) > 0:
reference_nd2_file_for_metadata = scan_result.nd2_paths[0]
target_mea = max(scan_result.dim[0], scan_result.dim[1])
if not utils.is_power_of_2(target_mea):
new_dim = utils.next_power_of_2(target_mea)
_convert_message(target_mea, new_dim)
target_mea = new_dim
def clamp_fields(n_fields_true: int) -> Tuple[int, int]:
n_fields = n_fields_true
n_fields_limit = ims_import_params.get("n_fields_limit")
if n_fields_limit is not None:
n_fields = n_fields_limit
start_field = ims_import_params.get("start_field", 0)
if start_field + n_fields > n_fields_true:
n_fields = n_fields_true - start_field
return start_field, n_fields
def clamp_cycles(n_cycles_true: int) -> Tuple[int, int]:
n_cycles = n_cycles_true
n_cycles_limit = ims_import_params.get("n_cycles_limit")
if n_cycles_limit is not None:
n_cycles = n_cycles_limit
start_cycle = ims_import_params.get("start_cycle", 0)
if start_cycle is None:
start_cycle = 0
if start_cycle + n_cycles > n_cycles_true:
n_cycles = n_cycles_true - start_cycle
return start_cycle, n_cycles
tsv_data = tsv.load_tsv_for_folder(src_dir)
# ALLOCATE the ImsImportResult
ims_import_result = ImsImportResult(params=ims_import_params,
tsv_data=Munch(tsv_data))
dst_ch_i_to_src_ch_i = ims_import_params.dst_ch_i_to_src_ch_i
if dst_ch_i_to_src_ch_i is None:
dst_ch_i_to_src_ch_i = [i for i in range(scan_result.n_channels)]
n_out_channels = len(dst_ch_i_to_src_ch_i)
# Sanity check that we didn't end up with any src_channels outside of the channel range
assert all([
0 <= src_ch_i < scan_result.n_channels
for src_ch_i in dst_ch_i_to_src_ch_i
])
if ims_import_params.is_z_stack_single_file:
field_iz, n_cycles_found = _z_stack_import(
scan_result.nd2_paths[0],
target_mea,
ims_import_result,
dst_ch_i_to_src_ch_i,
ims_import_params.z_stack_n_slices_per_field,
)
n_cycles = ims_import_params.z_stack_n_slices_per_field
elif ims_import_params.is_movie:
if scan_result.mode == ScanFileMode.nd2:
# "Movie mode" means that there aren't any chemical cycles, but rather we are using "cycles" to represent different images in a zstack
start_field, n_fields = clamp_fields(len(scan_result.nd2_paths))
# In movie mode, the n_fields from the .nd2 file is becoming n_cycles
scan_result.n_cycles = scan_result.n_fields
start_cycle, n_cycles = clamp_cycles(scan_result.n_cycles)
with zap.Context(progress=progress):
field_iz, n_cycles_found = zap.arrays(
_do_movie_import_nd2,
dict(
input_field_i=list(
range(start_field, start_field + n_fields)),
output_field_i=list(range(n_fields)),
),
_stack=True,
scan_result=scan_result,
start_cycle=start_cycle,
n_cycles=n_cycles,
target_mea=target_mea,
import_result=ims_import_result,
dst_ch_i_to_src_ch_i=dst_ch_i_to_src_ch_i,
)
elif scan_result.mode == ScanFileMode.npy:
start_field, n_fields = clamp_fields(scan_result.n_fields)
start_cycle, n_cycles = clamp_cycles(scan_result.n_cycles)
with zap.Context(progress=progress):
field_iz, n_cycles_found = zap.arrays(
_do_movie_import_npy,
dict(
input_field_i=list(
range(start_field, start_field + n_fields)),
output_field_i=list(range(n_fields)),
),
_stack=True,
scan_result=scan_result,
start_cycle=start_cycle,
n_cycles=n_cycles,
target_mea=target_mea,
import_result=ims_import_result,
dst_ch_i_to_src_ch_i=dst_ch_i_to_src_ch_i,
)
else:
raise NotImplementedError()
else:
start_field, n_fields = clamp_fields(scan_result.n_fields)
if pipeline:
pipeline.set_phase(0, 2)
if scan_result.mode == ScanFileMode.nd2:
scan_result.n_cycles = len(scan_result.nd2_paths)
# SCATTER
with zap.Context(mode="thread", progress=progress):
zap.arrays(
_do_nd2_scatter,
dict(
cycle_i=list(range(len(scan_result.nd2_paths))),
src_path=scan_result.nd2_paths,
),
_stack=True,
start_field=start_field,
n_fields=n_fields,
n_channels=scan_result.n_channels,
target_mea=target_mea,
)
elif scan_result.mode == ScanFileMode.tif:
# SCATTER
work_orders = [
Munch(field_i=k[0], channel_i=k[1], cycle_i=k[2], path=path)
for k, path in
scan_result.tif_paths_by_field_channel_cycle.items()
]
with zap.Context(trap_exceptions=False):
results = zap.work_orders(_do_tif_scatter, work_orders)
# CHECK that every file exists
for f in range(n_fields):
for ch in range(scan_result.n_channels):
for cy in range(scan_result.n_cycles):
expected = f"__{f:03d}-{ch:02d}-{cy:02d}.npy"
if expected not in results:
raise FileNotFoundError(
f"File is missing in tif pattern: {expected}")
elif scan_result.mode == ScanFileMode.npy:
# In npy mode there's no scatter as the files are already fully scattered
pass
else:
raise ValueError(f"Unknown im import mode {scan_result.mode}")
if pipeline:
pipeline.set_phase(1, 2)
# GATHER
start_cycle, n_cycles = clamp_cycles(scan_result.n_cycles)
with zap.Context(progress=progress):
field_iz = zap.arrays(
_do_gather,
dict(
input_field_i=list(
range(start_field, start_field + n_fields)),
output_field_i=list(range(0, n_fields)),
),
_stack=True,
start_cycle=start_cycle,
n_cycles=n_cycles,
dim=target_mea,
import_result=ims_import_result,
mode=scan_result.mode,
npy_paths_by_field_channel_cycle=scan_result.
npy_paths_by_field_channel_cycle,
dst_ch_i_to_src_ch_i=dst_ch_i_to_src_ch_i,
)
if reference_nd2_file_for_metadata:
with _nd2(reference_nd2_file_for_metadata) as nd2:
if hasattr(nd2, "metadata"):
full = Munch(
metadata=nd2.metadata,
metadata_seq=nd2.metadata_seq,
)
ims_import_result._nd2_metadata_full = full
def me(block_name, default=None):
return utils.block_search(full.metadata.SLxExperiment,
block_name, default)
def mp(block_name, default=None):
return utils.block_search(
full.metadata_seq.SLxPictureMetadata, block_name,
default)
n_channels = mp("sPicturePlanes.uiSampleCount", 1)
ims_import_result._nd2_metadata = Munch(
calibrated_pixel_size=mp("dCalibration"),
experiment_type="movie" if me("eType") == 1 else "edman",
n_cycles=me("uLoopPars.uiCount"),
cmd_before=me("wsCommandBeforeCapture"),
cmd_after=me("wsCommandAfterCapture"),
n_channels=n_channels,
)
per_channel = []
for ch_i in range(n_channels):
laser_wavelength = None
laser_power = None
n_lasers = mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pDeviceSetting.m_uiMultiLaserLines0",
0,
)
for i in range(n_lasers):
is_used = mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pDeviceSetting.m_bMultiLaserLineUsed0-{i:02d}",
0,
)
if is_used == 1:
laser_wavelength = mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pDeviceSetting.m_uiMultiLaserLineWavelength0-{i:02d}",
0,
)
laser_power = mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pDeviceSetting.m_dMultiLaserLinePower0-{i:02d}",
0,
)
ch_munch = Munch(
laser_wavelength=laser_wavelength,
laser_power=laser_power,
camera_name=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pCameraSetting.CameraUniqueName"
),
sensor_pixels_x=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pCameraSetting.FormatQuality.fmtDesc.sizeSensorPixels.cx"
),
sensor_pixels_y=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pCameraSetting.FormatQuality.fmtDesc.sizeSensorPixels.cy"
),
sensor_microns_x=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pCameraSetting.FormatQuality.fmtDesc.sizeSensorMicrons.cx"
),
sensor_microns_y=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pCameraSetting.FormatQuality.fmtDesc.sizeSensorMicrons.cy"
),
bin_x=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pCameraSetting.FormatQuality.fmtDesc.dBinningX"
),
bin_y=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pCameraSetting.FormatQuality.fmtDesc.dBinningY"
),
format=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pCameraSetting.FormatQuality.fmtDesc.wszFormatDesc"
),
roi_l=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pCameraSetting.FormatQuality.rectSensorUser.left"
),
roi_r=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pCameraSetting.FormatQuality.rectSensorUser.right"
),
roi_t=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pCameraSetting.FormatQuality.rectSensorUser.top"
),
roi_b=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pCameraSetting.FormatQuality.rectSensorUser.bottom"
),
averaging=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pCameraSetting.PropertiesQuality.Average"
),
integration=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pCameraSetting.PropertiesQuality.Integrate"
),
name=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pCameraSetting.Metadata.Channels.Channel_0.Name"
),
dichroic_filter=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pDeviceSetting.m_sFilterName0"
),
emission_filter=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pDeviceSetting.m_sFilterName1"
),
optivar=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pDeviceSetting.m_dZoomPosition"
),
tirf_focus=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pDeviceSetting.m_dTIRFPositionFocus"
),
tirf_align_x=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pDeviceSetting.m_dTIRFPositionX"
),
tirf_align_y=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pDeviceSetting.m_dTIRFPositionY"
),
objective_mag=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pObjectiveSetting.dObjectiveMag"
),
objective_na=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pObjectiveSetting.dObjectiveNA"
),
objective_refractive_index=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.pObjectiveSetting.dRefractIndex"
),
settings_name=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.sOpticalConfigs.\x02.sOpticalConfigName"
),
readout_mode=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.sSpecSettings.Readout Mode"
),
readout_rate=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.sSpecSettings.Readout Rate"
),
noise_filter=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.sSpecSettings.Noise Filter"
),
temperature=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.sSpecSettings.Temperature"
),
exposure=mp(
f"sPicturePlanes.sSampleSetting.a{ch_i}.dExposureTime"
),
)
per_channel += [ch_munch]
ims_import_result._nd2_metadata.update(**Munch(
per_channel=per_channel))
if me("eType") == 1:
# Movie mode
ims_import_result._nd2_metadata.update(**Munch(
movie_start=me("dStart"),
movie_period=me("dPeriod"),
movie_duration=me("dDuration"),
movie_duration_pref=me("bDurationPref"),
movie_max_period_diff=me("dMaxPeriodDiff"),
movie_min_period_diff=me("dMinPeriodDiff"),
movie_avg_period_diff=me("dAvgPeriodDiff"),
))
ims_import_result.n_fields = len(field_iz)
ims_import_result.n_channels = n_out_channels
ims_import_result.n_cycles = n_cycles
ims_import_result.dim = target_mea
ims_import_result.dtype = np.dtype(OUTPUT_NP_TYPE).name
ims_import_result.src_dir = src_dir
# CLEAN
for file in local.cwd // "__*":
file.delete()
return ims_import_result
def main(self, job_folder=None):
switches = utils.plumbum_switches(self)
if job_folder is None:
error(f"No job_folder was specified")
return 1
important(
f"Plaster run {job_folder} limit={self.limit} started at {arrow.utcnow().format()}"
)
job_folder = assets.validate_job_folder_return_path(
job_folder, allow_run_folders=True)
if not job_folder.exists():
error(f"Unable to find the path {job_folder}")
return 1
# Find all the plaster_run.yaml files. They might be in run subfolders
found = list(
job_folder.walk(filter=lambda p: p.name == "plaster_run.yaml"))
run_dirs = [p.dirname for p in found]
if len(run_dirs) == 0:
error(
"Plaster: Nothing to do because no run_dirs have plaster_run.yaml files"
)
return 1
# A normal run where all happens in this process
failure_count = 0
for run_dir_i, run_dir in enumerate(sorted(run_dirs)):
metrics(
_type="plaster_start",
run_dir=run_dir,
run_dir_i=run_dir_i,
run_dir_n=len(run_dirs),
**switches,
)
important(
f"Starting run subdirectory {run_dir}. {run_dir_i + 1} of {len(run_dirs)}"
)
try:
with zap.Context(cpu_limit=self.cpu_limit,
debug_mode=self.debug_mode):
run = RunExecutor(run_dir).load()
if "_erisyon" in run.config:
metrics(_type="erisyon_block", **run.config._erisyon)
failure_count += run.execute(
force=self.force,
limit=self.limit.split(",") if self.limit else None,
clean=self.clean,
n_fields_limit=self.n_fields_limit,
skip_s3=self.skip_s3,
)
except Exception as e:
failure_count += 1
if not self.continue_on_error:
raise e
if (failure_count == 0 and self.limit is None and not self.clean
and not self.skip_reports):
# RUN reports
report_src_path = job_folder / "report.ipynb"
report_dst_path = job_folder / "report.html"
if (self.force or report_src_path.exists()
and utils.out_of_date(report_src_path, report_dst_path)):
self.run_ipynb(report_src_path)
return 0
return failure_count
