def it_returns_non_none_if_old_and_new_different():
new_config = Munch(a=1, b=2)
utils.json_save("/tmp/config.json", new_config)
new_config["c"] = 3
assert (
PipelineTask._config_dirty(new_config, "/tmp/config.json") is not None
)
def success(self):
"""Write the success state without over-writing"""
with local.cwd(self.dst_dir):
if self.wrote_state:
# Do not double write
return
self.wrote_state = True
local.path(PipelineState.success).touch()
utils.json_save(local.path("config.json"), self.config)
def _do_nd2_scatter(src_path, start_field, n_fields, cycle_i, n_channels,
target_dim):
"""
Scatter a cycle .nd2 into individual numpy files.
target_dim is a scalar. The target will be put into this square form.
"""
nd2 = _nd2(src_path)
ims = nd2.get_fields()
_n_channels = ims.shape[1]
actual_dim = ims.shape[2:4]
assert n_channels == _n_channels
check.affirm(
actual_dim[0] <= target_dim and actual_dim[1] <= target_dim,
f"nd2 scatter requested {target_dim} which is smaller than {actual_dim}",
)
if actual_dim[0] != target_dim or actual_dim[1] != target_dim:
# CONVERT into a zero pad
new_ims = np.zeros((n_fields, _n_channels, target_dim, target_dim),
dtype=ims.dtype)
new_ims[:, :, 0:actual_dim[0], 0:actual_dim[1]] = ims[:, :, :, :]
ims = new_ims
dst_files = []
for field_i in range(start_field, start_field + n_fields):
info = Munch(
x=nd2.x[field_i],
y=nd2.y[field_i],
z=nd2.z[field_i],
pfs_status=nd2.pfs_status[field_i],
pfs_offset=nd2.pfs_offset[field_i],
exposure_time=nd2.exposure_time[field_i],
camera_temp=nd2.camera_temp[field_i],
cycle_i=cycle_i,
field_i=field_i,
)
info_dst_file = _metadata_filename_by_field_cycle(field_i, cycle_i)
utils.json_save(info_dst_file, info)
for channel_i in range(n_channels):
dst_file = _npy_filename_by_field_channel_cycle(
field_i, channel_i, cycle_i)
dst_files += [dst_file]
np.save(dst_file, ims[field_i, channel_i])
return dst_files
def _do_nd2_scatter(src_path, start_field, n_fields, cycle_i, n_channels,
target_mea):
"""
Scatter a cycle .nd2 into individual numpy files.
target_mea is a scalar. The target will be put into this square form.
"""
working_im = np.zeros((target_mea, target_mea), np.uint16)
with _nd2(src_path) as nd2:
dst_files = []
for field_i in range(start_field, start_field + n_fields):
info = Munch(
x=nd2.x[field_i],
y=nd2.y[field_i],
z=nd2.z[field_i],
pfs_status=nd2.pfs_status[field_i],
pfs_offset=nd2.pfs_offset[field_i],
exposure_time=nd2.exposure_time[field_i],
camera_temp=nd2.camera_temp[field_i],
cycle_i=cycle_i,
field_i=field_i,
)
info_dst_file = _metadata_filename_by_field_cycle(field_i, cycle_i)
utils.json_save(info_dst_file, info)
for channel_i in range(n_channels):
im = nd2.get_field(field_i, channel_i)
if im.shape[0] != target_mea or im.shape[1] != target_mea:
working_im[0:im.shape[0], 0:im.shape[1]] = im[:, :]
im = working_im
dst_file = _npy_filename_by_field_channel_cycle(
field_i, channel_i, cycle_i)
dst_files += [dst_file]
np.save(dst_file, im)
return dst_files
def it_returns_none_if_old_and_new_differ_only_by_underscore_blocks():
new_config = Munch(a=1, b=2, _c=3)
utils.json_save("/tmp/config.json", new_config)
new_config["_c"] = 4
assert PipelineTask._config_dirty(new_config, "/tmp/config.json") is None
def it_returns_none_if_old_and_new_identical():
new_config = Munch(a=1, b=2)
utils.json_save("/tmp/config.json", new_config)
assert PipelineTask._config_dirty(new_config, "/tmp/config.json") is None
def main(self):
if self.construct_fail:
return
job_folder = self.validate_job_name_and_folder()
schema = self.generator_klass.schema
defaults = self.generator_klass.defaults
requirements = schema.requirements()
# APPLY defaults and then ask user for any elements that are not declared
generator_args = {}
switches = self._switches_by_name
if self.protein_random is not None:
tell(
f"Sampling {self.protein_random} random proteins from imported set"
)
n = len(self.derived_vals.protein)
assert n >= self.protein_random
self.derived_vals.protein = stats.subsample(
self.derived_vals.protein, self.protein_random)
assert len(self.derived_vals.protein) == self.protein_random
for arg_name, arg_type, _, arg_userdata in requirements:
if (arg_name in self.derived_vals
and self.derived_vals.get(arg_name) is not None):
# Load from a derived switch (eg: protein)
generator_args[arg_name] = self.derived_vals[arg_name]
elif arg_name in switches and switches.get(arg_name) is not None:
# Load from a switch
generator_args[arg_name] = getattr(self, arg_name)
else:
# If the schema allows the user to enter manually
if arg_userdata.get("allowed_to_be_entered_manually"):
generator_args[arg_name] = self._request_field_from_user(
arg_name, arg_type, default=defaults.get(arg_name))
# Download sigproc sources and replace with local path before handing to generator
if "sigproc_source" in generator_args:
source = generator_args["sigproc_source"]
if source is not None and source.startswith("s3:"):
generator_args["sigproc_source"] = self._cache_s3_reference(
source)
# Intentionally run the generate before the job folder is written
# so that if generate fails it doesn't leave around a partial job.
gen_sources_folder = job_folder / "_gen_sources"
try:
generator_args["force_run_name"] = self.run_name
generator_args[
"local_sources_tmp_folder"] = self.local_sources_tmp_folder
generator_args["gen_sources_folder"] = gen_sources_folder
generator = self.generator_klass(**generator_args)
run_descs = generator.generate()
except (SchemaValidationFailed, ValidationError) as e:
# Emit clean failure and exit 1
log.exception("Gen failed")
return 1
# WRITE the job & copy any file sources
self._write_runs(job_folder, run_descs, props=self.prop)
gen_sources_folder.delete()
self.local_sources_tmp_folder.move(gen_sources_folder)
if not self.skip_report:
for report_name, report_builder in generator.reports.items():
report = report_builder.report_assemble()
if report is not None:
utils.json_save(job_folder / f"{report_name}.ipynb",
report)
(job_folder / "reports_archive").mkdir()
for report_name in generator.static_reports:
if report_name is not None:
report_name = f"{report_name}.ipynb"
src = local.path(__file__) / "../../reports" / report_name
dst_folder = job_folder / "_reports"
dst_folder.mkdir()
dst = dst_folder / report_name
src.copy(dst)
utils.yaml_write(
job_folder / "job_manifest.yaml",
uuid=self.job_uuid,
localtime=time.strftime("%Y-%m-%d, %H:%M:%S", time.localtime()),
# Note: it seems localtime inside our container is UTC
who=local.env.get("RUN_USER", "Unknown"),
cmdline_args=sys.argv,
)
def main(self):
if self.construct_fail:
return
with local.cwd("/erisyon"):
assert local.path("erisyon_root").exists()
job_folder = self.validate_job_name_and_folder()
schema = self.generator_klass.schema
defaults = self.generator_klass.defaults
requirements = schema.requirements()
# APPLY defaults and then ask user for any elements that are not declared
generator_args = {}
switches = self._switches_by_name
if self.protein_random is not None:
info(
f"Sampling {self.protein_random} random proteins from imported set"
)
n = len(self.derived_vals.protein)
assert n >= self.protein_random
self.derived_vals.protein = data.subsample(
self.derived_vals.protein, self.protein_random
)
assert len(self.derived_vals.protein) == self.protein_random
for arg_name, arg_type, arg_help, arg_userdata in requirements:
if (
arg_name in self.derived_vals
and self.derived_vals.get(arg_name) is not None
):
# Load from a derived switch (eg: protein)
generator_args[arg_name] = self.derived_vals[arg_name]
elif arg_name in switches and switches.get(arg_name) is not None:
# Load from a switch
generator_args[arg_name] = getattr(self, arg_name)
else:
# If the schema allows the user to enter manually
if arg_userdata.get("allowed_to_be_entered_manually"):
generator_args[arg_name] = self._request_field_from_user(
arg_name, arg_type, default=defaults.get(arg_name)
)
# Intentionally run the generate before the job folder is written
# so that if generate fails it doesn't leave around a partial job.
try:
generator_args["force_run_name"] = self.run_name
generator = self.generator_klass(**generator_args)
run_descs = generator.generate()
except (SchemaValidationFailed, ValidationError) as e:
# Emit clean failure and exit 1
error(str(e))
return 1
# WRITE the job & copy any file sources
self._write_runs(job_folder, run_descs, props=self.prop)
(job_folder / "_gen_sources").delete()
self.local_sources_tmp_folder.move(job_folder / "_gen_sources")
if not self.skip_report:
report = generator.report_assemble()
utils.json_save(job_folder / "report.ipynb", report)
utils.yaml_write(
job_folder / "job_manifest.yaml",
uuid=self.job_uuid,
localtime=time.strftime("%Y-%m-%d, %H:%M:%S", time.localtime()),
# Note: it seems localtime inside our container is UTC
who=local.env.get("RUN_USER", "Unknown"),
cmdline_args=sys.argv,
)