def it_returns_non_none_if_old_and_new_different(): new_config = Munch(a=1, b=2) utils.json_save("/tmp/config.json", new_config) new_config["c"] = 3 assert ( PipelineTask._config_dirty(new_config, "/tmp/config.json") is not None )
def success(self): """Write the success state without over-writing""" with local.cwd(self.dst_dir): if self.wrote_state: # Do not double write return self.wrote_state = True local.path(PipelineState.success).touch() utils.json_save(local.path("config.json"), self.config)
def _do_nd2_scatter(src_path, start_field, n_fields, cycle_i, n_channels, target_dim): """ Scatter a cycle .nd2 into individual numpy files. target_dim is a scalar. The target will be put into this square form. """ nd2 = _nd2(src_path) ims = nd2.get_fields() _n_channels = ims.shape[1] actual_dim = ims.shape[2:4] assert n_channels == _n_channels check.affirm( actual_dim[0] <= target_dim and actual_dim[1] <= target_dim, f"nd2 scatter requested {target_dim} which is smaller than {actual_dim}", ) if actual_dim[0] != target_dim or actual_dim[1] != target_dim: # CONVERT into a zero pad new_ims = np.zeros((n_fields, _n_channels, target_dim, target_dim), dtype=ims.dtype) new_ims[:, :, 0:actual_dim[0], 0:actual_dim[1]] = ims[:, :, :, :] ims = new_ims dst_files = [] for field_i in range(start_field, start_field + n_fields): info = Munch( x=nd2.x[field_i], y=nd2.y[field_i], z=nd2.z[field_i], pfs_status=nd2.pfs_status[field_i], pfs_offset=nd2.pfs_offset[field_i], exposure_time=nd2.exposure_time[field_i], camera_temp=nd2.camera_temp[field_i], cycle_i=cycle_i, field_i=field_i, ) info_dst_file = _metadata_filename_by_field_cycle(field_i, cycle_i) utils.json_save(info_dst_file, info) for channel_i in range(n_channels): dst_file = _npy_filename_by_field_channel_cycle( field_i, channel_i, cycle_i) dst_files += [dst_file] np.save(dst_file, ims[field_i, channel_i]) return dst_files
def _do_nd2_scatter(src_path, start_field, n_fields, cycle_i, n_channels, target_mea): """ Scatter a cycle .nd2 into individual numpy files. target_mea is a scalar. The target will be put into this square form. """ working_im = np.zeros((target_mea, target_mea), np.uint16) with _nd2(src_path) as nd2: dst_files = [] for field_i in range(start_field, start_field + n_fields): info = Munch( x=nd2.x[field_i], y=nd2.y[field_i], z=nd2.z[field_i], pfs_status=nd2.pfs_status[field_i], pfs_offset=nd2.pfs_offset[field_i], exposure_time=nd2.exposure_time[field_i], camera_temp=nd2.camera_temp[field_i], cycle_i=cycle_i, field_i=field_i, ) info_dst_file = _metadata_filename_by_field_cycle(field_i, cycle_i) utils.json_save(info_dst_file, info) for channel_i in range(n_channels): im = nd2.get_field(field_i, channel_i) if im.shape[0] != target_mea or im.shape[1] != target_mea: working_im[0:im.shape[0], 0:im.shape[1]] = im[:, :] im = working_im dst_file = _npy_filename_by_field_channel_cycle( field_i, channel_i, cycle_i) dst_files += [dst_file] np.save(dst_file, im) return dst_files
def it_returns_none_if_old_and_new_differ_only_by_underscore_blocks(): new_config = Munch(a=1, b=2, _c=3) utils.json_save("/tmp/config.json", new_config) new_config["_c"] = 4 assert PipelineTask._config_dirty(new_config, "/tmp/config.json") is None
def it_returns_none_if_old_and_new_identical(): new_config = Munch(a=1, b=2) utils.json_save("/tmp/config.json", new_config) assert PipelineTask._config_dirty(new_config, "/tmp/config.json") is None
def main(self): if self.construct_fail: return job_folder = self.validate_job_name_and_folder() schema = self.generator_klass.schema defaults = self.generator_klass.defaults requirements = schema.requirements() # APPLY defaults and then ask user for any elements that are not declared generator_args = {} switches = self._switches_by_name if self.protein_random is not None: tell( f"Sampling {self.protein_random} random proteins from imported set" ) n = len(self.derived_vals.protein) assert n >= self.protein_random self.derived_vals.protein = stats.subsample( self.derived_vals.protein, self.protein_random) assert len(self.derived_vals.protein) == self.protein_random for arg_name, arg_type, _, arg_userdata in requirements: if (arg_name in self.derived_vals and self.derived_vals.get(arg_name) is not None): # Load from a derived switch (eg: protein) generator_args[arg_name] = self.derived_vals[arg_name] elif arg_name in switches and switches.get(arg_name) is not None: # Load from a switch generator_args[arg_name] = getattr(self, arg_name) else: # If the schema allows the user to enter manually if arg_userdata.get("allowed_to_be_entered_manually"): generator_args[arg_name] = self._request_field_from_user( arg_name, arg_type, default=defaults.get(arg_name)) # Download sigproc sources and replace with local path before handing to generator if "sigproc_source" in generator_args: source = generator_args["sigproc_source"] if source is not None and source.startswith("s3:"): generator_args["sigproc_source"] = self._cache_s3_reference( source) # Intentionally run the generate before the job folder is written # so that if generate fails it doesn't leave around a partial job. gen_sources_folder = job_folder / "_gen_sources" try: generator_args["force_run_name"] = self.run_name generator_args[ "local_sources_tmp_folder"] = self.local_sources_tmp_folder generator_args["gen_sources_folder"] = gen_sources_folder generator = self.generator_klass(**generator_args) run_descs = generator.generate() except (SchemaValidationFailed, ValidationError) as e: # Emit clean failure and exit 1 log.exception("Gen failed") return 1 # WRITE the job & copy any file sources self._write_runs(job_folder, run_descs, props=self.prop) gen_sources_folder.delete() self.local_sources_tmp_folder.move(gen_sources_folder) if not self.skip_report: for report_name, report_builder in generator.reports.items(): report = report_builder.report_assemble() if report is not None: utils.json_save(job_folder / f"{report_name}.ipynb", report) (job_folder / "reports_archive").mkdir() for report_name in generator.static_reports: if report_name is not None: report_name = f"{report_name}.ipynb" src = local.path(__file__) / "../../reports" / report_name dst_folder = job_folder / "_reports" dst_folder.mkdir() dst = dst_folder / report_name src.copy(dst) utils.yaml_write( job_folder / "job_manifest.yaml", uuid=self.job_uuid, localtime=time.strftime("%Y-%m-%d, %H:%M:%S", time.localtime()), # Note: it seems localtime inside our container is UTC who=local.env.get("RUN_USER", "Unknown"), cmdline_args=sys.argv, )
def main(self): if self.construct_fail: return with local.cwd("/erisyon"): assert local.path("erisyon_root").exists() job_folder = self.validate_job_name_and_folder() schema = self.generator_klass.schema defaults = self.generator_klass.defaults requirements = schema.requirements() # APPLY defaults and then ask user for any elements that are not declared generator_args = {} switches = self._switches_by_name if self.protein_random is not None: info( f"Sampling {self.protein_random} random proteins from imported set" ) n = len(self.derived_vals.protein) assert n >= self.protein_random self.derived_vals.protein = data.subsample( self.derived_vals.protein, self.protein_random ) assert len(self.derived_vals.protein) == self.protein_random for arg_name, arg_type, arg_help, arg_userdata in requirements: if ( arg_name in self.derived_vals and self.derived_vals.get(arg_name) is not None ): # Load from a derived switch (eg: protein) generator_args[arg_name] = self.derived_vals[arg_name] elif arg_name in switches and switches.get(arg_name) is not None: # Load from a switch generator_args[arg_name] = getattr(self, arg_name) else: # If the schema allows the user to enter manually if arg_userdata.get("allowed_to_be_entered_manually"): generator_args[arg_name] = self._request_field_from_user( arg_name, arg_type, default=defaults.get(arg_name) ) # Intentionally run the generate before the job folder is written # so that if generate fails it doesn't leave around a partial job. try: generator_args["force_run_name"] = self.run_name generator = self.generator_klass(**generator_args) run_descs = generator.generate() except (SchemaValidationFailed, ValidationError) as e: # Emit clean failure and exit 1 error(str(e)) return 1 # WRITE the job & copy any file sources self._write_runs(job_folder, run_descs, props=self.prop) (job_folder / "_gen_sources").delete() self.local_sources_tmp_folder.move(job_folder / "_gen_sources") if not self.skip_report: report = generator.report_assemble() utils.json_save(job_folder / "report.ipynb", report) utils.yaml_write( job_folder / "job_manifest.yaml", uuid=self.job_uuid, localtime=time.strftime("%Y-%m-%d, %H:%M:%S", time.localtime()), # Note: it seems localtime inside our container is UTC who=local.env.get("RUN_USER", "Unknown"), cmdline_args=sys.argv, )