def test_invalid_config(self):
with tempfile.TemporaryDirectory() as td:
# Config files with unknown config values
yaml_path = pathlib.Path(td).joinpath("garbled.yaml")
yaml_path.write_text('foo: bar\n')
json_path = pathlib.Path(td).joinpath("garbled.json")
json_path.write_text('{ "foo": "bar",\n "bar": 1}')
sys_parser = cfg_p.ConfigParser(cfg.RuntimeConfig)
job_parser = cfg_p.ConfigParser(cfg.JobConfig)
sys_yaml_config = sys_parser.load_raw_config(yaml_path)
sys_json_config = sys_parser.load_raw_config(json_path)
job_yaml_config = job_parser.load_raw_config(yaml_path)
job_json_config = job_parser.load_raw_config(json_path)
self.assertRaises(ex.EConfigParse,
lambda: sys_parser.parse(sys_yaml_config))
self.assertRaises(ex.EConfigParse,
lambda: sys_parser.parse(sys_json_config))
self.assertRaises(ex.EConfigParse,
lambda: job_parser.parse(job_yaml_config))
self.assertRaises(ex.EConfigParse,
lambda: job_parser.parse(job_json_config))
def test_config_file_not_found(self):
nonexistent_path = PYTHON_EXAMPLES_DIR.joinpath("nonexistent.yaml")
parser = cfg_p.ConfigParser(cfg.RuntimeConfig)
self.assertRaises(ex.EConfigLoad,
lambda: parser.load_raw_config(nonexistent_path))
def pre_start(self):
try:
self._log.info(f"Beginning pre-start sequence...")
# Plugins will be loaded here, before config
# Load sys and job config (or use embedded)
if self._sys_config is None:
sys_config_dev_mode = _dev_mode.DEV_MODE_SYS_CONFIG if self._dev_mode else None
sys_config_parser = _cparse.ConfigParser(
_cfg.RuntimeConfig, sys_config_dev_mode)
sys_config_raw = sys_config_parser.load_raw_config(
self._sys_config_path, config_file_name="system")
self._sys_config = sys_config_parser.parse(
sys_config_raw, self._sys_config_path)
else:
self._log.info("Using embedded system config")
if self._job_config is None and self._batch_mode:
job_config_dev_mode = _dev_mode.DEV_MODE_JOB_CONFIG if self._dev_mode else None
job_config_parser = _cparse.ConfigParser(
_cfg.JobConfig, job_config_dev_mode)
job_config_raw = job_config_parser.load_raw_config(
self._job_config_path, config_file_name="job")
self._job_config = job_config_parser.parse(
job_config_raw, self._job_config_path)
elif self._batch_mode:
self._log.info("Using embedded job config")
# Dev mode translation is controlled by the dev mode flag
# I.e. it can be applied to embedded configs
if self._dev_mode:
job_config, sys_config = _dev_mode.DevModeTranslator.translate_dev_mode_config(
self._sys_config, self._job_config, self._sys_config_path,
self._job_config_path, self._model_class)
self._job_config = job_config
self._sys_config = sys_config
except Exception as e:
self._handle_startup_error(e)
def test_example_sys_config_ok(self):
parser = cfg_p.ConfigParser(cfg.RuntimeConfig)
raw_config_path = PYTHON_EXAMPLES_DIR.joinpath("sys_config.yaml")
raw_config = parser.load_raw_config(raw_config_path, "system")
sys_config = parser.parse(raw_config, raw_config_path.name)
self.assertIsInstance(sys_config, cfg.RuntimeConfig)
def test_example_job_config_ok(self):
# Sample job config uses dev mode configuration, so supply DEV_MODE_JOB_CONFIG
parser = cfg_p.ConfigParser(cfg.JobConfig,
dev_mode.DEV_MODE_JOB_CONFIG)
raw_config_path = PYTHON_EXAMPLES_DIR.joinpath(
"using_data/using_data.yaml")
raw_config = parser.load_raw_config(raw_config_path, "job")
job_config = parser.parse(raw_config, raw_config_path.name)
self.assertIsInstance(job_config, cfg.JobConfig)
def test_config_file_wrong_format(self):
parser = cfg_p.ConfigParser(cfg.RuntimeConfig)
with tempfile.TemporaryDirectory() as td:
# Write YAML into a JSON file
json_path = pathlib.Path(td).joinpath("garbled.json")
json_path.write_text('foo: bar\n')
self.assertRaises(ex.EConfigParse,
lambda: parser.load_raw_config(json_path))
def test_invalid_path(self):
parser = cfg_p.ConfigParser(cfg.RuntimeConfig)
self.assertRaises(ex.EConfigLoad,
lambda: parser.load_raw_config(None)) # noqa
self.assertRaises(ex.EConfigLoad,
lambda: parser.load_raw_config(object())) # noqa
self.assertRaises(ex.EConfigLoad, lambda: parser.load_raw_config(""))
self.assertRaises(ex.EConfigLoad,
lambda: parser.load_raw_config("$%^&*--`!"))
self.assertRaises(
ex.EConfigLoad, lambda: parser.load_raw_config(
"pigeon://pigeon-svr/lovely-pigeon.pg"))
def test_empty_job_config_ok(self):
parser = cfg_p.ConfigParser(cfg.JobConfig)
with tempfile.TemporaryDirectory() as td:
yaml_path = pathlib.Path(td).joinpath("empty.yaml")
yaml_path.touch()
raw_config = parser.load_raw_config(yaml_path, "job")
job_config = parser.parse(raw_config, yaml_path.name)
self.assertIsInstance(job_config, cfg.JobConfig)
json_path = pathlib.Path(td).joinpath("empty.json")
json_path.write_text("{}")
raw_config = parser.load_raw_config(json_path, "job")
job_config = parser.parse(raw_config, yaml_path.name)
self.assertIsInstance(job_config, cfg.JobConfig)
def test_config_file_garbled(self):
parser = cfg_p.ConfigParser(cfg.RuntimeConfig)
noise_bytes = 256
noise = bytearray(random.getrandbits(8) for _ in range(noise_bytes))
with tempfile.TemporaryDirectory() as td:
yaml_path = pathlib.Path(td).joinpath("garbled.yaml")
yaml_path.write_bytes(noise)
self.assertRaises(ex.EConfigParse,
lambda: parser.load_raw_config(yaml_path))
json_path = pathlib.Path(td).joinpath("garbled.json")
json_path.write_bytes(noise)
self.assertRaises(ex.EConfigParse,
lambda: parser.load_raw_config(json_path))
def _expand_flow_definition(
cls, job_config: _cfg.JobConfig, sys_config: _cfg.RuntimeConfig,
job_config_dir: pathlib.Path) \
-> (_cfg.JobConfig, _cfg.RuntimeConfig):
flow_details = job_config.job.runFlow.flow
# The full specification for a flow is as a tag selector for a valid job resource
# This is still allowed in dev mode, in which case dev mode translation is not applied
if isinstance(flow_details, _meta.TagHeader) or isinstance(flow_details, _meta.TagSelector):
_util.get_job_resource(flow_details, job_config, optional=False)
return job_config, sys_config
# Otherwise, flow is specified as the path to dev-mode flow definition
if not isinstance(flow_details, str):
err = f"Invalid config value for [job.runFlow.flow]: Expected path or tag selector, got [{flow_details}])"
cls._log.error(err)
raise _ex.EConfigParse(err)
flow_id = _util.new_object_id(_meta.ObjectType.FLOW)
flow_key = _util.object_key(flow_id)
cls._log.info(f"Generating flow definition for [{flow_details}] with ID = [{flow_key}]")
flow_path = job_config_dir.joinpath(flow_details)
flow_parser = _cfg_p.ConfigParser(_meta.FlowDefinition)
flow_raw_data = flow_parser.load_raw_config(flow_path, flow_path.name)
flow_def = flow_parser.parse(flow_raw_data, flow_path.name)
flow_def = cls._autowire_flow(flow_def)
flow_def = cls._generate_flow_parameters(flow_def, job_config)
flow_def = cls._generate_flow_inputs(flow_def, job_config)
flow_def = cls._generate_flow_outputs(flow_def, job_config)
flow_object = _meta.ObjectDefinition(
objectType=_meta.ObjectType.FLOW,
flow=flow_def)
return flow_id, flow_object
def test_config_file_is_a_folder(self):
parser = cfg_p.ConfigParser(cfg.RuntimeConfig)
self.assertRaises(ex.EConfigLoad,
lambda: parser.load_raw_config(PYTHON_EXAMPLES_DIR))