def auto_config(extension, inputs, job_post_process_config_file, config_file, verbose): """Automatically create a configuration.""" level = logging.DEBUG if verbose else logging.WARNING setup_logging("auto_config", None, console_level=level) if job_post_process_config_file is not None: module, class_name, data = JobPostProcess.load_config_from_file( job_post_process_config_file) JobPostProcess(module, class_name, data) # ensure everything ok job_post_process_config = { "module": module, "class": class_name, "data": data } else: job_post_process_config = None # User extension registry = Registry() if not registry.is_registered(extension): raise InvalidExtension(f"Extension '{extension}' is not registered.") cli = registry.get_extension_class(extension, ExtensionClassType.CLI) config = cli.auto_config(*inputs, job_post_process_config=job_post_process_config) print(f"Created configuration with {config.get_num_jobs()} jobs.") config.dump(config_file) print(f"Dumped configuration to {config_file}.\n")
def test_registry__is_registered(registry_fixture): registry = Registry(registry_filename=TEST_FILENAME) registry.reset_defaults() assert registry.is_registered(DEFAULT_REGISTRY["extensions"][0]["name"])
def run(extension, **kwargs): """Runs individual job.""" registry = Registry() if not registry.is_registered(extension): raise InvalidExtension(f"Extension '{extension}' is not registered.") # Parse Argument config_file = kwargs["config_file"] name = kwargs["name"] output = kwargs["output"] output_format = kwargs["output_format"] verbose = kwargs["verbose"] level = logging.DEBUG if verbose else logging.INFO # Create directory for current job job_dir = os.path.join(output, name) os.makedirs(job_dir, exist_ok=True) # Structural logging setup event_file = os.path.join(job_dir, "events.log") setup_event_logging(event_file) # General logging setup log_file = os.path.join(job_dir, "run.log") general_logger = setup_logging( extension, log_file, console_level=logging.ERROR, file_level=level, ) general_logger.info(get_cli_string()) # Create config for run try: cli = registry.get_extension_class(extension, ExtensionClassType.CLI) ret = cli.run(config_file, name, output, output_format, verbose) except Exception as err: msg = f"unexpected exception in run '{extension}' job={name} - {err}" general_logger.exception(msg) event = StructuredErrorLogEvent( source=name, category=EVENT_CATEGORY_ERROR, name=EVENT_NAME_UNHANDLED_ERROR, message=msg, ) log_event(event) ret = 1 if ret == 0: try: config = load_data(config_file) if "job_post_process_config" in config.keys(): post_process = JobPostProcess( module_name=config["job_post_process_config"]["module"], class_name=config["job_post_process_config"]["class"], data=config["job_post_process_config"]["data"], job_name=name, output=output, ) post_process.run(config_file=config_file, output=output) except Exception as err: msg = f"unexpected exception in post-process '{extension}' job={name} - {err}" general_logger.exception(msg) event = StructuredErrorLogEvent( source=name, category=EVENT_CATEGORY_ERROR, name=EVENT_NAME_UNHANDLED_ERROR, message=msg, ) log_event(event) ret = 1 sys.exit(ret)
import os import sys import pytest from jade.extensions.registry import Registry if os.environ.get("LOCAL_SUBMITTER") is not None: print("You must unset the environment variable LOCAL_SUBMITTER.") sys.exit(1) registry = Registry() if not registry.is_registered("demo"): registry.register_demo_extension() @pytest.fixture def test_data_dir(): """The path to the directory that contains the fixture data""" return os.path.join(os.path.dirname(__file__), "data") @pytest.fixture def example_output(): return os.path.join(os.path.dirname(__file__), "data", "example_output")