def auto_config(extension, inputs, job_post_process_config_file, config_file,
verbose):
"""Automatically create a configuration."""
level = logging.DEBUG if verbose else logging.WARNING
setup_logging("auto_config", None, console_level=level)
if job_post_process_config_file is not None:
module, class_name, data = JobPostProcess.load_config_from_file(
job_post_process_config_file)
JobPostProcess(module, class_name, data) # ensure everything ok
job_post_process_config = {
"module": module,
"class": class_name,
"data": data
}
else:
job_post_process_config = None
# User extension
registry = Registry()
if not registry.is_registered(extension):
raise InvalidExtension(f"Extension '{extension}' is not registered.")
cli = registry.get_extension_class(extension, ExtensionClassType.CLI)
config = cli.auto_config(*inputs,
job_post_process_config=job_post_process_config)
print(f"Created configuration with {config.get_num_jobs()} jobs.")
config.dump(config_file)
print(f"Dumped configuration to {config_file}.\n")
def test_registry__is_registered(registry_fixture):
registry = Registry(registry_filename=TEST_FILENAME)
registry.reset_defaults()
assert registry.is_registered(DEFAULT_REGISTRY["extensions"][0]["name"])
def run(extension, **kwargs):
"""Runs individual job."""
registry = Registry()
if not registry.is_registered(extension):
raise InvalidExtension(f"Extension '{extension}' is not registered.")
# Parse Argument
config_file = kwargs["config_file"]
name = kwargs["name"]
output = kwargs["output"]
output_format = kwargs["output_format"]
verbose = kwargs["verbose"]
level = logging.DEBUG if verbose else logging.INFO
# Create directory for current job
job_dir = os.path.join(output, name)
os.makedirs(job_dir, exist_ok=True)
# Structural logging setup
event_file = os.path.join(job_dir, "events.log")
setup_event_logging(event_file)
# General logging setup
log_file = os.path.join(job_dir, "run.log")
general_logger = setup_logging(
extension,
log_file,
console_level=logging.ERROR,
file_level=level,
)
general_logger.info(get_cli_string())
# Create config for run
try:
cli = registry.get_extension_class(extension, ExtensionClassType.CLI)
ret = cli.run(config_file, name, output, output_format, verbose)
except Exception as err:
msg = f"unexpected exception in run '{extension}' job={name} - {err}"
general_logger.exception(msg)
event = StructuredErrorLogEvent(
source=name,
category=EVENT_CATEGORY_ERROR,
name=EVENT_NAME_UNHANDLED_ERROR,
message=msg,
)
log_event(event)
ret = 1
if ret == 0:
try:
config = load_data(config_file)
if "job_post_process_config" in config.keys():
post_process = JobPostProcess(
module_name=config["job_post_process_config"]["module"],
class_name=config["job_post_process_config"]["class"],
data=config["job_post_process_config"]["data"],
job_name=name,
output=output,
)
post_process.run(config_file=config_file, output=output)
except Exception as err:
msg = f"unexpected exception in post-process '{extension}' job={name} - {err}"
general_logger.exception(msg)
event = StructuredErrorLogEvent(
source=name,
category=EVENT_CATEGORY_ERROR,
name=EVENT_NAME_UNHANDLED_ERROR,
message=msg,
)
log_event(event)
ret = 1
sys.exit(ret)
import os
import sys
import pytest
from jade.extensions.registry import Registry
if os.environ.get("LOCAL_SUBMITTER") is not None:
print("You must unset the environment variable LOCAL_SUBMITTER.")
sys.exit(1)
registry = Registry()
if not registry.is_registered("demo"):
registry.register_demo_extension()
@pytest.fixture
def test_data_dir():
"""The path to the directory that contains the fixture data"""
return os.path.join(os.path.dirname(__file__), "data")
@pytest.fixture
def example_output():
return os.path.join(os.path.dirname(__file__), "data", "example_output")