def async_cmd():
"""Async CLI command fixture"""
job = mock.MagicMock()
job.name = "Test-Job"
cmd = "echo 'Hello World'"
output = os.path.join(tempfile.gettempdir(), "jade-test-async-cli-job")
os.makedirs(output, exist_ok=True)
os.makedirs(os.path.join(output, RESULTS_DIR), exist_ok=True)
ResultsAggregator.create(output)
cmd = AsyncCliCommand(job, cmd, output, 1, True, "0")
yield cmd
shutil.rmtree(output)
def test_results_aggregator(cleanup):
"""Test ResultsAggregator"""
if os.path.exists(OUTPUT):
shutil.rmtree(OUTPUT)
results = [create_result(i) for i in range(100)]
os.makedirs(OUTPUT)
pytest.aggregator = ResultsAggregator.create(OUTPUT)
assert os.path.exists(pytest.aggregator._filename)
for result in results:
if int(result.name) % 2 == 0:
pytest.aggregator.append_result(result)
final_results = pytest.aggregator.get_results()
final_results.sort(key=lambda x: int(x.name))
expected = [x for x in results if int(x.name) % 2 == 0]
assert final_results == expected
def submit_jobs(self, cluster, force_local=False):
"""Submit simulations. Auto-detect whether the current system is an HPC
and submit to its queue. Otherwise, run locally.
Parameters
----------
cluster : Cluster
force_local : bool
If on HPC, run jobs through subprocess as if local.
Returns
-------
Status
"""
if self._is_new:
logger.info("Submit %s jobs for execution.",
self._config.get_num_jobs())
logger.info("JADE version %s", jade.version.__version__)
registry = Registry()
loggers = registry.list_loggers()
logger.info("Registered modules for logging: %s",
", ".join(loggers))
self._save_repository_info(registry)
ResultsAggregator.create(self._output)
# If an events summary file exists, it is invalid.
events_file = os.path.join(self._output, EVENTS_FILENAME)
if os.path.exists(events_file):
os.remove(events_file)
event = StructuredLogEvent(
source="submitter",
category=EVENT_CATEGORY_RESOURCE_UTIL,
name=EVENT_NAME_SUBMIT_COMPLETED,
message="job submission started",
num_jobs=self.get_num_jobs(),
)
log_event(event)
os.environ["JADE_RUNTIME_OUTPUT"] = self._output
if self._config.setup_command is not None:
cmd = f"JADE_RUNTIME_OUTPUT={self._output} {self._config.setup_command}"
logger.info("Running setup command: %s", cmd)
check_run_command(self._config.setup_command)
else:
self._handle_submission_groups()
result = Status.IN_PROGRESS
group = self._config.get_default_submission_group()
groups = make_submission_group_lookup(cluster.config.submission_groups)
self._hpc = HpcManager(groups, self._output)
if self._hpc.hpc_type == HpcType.LOCAL or force_local:
runner = JobRunner(self._config_file, output=self._output)
num_processes = group.submitter_params.num_processes
verbose = group.submitter_params.verbose
result = runner.run_jobs(verbose=verbose,
num_processes=num_processes)
agg = ResultsAggregator.load(self._output)
agg.process_results()
is_complete = True
else:
is_complete = self._submit_to_hpc(cluster)
if is_complete:
result = self._handle_completion(cluster)
return result