def test_runCommand_dryRun(self):
"""With dry_run, testdir should be empty"""
job_launcher = create_job_launcher(
job_launcher_type=JOB_LAUNCHER_NOBATCH)
job_launcher.run_command(
command=['echo', 'hello', 'world'],
stdout_path=os.path.join(self._testdir, 'stdout'),
stderr_path=os.path.join(self._testdir, 'stderr'),
dry_run=True)
self.assertEqual(os.listdir(self._testdir), [])
def test_runCommand(self):
"""Test that the given command gets executed"""
job_launcher = create_job_launcher(
job_launcher_type=JOB_LAUNCHER_NOBATCH)
stdout = os.path.join(self._testdir, 'stdout')
job_launcher.run_command(command=['echo', 'hello', 'world'],
stdout_path=stdout,
stderr_path=os.path.join(
self._testdir, 'stderr'))
self.assertTrue(os.path.isfile(stdout))
self.assertFileContentsEqual('hello world\n', stdout)
def test_runCommand_dryRun(self):
"""With dry_run, testdir should be empty"""
job_launcher = create_job_launcher(
job_launcher_type=JOB_LAUNCHER_NOBATCH)
job_launcher.run_command(
command=['echo', 'hello', 'world'],
stdout_path=os.path.join(self._testdir, 'stdout'),
stderr_path=os.path.join(self._testdir, 'stderr'),
dry_run=True)
# There shouldn't be a "last process", but in case there is, wait for it to
# complete so we can be confident that the test isn't passing simply because the
# process hasn't completed yet. (This relies on there being logic in
# wait_for_last_process_to_complete so that it succeeds even if there is no "last
# process".)
job_launcher.wait_for_last_process_to_complete()
self.assertEqual(os.listdir(self._testdir), [])
def create_machine(machine_name,
defaults,
job_launcher_type=None,
scratch_dir=None,
account=None,
job_launcher_queue=None,
job_launcher_walltime=None,
job_launcher_nice_level=None,
job_launcher_extra_args=None,
allow_missing_entries=False):
"""Create a machine object (of type Machine, as given above)
This uses the provided (non-None) arguments to override any defaults provided via the
'defaults' argument.
Args:
machine_name (str): name of machine; this is used to index into the 'defaults'
argument, and is used as the machine name in the returned object
defaults: dict of MachineDefaults (as defined in machine_defaults)
scratch_dir: path to scratch directory (if not provided, will attempt to get it from
machine defaults)
account: account to use for job submission to a queue (if not provided, will attempt
to determine it automatically using the CIME method)
job_launcher_type: one of the JOB_LAUNCHER constants defined in job_launcher_factory,
or None. If None, we pick the default for this machine.
job_launcher_queue (str or None): Queue to use (not applicable for
JOB_LAUNCHER_NOBATCH and JOB_LAUNCHER_FAKE)
job_launcher_walltime (str or None): Walltime to use (not applicable for
JOB_LAUNCHER_NOBATCH and JOB_LAUNCHER_FAKE)
job_launcher_nice_level (int or None): Level used for the nice command; only
applicable for JOB_LAUNCHER_NOBATCH
job_launcher_extra_args (str or None): Arguments to the job launcher that can be
overridden by the user. Not applicable for JOB_LAUNCHER_NOBATCH and
JOB_LAUNCHER_FAKE
allow_missing_entries (bool): For a machine that generally requires certain entries
(e.g., account): If allow_missing_entries is True, then we proceed even if these
entries are missing. This is intended for when create_machine is just called for
the sake of getting default values.
"""
# ------------------------------------------------------------------------
# Settings that are independent of both machine and job launcher type
# ------------------------------------------------------------------------
if account is None:
account = _get_account()
# ------------------------------------------------------------------------
# Settings that depend on machine
# ------------------------------------------------------------------------
mach_defaults = defaults.get(machine_name)
baseline_dir = None
if mach_defaults is not None:
if job_launcher_type is None:
job_launcher_type = mach_defaults.job_launcher_type
if scratch_dir is None:
scratch_dir = mach_defaults.scratch_dir
# NOTE(wjs, 2019-05-17) Note that we don't provide a way to override the default
# baseline_dir. The idea is that this should always provide the standard baseline
# directory for this machine, even if a particular application wants to use
# something different (in contrast to, say, scratch_dir, for which the value in
# the machines object can itself be overridden). I think this will be smoother and
# more intuitive if different baseline directories are needed for different
# purposes in a single application (e.g., different baseline directories for
# generation and comparison, or making a link in some temporary location that
# points to the standard baselines).
baseline_dir = mach_defaults.baseline_dir
if account is None and mach_defaults.account_required and not allow_missing_entries:
raise RuntimeError("Could not find an account code")
else:
if not allow_missing_entries:
# This isn't exactly a missing entry, but the times we don't care about this
# warning tend to be the same as the times when allow_missing_entries is true
logger.warning(
"machine %s not recognized; using generic no-batch settings",
machine_name)
if job_launcher_type is None:
job_launcher_type = JOB_LAUNCHER_NOBATCH
# ------------------------------------------------------------------------
# Settings that depend on both machine and job launcher type
# ------------------------------------------------------------------------
# Required args are ones that cannot be overridden by the user. So it doesn't make
# sense to have these in the argument list for this function: they should only come
# from the defaults structure. If the user wants to provide their own arguments, those
# should be provided via extra_args.
job_launcher_required_args = ''
if mach_defaults is not None:
these_defaults = mach_defaults.job_launcher_defaults.get(
job_launcher_type)
if these_defaults is not None:
if job_launcher_queue is None:
job_launcher_queue = these_defaults.queue
if job_launcher_walltime is None:
job_launcher_walltime = these_defaults.walltime
if job_launcher_extra_args is None:
job_launcher_extra_args = these_defaults.extra_args
job_launcher_required_args = these_defaults.required_args
# ------------------------------------------------------------------------
# Create the job launcher and the full machine object
# ------------------------------------------------------------------------
job_launcher = create_job_launcher(
job_launcher_type=job_launcher_type,
account=account,
queue=job_launcher_queue,
walltime=job_launcher_walltime,
nice_level=job_launcher_nice_level,
required_args=job_launcher_required_args,
extra_args=job_launcher_extra_args,
allow_missing_entries=allow_missing_entries)
return Machine(name=machine_name,
scratch_dir=scratch_dir,
baseline_dir=baseline_dir,
account=account,
job_launcher=job_launcher)