def _create_newcase_phase(self, test_name):
###########################################################################
test_dir = self._get_test_dir(test_name)
test_case, case_opts, grid, compset, machine, compiler, test_mods = acme_util.parse_test_name(test_name)
if (compiler != self._compiler):
raise StandardError("Test '%s' has compiler that does not match instance compliler '%s'" % (test_name, self._compiler))
if (self._parallel_jobs == 1):
scratch_dir = acme_util.get_machine_info("CESMSCRATCHROOT", machine=machine, project=self._project)
sharedlibroot = os.path.join(scratch_dir, "sharedlibroot.%s" % self._test_id)
else:
# Parallelizing builds introduces potential sync problems with sharedlibroot
# Just let every case build it's own
sharedlibroot = os.path.join(test_dir, "sharedlibroot.%s" % self._test_id)
create_newcase_cmd = "%s -silent -case %s -res %s -mach %s -compiler %s -compset %s -testname %s -project %s -nosavetiming -sharedlibroot %s" % \
(os.path.join(self._cime_root, "scripts", "create_newcase"),
test_dir, grid, machine, compiler, compset, test_case, self._project,
sharedlibroot)
if (case_opts is not None):
create_newcase_cmd += " -confopts _%s" % ("_".join(case_opts))
if (test_mods is not None):
test_mod_file = os.path.join(self._cime_root, "scripts", "Testing", "Testlistxml", "testmods_dirs", test_mods)
if (not os.path.exists(test_mod_file)):
self._log_output(test_name, "Missing testmod file '%s'" % test_mod_file)
return False
create_newcase_cmd += " -user_mods_dir %s" % test_mod_file
return self._run_phase_command(test_name, create_newcase_cmd, CREATE_NEWCASE_PHASE)
def _create_newcase_phase(self, test_name):
###########################################################################
test_dir = self._get_test_dir(test_name)
test_case, case_opts, grid, compset, machine, compiler, test_mods = acme_util.parse_test_name(test_name)
if (compiler != self._compiler):
raise StandardError("Test '%s' has compiler that does not match instance compliler '%s'" % (test_name, self._compiler))
if (self._parallel_jobs == 1):
scratch_dir = acme_util.get_machine_info("CESMSCRATCHROOT", machine=machine, project=self._project)
sharedlibroot = os.path.join(scratch_dir, "sharedlibroot.%s" % self._test_id)
else:
# Parallelizing builds introduces potential sync problems with sharedlibroot
# Just let every case build it's own
sharedlibroot = os.path.join(test_dir, "sharedlibroot.%s" % self._test_id)
create_newcase_cmd = "%s -silent -case %s -res %s -mach %s -compiler %s -compset %s -testname %s -project %s -nosavetiming -sharedlibroot %s" % \
(os.path.join(self._cime_root, "scripts", "create_newcase"),
test_dir, grid, machine, compiler, compset, test_case, self._project,
sharedlibroot)
if (case_opts is not None):
create_newcase_cmd += " -confopts _%s" % ("_".join(case_opts))
if (test_mods is not None):
test_mod_file = os.path.join(self._cime_root, "scripts", "Testing", "Testlistxml", "testmods_dirs", test_mods)
if (not os.path.exists(test_mod_file)):
self._log_output(test_name, "Missing testmod file '%s'" % test_mod_file)
return False
create_newcase_cmd += " -user_mods_dir %s" % test_mod_file
return self._run_phase_command(test_name, create_newcase_cmd, CREATE_NEWCASE_PHASE)
def find_all_supported_platforms():
###############################################################################
"""
Returns a set of all ACME supported platforms as defined in the
XML configuration file config_machines.xml in the ACME source
tree. A platform is defined by a triple (machine name, compiler,
mpi library).
"""
machines = acme_util.get_machines()
platform_set = set()
for machine in machines:
compilers, mpilibs = acme_util.get_machine_info(["COMPILERS", "MPILIBS"], machine=machine)
for compiler in compilers:
for mpilib in mpilibs:
platform_set.add((machine, compiler, mpilib))
return list(platform_set)
def find_all_supported_platforms():
###############################################################################
"""
Returns a set of all ACME supported platforms as defined in the
XML configuration file config_machines.xml in the ACME source
tree. A platform is defined by a triple (machine name, compiler,
mpi library).
"""
machines = acme_util.get_machines()
platform_set = set()
for machine in machines:
compilers, mpilibs = acme_util.get_machine_info(["COMPILERS", "MPILIBS"], machine=machine)
for compiler in compilers:
for mpilib in mpilibs:
platform_set.add((machine, compiler, mpilib))
return list(platform_set)
def get_test_suite(suite, machine=None, compiler=None):
###############################################################################
"""
Return a list of FULL test names for a suite.
"""
expect(suite in _TEST_SUITES, "Unknown test suite: '%s'" % suite)
machine = acme_util.probe_machine_name() if machine is None else machine
compiler = acme_util.get_machine_info("COMPILERS", machine=machine)[0] if compiler is None else compiler
inherits_from, tests_raw = _TEST_SUITES[suite]
tests = []
for item in tests_raw:
test_mod = None
if (isinstance(item, str)):
test_name = item
else:
expect(isinstance(item, tuple), "Bad item type for item '%s'" % str(item))
expect(len(item) in [2, 3], "Expected two or three items in item '%s'" % str(item))
expect(isinstance(item[0], str), "Expected string in first field of item '%s'" % str(item))
expect(isinstance(item[1], str), "Expected string in second field of item '%s'" % str(item))
test_name = item[0]
if (len(item) == 2):
test_mod = item[1]
else:
expect(type(item[2]) in [str, tuple], "Expected string or tuple for third field of item '%s'" % str(item))
test_mod_machines = [item[2]] if isinstance(item[2], str) else item[2]
if (machine in test_mod_machines):
test_mod = item[1]
tests.append(acme_util.get_full_test_name(test_name, machine, compiler, testmod=test_mod))
if (inherits_from is not None):
inherited_tests = get_test_suite(inherits_from, machine, compiler)
expect(len(set(tests) & set(inherited_tests)) == 0,
"Tests %s defined in multiple suites" % ", ".join(set(tests) & set(inherited_tests)))
tests.extend(inherited_tests)
return tests
def get_test_suite(suite, machine=None, compiler=None):
###############################################################################
"""
Return a list of FULL test names for a suite.
"""
expect(suite in _TEST_SUITES, "Unknown test suite: '%s'" % suite)
machine = acme_util.probe_machine_name() if machine is None else machine
compiler = acme_util.get_machine_info("COMPILERS", machine=machine)[0] if compiler is None else compiler
inherits_from, tests_raw = _TEST_SUITES[suite]
tests = []
for item in tests_raw:
test_mod = None
if (isinstance(item, str)):
test_name = item
else:
expect(isinstance(item, tuple), "Bad item type for item '%s'" % str(item))
expect(len(item) in [2, 3], "Expected two or three items in item '%s'" % str(item))
expect(isinstance(item[0], str), "Expected string in first field of item '%s'" % str(item))
expect(isinstance(item[1], str), "Expected string in second field of item '%s'" % str(item))
test_name = item[0]
if (len(item) == 2):
test_mod = item[1]
else:
expect(type(item[2]) in [str, tuple], "Expected string or tuple for third field of item '%s'" % str(item))
test_mod_machines = [item[2]] if isinstance(item[2], str) else item[2]
if (machine in test_mod_machines):
test_mod = item[1]
tests.append(acme_util.get_full_test_name(test_name, machine, compiler, testmod=test_mod))
if (inherits_from is not None):
inherited_tests = get_test_suite(inherits_from, machine, compiler)
expect(len(set(tests) & set(inherited_tests)) == 0,
"Tests %s defined in multiple suites" % ", ".join(set(tests) & set(inherited_tests)))
tests.extend(inherited_tests)
return tests
def __init__(self, test_names,
no_run=False, no_build=False, no_batch=None,
test_root=None, test_id=None,
compiler=None,
baseline_root=None, baseline_name=None,
clean=False,
compare=False, generate=False, namelists_only=False,
project=None, parallel_jobs=None):
###########################################################################
self._cime_root = acme_util.get_cime_root()
self._test_names = test_names
self._no_build = no_build if not namelists_only else True
self._no_run = no_run if not self._no_build else True
self._no_batch = no_batch if no_batch is not None else not acme_util.does_machine_have_batch()
self._test_root = test_root if test_root is not None else acme_util.get_machine_info("CESMSCRATCHROOT")
self._test_id = test_id if test_id is not None else acme_util.get_utc_timestamp()
self._project = project if project is not None else acme_util.get_machine_project()
self._baseline_root = baseline_root if baseline_root is not None else acme_util.get_machine_info("CCSM_BASELINE", project=self._project)
self._baseline_name = None
self._compiler = compiler if compiler is not None else acme_util.get_machine_info("COMPILERS")[0]
self._clean = clean
self._compare = compare
self._generate = generate
self._namelists_only = namelists_only
self._parallel_jobs = parallel_jobs if parallel_jobs is not None else min(len(self._test_names), int(acme_util.get_machine_info("MAX_TASKS_PER_NODE")))
# Oversubscribe by 1/4
pes = int(acme_util.get_machine_info("MAX_TASKS_PER_NODE"))
# This is the only data that multiple threads will simultaneously access
# Each test has it's own index and setting/retrieving items from a list
# is atomic, so this should be fine to use without mutex
self._test_states = [ (INITIAL_PHASE, TEST_PASS_STATUS) ] * len(test_names)
self._proc_pool = int(pes * 1.25)
# Since the name-list phase can fail without aborting later phases, we
# need some extra state to remember tests that had namelist problems
self._tests_with_nl_problems = [None] * len(test_names)
# Setup phases
self._phases = list(PHASES)
if (no_build):
self._phases.remove(BUILD_PHASE)
if (no_run):
self._phases.remove(RUN_PHASE)
if (not self._compare and not self._generate):
self._phases.remove(NAMELIST_PHASE)
else:
if (baseline_name is None):
branch_name = acme_util.get_current_branch(repo=self._cime_root)
expect(branch_name is not None, "Could not determine baseline name from branch, please use -b option")
self._baseline_name = os.path.join(self._compiler, branch_name)
else:
self._baseline_name = baseline_name
if (not self._baseline_name.startswith("%s/" % self._compiler)):
self._baseline_name = os.path.join(self._compiler, self._baseline_name)
# Validate any assumptions that were not caught by the arg parser
# None of the test directories should already exist.
for test in self._test_names:
expect(not os.path.exists(self._get_test_dir(test)),
"Cannot create new case in directory '%s', it already exists. Pick a different test-id" % self._get_test_dir(test))
def __init__(self,
test_names,
no_run=False,
no_build=False,
no_batch=None,
test_root=None,
test_id=None,
compiler=None,
baseline_root=None,
baseline_name=None,
clean=False,
compare=False,
generate=False,
namelists_only=False,
project=None,
parallel_jobs=None):
###########################################################################
self._cime_root = acme_util.get_cime_root()
self._test_names = test_names
self._no_build = no_build if not namelists_only else True
self._no_run = no_run if not self._no_build else True
self._no_batch = no_batch if no_batch is not None else not acme_util.does_machine_have_batch(
)
self._test_root = test_root if test_root is not None else acme_util.get_machine_info(
"CESMSCRATCHROOT")
self._test_id = test_id if test_id is not None else acme_util.get_utc_timestamp(
)
self._project = project if project is not None else acme_util.get_machine_project(
)
self._baseline_root = baseline_root if baseline_root is not None else acme_util.get_machine_info(
"CCSM_BASELINE", project=self._project)
self._baseline_name = None
self._compiler = compiler if compiler is not None else acme_util.get_machine_info(
"COMPILERS")[0]
self._clean = clean
self._compare = compare
self._generate = generate
self._namelists_only = namelists_only
self._parallel_jobs = parallel_jobs if parallel_jobs is not None else min(
len(self._test_names),
int(acme_util.get_machine_info("MAX_TASKS_PER_NODE")))
# Oversubscribe by 1/4
pes = int(acme_util.get_machine_info("MAX_TASKS_PER_NODE"))
# This is the only data that multiple threads will simultaneously access
# Each test has it's own index and setting/retrieving items from a list
# is atomic, so this should be fine to use without mutex
self._test_states = [(INITIAL_PHASE, TEST_PASS_STATUS)
] * len(test_names)
self._proc_pool = int(pes * 1.25)
# Since the name-list phase can fail without aborting later phases, we
# need some extra state to remember tests that had namelist problems
self._tests_with_nl_problems = [None] * len(test_names)
# Setup phases
self._phases = list(PHASES)
if (no_build):
self._phases.remove(BUILD_PHASE)
if (no_run):
self._phases.remove(RUN_PHASE)
if (not self._compare and not self._generate):
self._phases.remove(NAMELIST_PHASE)
else:
if (baseline_name is None):
branch_name = acme_util.get_current_branch(
repo=self._cime_root)
expect(
branch_name is not None,
"Could not determine baseline name from branch, please use -b option"
)
self._baseline_name = os.path.join(self._compiler, branch_name)
else:
self._baseline_name = baseline_name
if (not self._baseline_name.startswith(
"%s/" % self._compiler)):
self._baseline_name = os.path.join(self._compiler,
self._baseline_name)
# Validate any assumptions that were not caught by the arg parser
# None of the test directories should already exist.
for test in self._test_names:
expect(
not os.path.exists(self._get_test_dir(test)),
"Cannot create new case in directory '%s', it already exists. Pick a different test-id"
% self._get_test_dir(test))