def wait_for_tests_impl(test_paths, no_wait=False, check_throughput=False, check_memory=False, ignore_namelists=False):
###############################################################################
results = Queue.Queue()
for test_path in test_paths:
t = threading.Thread(target=wait_for_test, args=(test_path, results, not no_wait, check_throughput, check_memory, ignore_namelists))
t.daemon = True
t.start()
while threading.active_count() > 1:
time.sleep(1)
test_results = {}
completed_test_paths = []
while (not results.empty()):
test_name, test_path, test_status = results.get()
if (test_name in test_results):
prior_path, prior_status = test_results[test_name]
if (test_status == prior_status):
warning("Test name '%s' was found in both '%s' and '%s'" %
(test_name, test_path, prior_path))
else:
raise SystemExit("Test name '%s' was found in both '%s' and '%s' with different results" %
(test_name, test_path, prior_path))
test_results[test_name] = (test_path, test_status)
completed_test_paths.append(test_path)
expect(set(test_paths) == set(completed_test_paths),
"Missing results for test paths: %s" % (set(test_paths) - set(completed_test_paths)) )
return test_results
def _update_test_status(self, test_name, phase, status):
###########################################################################
state_idx = self._test_names.index(test_name)
phase_idx = self._phases.index(phase)
old_phase, old_status = self._test_states[state_idx]
if (old_phase == phase):
expect(
old_status == TEST_PENDING_STATUS,
"Only valid to transition from PENDING to something else, found '%s'"
% old_status)
expect(status != TEST_PENDING_STATUS,
"Cannot transition from PEND -> PEND")
else:
expect(
old_status in CONTINUE,
"Why did we move on to next phase when prior phase did not pass?"
)
expect(status == TEST_PENDING_STATUS,
"New phase should be set to pending status")
expect(
self._phases.index(old_phase) == phase_idx - 1,
"Skipped phase?")
self._test_states[state_idx] = (phase, status)
def compare_values(namelist, name, gold_value, comp_value, case):
###############################################################################
"""
Compare values for a specific variable in a namelist.
"""
if (type(gold_value) != type(comp_value)):
print "In namelist '%s', variable '%s' did not have expected type '%s', instead is type '%s'" % \
(namelist, name, type(gold_value), type(comp_value))
return False
rv = True
if (type(gold_value) is list):
# Note, list values remain order sensitive
for idx, gold_value_list_item in enumerate(gold_value):
if (idx < len(comp_value)):
rv &= compare_values(namelist, "%s list item %d" % (name, idx),
gold_value_list_item, comp_value[idx],
case)
else:
rv = False
print "In namelist '%s', list variable '%s' missing value %s" % (
namelist, name, gold_value_list_item)
if (len(comp_value) > len(gold_value)):
for comp_value_list_item in comp_value[len(gold_value):]:
rv = False
print "In namelist '%s', list variable '%s' has extra value %s" % (
namelist, name, comp_value_list_item)
elif (type(gold_value) is dict):
for key, gold_value_dict_item in gold_value.iteritems():
if (key in comp_value):
rv &= compare_values(namelist, "%s dict item %s" % (name, key),
gold_value_dict_item, comp_value[key],
case)
else:
rv = False
print "In namelist '%s', dict variable '%s' missing key %s" % (
namelist, name, key)
for key in comp_value:
if (key not in gold_value):
rv = False
print "In namelist '%s', dict variable '%s' has extra key %s" % (
namelist, name, key)
else:
expect(
type(gold_value) is str,
"Unexpected type found: '%s'" % type(gold_value))
norm_gold_value = normalize_string_value(name, gold_value, case)
norm_comp_value = normalize_string_value(name, comp_value, case)
if (norm_gold_value != norm_comp_value):
rv = False
print "In namelist '%s', '%s' has inequivalent values %s != %s" % (
namelist, name, gold_value, comp_value)
print " NORMALIZED: %s != %s" % (norm_gold_value, norm_comp_value)
return rv
def compare_files(gold_file, compare_file, case=None):
###############################################################################
expect(os.path.exists(gold_file), "File not found: %s" % gold_file)
expect(os.path.exists(compare_file), "File not found: %s" % compare_file)
return compare_data(open(gold_file, "r").readlines(),
open(compare_file, "r").readlines(),
case)
def compare_namelist_files(gold_file, compare_file, case=None):
###############################################################################
expect(os.path.exists(gold_file), "File not found: %s" % gold_file)
expect(os.path.exists(compare_file), "File not found: %s" % compare_file)
gold_namelists = parse_namelists(open(gold_file, "r").readlines(), gold_file)
comp_namelists = parse_namelists(open(compare_file, "r").readlines(), compare_file)
return compare_namelists(gold_namelists, comp_namelists, case)
def create_test(self):
###########################################################################
"""
Main API for this class.
Return True if all tests passed.
"""
start_time = time.time()
# Tell user what will be run
print "RUNNING TESTS:"
for test_name in self._test_names:
print " ", test_name
# TODO - documentation
self._producer()
expect(threading.active_count() == 1, "Leftover threads?")
# Setup cs files
self._setup_cs_files()
# Return True if all tests passed
print "At create_test close, state is:"
rv = True
for idx, test_name in enumerate(self._test_names):
phase, status = self._test_states[idx]
if (status == TEST_PASS_STATUS and phase == RUN_PHASE):
# Be cautious about telling the user that the test passed. This
# status should match what they would see on the dashboard. Our
# self._test_states does not include comparison fail information,
# so we need to parse test status.
test_status_file = os.path.join(self._get_test_dir(test_name),
TEST_STATUS_FILENAME)
status = wait_for_tests.interpret_status_file(
test_status_file)[1]
if (status not in [TEST_PASS_STATUS, TEST_PENDING_STATUS]):
print "%s %s (phase %s)" % (status, test_name, phase)
rv = False
elif (test_name in self._tests_with_nl_problems):
print "%s %s (but otherwise OK)" % (NAMELIST_FAIL_STATUS,
test_name)
rv = False
else:
print status, test_name, phase
print " Case dir: %s" % self._get_test_dir(test_name)
print "create_test took", time.time() - start_time, "seconds"
return rv
def create_test(self):
###########################################################################
"""
Main API for this class.
Return True if all tests passed.
"""
start_time = time.time()
# Tell user what will be run
print "RUNNING TESTS:"
for test_name in self._test_names:
print " ", test_name
# TODO - documentation
self._producer()
expect(threading.active_count() == 1, "Leftover threads?")
# Setup cs files
self._setup_cs_files()
# Return True if all tests passed
print "At create_test close, state is:"
rv = True
for idx, test_name in enumerate(self._test_names):
phase, status = self._test_states[idx]
if (status == TEST_PASS_STATUS and phase == RUN_PHASE):
# Be cautious about telling the user that the test passed. This
# status should match what they would see on the dashboard. Our
# self._test_states does not include comparison fail information,
# so we need to parse test status.
test_status_file = os.path.join(self._get_test_dir(test_name), TEST_STATUS_FILENAME)
status = wait_for_tests.interpret_status_file(test_status_file)[1]
if (status not in [TEST_PASS_STATUS, TEST_PENDING_STATUS]):
print "%s %s (phase %s)" % (status, test_name, phase)
rv = False
elif (test_name in self._tests_with_nl_problems):
print "%s %s (but otherwise OK)" % (NAMELIST_FAIL_STATUS, test_name)
rv = False
else:
print status, test_name, phase
print " Case dir: %s" % self._get_test_dir(test_name)
print "create_test took", time.time() - start_time, "seconds"
return rv
def _get_test_status(self, test_name, phase=None):
###########################################################################
curr_phase = self._get_test_phase(test_name)
if (phase == NAMELIST_PHASE and test_name in self._tests_with_nl_problems):
return NAMELIST_FAIL_STATUS
elif (phase is None or phase == curr_phase):
return self._get_test_data(test_name)[1]
else:
expect(phase is None or self._phases.index(phase) < self._phases.index(curr_phase),
"Tried to see the future")
# Assume all older phases PASSed
return TEST_PASS_STATUS
def _get_test_status(self, test_name, phase=None):
###########################################################################
curr_phase = self._get_test_phase(test_name)
if (phase == NAMELIST_PHASE and test_name in self._tests_with_nl_problems):
return NAMELIST_FAIL_STATUS
elif (phase is None or phase == curr_phase):
return self._get_test_data(test_name)[1]
else:
expect(phase is None or self._phases.index(phase) < self._phases.index(curr_phase),
"Tried to see the future")
# Assume all older phases PASSed
return TEST_PASS_STATUS
def _producer(self):
###########################################################################
threads_in_flight = {} # test-name -> (thread, procs)
while (True):
work_to_do = False
num_threads_launched_this_iteration = 0
for test_name in self._test_names:
# If we have no workers available, immediately wait
if (len(threads_in_flight) == self._parallel_jobs):
self._wait_for_something_to_finish(threads_in_flight)
if (self._work_remains(test_name)):
work_to_do = True
if (test_name not in threads_in_flight):
test_phase, test_status = self._get_test_data(
test_name)
expect(test_status != TEST_PENDING_STATUS, test_name)
next_phase = self._phases[
self._phases.index(test_phase) + 1]
procs_needed = self._get_procs_needed(
test_name, next_phase)
if (procs_needed <= self._proc_pool):
self._proc_pool -= procs_needed
# Necessary to print this way when multiple threads printing
sys.stdout.write(
"Starting %s for test %s with %d procs\n" %
(next_phase, test_name, procs_needed))
self._update_test_status(test_name, next_phase,
TEST_PENDING_STATUS)
t = threading.Thread(
target=self._consumer,
args=(test_name, next_phase,
getattr(self, "_%s_phase" %
next_phase.lower())))
threads_in_flight[test_name] = (t, procs_needed)
t.start()
num_threads_launched_this_iteration += 1
if (not work_to_do):
break
if (num_threads_launched_this_iteration == 0):
# No free resources, wait for something in flight to finish
self._wait_for_something_to_finish(threads_in_flight)
for thread_info in threads_in_flight.values():
thread_info[0].join()
def compare_values(namelist, name, gold_value, comp_value, case):
###############################################################################
"""
Compare values for a specific variable in a namelist.
"""
if (type(gold_value) != type(comp_value)):
print "In namelist '%s', variable '%s' did not have expected type '%s', instead is type '%s'" % \
(namelist, name, type(gold_value), type(comp_value))
return False
rv = True
if (type(gold_value) is list):
# Note, list values remain order sensitive
for idx, gold_value_list_item in enumerate(gold_value):
if (idx < len(comp_value)):
rv &= compare_values(namelist, "%s list item %d" % (name, idx), gold_value_list_item, comp_value[idx], case)
else:
rv = False
print "In namelist '%s', list variable '%s' missing value %s" % (namelist, name, gold_value_list_item)
if (len(comp_value) > len(gold_value)):
for comp_value_list_item in comp_value[len(gold_value):]:
rv = False
print "In namelist '%s', list variable '%s' has extra value %s" % (namelist, name, comp_value_list_item)
elif (type(gold_value) is dict):
for key, gold_value_dict_item in gold_value.iteritems():
if (key in comp_value):
rv &= compare_values(namelist, "%s dict item %s" % (name, key), gold_value_dict_item, comp_value[key], case)
else:
rv = False
print "In namelist '%s', dict variable '%s' missing key %s" % (namelist, name, key)
for key in comp_value:
if (key not in gold_value):
rv = False
print "In namelist '%s', dict variable '%s' has extra key %s" % (namelist, name, key)
else:
expect(type(gold_value) is str, "Unexpected type found: '%s'" % type(gold_value))
norm_gold_value = normalize_string_value(name, gold_value, case)
norm_comp_value = normalize_string_value(name, comp_value, case)
if (norm_gold_value != norm_comp_value):
rv = False
print "In namelist '%s', '%s' has inequivalent values %s != %s" % (namelist, name, gold_value, comp_value)
print " NORMALIZED: %s != %s" % (norm_gold_value, norm_comp_value)
return rv
def _wait_for_something_to_finish(self, threads_in_flight):
###########################################################################
expect(len(threads_in_flight) <= self._parallel_jobs, "Oversubscribed?")
finished_tests = []
while (not finished_tests):
for test_name, thread_info in threads_in_flight.iteritems():
if (not thread_info[0].is_alive()):
finished_tests.append( (test_name, thread_info[1]) )
if (not finished_tests):
time.sleep(0.2)
for finished_test, procs_needed in finished_tests:
self._proc_pool += procs_needed
del threads_in_flight[finished_test]
def _wait_for_something_to_finish(self, threads_in_flight):
###########################################################################
expect(len(threads_in_flight) <= self._parallel_jobs, "Oversubscribed?")
finished_tests = []
while (not finished_tests):
for test_name, thread_info in threads_in_flight.iteritems():
if (not thread_info[0].is_alive()):
finished_tests.append( (test_name, thread_info[1]) )
if (not finished_tests):
time.sleep(0.2)
for finished_test, procs_needed in finished_tests:
self._proc_pool += procs_needed
del threads_in_flight[finished_test]
def wait_for_tests_impl(test_paths,
no_wait=False,
check_throughput=False,
check_memory=False,
ignore_namelists=False):
###############################################################################
results = Queue.Queue()
for test_path in test_paths:
t = threading.Thread(target=wait_for_test,
args=(test_path, results, not no_wait,
check_throughput, check_memory,
ignore_namelists))
t.daemon = True
t.start()
while threading.active_count() > 1:
time.sleep(1)
test_results = {}
completed_test_paths = []
while (not results.empty()):
test_name, test_path, test_status = results.get()
if (test_name in test_results):
prior_path, prior_status = test_results[test_name]
if (test_status == prior_status):
warning("Test name '%s' was found in both '%s' and '%s'" %
(test_name, test_path, prior_path))
else:
raise SystemExit(
"Test name '%s' was found in both '%s' and '%s' with different results"
% (test_name, test_path, prior_path))
test_results[test_name] = (test_path, test_status)
completed_test_paths.append(test_path)
expect(
set(test_paths) == set(completed_test_paths),
"Missing results for test paths: %s" %
(set(test_paths) - set(completed_test_paths)))
return test_results
def _producer(self):
###########################################################################
threads_in_flight = {} # test-name -> (thread, procs)
while (True):
work_to_do = False
num_threads_launched_this_iteration = 0
for test_name in self._test_names:
# If we have no workers available, immediately wait
if (len(threads_in_flight) == self._parallel_jobs):
self._wait_for_something_to_finish(threads_in_flight)
if (self._work_remains(test_name)):
work_to_do = True
if (test_name not in threads_in_flight):
test_phase, test_status = self._get_test_data(test_name)
expect(test_status != TEST_PENDING_STATUS, test_name)
next_phase = self._phases[self._phases.index(test_phase) + 1]
procs_needed = self._get_procs_needed(test_name, next_phase)
if (procs_needed <= self._proc_pool):
self._proc_pool -= procs_needed
# Necessary to print this way when multiple threads printing
sys.stdout.write("Starting %s for test %s with %d procs\n" % (next_phase, test_name, procs_needed))
self._update_test_status(test_name, next_phase, TEST_PENDING_STATUS)
t = threading.Thread(target=self._consumer,
args=(test_name, next_phase, getattr(self, "_%s_phase" % next_phase.lower()) ))
threads_in_flight[test_name] = (t, procs_needed)
t.start()
num_threads_launched_this_iteration += 1
if (not work_to_do):
break
if (num_threads_launched_this_iteration == 0):
# No free resources, wait for something in flight to finish
self._wait_for_something_to_finish(threads_in_flight)
for thread_info in threads_in_flight.values():
thread_info[0].join()
def _update_test_status(self, test_name, phase, status):
###########################################################################
state_idx = self._test_names.index(test_name)
phase_idx = self._phases.index(phase)
old_phase, old_status = self._test_states[state_idx]
if (old_phase == phase):
expect(old_status == TEST_PENDING_STATUS,
"Only valid to transition from PENDING to something else, found '%s'" % old_status)
expect(status != TEST_PENDING_STATUS,
"Cannot transition from PEND -> PEND")
else:
expect(old_status in CONTINUE,
"Why did we move on to next phase when prior phase did not pass?")
expect(status == TEST_PENDING_STATUS,
"New phase should be set to pending status")
expect(self._phases.index(old_phase) == phase_idx - 1,
"Skipped phase?")
self._test_states[state_idx] = (phase, status)
def update_acme_tests(xml_file, categories, platform=None):
###############################################################################
# Retrieve all supported ACME platforms, killing the third entry (MPI lib)
# for the moment.
supported_platforms = [p[:2] for p in find_all_supported_platforms()]
# Fish all of the existing machine/compiler combos out of the XML file.
if (platform is not None):
platforms = [tuple(platform.split(","))]
else:
platforms = find_all_platforms(xml_file)
# Prune the non-supported platforms from our list.
for p in platforms:
if p not in supported_platforms:
acme_util.verbose_print("pruning unsupported platform %s"%repr(p))
platforms = [p for p in platforms if p in supported_platforms]
manage_xml_entries = os.path.join(acme_util.get_cime_root(), "scripts", "manage_testlists")
expect(os.path.isfile(manage_xml_entries),
"Couldn't find manage_testlists, expected it to be here: '%s'" % manage_xml_entries)
for category in categories:
# Remove any existing acme test category from the file.
if (platform is None):
acme_util.run_cmd("%s -component allactive -removetests -category %s" % (manage_xml_entries, category))
else:
acme_util.run_cmd("%s -component allactive -removetests -category %s -machine %s -compiler %s"
% (manage_xml_entries, category, platforms[0][0], platforms[0][1]))
# Generate a list of test entries corresponding to our suite at the top
# of the file.
new_test_file = generate_acme_test_entries(category, platforms)
acme_util.run_cmd("%s -component allactive -addlist -file %s -category %s" %
(manage_xml_entries, new_test_file, category))
os.unlink(new_test_file)
print "SUCCESS"
def update_acme_tests(xml_file, categories, platform=None):
###############################################################################
# Retrieve all supported ACME platforms, killing the third entry (MPI lib)
# for the moment.
supported_platforms = [p[:2] for p in find_all_supported_platforms()]
# Fish all of the existing machine/compiler combos out of the XML file.
if (platform is not None):
platforms = [tuple(platform.split(","))]
else:
platforms = find_all_platforms(xml_file)
# Prune the non-supported platforms from our list.
for p in platforms:
if p not in supported_platforms:
acme_util.verbose_print("pruning unsupported platform %s"%repr(p))
platforms = [p for p in platforms if p in supported_platforms]
manage_xml_entries = os.path.join(acme_util.get_cime_root(), "scripts", "manage_testlists")
expect(os.path.isfile(manage_xml_entries),
"Couldn't find manage_testlists, expected it to be here: '%s'" % manage_xml_entries)
for category in categories:
# Remove any existing acme test category from the file.
if (platform is None):
acme_util.run_cmd("%s -component allactive -removetests -category %s" % (manage_xml_entries, category))
else:
acme_util.run_cmd("%s -component allactive -removetests -category %s -machine %s -compiler %s"
% (manage_xml_entries, category, platforms[0][0], platforms[0][1]))
# Generate a list of test entries corresponding to our suite at the top
# of the file.
new_test_file = generate_acme_test_entries(category, platforms)
acme_util.run_cmd("%s -component allactive -addlist -file %s -category %s" %
(manage_xml_entries, new_test_file, category))
os.unlink(new_test_file)
print "SUCCESS"
def get_test_suite(suite, machine=None, compiler=None):
###############################################################################
"""
Return a list of FULL test names for a suite.
"""
expect(suite in _TEST_SUITES, "Unknown test suite: '%s'" % suite)
machine = acme_util.probe_machine_name() if machine is None else machine
compiler = acme_util.get_machine_info("COMPILERS", machine=machine)[0] if compiler is None else compiler
inherits_from, tests_raw = _TEST_SUITES[suite]
tests = []
for item in tests_raw:
test_mod = None
if (isinstance(item, str)):
test_name = item
else:
expect(isinstance(item, tuple), "Bad item type for item '%s'" % str(item))
expect(len(item) in [2, 3], "Expected two or three items in item '%s'" % str(item))
expect(isinstance(item[0], str), "Expected string in first field of item '%s'" % str(item))
expect(isinstance(item[1], str), "Expected string in second field of item '%s'" % str(item))
test_name = item[0]
if (len(item) == 2):
test_mod = item[1]
else:
expect(type(item[2]) in [str, tuple], "Expected string or tuple for third field of item '%s'" % str(item))
test_mod_machines = [item[2]] if isinstance(item[2], str) else item[2]
if (machine in test_mod_machines):
test_mod = item[1]
tests.append(acme_util.get_full_test_name(test_name, machine, compiler, testmod=test_mod))
if (inherits_from is not None):
inherited_tests = get_test_suite(inherits_from, machine, compiler)
expect(len(set(tests) & set(inherited_tests)) == 0,
"Tests %s defined in multiple suites" % ", ".join(set(tests) & set(inherited_tests)))
tests.extend(inherited_tests)
return tests
def get_test_suite(suite, machine=None, compiler=None):
###############################################################################
"""
Return a list of FULL test names for a suite.
"""
expect(suite in _TEST_SUITES, "Unknown test suite: '%s'" % suite)
machine = acme_util.probe_machine_name() if machine is None else machine
compiler = acme_util.get_machine_info("COMPILERS", machine=machine)[0] if compiler is None else compiler
inherits_from, tests_raw = _TEST_SUITES[suite]
tests = []
for item in tests_raw:
test_mod = None
if (isinstance(item, str)):
test_name = item
else:
expect(isinstance(item, tuple), "Bad item type for item '%s'" % str(item))
expect(len(item) in [2, 3], "Expected two or three items in item '%s'" % str(item))
expect(isinstance(item[0], str), "Expected string in first field of item '%s'" % str(item))
expect(isinstance(item[1], str), "Expected string in second field of item '%s'" % str(item))
test_name = item[0]
if (len(item) == 2):
test_mod = item[1]
else:
expect(type(item[2]) in [str, tuple], "Expected string or tuple for third field of item '%s'" % str(item))
test_mod_machines = [item[2]] if isinstance(item[2], str) else item[2]
if (machine in test_mod_machines):
test_mod = item[1]
tests.append(acme_util.get_full_test_name(test_name, machine, compiler, testmod=test_mod))
if (inherits_from is not None):
inherited_tests = get_test_suite(inherits_from, machine, compiler)
expect(len(set(tests) & set(inherited_tests)) == 0,
"Tests %s defined in multiple suites" % ", ".join(set(tests) & set(inherited_tests)))
tests.extend(inherited_tests)
return tests
elif (cidx == cnum):
print "Missing lines"
print "\n".join(gold_lines[gidx:1])
return False
gold_value = gold_lines[gidx].strip()
comp_value = comp_lines[cidx].strip()
norm_gold_value = normalize_string_value(gold_value, case)
norm_comp_value = normalize_string_value(comp_value, case)
if (norm_gold_value != norm_comp_value):
rv = False
print "Inequivalent lines %s != %s" % (gold_value, comp_value)
print " NORMALIZED: %s != %s" % (norm_gold_value, norm_comp_value)
gidx += 1
cidx += 1
return rv
###############################################################################
def compare_files(gold_file, compare_file, case=None):
###############################################################################
expect(os.path.exists(gold_file), "File not found: %s" % gold_file)
expect(os.path.exists(compare_file), "File not found: %s" % compare_file)
return compare_data(open(gold_file, "r").readlines(),
open(compare_file, "r").readlines(),
case)
def __init__(self, test_names,
no_run=False, no_build=False, no_batch=None,
test_root=None, test_id=None,
compiler=None,
baseline_root=None, baseline_name=None,
clean=False,
compare=False, generate=False, namelists_only=False,
project=None, parallel_jobs=None):
###########################################################################
self._cime_root = acme_util.get_cime_root()
self._test_names = test_names
self._no_build = no_build if not namelists_only else True
self._no_run = no_run if not self._no_build else True
self._no_batch = no_batch if no_batch is not None else not acme_util.does_machine_have_batch()
self._test_root = test_root if test_root is not None else acme_util.get_machine_info("CESMSCRATCHROOT")
self._test_id = test_id if test_id is not None else acme_util.get_utc_timestamp()
self._project = project if project is not None else acme_util.get_machine_project()
self._baseline_root = baseline_root if baseline_root is not None else acme_util.get_machine_info("CCSM_BASELINE", project=self._project)
self._baseline_name = None
self._compiler = compiler if compiler is not None else acme_util.get_machine_info("COMPILERS")[0]
self._clean = clean
self._compare = compare
self._generate = generate
self._namelists_only = namelists_only
self._parallel_jobs = parallel_jobs if parallel_jobs is not None else min(len(self._test_names), int(acme_util.get_machine_info("MAX_TASKS_PER_NODE")))
# Oversubscribe by 1/4
pes = int(acme_util.get_machine_info("MAX_TASKS_PER_NODE"))
# This is the only data that multiple threads will simultaneously access
# Each test has it's own index and setting/retrieving items from a list
# is atomic, so this should be fine to use without mutex
self._test_states = [ (INITIAL_PHASE, TEST_PASS_STATUS) ] * len(test_names)
self._proc_pool = int(pes * 1.25)
# Since the name-list phase can fail without aborting later phases, we
# need some extra state to remember tests that had namelist problems
self._tests_with_nl_problems = [None] * len(test_names)
# Setup phases
self._phases = list(PHASES)
if (no_build):
self._phases.remove(BUILD_PHASE)
if (no_run):
self._phases.remove(RUN_PHASE)
if (not self._compare and not self._generate):
self._phases.remove(NAMELIST_PHASE)
else:
if (baseline_name is None):
branch_name = acme_util.get_current_branch(repo=self._cime_root)
expect(branch_name is not None, "Could not determine baseline name from branch, please use -b option")
self._baseline_name = os.path.join(self._compiler, branch_name)
else:
self._baseline_name = baseline_name
if (not self._baseline_name.startswith("%s/" % self._compiler)):
self._baseline_name = os.path.join(self._compiler, self._baseline_name)
# Validate any assumptions that were not caught by the arg parser
# None of the test directories should already exist.
for test in self._test_names:
expect(not os.path.exists(self._get_test_dir(test)),
"Cannot create new case in directory '%s', it already exists. Pick a different test-id" % self._get_test_dir(test))
def parse_namelists(namelist_lines, filename):
###############################################################################
"""
Return data in form: {namelist -> {key -> value} }.
value can be an int, string, list, or dict
>>> teststr = '''&nml
... val = 'foo'
... aval = 'one','two', 'three'
... maval = 'one', 'two',
... 'three', 'four'
... dval = 'one->two', 'three -> four'
... mdval = 'one -> two',
... 'three -> four',
... 'five -> six'
... nval = 1850
... /
...
... # Hello
...
... &nml2
... val2 = .false.
... /
... '''
>>> parse_namelists(teststr.splitlines(), 'foo')
{'nml': {'dval': {'three': 'four', 'one': 'two'}, 'val': "'foo'", 'maval': ["'one'", "'two'", "'three'", "'four'"], 'aval': ["'one'", "'two'", "'three'"], 'nval': '1850', 'mdval': {'five': 'six', 'three': 'four', 'one': 'two'}}, 'nml2': {'val2': '.false.'}}
>>> parse_namelists('blah', 'foo')
Traceback (most recent call last):
...
SystemExit: FAIL: File 'foo' does not appear to be a namelist file, skipping
>>> teststr = '''&nml
... val = 'one', 'two',
... val2 = 'three'
... /'''
>>> parse_namelists(teststr.splitlines(), 'foo')
Traceback (most recent call last):
...
SystemExit: FAIL: In file 'foo', Incomplete multiline variable: 'val'
>>> teststr = '''&nml
... val = 'one', 'two',
... /'''
>>> parse_namelists(teststr.splitlines(), 'foo')
Traceback (most recent call last):
...
SystemExit: FAIL: In file 'foo', Incomplete multiline variable: 'val'
>>> teststr = '''&nml
... val = 'one', 'two',
... 'three -> four'
... /'''
>>> parse_namelists(teststr.splitlines(), 'foo')
Traceback (most recent call last):
...
SystemExit: FAIL: In file 'foo', multiline list variable 'val' had dict entries
"""
comment_re = re.compile(r'^[#!]')
namelist_re = re.compile(r'^&(\S+)$')
name_re = re.compile(r"^([^\s=']+)\s*=\s*(.+)$")
dict_re = re.compile(r"^'(\S+)\s*->\s*(\S+)'")
comma_re = re.compile(r'\s*,\s*')
rv = {}
current_namelist = None
multiline_variable = None # (name, value)
for line in namelist_lines:
line = line.strip()
verbose_print("Parsing line: '%s'" % line)
if (line == "" or comment_re.match(line)):
verbose_print(" Line was whitespace or comment, skipping.")
continue
if (current_namelist is None):
# Must start a namelist
expect(
multiline_variable is None,
"In file '%s', Incomplete multiline variable: '%s'" %
(filename, multiline_variable[0]
if multiline_variable is not None else ""))
# Unfornately, other tools were using the old compare_namelists.pl script
# to compare files that are not namelist files. We need a special error
# to signify this event
if (namelist_re.match(line) is None):
expect(
rv != {},
"File '%s' does not appear to be a namelist file, skipping"
% filename)
expect(
False,
"In file '%s', Line '%s' did not begin a namelist as expected"
% (filename, line))
current_namelist = namelist_re.match(line).groups()[0]
expect(
current_namelist not in rv,
"In file '%s', Duplicate namelist '%s'" %
(filename, current_namelist))
rv[current_namelist] = {}
verbose_print(" Starting namelist '%s'" % current_namelist)
elif (line == "/"):
# Ends a namelist
verbose_print(" Ending namelist '%s'" % current_namelist)
expect(
multiline_variable is None,
"In file '%s', Incomplete multiline variable: '%s'" %
(filename, multiline_variable[0]
if multiline_variable is not None else ""))
current_namelist = None
elif (name_re.match(line)):
# Defining a variable (AKA name)
name, value = name_re.match(line).groups()
verbose_print(" Parsing variable '%s' with data '%s'" %
(name, value))
expect(
multiline_variable is None,
"In file '%s', Incomplete multiline variable: '%s'" %
(filename, multiline_variable[0]
if multiline_variable is not None else ""))
expect(name not in rv[current_namelist],
"In file '%s', Duplicate name: '%s'" % (filename, name))
tokens = [
item.strip() for item in comma_re.split(value)
if item.strip() != ""
]
if ("->" in value):
# dict
rv[current_namelist][name] = {}
for token in tokens:
m = dict_re.match(token)
expect(
m is not None,
"In file '%s', Dict entry '%s' does not match expected format"
% (filename, token))
k, v = m.groups()
rv[current_namelist][name][k] = v
verbose_print(" Adding dict entry '%s' -> '%s'" %
(k, v))
elif ("," in value):
# list
rv[current_namelist][name] = tokens
verbose_print(" Adding list entries: %s" %
", ".join(tokens))
else:
rv[current_namelist][name] = value
verbose_print(" Setting to value '%s'" % value)
if (line.endswith(",")):
# Value will continue on in subsequent lines
multiline_variable = (name, rv[current_namelist][name])
verbose_print(" Var is multiline...")
elif (multiline_variable is not None):
# Continuation of list or dict variable
current_value = multiline_variable[1]
verbose_print(
" Continuing multiline variable '%s' with data '%s'" %
(multiline_variable[0], line))
tokens = [
item.strip() for item in comma_re.split(line)
if item.strip() != ""
]
if (type(current_value) is list):
expect(
"->" not in line,
"In file '%s', multiline list variable '%s' had dict entries"
% (filename, multiline_variable[0]))
current_value.extend(tokens)
verbose_print(" Adding list entries: %s" %
", ".join(tokens))
elif (type(current_value) is dict):
for token in tokens:
m = dict_re.match(token)
expect(
m is not None,
"In file '%s', Dict entry '%s' does not match expected format"
% (filename, token))
k, v = m.groups()
current_value[k] = v
verbose_print(" Adding dict entry '%s' -> '%s'" %
(k, v))
else:
expect(
False,
"In file '%s', Continuation should have been for list or dict, instead it was: '%s'"
% (filename, type(current_value)))
if (not line.endswith(",")):
# Completed
multiline_variable = None
verbose_print(" Terminating multiline variable")
else:
expect(False,
"In file '%s', Unrecognized line: '%s'" % (filename, line))
return rv
def parse_namelists(namelist_lines, filename):
###############################################################################
"""
Return data in form: {namelist -> {key -> value} }.
value can be an int, string, list, or dict
>>> teststr = '''&nml
... val = 'foo'
... aval = 'one','two', 'three'
... maval = 'one', 'two',
... 'three', 'four'
... dval = 'one->two', 'three -> four'
... mdval = 'one -> two',
... 'three -> four',
... 'five -> six'
... nval = 1850
... /
...
... # Hello
...
... &nml2
... val2 = .false.
... /
... '''
>>> parse_namelists(teststr.splitlines(), 'foo')
{'nml': {'dval': {'three': 'four', 'one': 'two'}, 'val': "'foo'", 'maval': ["'one'", "'two'", "'three'", "'four'"], 'aval': ["'one'", "'two'", "'three'"], 'nval': '1850', 'mdval': {'five': 'six', 'three': 'four', 'one': 'two'}}, 'nml2': {'val2': '.false.'}}
>>> parse_namelists('blah', 'foo')
Traceback (most recent call last):
...
SystemExit: FAIL: File 'foo' does not appear to be a namelist file, skipping
>>> teststr = '''&nml
... val = 'one', 'two',
... val2 = 'three'
... /'''
>>> parse_namelists(teststr.splitlines(), 'foo')
Traceback (most recent call last):
...
SystemExit: FAIL: In file 'foo', Incomplete multiline variable: 'val'
>>> teststr = '''&nml
... val = 'one', 'two',
... /'''
>>> parse_namelists(teststr.splitlines(), 'foo')
Traceback (most recent call last):
...
SystemExit: FAIL: In file 'foo', Incomplete multiline variable: 'val'
>>> teststr = '''&nml
... val = 'one', 'two',
... 'three -> four'
... /'''
>>> parse_namelists(teststr.splitlines(), 'foo')
Traceback (most recent call last):
...
SystemExit: FAIL: In file 'foo', multiline list variable 'val' had dict entries
"""
comment_re = re.compile(r'^[#!]')
namelist_re = re.compile(r'^&(\S+)$')
name_re = re.compile(r"^([^\s=']+)\s*=\s*(.+)$")
dict_re = re.compile(r"^'(\S+)\s*->\s*(\S+)'")
comma_re = re.compile(r'\s*,\s*')
rv = {}
current_namelist = None
multiline_variable = None # (name, value)
for line in namelist_lines:
line = line.strip()
verbose_print("Parsing line: '%s'" % line)
if (line == "" or comment_re.match(line)):
verbose_print(" Line was whitespace or comment, skipping.")
continue
if (current_namelist is None):
# Must start a namelist
expect(multiline_variable is None,
"In file '%s', Incomplete multiline variable: '%s'" % (filename, multiline_variable[0] if multiline_variable is not None else ""))
# Unfornately, other tools were using the old compare_namelists.pl script
# to compare files that are not namelist files. We need a special error
# to signify this event
if (namelist_re.match(line) is None):
expect(rv != {},
"File '%s' does not appear to be a namelist file, skipping" % filename)
expect(False,
"In file '%s', Line '%s' did not begin a namelist as expected" % (filename, line))
current_namelist = namelist_re.match(line).groups()[0]
expect(current_namelist not in rv,
"In file '%s', Duplicate namelist '%s'" % (filename, current_namelist))
rv[current_namelist] = {}
verbose_print(" Starting namelist '%s'" % current_namelist)
elif (line == "/"):
# Ends a namelist
verbose_print(" Ending namelist '%s'" % current_namelist)
expect(multiline_variable is None,
"In file '%s', Incomplete multiline variable: '%s'" % (filename, multiline_variable[0] if multiline_variable is not None else ""))
current_namelist = None
elif (name_re.match(line)):
# Defining a variable (AKA name)
name, value = name_re.match(line).groups()
verbose_print(" Parsing variable '%s' with data '%s'" % (name, value))
expect(multiline_variable is None,
"In file '%s', Incomplete multiline variable: '%s'" % (filename, multiline_variable[0] if multiline_variable is not None else ""))
expect(name not in rv[current_namelist], "In file '%s', Duplicate name: '%s'" % (filename, name))
tokens = [item.strip() for item in comma_re.split(value) if item.strip() != ""]
if ("->" in value):
# dict
rv[current_namelist][name] = {}
for token in tokens:
m = dict_re.match(token)
expect(m is not None, "In file '%s', Dict entry '%s' does not match expected format" % (filename, token))
k, v = m.groups()
rv[current_namelist][name][k] = v
verbose_print(" Adding dict entry '%s' -> '%s'" % (k, v))
elif ("," in value):
# list
rv[current_namelist][name] = tokens
verbose_print(" Adding list entries: %s" % ", ".join(tokens))
else:
rv[current_namelist][name] = value
verbose_print(" Setting to value '%s'" % value)
if (line.endswith(",")):
# Value will continue on in subsequent lines
multiline_variable = (name, rv[current_namelist][name])
verbose_print(" Var is multiline...")
elif (multiline_variable is not None):
# Continuation of list or dict variable
current_value = multiline_variable[1]
verbose_print(" Continuing multiline variable '%s' with data '%s'" % (multiline_variable[0], line))
tokens = [item.strip() for item in comma_re.split(line) if item.strip() != ""]
if (type(current_value) is list):
expect("->" not in line, "In file '%s', multiline list variable '%s' had dict entries" % (filename, multiline_variable[0]))
current_value.extend(tokens)
verbose_print(" Adding list entries: %s" % ", ".join(tokens))
elif (type(current_value) is dict):
for token in tokens:
m = dict_re.match(token)
expect(m is not None, "In file '%s', Dict entry '%s' does not match expected format" % (filename, token))
k, v = m.groups()
current_value[k] = v
verbose_print(" Adding dict entry '%s' -> '%s'" % (k, v))
else:
expect(False, "In file '%s', Continuation should have been for list or dict, instead it was: '%s'" % (filename, type(current_value)))
if (not line.endswith(",")):
# Completed
multiline_variable = None
verbose_print(" Terminating multiline variable")
else:
expect(False, "In file '%s', Unrecognized line: '%s'" % (filename, line))
return rv
def __init__(self,
test_names,
no_run=False,
no_build=False,
no_batch=None,
test_root=None,
test_id=None,
compiler=None,
baseline_root=None,
baseline_name=None,
clean=False,
compare=False,
generate=False,
namelists_only=False,
project=None,
parallel_jobs=None):
###########################################################################
self._cime_root = acme_util.get_cime_root()
self._test_names = test_names
self._no_build = no_build if not namelists_only else True
self._no_run = no_run if not self._no_build else True
self._no_batch = no_batch if no_batch is not None else not acme_util.does_machine_have_batch(
)
self._test_root = test_root if test_root is not None else acme_util.get_machine_info(
"CESMSCRATCHROOT")
self._test_id = test_id if test_id is not None else acme_util.get_utc_timestamp(
)
self._project = project if project is not None else acme_util.get_machine_project(
)
self._baseline_root = baseline_root if baseline_root is not None else acme_util.get_machine_info(
"CCSM_BASELINE", project=self._project)
self._baseline_name = None
self._compiler = compiler if compiler is not None else acme_util.get_machine_info(
"COMPILERS")[0]
self._clean = clean
self._compare = compare
self._generate = generate
self._namelists_only = namelists_only
self._parallel_jobs = parallel_jobs if parallel_jobs is not None else min(
len(self._test_names),
int(acme_util.get_machine_info("MAX_TASKS_PER_NODE")))
# Oversubscribe by 1/4
pes = int(acme_util.get_machine_info("MAX_TASKS_PER_NODE"))
# This is the only data that multiple threads will simultaneously access
# Each test has it's own index and setting/retrieving items from a list
# is atomic, so this should be fine to use without mutex
self._test_states = [(INITIAL_PHASE, TEST_PASS_STATUS)
] * len(test_names)
self._proc_pool = int(pes * 1.25)
# Since the name-list phase can fail without aborting later phases, we
# need some extra state to remember tests that had namelist problems
self._tests_with_nl_problems = [None] * len(test_names)
# Setup phases
self._phases = list(PHASES)
if (no_build):
self._phases.remove(BUILD_PHASE)
if (no_run):
self._phases.remove(RUN_PHASE)
if (not self._compare and not self._generate):
self._phases.remove(NAMELIST_PHASE)
else:
if (baseline_name is None):
branch_name = acme_util.get_current_branch(
repo=self._cime_root)
expect(
branch_name is not None,
"Could not determine baseline name from branch, please use -b option"
)
self._baseline_name = os.path.join(self._compiler, branch_name)
else:
self._baseline_name = baseline_name
if (not self._baseline_name.startswith(
"%s/" % self._compiler)):
self._baseline_name = os.path.join(self._compiler,
self._baseline_name)
# Validate any assumptions that were not caught by the arg parser
# None of the test directories should already exist.
for test in self._test_names:
expect(
not os.path.exists(self._get_test_dir(test)),
"Cannot create new case in directory '%s', it already exists. Pick a different test-id"
% self._get_test_dir(test))
