def wait_for_tests(test_paths,
no_wait=False,
check_throughput=False,
check_memory=False,
ignore_namelists=False,
cdash_build_name=None,
cdash_project=ACME_MAIN_CDASH,
cdash_build_group=CDASH_DEFAULT_BUILD_GROUP,
start_time=None):
###############################################################################
# Set up signal handling, we want to print results before the program
# is terminated
set_up_signal_handlers()
test_results = wait_for_tests_impl(test_paths, no_wait, check_throughput, check_memory, ignore_namelists)
all_pass = True
for test_name, test_data in sorted(test_results.iteritems()):
test_path, test_status = test_data
print "Test '%s' finished with status '%s'" % (test_name, test_status)
verbose_print(" Path: %s" % test_path)
all_pass &= test_status == TEST_PASS_STATUS
if (cdash_build_name):
create_cdash_xml(test_results, cdash_build_name, cdash_project, cdash_build_group, start_time)
return all_pass
def wait_for_tests(test_paths,
no_wait=False,
check_throughput=False,
check_memory=False,
ignore_namelists=False,
cdash_build_name=None,
cdash_project=ACME_MAIN_CDASH,
cdash_build_group=CDASH_DEFAULT_BUILD_GROUP,
start_time=None):
###############################################################################
# Set up signal handling, we want to print results before the program
# is terminated
set_up_signal_handlers()
test_results = wait_for_tests_impl(test_paths, no_wait, check_throughput,
check_memory, ignore_namelists)
all_pass = True
for test_name, test_data in sorted(test_results.iteritems()):
test_path, test_status = test_data
print "Test '%s' finished with status '%s'" % (test_name, test_status)
verbose_print(" Path: %s" % test_path)
all_pass &= test_status == TEST_PASS_STATUS
if (cdash_build_name):
create_cdash_xml(test_results, cdash_build_name, cdash_project,
cdash_build_group, start_time)
return all_pass
def wait_for_test(test_path, results, wait, check_throughput, check_memory, ignore_namelists):
###############################################################################
if (os.path.isdir(test_path)):
test_status_filepath = os.path.join(test_path, TEST_STATUS_FILENAME)
else:
test_status_filepath = test_path
verbose_print("Watching file: '%s'" % test_status_filepath)
while (True):
if (os.path.exists(test_status_filepath)):
test_name, test_status = interpret_status_file(test_status_filepath, check_throughput, check_memory, ignore_namelists)
if (test_status == TEST_PENDING_STATUS and (wait and not SIGNAL_RECEIVED)):
time.sleep(SLEEP_INTERVAL_SEC)
verbose_print("Waiting for test to finish")
else:
results.put( (test_name, test_path, test_status) )
break
else:
if (wait and not SIGNAL_RECEIVED):
verbose_print("File '%s' does not yet exist" % test_status_filepath)
time.sleep(SLEEP_INTERVAL_SEC)
else:
test_name = os.path.abspath(test_status_filepath).split("/")[-2]
results.put( (test_name, test_path, "File '%s' doesn't exist" % test_status_filepath) )
break
def wait_for_test(test_path, results, wait, check_throughput, check_memory,
ignore_namelists):
###############################################################################
if (os.path.isdir(test_path)):
test_status_filepath = os.path.join(test_path, TEST_STATUS_FILENAME)
else:
test_status_filepath = test_path
verbose_print("Watching file: '%s'" % test_status_filepath)
while (True):
if (os.path.exists(test_status_filepath)):
test_name, test_status = interpret_status_file(
test_status_filepath, check_throughput, check_memory,
ignore_namelists)
if (test_status == TEST_PENDING_STATUS
and (wait and not SIGNAL_RECEIVED)):
time.sleep(SLEEP_INTERVAL_SEC)
verbose_print("Waiting for test to finish")
else:
results.put((test_name, test_path, test_status))
break
else:
if (wait and not SIGNAL_RECEIVED):
verbose_print("File '%s' does not yet exist" %
test_status_filepath)
time.sleep(SLEEP_INTERVAL_SEC)
else:
test_name = os.path.abspath(test_status_filepath).split(
"/")[-2]
results.put((test_name, test_path,
"File '%s' doesn't exist" % test_status_filepath))
break
def update_acme_tests(xml_file, categories, platform=None):
###############################################################################
# Retrieve all supported ACME platforms, killing the third entry (MPI lib)
# for the moment.
supported_platforms = [p[:2] for p in find_all_supported_platforms()]
# Fish all of the existing machine/compiler combos out of the XML file.
if (platform is not None):
platforms = [tuple(platform.split(","))]
else:
platforms = find_all_platforms(xml_file)
# Prune the non-supported platforms from our list.
for p in platforms:
if p not in supported_platforms:
acme_util.verbose_print("pruning unsupported platform %s"%repr(p))
platforms = [p for p in platforms if p in supported_platforms]
manage_xml_entries = os.path.join(acme_util.get_cime_root(), "scripts", "manage_testlists")
expect(os.path.isfile(manage_xml_entries),
"Couldn't find manage_testlists, expected it to be here: '%s'" % manage_xml_entries)
for category in categories:
# Remove any existing acme test category from the file.
if (platform is None):
acme_util.run_cmd("%s -component allactive -removetests -category %s" % (manage_xml_entries, category))
else:
acme_util.run_cmd("%s -component allactive -removetests -category %s -machine %s -compiler %s"
% (manage_xml_entries, category, platforms[0][0], platforms[0][1]))
# Generate a list of test entries corresponding to our suite at the top
# of the file.
new_test_file = generate_acme_test_entries(category, platforms)
acme_util.run_cmd("%s -component allactive -addlist -file %s -category %s" %
(manage_xml_entries, new_test_file, category))
os.unlink(new_test_file)
print "SUCCESS"
def update_acme_tests(xml_file, categories, platform=None):
###############################################################################
# Retrieve all supported ACME platforms, killing the third entry (MPI lib)
# for the moment.
supported_platforms = [p[:2] for p in find_all_supported_platforms()]
# Fish all of the existing machine/compiler combos out of the XML file.
if (platform is not None):
platforms = [tuple(platform.split(","))]
else:
platforms = find_all_platforms(xml_file)
# Prune the non-supported platforms from our list.
for p in platforms:
if p not in supported_platforms:
acme_util.verbose_print("pruning unsupported platform %s"%repr(p))
platforms = [p for p in platforms if p in supported_platforms]
manage_xml_entries = os.path.join(acme_util.get_cime_root(), "scripts", "manage_testlists")
expect(os.path.isfile(manage_xml_entries),
"Couldn't find manage_testlists, expected it to be here: '%s'" % manage_xml_entries)
for category in categories:
# Remove any existing acme test category from the file.
if (platform is None):
acme_util.run_cmd("%s -component allactive -removetests -category %s" % (manage_xml_entries, category))
else:
acme_util.run_cmd("%s -component allactive -removetests -category %s -machine %s -compiler %s"
% (manage_xml_entries, category, platforms[0][0], platforms[0][1]))
# Generate a list of test entries corresponding to our suite at the top
# of the file.
new_test_file = generate_acme_test_entries(category, platforms)
acme_util.run_cmd("%s -component allactive -addlist -file %s -category %s" %
(manage_xml_entries, new_test_file, category))
os.unlink(new_test_file)
print "SUCCESS"
def parse_namelists(namelist_lines, filename):
###############################################################################
"""
Return data in form: {namelist -> {key -> value} }.
value can be an int, string, list, or dict
>>> teststr = '''&nml
... val = 'foo'
... aval = 'one','two', 'three'
... maval = 'one', 'two',
... 'three', 'four'
... dval = 'one->two', 'three -> four'
... mdval = 'one -> two',
... 'three -> four',
... 'five -> six'
... nval = 1850
... /
...
... # Hello
...
... &nml2
... val2 = .false.
... /
... '''
>>> parse_namelists(teststr.splitlines(), 'foo')
{'nml': {'dval': {'three': 'four', 'one': 'two'}, 'val': "'foo'", 'maval': ["'one'", "'two'", "'three'", "'four'"], 'aval': ["'one'", "'two'", "'three'"], 'nval': '1850', 'mdval': {'five': 'six', 'three': 'four', 'one': 'two'}}, 'nml2': {'val2': '.false.'}}
>>> parse_namelists('blah', 'foo')
Traceback (most recent call last):
...
SystemExit: FAIL: File 'foo' does not appear to be a namelist file, skipping
>>> teststr = '''&nml
... val = 'one', 'two',
... val2 = 'three'
... /'''
>>> parse_namelists(teststr.splitlines(), 'foo')
Traceback (most recent call last):
...
SystemExit: FAIL: In file 'foo', Incomplete multiline variable: 'val'
>>> teststr = '''&nml
... val = 'one', 'two',
... /'''
>>> parse_namelists(teststr.splitlines(), 'foo')
Traceback (most recent call last):
...
SystemExit: FAIL: In file 'foo', Incomplete multiline variable: 'val'
>>> teststr = '''&nml
... val = 'one', 'two',
... 'three -> four'
... /'''
>>> parse_namelists(teststr.splitlines(), 'foo')
Traceback (most recent call last):
...
SystemExit: FAIL: In file 'foo', multiline list variable 'val' had dict entries
"""
comment_re = re.compile(r'^[#!]')
namelist_re = re.compile(r'^&(\S+)$')
name_re = re.compile(r"^([^\s=']+)\s*=\s*(.+)$")
dict_re = re.compile(r"^'(\S+)\s*->\s*(\S+)'")
comma_re = re.compile(r'\s*,\s*')
rv = {}
current_namelist = None
multiline_variable = None # (name, value)
for line in namelist_lines:
line = line.strip()
verbose_print("Parsing line: '%s'" % line)
if (line == "" or comment_re.match(line)):
verbose_print(" Line was whitespace or comment, skipping.")
continue
if (current_namelist is None):
# Must start a namelist
expect(multiline_variable is None,
"In file '%s', Incomplete multiline variable: '%s'" % (filename, multiline_variable[0] if multiline_variable is not None else ""))
# Unfornately, other tools were using the old compare_namelists.pl script
# to compare files that are not namelist files. We need a special error
# to signify this event
if (namelist_re.match(line) is None):
expect(rv != {},
"File '%s' does not appear to be a namelist file, skipping" % filename)
expect(False,
"In file '%s', Line '%s' did not begin a namelist as expected" % (filename, line))
current_namelist = namelist_re.match(line).groups()[0]
expect(current_namelist not in rv,
"In file '%s', Duplicate namelist '%s'" % (filename, current_namelist))
rv[current_namelist] = {}
verbose_print(" Starting namelist '%s'" % current_namelist)
elif (line == "/"):
# Ends a namelist
verbose_print(" Ending namelist '%s'" % current_namelist)
expect(multiline_variable is None,
"In file '%s', Incomplete multiline variable: '%s'" % (filename, multiline_variable[0] if multiline_variable is not None else ""))
current_namelist = None
elif (name_re.match(line)):
# Defining a variable (AKA name)
name, value = name_re.match(line).groups()
verbose_print(" Parsing variable '%s' with data '%s'" % (name, value))
expect(multiline_variable is None,
"In file '%s', Incomplete multiline variable: '%s'" % (filename, multiline_variable[0] if multiline_variable is not None else ""))
expect(name not in rv[current_namelist], "In file '%s', Duplicate name: '%s'" % (filename, name))
tokens = [item.strip() for item in comma_re.split(value) if item.strip() != ""]
if ("->" in value):
# dict
rv[current_namelist][name] = {}
for token in tokens:
m = dict_re.match(token)
expect(m is not None, "In file '%s', Dict entry '%s' does not match expected format" % (filename, token))
k, v = m.groups()
rv[current_namelist][name][k] = v
verbose_print(" Adding dict entry '%s' -> '%s'" % (k, v))
elif ("," in value):
# list
rv[current_namelist][name] = tokens
verbose_print(" Adding list entries: %s" % ", ".join(tokens))
else:
rv[current_namelist][name] = value
verbose_print(" Setting to value '%s'" % value)
if (line.endswith(",")):
# Value will continue on in subsequent lines
multiline_variable = (name, rv[current_namelist][name])
verbose_print(" Var is multiline...")
elif (multiline_variable is not None):
# Continuation of list or dict variable
current_value = multiline_variable[1]
verbose_print(" Continuing multiline variable '%s' with data '%s'" % (multiline_variable[0], line))
tokens = [item.strip() for item in comma_re.split(line) if item.strip() != ""]
if (type(current_value) is list):
expect("->" not in line, "In file '%s', multiline list variable '%s' had dict entries" % (filename, multiline_variable[0]))
current_value.extend(tokens)
verbose_print(" Adding list entries: %s" % ", ".join(tokens))
elif (type(current_value) is dict):
for token in tokens:
m = dict_re.match(token)
expect(m is not None, "In file '%s', Dict entry '%s' does not match expected format" % (filename, token))
k, v = m.groups()
current_value[k] = v
verbose_print(" Adding dict entry '%s' -> '%s'" % (k, v))
else:
expect(False, "In file '%s', Continuation should have been for list or dict, instead it was: '%s'" % (filename, type(current_value)))
if (not line.endswith(",")):
# Completed
multiline_variable = None
verbose_print(" Terminating multiline variable")
else:
expect(False, "In file '%s', Unrecognized line: '%s'" % (filename, line))
return rv
def parse_namelists(namelist_lines, filename):
###############################################################################
"""
Return data in form: {namelist -> {key -> value} }.
value can be an int, string, list, or dict
>>> teststr = '''&nml
... val = 'foo'
... aval = 'one','two', 'three'
... maval = 'one', 'two',
... 'three', 'four'
... dval = 'one->two', 'three -> four'
... mdval = 'one -> two',
... 'three -> four',
... 'five -> six'
... nval = 1850
... /
...
... # Hello
...
... &nml2
... val2 = .false.
... /
... '''
>>> parse_namelists(teststr.splitlines(), 'foo')
{'nml': {'dval': {'three': 'four', 'one': 'two'}, 'val': "'foo'", 'maval': ["'one'", "'two'", "'three'", "'four'"], 'aval': ["'one'", "'two'", "'three'"], 'nval': '1850', 'mdval': {'five': 'six', 'three': 'four', 'one': 'two'}}, 'nml2': {'val2': '.false.'}}
>>> parse_namelists('blah', 'foo')
Traceback (most recent call last):
...
SystemExit: FAIL: File 'foo' does not appear to be a namelist file, skipping
>>> teststr = '''&nml
... val = 'one', 'two',
... val2 = 'three'
... /'''
>>> parse_namelists(teststr.splitlines(), 'foo')
Traceback (most recent call last):
...
SystemExit: FAIL: In file 'foo', Incomplete multiline variable: 'val'
>>> teststr = '''&nml
... val = 'one', 'two',
... /'''
>>> parse_namelists(teststr.splitlines(), 'foo')
Traceback (most recent call last):
...
SystemExit: FAIL: In file 'foo', Incomplete multiline variable: 'val'
>>> teststr = '''&nml
... val = 'one', 'two',
... 'three -> four'
... /'''
>>> parse_namelists(teststr.splitlines(), 'foo')
Traceback (most recent call last):
...
SystemExit: FAIL: In file 'foo', multiline list variable 'val' had dict entries
"""
comment_re = re.compile(r'^[#!]')
namelist_re = re.compile(r'^&(\S+)$')
name_re = re.compile(r"^([^\s=']+)\s*=\s*(.+)$")
dict_re = re.compile(r"^'(\S+)\s*->\s*(\S+)'")
comma_re = re.compile(r'\s*,\s*')
rv = {}
current_namelist = None
multiline_variable = None # (name, value)
for line in namelist_lines:
line = line.strip()
verbose_print("Parsing line: '%s'" % line)
if (line == "" or comment_re.match(line)):
verbose_print(" Line was whitespace or comment, skipping.")
continue
if (current_namelist is None):
# Must start a namelist
expect(
multiline_variable is None,
"In file '%s', Incomplete multiline variable: '%s'" %
(filename, multiline_variable[0]
if multiline_variable is not None else ""))
# Unfornately, other tools were using the old compare_namelists.pl script
# to compare files that are not namelist files. We need a special error
# to signify this event
if (namelist_re.match(line) is None):
expect(
rv != {},
"File '%s' does not appear to be a namelist file, skipping"
% filename)
expect(
False,
"In file '%s', Line '%s' did not begin a namelist as expected"
% (filename, line))
current_namelist = namelist_re.match(line).groups()[0]
expect(
current_namelist not in rv,
"In file '%s', Duplicate namelist '%s'" %
(filename, current_namelist))
rv[current_namelist] = {}
verbose_print(" Starting namelist '%s'" % current_namelist)
elif (line == "/"):
# Ends a namelist
verbose_print(" Ending namelist '%s'" % current_namelist)
expect(
multiline_variable is None,
"In file '%s', Incomplete multiline variable: '%s'" %
(filename, multiline_variable[0]
if multiline_variable is not None else ""))
current_namelist = None
elif (name_re.match(line)):
# Defining a variable (AKA name)
name, value = name_re.match(line).groups()
verbose_print(" Parsing variable '%s' with data '%s'" %
(name, value))
expect(
multiline_variable is None,
"In file '%s', Incomplete multiline variable: '%s'" %
(filename, multiline_variable[0]
if multiline_variable is not None else ""))
expect(name not in rv[current_namelist],
"In file '%s', Duplicate name: '%s'" % (filename, name))
tokens = [
item.strip() for item in comma_re.split(value)
if item.strip() != ""
]
if ("->" in value):
# dict
rv[current_namelist][name] = {}
for token in tokens:
m = dict_re.match(token)
expect(
m is not None,
"In file '%s', Dict entry '%s' does not match expected format"
% (filename, token))
k, v = m.groups()
rv[current_namelist][name][k] = v
verbose_print(" Adding dict entry '%s' -> '%s'" %
(k, v))
elif ("," in value):
# list
rv[current_namelist][name] = tokens
verbose_print(" Adding list entries: %s" %
", ".join(tokens))
else:
rv[current_namelist][name] = value
verbose_print(" Setting to value '%s'" % value)
if (line.endswith(",")):
# Value will continue on in subsequent lines
multiline_variable = (name, rv[current_namelist][name])
verbose_print(" Var is multiline...")
elif (multiline_variable is not None):
# Continuation of list or dict variable
current_value = multiline_variable[1]
verbose_print(
" Continuing multiline variable '%s' with data '%s'" %
(multiline_variable[0], line))
tokens = [
item.strip() for item in comma_re.split(line)
if item.strip() != ""
]
if (type(current_value) is list):
expect(
"->" not in line,
"In file '%s', multiline list variable '%s' had dict entries"
% (filename, multiline_variable[0]))
current_value.extend(tokens)
verbose_print(" Adding list entries: %s" %
", ".join(tokens))
elif (type(current_value) is dict):
for token in tokens:
m = dict_re.match(token)
expect(
m is not None,
"In file '%s', Dict entry '%s' does not match expected format"
% (filename, token))
k, v = m.groups()
current_value[k] = v
verbose_print(" Adding dict entry '%s' -> '%s'" %
(k, v))
else:
expect(
False,
"In file '%s', Continuation should have been for list or dict, instead it was: '%s'"
% (filename, type(current_value)))
if (not line.endswith(",")):
# Completed
multiline_variable = None
verbose_print(" Terminating multiline variable")
else:
expect(False,
"In file '%s', Unrecognized line: '%s'" % (filename, line))
return rv