def wait_for_tests(test_paths, no_wait=False, check_throughput=False, check_memory=False, ignore_namelists=False, cdash_build_name=None, cdash_project=ACME_MAIN_CDASH, cdash_build_group=CDASH_DEFAULT_BUILD_GROUP, start_time=None): ############################################################################### # Set up signal handling, we want to print results before the program # is terminated set_up_signal_handlers() test_results = wait_for_tests_impl(test_paths, no_wait, check_throughput, check_memory, ignore_namelists) all_pass = True for test_name, test_data in sorted(test_results.iteritems()): test_path, test_status = test_data print "Test '%s' finished with status '%s'" % (test_name, test_status) verbose_print(" Path: %s" % test_path) all_pass &= test_status == TEST_PASS_STATUS if (cdash_build_name): create_cdash_xml(test_results, cdash_build_name, cdash_project, cdash_build_group, start_time) return all_pass
def wait_for_test(test_path, results, wait, check_throughput, check_memory, ignore_namelists): ############################################################################### if (os.path.isdir(test_path)): test_status_filepath = os.path.join(test_path, TEST_STATUS_FILENAME) else: test_status_filepath = test_path verbose_print("Watching file: '%s'" % test_status_filepath) while (True): if (os.path.exists(test_status_filepath)): test_name, test_status = interpret_status_file(test_status_filepath, check_throughput, check_memory, ignore_namelists) if (test_status == TEST_PENDING_STATUS and (wait and not SIGNAL_RECEIVED)): time.sleep(SLEEP_INTERVAL_SEC) verbose_print("Waiting for test to finish") else: results.put( (test_name, test_path, test_status) ) break else: if (wait and not SIGNAL_RECEIVED): verbose_print("File '%s' does not yet exist" % test_status_filepath) time.sleep(SLEEP_INTERVAL_SEC) else: test_name = os.path.abspath(test_status_filepath).split("/")[-2] results.put( (test_name, test_path, "File '%s' doesn't exist" % test_status_filepath) ) break
def wait_for_test(test_path, results, wait, check_throughput, check_memory, ignore_namelists): ############################################################################### if (os.path.isdir(test_path)): test_status_filepath = os.path.join(test_path, TEST_STATUS_FILENAME) else: test_status_filepath = test_path verbose_print("Watching file: '%s'" % test_status_filepath) while (True): if (os.path.exists(test_status_filepath)): test_name, test_status = interpret_status_file( test_status_filepath, check_throughput, check_memory, ignore_namelists) if (test_status == TEST_PENDING_STATUS and (wait and not SIGNAL_RECEIVED)): time.sleep(SLEEP_INTERVAL_SEC) verbose_print("Waiting for test to finish") else: results.put((test_name, test_path, test_status)) break else: if (wait and not SIGNAL_RECEIVED): verbose_print("File '%s' does not yet exist" % test_status_filepath) time.sleep(SLEEP_INTERVAL_SEC) else: test_name = os.path.abspath(test_status_filepath).split( "/")[-2] results.put((test_name, test_path, "File '%s' doesn't exist" % test_status_filepath)) break
def update_acme_tests(xml_file, categories, platform=None): ############################################################################### # Retrieve all supported ACME platforms, killing the third entry (MPI lib) # for the moment. supported_platforms = [p[:2] for p in find_all_supported_platforms()] # Fish all of the existing machine/compiler combos out of the XML file. if (platform is not None): platforms = [tuple(platform.split(","))] else: platforms = find_all_platforms(xml_file) # Prune the non-supported platforms from our list. for p in platforms: if p not in supported_platforms: acme_util.verbose_print("pruning unsupported platform %s"%repr(p)) platforms = [p for p in platforms if p in supported_platforms] manage_xml_entries = os.path.join(acme_util.get_cime_root(), "scripts", "manage_testlists") expect(os.path.isfile(manage_xml_entries), "Couldn't find manage_testlists, expected it to be here: '%s'" % manage_xml_entries) for category in categories: # Remove any existing acme test category from the file. if (platform is None): acme_util.run_cmd("%s -component allactive -removetests -category %s" % (manage_xml_entries, category)) else: acme_util.run_cmd("%s -component allactive -removetests -category %s -machine %s -compiler %s" % (manage_xml_entries, category, platforms[0][0], platforms[0][1])) # Generate a list of test entries corresponding to our suite at the top # of the file. new_test_file = generate_acme_test_entries(category, platforms) acme_util.run_cmd("%s -component allactive -addlist -file %s -category %s" % (manage_xml_entries, new_test_file, category)) os.unlink(new_test_file) print "SUCCESS"
def parse_namelists(namelist_lines, filename): ############################################################################### """ Return data in form: {namelist -> {key -> value} }. value can be an int, string, list, or dict >>> teststr = '''&nml ... val = 'foo' ... aval = 'one','two', 'three' ... maval = 'one', 'two', ... 'three', 'four' ... dval = 'one->two', 'three -> four' ... mdval = 'one -> two', ... 'three -> four', ... 'five -> six' ... nval = 1850 ... / ... ... # Hello ... ... &nml2 ... val2 = .false. ... / ... ''' >>> parse_namelists(teststr.splitlines(), 'foo') {'nml': {'dval': {'three': 'four', 'one': 'two'}, 'val': "'foo'", 'maval': ["'one'", "'two'", "'three'", "'four'"], 'aval': ["'one'", "'two'", "'three'"], 'nval': '1850', 'mdval': {'five': 'six', 'three': 'four', 'one': 'two'}}, 'nml2': {'val2': '.false.'}} >>> parse_namelists('blah', 'foo') Traceback (most recent call last): ... SystemExit: FAIL: File 'foo' does not appear to be a namelist file, skipping >>> teststr = '''&nml ... val = 'one', 'two', ... val2 = 'three' ... /''' >>> parse_namelists(teststr.splitlines(), 'foo') Traceback (most recent call last): ... SystemExit: FAIL: In file 'foo', Incomplete multiline variable: 'val' >>> teststr = '''&nml ... val = 'one', 'two', ... /''' >>> parse_namelists(teststr.splitlines(), 'foo') Traceback (most recent call last): ... SystemExit: FAIL: In file 'foo', Incomplete multiline variable: 'val' >>> teststr = '''&nml ... val = 'one', 'two', ... 'three -> four' ... /''' >>> parse_namelists(teststr.splitlines(), 'foo') Traceback (most recent call last): ... SystemExit: FAIL: In file 'foo', multiline list variable 'val' had dict entries """ comment_re = re.compile(r'^[#!]') namelist_re = re.compile(r'^&(\S+)$') name_re = re.compile(r"^([^\s=']+)\s*=\s*(.+)$") dict_re = re.compile(r"^'(\S+)\s*->\s*(\S+)'") comma_re = re.compile(r'\s*,\s*') rv = {} current_namelist = None multiline_variable = None # (name, value) for line in namelist_lines: line = line.strip() verbose_print("Parsing line: '%s'" % line) if (line == "" or comment_re.match(line)): verbose_print(" Line was whitespace or comment, skipping.") continue if (current_namelist is None): # Must start a namelist expect(multiline_variable is None, "In file '%s', Incomplete multiline variable: '%s'" % (filename, multiline_variable[0] if multiline_variable is not None else "")) # Unfornately, other tools were using the old compare_namelists.pl script # to compare files that are not namelist files. We need a special error # to signify this event if (namelist_re.match(line) is None): expect(rv != {}, "File '%s' does not appear to be a namelist file, skipping" % filename) expect(False, "In file '%s', Line '%s' did not begin a namelist as expected" % (filename, line)) current_namelist = namelist_re.match(line).groups()[0] expect(current_namelist not in rv, "In file '%s', Duplicate namelist '%s'" % (filename, current_namelist)) rv[current_namelist] = {} verbose_print(" Starting namelist '%s'" % current_namelist) elif (line == "/"): # Ends a namelist verbose_print(" Ending namelist '%s'" % current_namelist) expect(multiline_variable is None, "In file '%s', Incomplete multiline variable: '%s'" % (filename, multiline_variable[0] if multiline_variable is not None else "")) current_namelist = None elif (name_re.match(line)): # Defining a variable (AKA name) name, value = name_re.match(line).groups() verbose_print(" Parsing variable '%s' with data '%s'" % (name, value)) expect(multiline_variable is None, "In file '%s', Incomplete multiline variable: '%s'" % (filename, multiline_variable[0] if multiline_variable is not None else "")) expect(name not in rv[current_namelist], "In file '%s', Duplicate name: '%s'" % (filename, name)) tokens = [item.strip() for item in comma_re.split(value) if item.strip() != ""] if ("->" in value): # dict rv[current_namelist][name] = {} for token in tokens: m = dict_re.match(token) expect(m is not None, "In file '%s', Dict entry '%s' does not match expected format" % (filename, token)) k, v = m.groups() rv[current_namelist][name][k] = v verbose_print(" Adding dict entry '%s' -> '%s'" % (k, v)) elif ("," in value): # list rv[current_namelist][name] = tokens verbose_print(" Adding list entries: %s" % ", ".join(tokens)) else: rv[current_namelist][name] = value verbose_print(" Setting to value '%s'" % value) if (line.endswith(",")): # Value will continue on in subsequent lines multiline_variable = (name, rv[current_namelist][name]) verbose_print(" Var is multiline...") elif (multiline_variable is not None): # Continuation of list or dict variable current_value = multiline_variable[1] verbose_print(" Continuing multiline variable '%s' with data '%s'" % (multiline_variable[0], line)) tokens = [item.strip() for item in comma_re.split(line) if item.strip() != ""] if (type(current_value) is list): expect("->" not in line, "In file '%s', multiline list variable '%s' had dict entries" % (filename, multiline_variable[0])) current_value.extend(tokens) verbose_print(" Adding list entries: %s" % ", ".join(tokens)) elif (type(current_value) is dict): for token in tokens: m = dict_re.match(token) expect(m is not None, "In file '%s', Dict entry '%s' does not match expected format" % (filename, token)) k, v = m.groups() current_value[k] = v verbose_print(" Adding dict entry '%s' -> '%s'" % (k, v)) else: expect(False, "In file '%s', Continuation should have been for list or dict, instead it was: '%s'" % (filename, type(current_value))) if (not line.endswith(",")): # Completed multiline_variable = None verbose_print(" Terminating multiline variable") else: expect(False, "In file '%s', Unrecognized line: '%s'" % (filename, line)) return rv
def parse_namelists(namelist_lines, filename): ############################################################################### """ Return data in form: {namelist -> {key -> value} }. value can be an int, string, list, or dict >>> teststr = '''&nml ... val = 'foo' ... aval = 'one','two', 'three' ... maval = 'one', 'two', ... 'three', 'four' ... dval = 'one->two', 'three -> four' ... mdval = 'one -> two', ... 'three -> four', ... 'five -> six' ... nval = 1850 ... / ... ... # Hello ... ... &nml2 ... val2 = .false. ... / ... ''' >>> parse_namelists(teststr.splitlines(), 'foo') {'nml': {'dval': {'three': 'four', 'one': 'two'}, 'val': "'foo'", 'maval': ["'one'", "'two'", "'three'", "'four'"], 'aval': ["'one'", "'two'", "'three'"], 'nval': '1850', 'mdval': {'five': 'six', 'three': 'four', 'one': 'two'}}, 'nml2': {'val2': '.false.'}} >>> parse_namelists('blah', 'foo') Traceback (most recent call last): ... SystemExit: FAIL: File 'foo' does not appear to be a namelist file, skipping >>> teststr = '''&nml ... val = 'one', 'two', ... val2 = 'three' ... /''' >>> parse_namelists(teststr.splitlines(), 'foo') Traceback (most recent call last): ... SystemExit: FAIL: In file 'foo', Incomplete multiline variable: 'val' >>> teststr = '''&nml ... val = 'one', 'two', ... /''' >>> parse_namelists(teststr.splitlines(), 'foo') Traceback (most recent call last): ... SystemExit: FAIL: In file 'foo', Incomplete multiline variable: 'val' >>> teststr = '''&nml ... val = 'one', 'two', ... 'three -> four' ... /''' >>> parse_namelists(teststr.splitlines(), 'foo') Traceback (most recent call last): ... SystemExit: FAIL: In file 'foo', multiline list variable 'val' had dict entries """ comment_re = re.compile(r'^[#!]') namelist_re = re.compile(r'^&(\S+)$') name_re = re.compile(r"^([^\s=']+)\s*=\s*(.+)$") dict_re = re.compile(r"^'(\S+)\s*->\s*(\S+)'") comma_re = re.compile(r'\s*,\s*') rv = {} current_namelist = None multiline_variable = None # (name, value) for line in namelist_lines: line = line.strip() verbose_print("Parsing line: '%s'" % line) if (line == "" or comment_re.match(line)): verbose_print(" Line was whitespace or comment, skipping.") continue if (current_namelist is None): # Must start a namelist expect( multiline_variable is None, "In file '%s', Incomplete multiline variable: '%s'" % (filename, multiline_variable[0] if multiline_variable is not None else "")) # Unfornately, other tools were using the old compare_namelists.pl script # to compare files that are not namelist files. We need a special error # to signify this event if (namelist_re.match(line) is None): expect( rv != {}, "File '%s' does not appear to be a namelist file, skipping" % filename) expect( False, "In file '%s', Line '%s' did not begin a namelist as expected" % (filename, line)) current_namelist = namelist_re.match(line).groups()[0] expect( current_namelist not in rv, "In file '%s', Duplicate namelist '%s'" % (filename, current_namelist)) rv[current_namelist] = {} verbose_print(" Starting namelist '%s'" % current_namelist) elif (line == "/"): # Ends a namelist verbose_print(" Ending namelist '%s'" % current_namelist) expect( multiline_variable is None, "In file '%s', Incomplete multiline variable: '%s'" % (filename, multiline_variable[0] if multiline_variable is not None else "")) current_namelist = None elif (name_re.match(line)): # Defining a variable (AKA name) name, value = name_re.match(line).groups() verbose_print(" Parsing variable '%s' with data '%s'" % (name, value)) expect( multiline_variable is None, "In file '%s', Incomplete multiline variable: '%s'" % (filename, multiline_variable[0] if multiline_variable is not None else "")) expect(name not in rv[current_namelist], "In file '%s', Duplicate name: '%s'" % (filename, name)) tokens = [ item.strip() for item in comma_re.split(value) if item.strip() != "" ] if ("->" in value): # dict rv[current_namelist][name] = {} for token in tokens: m = dict_re.match(token) expect( m is not None, "In file '%s', Dict entry '%s' does not match expected format" % (filename, token)) k, v = m.groups() rv[current_namelist][name][k] = v verbose_print(" Adding dict entry '%s' -> '%s'" % (k, v)) elif ("," in value): # list rv[current_namelist][name] = tokens verbose_print(" Adding list entries: %s" % ", ".join(tokens)) else: rv[current_namelist][name] = value verbose_print(" Setting to value '%s'" % value) if (line.endswith(",")): # Value will continue on in subsequent lines multiline_variable = (name, rv[current_namelist][name]) verbose_print(" Var is multiline...") elif (multiline_variable is not None): # Continuation of list or dict variable current_value = multiline_variable[1] verbose_print( " Continuing multiline variable '%s' with data '%s'" % (multiline_variable[0], line)) tokens = [ item.strip() for item in comma_re.split(line) if item.strip() != "" ] if (type(current_value) is list): expect( "->" not in line, "In file '%s', multiline list variable '%s' had dict entries" % (filename, multiline_variable[0])) current_value.extend(tokens) verbose_print(" Adding list entries: %s" % ", ".join(tokens)) elif (type(current_value) is dict): for token in tokens: m = dict_re.match(token) expect( m is not None, "In file '%s', Dict entry '%s' does not match expected format" % (filename, token)) k, v = m.groups() current_value[k] = v verbose_print(" Adding dict entry '%s' -> '%s'" % (k, v)) else: expect( False, "In file '%s', Continuation should have been for list or dict, instead it was: '%s'" % (filename, type(current_value))) if (not line.endswith(",")): # Completed multiline_variable = None verbose_print(" Terminating multiline variable") else: expect(False, "In file '%s', Unrecognized line: '%s'" % (filename, line)) return rv