def wait_for_tests_impl(test_paths, no_wait=False, check_throughput=False, check_memory=False, ignore_namelists=False): ############################################################################### results = Queue.Queue() for test_path in test_paths: t = threading.Thread(target=wait_for_test, args=(test_path, results, not no_wait, check_throughput, check_memory, ignore_namelists)) t.daemon = True t.start() while threading.active_count() > 1: time.sleep(1) test_results = {} completed_test_paths = [] while (not results.empty()): test_name, test_path, test_status = results.get() if (test_name in test_results): prior_path, prior_status = test_results[test_name] if (test_status == prior_status): warning("Test name '%s' was found in both '%s' and '%s'" % (test_name, test_path, prior_path)) else: raise SystemExit("Test name '%s' was found in both '%s' and '%s' with different results" % (test_name, test_path, prior_path)) test_results[test_name] = (test_path, test_status) completed_test_paths.append(test_path) expect(set(test_paths) == set(completed_test_paths), "Missing results for test paths: %s" % (set(test_paths) - set(completed_test_paths)) ) return test_results
def _update_test_status(self, test_name, phase, status): ########################################################################### state_idx = self._test_names.index(test_name) phase_idx = self._phases.index(phase) old_phase, old_status = self._test_states[state_idx] if (old_phase == phase): expect( old_status == TEST_PENDING_STATUS, "Only valid to transition from PENDING to something else, found '%s'" % old_status) expect(status != TEST_PENDING_STATUS, "Cannot transition from PEND -> PEND") else: expect( old_status in CONTINUE, "Why did we move on to next phase when prior phase did not pass?" ) expect(status == TEST_PENDING_STATUS, "New phase should be set to pending status") expect( self._phases.index(old_phase) == phase_idx - 1, "Skipped phase?") self._test_states[state_idx] = (phase, status)
def compare_values(namelist, name, gold_value, comp_value, case): ############################################################################### """ Compare values for a specific variable in a namelist. """ if (type(gold_value) != type(comp_value)): print "In namelist '%s', variable '%s' did not have expected type '%s', instead is type '%s'" % \ (namelist, name, type(gold_value), type(comp_value)) return False rv = True if (type(gold_value) is list): # Note, list values remain order sensitive for idx, gold_value_list_item in enumerate(gold_value): if (idx < len(comp_value)): rv &= compare_values(namelist, "%s list item %d" % (name, idx), gold_value_list_item, comp_value[idx], case) else: rv = False print "In namelist '%s', list variable '%s' missing value %s" % ( namelist, name, gold_value_list_item) if (len(comp_value) > len(gold_value)): for comp_value_list_item in comp_value[len(gold_value):]: rv = False print "In namelist '%s', list variable '%s' has extra value %s" % ( namelist, name, comp_value_list_item) elif (type(gold_value) is dict): for key, gold_value_dict_item in gold_value.iteritems(): if (key in comp_value): rv &= compare_values(namelist, "%s dict item %s" % (name, key), gold_value_dict_item, comp_value[key], case) else: rv = False print "In namelist '%s', dict variable '%s' missing key %s" % ( namelist, name, key) for key in comp_value: if (key not in gold_value): rv = False print "In namelist '%s', dict variable '%s' has extra key %s" % ( namelist, name, key) else: expect( type(gold_value) is str, "Unexpected type found: '%s'" % type(gold_value)) norm_gold_value = normalize_string_value(name, gold_value, case) norm_comp_value = normalize_string_value(name, comp_value, case) if (norm_gold_value != norm_comp_value): rv = False print "In namelist '%s', '%s' has inequivalent values %s != %s" % ( namelist, name, gold_value, comp_value) print " NORMALIZED: %s != %s" % (norm_gold_value, norm_comp_value) return rv
def compare_files(gold_file, compare_file, case=None): ############################################################################### expect(os.path.exists(gold_file), "File not found: %s" % gold_file) expect(os.path.exists(compare_file), "File not found: %s" % compare_file) return compare_data(open(gold_file, "r").readlines(), open(compare_file, "r").readlines(), case)
def compare_namelist_files(gold_file, compare_file, case=None): ############################################################################### expect(os.path.exists(gold_file), "File not found: %s" % gold_file) expect(os.path.exists(compare_file), "File not found: %s" % compare_file) gold_namelists = parse_namelists(open(gold_file, "r").readlines(), gold_file) comp_namelists = parse_namelists(open(compare_file, "r").readlines(), compare_file) return compare_namelists(gold_namelists, comp_namelists, case)
def create_test(self): ########################################################################### """ Main API for this class. Return True if all tests passed. """ start_time = time.time() # Tell user what will be run print "RUNNING TESTS:" for test_name in self._test_names: print " ", test_name # TODO - documentation self._producer() expect(threading.active_count() == 1, "Leftover threads?") # Setup cs files self._setup_cs_files() # Return True if all tests passed print "At create_test close, state is:" rv = True for idx, test_name in enumerate(self._test_names): phase, status = self._test_states[idx] if (status == TEST_PASS_STATUS and phase == RUN_PHASE): # Be cautious about telling the user that the test passed. This # status should match what they would see on the dashboard. Our # self._test_states does not include comparison fail information, # so we need to parse test status. test_status_file = os.path.join(self._get_test_dir(test_name), TEST_STATUS_FILENAME) status = wait_for_tests.interpret_status_file( test_status_file)[1] if (status not in [TEST_PASS_STATUS, TEST_PENDING_STATUS]): print "%s %s (phase %s)" % (status, test_name, phase) rv = False elif (test_name in self._tests_with_nl_problems): print "%s %s (but otherwise OK)" % (NAMELIST_FAIL_STATUS, test_name) rv = False else: print status, test_name, phase print " Case dir: %s" % self._get_test_dir(test_name) print "create_test took", time.time() - start_time, "seconds" return rv
def create_test(self): ########################################################################### """ Main API for this class. Return True if all tests passed. """ start_time = time.time() # Tell user what will be run print "RUNNING TESTS:" for test_name in self._test_names: print " ", test_name # TODO - documentation self._producer() expect(threading.active_count() == 1, "Leftover threads?") # Setup cs files self._setup_cs_files() # Return True if all tests passed print "At create_test close, state is:" rv = True for idx, test_name in enumerate(self._test_names): phase, status = self._test_states[idx] if (status == TEST_PASS_STATUS and phase == RUN_PHASE): # Be cautious about telling the user that the test passed. This # status should match what they would see on the dashboard. Our # self._test_states does not include comparison fail information, # so we need to parse test status. test_status_file = os.path.join(self._get_test_dir(test_name), TEST_STATUS_FILENAME) status = wait_for_tests.interpret_status_file(test_status_file)[1] if (status not in [TEST_PASS_STATUS, TEST_PENDING_STATUS]): print "%s %s (phase %s)" % (status, test_name, phase) rv = False elif (test_name in self._tests_with_nl_problems): print "%s %s (but otherwise OK)" % (NAMELIST_FAIL_STATUS, test_name) rv = False else: print status, test_name, phase print " Case dir: %s" % self._get_test_dir(test_name) print "create_test took", time.time() - start_time, "seconds" return rv
def _get_test_status(self, test_name, phase=None): ########################################################################### curr_phase = self._get_test_phase(test_name) if (phase == NAMELIST_PHASE and test_name in self._tests_with_nl_problems): return NAMELIST_FAIL_STATUS elif (phase is None or phase == curr_phase): return self._get_test_data(test_name)[1] else: expect(phase is None or self._phases.index(phase) < self._phases.index(curr_phase), "Tried to see the future") # Assume all older phases PASSed return TEST_PASS_STATUS
def _producer(self): ########################################################################### threads_in_flight = {} # test-name -> (thread, procs) while (True): work_to_do = False num_threads_launched_this_iteration = 0 for test_name in self._test_names: # If we have no workers available, immediately wait if (len(threads_in_flight) == self._parallel_jobs): self._wait_for_something_to_finish(threads_in_flight) if (self._work_remains(test_name)): work_to_do = True if (test_name not in threads_in_flight): test_phase, test_status = self._get_test_data( test_name) expect(test_status != TEST_PENDING_STATUS, test_name) next_phase = self._phases[ self._phases.index(test_phase) + 1] procs_needed = self._get_procs_needed( test_name, next_phase) if (procs_needed <= self._proc_pool): self._proc_pool -= procs_needed # Necessary to print this way when multiple threads printing sys.stdout.write( "Starting %s for test %s with %d procs\n" % (next_phase, test_name, procs_needed)) self._update_test_status(test_name, next_phase, TEST_PENDING_STATUS) t = threading.Thread( target=self._consumer, args=(test_name, next_phase, getattr(self, "_%s_phase" % next_phase.lower()))) threads_in_flight[test_name] = (t, procs_needed) t.start() num_threads_launched_this_iteration += 1 if (not work_to_do): break if (num_threads_launched_this_iteration == 0): # No free resources, wait for something in flight to finish self._wait_for_something_to_finish(threads_in_flight) for thread_info in threads_in_flight.values(): thread_info[0].join()
def compare_values(namelist, name, gold_value, comp_value, case): ############################################################################### """ Compare values for a specific variable in a namelist. """ if (type(gold_value) != type(comp_value)): print "In namelist '%s', variable '%s' did not have expected type '%s', instead is type '%s'" % \ (namelist, name, type(gold_value), type(comp_value)) return False rv = True if (type(gold_value) is list): # Note, list values remain order sensitive for idx, gold_value_list_item in enumerate(gold_value): if (idx < len(comp_value)): rv &= compare_values(namelist, "%s list item %d" % (name, idx), gold_value_list_item, comp_value[idx], case) else: rv = False print "In namelist '%s', list variable '%s' missing value %s" % (namelist, name, gold_value_list_item) if (len(comp_value) > len(gold_value)): for comp_value_list_item in comp_value[len(gold_value):]: rv = False print "In namelist '%s', list variable '%s' has extra value %s" % (namelist, name, comp_value_list_item) elif (type(gold_value) is dict): for key, gold_value_dict_item in gold_value.iteritems(): if (key in comp_value): rv &= compare_values(namelist, "%s dict item %s" % (name, key), gold_value_dict_item, comp_value[key], case) else: rv = False print "In namelist '%s', dict variable '%s' missing key %s" % (namelist, name, key) for key in comp_value: if (key not in gold_value): rv = False print "In namelist '%s', dict variable '%s' has extra key %s" % (namelist, name, key) else: expect(type(gold_value) is str, "Unexpected type found: '%s'" % type(gold_value)) norm_gold_value = normalize_string_value(name, gold_value, case) norm_comp_value = normalize_string_value(name, comp_value, case) if (norm_gold_value != norm_comp_value): rv = False print "In namelist '%s', '%s' has inequivalent values %s != %s" % (namelist, name, gold_value, comp_value) print " NORMALIZED: %s != %s" % (norm_gold_value, norm_comp_value) return rv
def _wait_for_something_to_finish(self, threads_in_flight): ########################################################################### expect(len(threads_in_flight) <= self._parallel_jobs, "Oversubscribed?") finished_tests = [] while (not finished_tests): for test_name, thread_info in threads_in_flight.iteritems(): if (not thread_info[0].is_alive()): finished_tests.append( (test_name, thread_info[1]) ) if (not finished_tests): time.sleep(0.2) for finished_test, procs_needed in finished_tests: self._proc_pool += procs_needed del threads_in_flight[finished_test]
def wait_for_tests_impl(test_paths, no_wait=False, check_throughput=False, check_memory=False, ignore_namelists=False): ############################################################################### results = Queue.Queue() for test_path in test_paths: t = threading.Thread(target=wait_for_test, args=(test_path, results, not no_wait, check_throughput, check_memory, ignore_namelists)) t.daemon = True t.start() while threading.active_count() > 1: time.sleep(1) test_results = {} completed_test_paths = [] while (not results.empty()): test_name, test_path, test_status = results.get() if (test_name in test_results): prior_path, prior_status = test_results[test_name] if (test_status == prior_status): warning("Test name '%s' was found in both '%s' and '%s'" % (test_name, test_path, prior_path)) else: raise SystemExit( "Test name '%s' was found in both '%s' and '%s' with different results" % (test_name, test_path, prior_path)) test_results[test_name] = (test_path, test_status) completed_test_paths.append(test_path) expect( set(test_paths) == set(completed_test_paths), "Missing results for test paths: %s" % (set(test_paths) - set(completed_test_paths))) return test_results
def _producer(self): ########################################################################### threads_in_flight = {} # test-name -> (thread, procs) while (True): work_to_do = False num_threads_launched_this_iteration = 0 for test_name in self._test_names: # If we have no workers available, immediately wait if (len(threads_in_flight) == self._parallel_jobs): self._wait_for_something_to_finish(threads_in_flight) if (self._work_remains(test_name)): work_to_do = True if (test_name not in threads_in_flight): test_phase, test_status = self._get_test_data(test_name) expect(test_status != TEST_PENDING_STATUS, test_name) next_phase = self._phases[self._phases.index(test_phase) + 1] procs_needed = self._get_procs_needed(test_name, next_phase) if (procs_needed <= self._proc_pool): self._proc_pool -= procs_needed # Necessary to print this way when multiple threads printing sys.stdout.write("Starting %s for test %s with %d procs\n" % (next_phase, test_name, procs_needed)) self._update_test_status(test_name, next_phase, TEST_PENDING_STATUS) t = threading.Thread(target=self._consumer, args=(test_name, next_phase, getattr(self, "_%s_phase" % next_phase.lower()) )) threads_in_flight[test_name] = (t, procs_needed) t.start() num_threads_launched_this_iteration += 1 if (not work_to_do): break if (num_threads_launched_this_iteration == 0): # No free resources, wait for something in flight to finish self._wait_for_something_to_finish(threads_in_flight) for thread_info in threads_in_flight.values(): thread_info[0].join()
def _update_test_status(self, test_name, phase, status): ########################################################################### state_idx = self._test_names.index(test_name) phase_idx = self._phases.index(phase) old_phase, old_status = self._test_states[state_idx] if (old_phase == phase): expect(old_status == TEST_PENDING_STATUS, "Only valid to transition from PENDING to something else, found '%s'" % old_status) expect(status != TEST_PENDING_STATUS, "Cannot transition from PEND -> PEND") else: expect(old_status in CONTINUE, "Why did we move on to next phase when prior phase did not pass?") expect(status == TEST_PENDING_STATUS, "New phase should be set to pending status") expect(self._phases.index(old_phase) == phase_idx - 1, "Skipped phase?") self._test_states[state_idx] = (phase, status)
def update_acme_tests(xml_file, categories, platform=None): ############################################################################### # Retrieve all supported ACME platforms, killing the third entry (MPI lib) # for the moment. supported_platforms = [p[:2] for p in find_all_supported_platforms()] # Fish all of the existing machine/compiler combos out of the XML file. if (platform is not None): platforms = [tuple(platform.split(","))] else: platforms = find_all_platforms(xml_file) # Prune the non-supported platforms from our list. for p in platforms: if p not in supported_platforms: acme_util.verbose_print("pruning unsupported platform %s"%repr(p)) platforms = [p for p in platforms if p in supported_platforms] manage_xml_entries = os.path.join(acme_util.get_cime_root(), "scripts", "manage_testlists") expect(os.path.isfile(manage_xml_entries), "Couldn't find manage_testlists, expected it to be here: '%s'" % manage_xml_entries) for category in categories: # Remove any existing acme test category from the file. if (platform is None): acme_util.run_cmd("%s -component allactive -removetests -category %s" % (manage_xml_entries, category)) else: acme_util.run_cmd("%s -component allactive -removetests -category %s -machine %s -compiler %s" % (manage_xml_entries, category, platforms[0][0], platforms[0][1])) # Generate a list of test entries corresponding to our suite at the top # of the file. new_test_file = generate_acme_test_entries(category, platforms) acme_util.run_cmd("%s -component allactive -addlist -file %s -category %s" % (manage_xml_entries, new_test_file, category)) os.unlink(new_test_file) print "SUCCESS"
def get_test_suite(suite, machine=None, compiler=None): ############################################################################### """ Return a list of FULL test names for a suite. """ expect(suite in _TEST_SUITES, "Unknown test suite: '%s'" % suite) machine = acme_util.probe_machine_name() if machine is None else machine compiler = acme_util.get_machine_info("COMPILERS", machine=machine)[0] if compiler is None else compiler inherits_from, tests_raw = _TEST_SUITES[suite] tests = [] for item in tests_raw: test_mod = None if (isinstance(item, str)): test_name = item else: expect(isinstance(item, tuple), "Bad item type for item '%s'" % str(item)) expect(len(item) in [2, 3], "Expected two or three items in item '%s'" % str(item)) expect(isinstance(item[0], str), "Expected string in first field of item '%s'" % str(item)) expect(isinstance(item[1], str), "Expected string in second field of item '%s'" % str(item)) test_name = item[0] if (len(item) == 2): test_mod = item[1] else: expect(type(item[2]) in [str, tuple], "Expected string or tuple for third field of item '%s'" % str(item)) test_mod_machines = [item[2]] if isinstance(item[2], str) else item[2] if (machine in test_mod_machines): test_mod = item[1] tests.append(acme_util.get_full_test_name(test_name, machine, compiler, testmod=test_mod)) if (inherits_from is not None): inherited_tests = get_test_suite(inherits_from, machine, compiler) expect(len(set(tests) & set(inherited_tests)) == 0, "Tests %s defined in multiple suites" % ", ".join(set(tests) & set(inherited_tests))) tests.extend(inherited_tests) return tests
elif (cidx == cnum): print "Missing lines" print "\n".join(gold_lines[gidx:1]) return False gold_value = gold_lines[gidx].strip() comp_value = comp_lines[cidx].strip() norm_gold_value = normalize_string_value(gold_value, case) norm_comp_value = normalize_string_value(comp_value, case) if (norm_gold_value != norm_comp_value): rv = False print "Inequivalent lines %s != %s" % (gold_value, comp_value) print " NORMALIZED: %s != %s" % (norm_gold_value, norm_comp_value) gidx += 1 cidx += 1 return rv ############################################################################### def compare_files(gold_file, compare_file, case=None): ############################################################################### expect(os.path.exists(gold_file), "File not found: %s" % gold_file) expect(os.path.exists(compare_file), "File not found: %s" % compare_file) return compare_data(open(gold_file, "r").readlines(), open(compare_file, "r").readlines(), case)
def __init__(self, test_names, no_run=False, no_build=False, no_batch=None, test_root=None, test_id=None, compiler=None, baseline_root=None, baseline_name=None, clean=False, compare=False, generate=False, namelists_only=False, project=None, parallel_jobs=None): ########################################################################### self._cime_root = acme_util.get_cime_root() self._test_names = test_names self._no_build = no_build if not namelists_only else True self._no_run = no_run if not self._no_build else True self._no_batch = no_batch if no_batch is not None else not acme_util.does_machine_have_batch() self._test_root = test_root if test_root is not None else acme_util.get_machine_info("CESMSCRATCHROOT") self._test_id = test_id if test_id is not None else acme_util.get_utc_timestamp() self._project = project if project is not None else acme_util.get_machine_project() self._baseline_root = baseline_root if baseline_root is not None else acme_util.get_machine_info("CCSM_BASELINE", project=self._project) self._baseline_name = None self._compiler = compiler if compiler is not None else acme_util.get_machine_info("COMPILERS")[0] self._clean = clean self._compare = compare self._generate = generate self._namelists_only = namelists_only self._parallel_jobs = parallel_jobs if parallel_jobs is not None else min(len(self._test_names), int(acme_util.get_machine_info("MAX_TASKS_PER_NODE"))) # Oversubscribe by 1/4 pes = int(acme_util.get_machine_info("MAX_TASKS_PER_NODE")) # This is the only data that multiple threads will simultaneously access # Each test has it's own index and setting/retrieving items from a list # is atomic, so this should be fine to use without mutex self._test_states = [ (INITIAL_PHASE, TEST_PASS_STATUS) ] * len(test_names) self._proc_pool = int(pes * 1.25) # Since the name-list phase can fail without aborting later phases, we # need some extra state to remember tests that had namelist problems self._tests_with_nl_problems = [None] * len(test_names) # Setup phases self._phases = list(PHASES) if (no_build): self._phases.remove(BUILD_PHASE) if (no_run): self._phases.remove(RUN_PHASE) if (not self._compare and not self._generate): self._phases.remove(NAMELIST_PHASE) else: if (baseline_name is None): branch_name = acme_util.get_current_branch(repo=self._cime_root) expect(branch_name is not None, "Could not determine baseline name from branch, please use -b option") self._baseline_name = os.path.join(self._compiler, branch_name) else: self._baseline_name = baseline_name if (not self._baseline_name.startswith("%s/" % self._compiler)): self._baseline_name = os.path.join(self._compiler, self._baseline_name) # Validate any assumptions that were not caught by the arg parser # None of the test directories should already exist. for test in self._test_names: expect(not os.path.exists(self._get_test_dir(test)), "Cannot create new case in directory '%s', it already exists. Pick a different test-id" % self._get_test_dir(test))
def parse_namelists(namelist_lines, filename): ############################################################################### """ Return data in form: {namelist -> {key -> value} }. value can be an int, string, list, or dict >>> teststr = '''&nml ... val = 'foo' ... aval = 'one','two', 'three' ... maval = 'one', 'two', ... 'three', 'four' ... dval = 'one->two', 'three -> four' ... mdval = 'one -> two', ... 'three -> four', ... 'five -> six' ... nval = 1850 ... / ... ... # Hello ... ... &nml2 ... val2 = .false. ... / ... ''' >>> parse_namelists(teststr.splitlines(), 'foo') {'nml': {'dval': {'three': 'four', 'one': 'two'}, 'val': "'foo'", 'maval': ["'one'", "'two'", "'three'", "'four'"], 'aval': ["'one'", "'two'", "'three'"], 'nval': '1850', 'mdval': {'five': 'six', 'three': 'four', 'one': 'two'}}, 'nml2': {'val2': '.false.'}} >>> parse_namelists('blah', 'foo') Traceback (most recent call last): ... SystemExit: FAIL: File 'foo' does not appear to be a namelist file, skipping >>> teststr = '''&nml ... val = 'one', 'two', ... val2 = 'three' ... /''' >>> parse_namelists(teststr.splitlines(), 'foo') Traceback (most recent call last): ... SystemExit: FAIL: In file 'foo', Incomplete multiline variable: 'val' >>> teststr = '''&nml ... val = 'one', 'two', ... /''' >>> parse_namelists(teststr.splitlines(), 'foo') Traceback (most recent call last): ... SystemExit: FAIL: In file 'foo', Incomplete multiline variable: 'val' >>> teststr = '''&nml ... val = 'one', 'two', ... 'three -> four' ... /''' >>> parse_namelists(teststr.splitlines(), 'foo') Traceback (most recent call last): ... SystemExit: FAIL: In file 'foo', multiline list variable 'val' had dict entries """ comment_re = re.compile(r'^[#!]') namelist_re = re.compile(r'^&(\S+)$') name_re = re.compile(r"^([^\s=']+)\s*=\s*(.+)$") dict_re = re.compile(r"^'(\S+)\s*->\s*(\S+)'") comma_re = re.compile(r'\s*,\s*') rv = {} current_namelist = None multiline_variable = None # (name, value) for line in namelist_lines: line = line.strip() verbose_print("Parsing line: '%s'" % line) if (line == "" or comment_re.match(line)): verbose_print(" Line was whitespace or comment, skipping.") continue if (current_namelist is None): # Must start a namelist expect( multiline_variable is None, "In file '%s', Incomplete multiline variable: '%s'" % (filename, multiline_variable[0] if multiline_variable is not None else "")) # Unfornately, other tools were using the old compare_namelists.pl script # to compare files that are not namelist files. We need a special error # to signify this event if (namelist_re.match(line) is None): expect( rv != {}, "File '%s' does not appear to be a namelist file, skipping" % filename) expect( False, "In file '%s', Line '%s' did not begin a namelist as expected" % (filename, line)) current_namelist = namelist_re.match(line).groups()[0] expect( current_namelist not in rv, "In file '%s', Duplicate namelist '%s'" % (filename, current_namelist)) rv[current_namelist] = {} verbose_print(" Starting namelist '%s'" % current_namelist) elif (line == "/"): # Ends a namelist verbose_print(" Ending namelist '%s'" % current_namelist) expect( multiline_variable is None, "In file '%s', Incomplete multiline variable: '%s'" % (filename, multiline_variable[0] if multiline_variable is not None else "")) current_namelist = None elif (name_re.match(line)): # Defining a variable (AKA name) name, value = name_re.match(line).groups() verbose_print(" Parsing variable '%s' with data '%s'" % (name, value)) expect( multiline_variable is None, "In file '%s', Incomplete multiline variable: '%s'" % (filename, multiline_variable[0] if multiline_variable is not None else "")) expect(name not in rv[current_namelist], "In file '%s', Duplicate name: '%s'" % (filename, name)) tokens = [ item.strip() for item in comma_re.split(value) if item.strip() != "" ] if ("->" in value): # dict rv[current_namelist][name] = {} for token in tokens: m = dict_re.match(token) expect( m is not None, "In file '%s', Dict entry '%s' does not match expected format" % (filename, token)) k, v = m.groups() rv[current_namelist][name][k] = v verbose_print(" Adding dict entry '%s' -> '%s'" % (k, v)) elif ("," in value): # list rv[current_namelist][name] = tokens verbose_print(" Adding list entries: %s" % ", ".join(tokens)) else: rv[current_namelist][name] = value verbose_print(" Setting to value '%s'" % value) if (line.endswith(",")): # Value will continue on in subsequent lines multiline_variable = (name, rv[current_namelist][name]) verbose_print(" Var is multiline...") elif (multiline_variable is not None): # Continuation of list or dict variable current_value = multiline_variable[1] verbose_print( " Continuing multiline variable '%s' with data '%s'" % (multiline_variable[0], line)) tokens = [ item.strip() for item in comma_re.split(line) if item.strip() != "" ] if (type(current_value) is list): expect( "->" not in line, "In file '%s', multiline list variable '%s' had dict entries" % (filename, multiline_variable[0])) current_value.extend(tokens) verbose_print(" Adding list entries: %s" % ", ".join(tokens)) elif (type(current_value) is dict): for token in tokens: m = dict_re.match(token) expect( m is not None, "In file '%s', Dict entry '%s' does not match expected format" % (filename, token)) k, v = m.groups() current_value[k] = v verbose_print(" Adding dict entry '%s' -> '%s'" % (k, v)) else: expect( False, "In file '%s', Continuation should have been for list or dict, instead it was: '%s'" % (filename, type(current_value))) if (not line.endswith(",")): # Completed multiline_variable = None verbose_print(" Terminating multiline variable") else: expect(False, "In file '%s', Unrecognized line: '%s'" % (filename, line)) return rv
def parse_namelists(namelist_lines, filename): ############################################################################### """ Return data in form: {namelist -> {key -> value} }. value can be an int, string, list, or dict >>> teststr = '''&nml ... val = 'foo' ... aval = 'one','two', 'three' ... maval = 'one', 'two', ... 'three', 'four' ... dval = 'one->two', 'three -> four' ... mdval = 'one -> two', ... 'three -> four', ... 'five -> six' ... nval = 1850 ... / ... ... # Hello ... ... &nml2 ... val2 = .false. ... / ... ''' >>> parse_namelists(teststr.splitlines(), 'foo') {'nml': {'dval': {'three': 'four', 'one': 'two'}, 'val': "'foo'", 'maval': ["'one'", "'two'", "'three'", "'four'"], 'aval': ["'one'", "'two'", "'three'"], 'nval': '1850', 'mdval': {'five': 'six', 'three': 'four', 'one': 'two'}}, 'nml2': {'val2': '.false.'}} >>> parse_namelists('blah', 'foo') Traceback (most recent call last): ... SystemExit: FAIL: File 'foo' does not appear to be a namelist file, skipping >>> teststr = '''&nml ... val = 'one', 'two', ... val2 = 'three' ... /''' >>> parse_namelists(teststr.splitlines(), 'foo') Traceback (most recent call last): ... SystemExit: FAIL: In file 'foo', Incomplete multiline variable: 'val' >>> teststr = '''&nml ... val = 'one', 'two', ... /''' >>> parse_namelists(teststr.splitlines(), 'foo') Traceback (most recent call last): ... SystemExit: FAIL: In file 'foo', Incomplete multiline variable: 'val' >>> teststr = '''&nml ... val = 'one', 'two', ... 'three -> four' ... /''' >>> parse_namelists(teststr.splitlines(), 'foo') Traceback (most recent call last): ... SystemExit: FAIL: In file 'foo', multiline list variable 'val' had dict entries """ comment_re = re.compile(r'^[#!]') namelist_re = re.compile(r'^&(\S+)$') name_re = re.compile(r"^([^\s=']+)\s*=\s*(.+)$") dict_re = re.compile(r"^'(\S+)\s*->\s*(\S+)'") comma_re = re.compile(r'\s*,\s*') rv = {} current_namelist = None multiline_variable = None # (name, value) for line in namelist_lines: line = line.strip() verbose_print("Parsing line: '%s'" % line) if (line == "" or comment_re.match(line)): verbose_print(" Line was whitespace or comment, skipping.") continue if (current_namelist is None): # Must start a namelist expect(multiline_variable is None, "In file '%s', Incomplete multiline variable: '%s'" % (filename, multiline_variable[0] if multiline_variable is not None else "")) # Unfornately, other tools were using the old compare_namelists.pl script # to compare files that are not namelist files. We need a special error # to signify this event if (namelist_re.match(line) is None): expect(rv != {}, "File '%s' does not appear to be a namelist file, skipping" % filename) expect(False, "In file '%s', Line '%s' did not begin a namelist as expected" % (filename, line)) current_namelist = namelist_re.match(line).groups()[0] expect(current_namelist not in rv, "In file '%s', Duplicate namelist '%s'" % (filename, current_namelist)) rv[current_namelist] = {} verbose_print(" Starting namelist '%s'" % current_namelist) elif (line == "/"): # Ends a namelist verbose_print(" Ending namelist '%s'" % current_namelist) expect(multiline_variable is None, "In file '%s', Incomplete multiline variable: '%s'" % (filename, multiline_variable[0] if multiline_variable is not None else "")) current_namelist = None elif (name_re.match(line)): # Defining a variable (AKA name) name, value = name_re.match(line).groups() verbose_print(" Parsing variable '%s' with data '%s'" % (name, value)) expect(multiline_variable is None, "In file '%s', Incomplete multiline variable: '%s'" % (filename, multiline_variable[0] if multiline_variable is not None else "")) expect(name not in rv[current_namelist], "In file '%s', Duplicate name: '%s'" % (filename, name)) tokens = [item.strip() for item in comma_re.split(value) if item.strip() != ""] if ("->" in value): # dict rv[current_namelist][name] = {} for token in tokens: m = dict_re.match(token) expect(m is not None, "In file '%s', Dict entry '%s' does not match expected format" % (filename, token)) k, v = m.groups() rv[current_namelist][name][k] = v verbose_print(" Adding dict entry '%s' -> '%s'" % (k, v)) elif ("," in value): # list rv[current_namelist][name] = tokens verbose_print(" Adding list entries: %s" % ", ".join(tokens)) else: rv[current_namelist][name] = value verbose_print(" Setting to value '%s'" % value) if (line.endswith(",")): # Value will continue on in subsequent lines multiline_variable = (name, rv[current_namelist][name]) verbose_print(" Var is multiline...") elif (multiline_variable is not None): # Continuation of list or dict variable current_value = multiline_variable[1] verbose_print(" Continuing multiline variable '%s' with data '%s'" % (multiline_variable[0], line)) tokens = [item.strip() for item in comma_re.split(line) if item.strip() != ""] if (type(current_value) is list): expect("->" not in line, "In file '%s', multiline list variable '%s' had dict entries" % (filename, multiline_variable[0])) current_value.extend(tokens) verbose_print(" Adding list entries: %s" % ", ".join(tokens)) elif (type(current_value) is dict): for token in tokens: m = dict_re.match(token) expect(m is not None, "In file '%s', Dict entry '%s' does not match expected format" % (filename, token)) k, v = m.groups() current_value[k] = v verbose_print(" Adding dict entry '%s' -> '%s'" % (k, v)) else: expect(False, "In file '%s', Continuation should have been for list or dict, instead it was: '%s'" % (filename, type(current_value))) if (not line.endswith(",")): # Completed multiline_variable = None verbose_print(" Terminating multiline variable") else: expect(False, "In file '%s', Unrecognized line: '%s'" % (filename, line)) return rv
def __init__(self, test_names, no_run=False, no_build=False, no_batch=None, test_root=None, test_id=None, compiler=None, baseline_root=None, baseline_name=None, clean=False, compare=False, generate=False, namelists_only=False, project=None, parallel_jobs=None): ########################################################################### self._cime_root = acme_util.get_cime_root() self._test_names = test_names self._no_build = no_build if not namelists_only else True self._no_run = no_run if not self._no_build else True self._no_batch = no_batch if no_batch is not None else not acme_util.does_machine_have_batch( ) self._test_root = test_root if test_root is not None else acme_util.get_machine_info( "CESMSCRATCHROOT") self._test_id = test_id if test_id is not None else acme_util.get_utc_timestamp( ) self._project = project if project is not None else acme_util.get_machine_project( ) self._baseline_root = baseline_root if baseline_root is not None else acme_util.get_machine_info( "CCSM_BASELINE", project=self._project) self._baseline_name = None self._compiler = compiler if compiler is not None else acme_util.get_machine_info( "COMPILERS")[0] self._clean = clean self._compare = compare self._generate = generate self._namelists_only = namelists_only self._parallel_jobs = parallel_jobs if parallel_jobs is not None else min( len(self._test_names), int(acme_util.get_machine_info("MAX_TASKS_PER_NODE"))) # Oversubscribe by 1/4 pes = int(acme_util.get_machine_info("MAX_TASKS_PER_NODE")) # This is the only data that multiple threads will simultaneously access # Each test has it's own index and setting/retrieving items from a list # is atomic, so this should be fine to use without mutex self._test_states = [(INITIAL_PHASE, TEST_PASS_STATUS) ] * len(test_names) self._proc_pool = int(pes * 1.25) # Since the name-list phase can fail without aborting later phases, we # need some extra state to remember tests that had namelist problems self._tests_with_nl_problems = [None] * len(test_names) # Setup phases self._phases = list(PHASES) if (no_build): self._phases.remove(BUILD_PHASE) if (no_run): self._phases.remove(RUN_PHASE) if (not self._compare and not self._generate): self._phases.remove(NAMELIST_PHASE) else: if (baseline_name is None): branch_name = acme_util.get_current_branch( repo=self._cime_root) expect( branch_name is not None, "Could not determine baseline name from branch, please use -b option" ) self._baseline_name = os.path.join(self._compiler, branch_name) else: self._baseline_name = baseline_name if (not self._baseline_name.startswith( "%s/" % self._compiler)): self._baseline_name = os.path.join(self._compiler, self._baseline_name) # Validate any assumptions that were not caught by the arg parser # None of the test directories should already exist. for test in self._test_names: expect( not os.path.exists(self._get_test_dir(test)), "Cannot create new case in directory '%s', it already exists. Pick a different test-id" % self._get_test_dir(test))