def get_value(self, name, attributes=None, resolved=True, subgroup=None): """ Get Value of fields in the config_machines.xml file """ expect(self.machine_node is not None, "Machine object has no machine defined") expect(subgroup is None, "This class does not support subgroups") value = None # COMPILER and MPILIB are special, if called without arguments they get the default value from the # COMPILERS and MPILIBS lists in the file. if name == "COMPILER": value = self.get_default_compiler() elif name == "MPILIB": value = self.get_default_MPIlib(attributes) else: node = self.get_optional_node(name, root=self.machine_node, attributes=attributes) if node is not None: value = node.text if value is None: # if all else fails value = GenericXML.get_value(self, name) if resolved: if value is not None: value = self.get_resolved_value(value) elif name in os.environ: value = os.environ[name] return value
def case_test(case, testname=None): if testname is None: testname = case.get_value("TESTCASE") expect(testname is not None, "testname argument not resolved") logging.warn("Running test for %s" % testname) _set_up_signal_handlers() try: # The following line can throw exceptions if the testname is # not found or the test constructor throws. We need to be # sure to leave TestStatus in the appropriate state if that # happens. test = find_system_test(testname, case)(case) except: caseroot = case.get_value("CASEROOT") with TestStatus(test_dir=caseroot) as ts: ts.set_status(RUN_PHASE, TEST_FAIL_STATUS, comments="failed to initialize") append_status(sys.exc_info()[1], sfile="TestStatus.log") raise success = test.run() return success
def update_shr_strdata_nml(self, config, stream, stream_path): """Updates values for the `shr_strdata_nml` namelist group. This should be done once per stream, and it shouldn't usually be called directly, since `create_stream_file` calls this method itself. """ assert config['stream'] == stream, \ "config stream is %s, but input stream is %s" % \ (config['stream'], stream) # Double-check the years for sanity. year_start = int(self.get_default("strm_year_start", config)) year_end = int(self.get_default("strm_year_end", config)) year_align = int(self.get_default("strm_year_align", config)) expect(year_end >= year_start, "Stream %s starts at year %d, but ends at earlier year %d." % (stream, year_start, year_end)) # Add to streams file. stream_string = "%s %d %d %d" % (os.path.basename(stream_path), year_align, year_start, year_end) self._streams_namelists["streams"].append(stream_string) for variable in self._streams_variables: default = self.get_default(variable, config) expect(len(default) == 1, "Stream %s had multiple settings for variable %s." % (stream, variable)) self._streams_namelists[variable].append(default[0])
def check_all_input_data(case): ############################################################################### success = check_input_data(case=case, download=True) expect(success, "Failed to download input data") get_refcase = case.get_value("GET_REFCASE") run_type = case.get_value("RUN_TYPE") continue_run = case.get_value("CONTINUE_RUN") # We do not fully populate the inputdata directory on every # machine and do not expect every user to download the 3TB+ of # data in our inputdata repository. This code checks for the # existence of inputdata in the local inputdata directory and # attempts to download data from the server if it's needed and # missing. if get_refcase and run_type != "startup" and not continue_run: din_loc_root = case.get_value("DIN_LOC_ROOT") run_refdate = case.get_value("RUN_REFDATE") run_refcase = case.get_value("RUN_REFCASE") run_refdir = case.get_value("RUN_REFDIR") rundir = case.get_value("RUNDIR") refdir = os.path.join(din_loc_root, run_refdir, run_refcase, run_refdate) expect(os.path.isdir(refdir), """ ***************************************************************** prestage ERROR: $refdir is not on local disk obtain this data from the svn input data repository > mkdir -p %s > cd %s > cd .. > svn export --force https://svn-ccsm-inputdata.cgd.ucar.edu/trunk/inputdata/%s or set GET_REFCASE to FALSE in env_run.xml and prestage the restart data to $RUNDIR manually *****************************************************************""" % (refdir, refdir, refdir)) logger.info(" - Prestaging REFCASE (%s) to %s" % (refdir, rundir)) # prestage the reference case's files. if (not os.path.exists(rundir)): logger.debug("Creating run directory: %s"%rundir) os.makedirs(rundir) for rcfile in glob.iglob(os.path.join(refdir,"*%s*"%run_refcase)): logger.debug("Staging file %s"%rcfile) rcbaseline = os.path.basename(rcfile) if not os.path.exists("%s/%s" % (rundir, rcbaseline)): os.symlink(rcfile, "%s/%s" % ((rundir, rcbaseline))) # copy the refcases' rpointer files to the run directory for rpointerfile in glob.iglob(os.path.join("%s","*rpointer*") % (refdir)): logger.debug("Copy rpointer %s"%rpointerfile) shutil.copy(rpointerfile, rundir) for cam2file in glob.iglob(os.path.join("%s","*.cam2.*") % rundir): camfile = cam2file.replace("cam2", "cam") os.symlink(cam2file, camfile)
def _write_input_files(self, input_data_list): """Write input data files to list.""" for group_name in self._namelist.get_group_names(): for variable_name in self._namelist.get_variable_names(group_name): input_pathname = self._definition.get_node_element_info(variable_name, "input_pathname") if input_pathname is not None: # This is where we end up for all variables that are paths # to input data files. literals = self._namelist.get_variable_value(group_name, variable_name) for literal in literals: file_path = character_literal_to_string(literal) if input_pathname == 'abs': # No further mangling needed for absolute paths. pass elif input_pathname.startswith('rel:'): # The part past "rel" is the name of a variable that # this variable specifies its path relative to. root_var = input_pathname[4:] root_dir = self.get_value(root_var) file_path = os.path.join(root_dir, file_path) else: expect(False, "Bad input_pathname value: %s." % input_pathname) # Write to the input data list. input_data_list.write("%s = %s\n" % (variable_name, file_path))
def _create_caseroot_tools(self): machines_dir = os.path.abspath(self.get_value("MACHDIR")) toolsdir = os.path.join(self.get_value("CIMEROOT"),"scripts","Tools") # setup executable files in caseroot/ exefiles = (os.path.join(toolsdir, "case.setup"), os.path.join(toolsdir, "case.build"), os.path.join(toolsdir, "case.submit"), os.path.join(toolsdir, "preview_namelists"), os.path.join(toolsdir, "check_input_data"), os.path.join(toolsdir, "check_case"), os.path.join(toolsdir, "archive_metadata.sh"), os.path.join(toolsdir, "xmlchange"), os.path.join(toolsdir, "xmlquery")) try: for exefile in exefiles: destfile = os.path.join(self._caseroot,os.path.basename(exefile)) os.symlink(exefile, destfile) except Exception as e: logger.warning("FAILED to set up exefiles: %s" % str(e)) # set up utility files in caseroot/Tools/ toolfiles = (os.path.join(toolsdir, "check_lockedfiles"), os.path.join(toolsdir, "lt_archive.sh"), os.path.join(toolsdir, "getTiming"), os.path.join(toolsdir, "save_provenance"), os.path.join(machines_dir,"Makefile"), os.path.join(machines_dir,"mkSrcfiles"), os.path.join(machines_dir,"mkDepends")) for toolfile in toolfiles: destfile = os.path.join(self._caseroot,"Tools",os.path.basename(toolfile)) expect(os.path.isfile(toolfile)," File %s does not exist"%toolfile) try: os.symlink(toolfile, destfile) except Exception as e: logger.warning("FAILED to set up toolfiles: %s %s %s" % (str(e), toolfile, destfile)) # Create Macros file. machine = self.get_value("MACH") files = Files() # Use config_build if the environment variable is set, or if there is no # config_compilers file. if os.getenv("CIME_USE_CONFIG_BUILD") == "TRUE" or \ files.get_value("COMPILERS_SPEC_FILE") is None: build_file = files.get_value("BUILD_SPEC_FILE") machobj = Machines(machine=machine, files=files) macro_maker = Build(machobj) macros_path = os.path.join(self._caseroot, "Macros") with open(macros_path, "w") as macros_file: macro_maker.write_macros('Makefile', build_file, macros_file) # Copy any system or compiler Depends files to the case. compiler = self.get_value("COMPILER") for dep in (machine, compiler): dfile = "Depends.%s"%dep if os.path.isfile(os.path.join(machines_dir,dfile)): shutil.copyfile(os.path.join(machines_dir,dfile), os.path.join(self._caseroot,dfile)) dfile = "Depends.%s.%s"%(machine,compiler) if os.path.isfile(os.path.join(machines_dir,dfile)): shutil.copyfile(os.path.join(machines_dir,dfile), os.path.join(self._caseroot, dfile))
def case_run(case): ############################################################################### # Set up the run, run the model, do the postrun steps run_with_submit = case.get_value("RUN_WITH_SUBMIT") expect (run_with_submit, "You are not calling the run script via the submit script. " "As a result, short-term archiving will not be called automatically." "Please submit your run using the submit script like so:" " ./case.submit") data_assimilation = case.get_value("DATA_ASSIMILATION") data_assimilation_cycles = case.get_value("DATA_ASSIMILATION_CYCLES") data_assimilation_script = case.get_value("DATA_ASSIMILATION_SCRIPT") # set up the LID lid = time.strftime("%y%m%d-%H%M%S") os.environ["LID"] = lid for _ in range(data_assimilation_cycles): preRunCheck(case) runModel(case) postRunCheck(case, lid) saveLogs(case, lid) # Copy log files back to caseroot getTimings(case, lid) # Run the getTiming script if data_assimilation: DoDataAssimilation(case, data_assimilation_script, lid) resubmitCheck(case) return True
def _write_input_files(self, input_data_list): """Write input data files to list.""" for group_name in self._namelist.get_group_names(): for variable_name in self._namelist.get_variable_names(group_name): input_pathname = self._definition.get_node_element_info(variable_name, "input_pathname") if input_pathname is not None: # This is where we end up for all variables that are paths # to input data files. literals = self._namelist.get_variable_value(group_name, variable_name) for literal in literals: file_path = character_literal_to_string(literal) # NOTE - these are hard-coded here and a better way is to make these extensible if file_path == 'UNSET' or file_path == 'idmap': continue if input_pathname == 'abs': # No further mangling needed for absolute paths. # At this point, there are overwrites that should be ignored if not os.path.isabs(file_path): continue else: pass elif input_pathname.startswith('rel:'): # The part past "rel" is the name of a variable that # this variable specifies its path relative to. root_var = input_pathname[4:] root_dir = self.get_value(root_var) file_path = os.path.join(root_dir, file_path) else: expect(False, "Bad input_pathname value: {}.".format(input_pathname)) # Write to the input data list. input_data_list.write("{} = {}\n".format(variable_name, file_path))
def get_resolved_value(self, raw_value): """ A value in the xml file may contain references to other xml variables or to environment variables. These are refered to in the perl style with $name and $ENV{name}. >>> obj = GenericXML() >>> os.environ["FOO"] = "BAR" >>> os.environ["BAZ"] = "BARF" >>> obj.get_resolved_value("one $ENV{FOO} two $ENV{BAZ} three") 'one BAR two BARF three' """ logging.debug("raw_value %s" % raw_value) reference_re = re.compile(r'\$(\w+)') env_ref_re = re.compile(r'\$ENV\{(\w+)\}') item_data = raw_value if (item_data is None): return None for m in env_ref_re.finditer(item_data): logging.debug("look for "+item_data+ " in env") env_var = m.groups()[0] expect(env_var in os.environ, "Undefined env var '%s'" % env_var) item_data = item_data.replace(m.group(), os.environ[env_var]) for m in reference_re.finditer(item_data): var = m.groups()[0] logging.debug("find: "+var) ref = self.get_value(var) if(ref is not None): logging.debug("resolve: "+ref) item_data = item_data.replace(m.group(), self.get_resolved_value(ref)) return item_data
def infer_machine_name_from_tests(testargs): ############################################################################### """ >>> infer_machine_name_from_tests(["NCK.f19_g16_rx1.A.melvin_gnu"]) 'melvin' >>> infer_machine_name_from_tests(["NCK.f19_g16_rx1.A"]) >>> infer_machine_name_from_tests(["NCK.f19_g16_rx1.A", "NCK.f19_g16_rx1.A.melvin_gnu"]) 'melvin' >>> infer_machine_name_from_tests(["NCK.f19_g16_rx1.A.melvin_gnu", "NCK.f19_g16_rx1.A.melvin_gnu"]) 'melvin' """ e3sm_test_suites = get_test_suites() machine = None for testarg in testargs: testarg = testarg.strip() if testarg.startswith("^"): testarg = testarg[1:] if testarg not in e3sm_test_suites: machine_for_this_test = parse_test_name(testarg)[4] if machine_for_this_test is not None: if machine is None: machine = machine_for_this_test else: expect(machine == machine_for_this_test, "Must have consistent machine '%s' != '%s'" % (machine, machine_for_this_test)) return machine
def get_recommended_test_time(test_full_name): ############################################################################### """ >>> get_recommended_test_time("ERS.f19_g16_rx1.A.melvin_gnu") '0:10:00' >>> get_recommended_test_time("TESTRUNPASS_P69.f19_g16_rx1.A.melvin_gnu.testmod") '0:13:00' >>> get_recommended_test_time("PET_Ln20.ne30_ne30.FC5.sandiatoss3_intel.cam-outfrq9s") >>> """ _, _, _, _, machine, compiler, _ = CIME.utils.parse_test_name(test_full_name) expect(machine is not None, "{} is not a full test name".format(test_full_name)) best_time = None suites = get_test_suites() for suite in suites: tests = get_test_suite(suite, machine=machine, compiler=compiler, skip_inherit=True) rec_time = get_test_data(suite)[1] if test_full_name in tests and rec_time is not None and \ (best_time is None or convert_to_seconds(rec_time) < convert_to_seconds(best_time)): best_time = rec_time return best_time
def run_phase(self): archive_dir = self._case.get_value('DOUT_S_ROOT') if os.path.isdir(archive_dir): shutil.rmtree(archive_dir) self.run_indv() # finished running, so all archive files should exist start_date = get_file_date(self._case.get_value('RUN_STARTDATE')) rest_dir = os.path.join(archive_dir, 'rest') delta_day = datetime.timedelta(1) current_date = start_date + delta_day next_datecheck = current_date days_left = self._case.get_value('STOP_N') final_date = start_date + delta_day * days_left while current_date < final_date: logger.info('Testing archiving with last date: {}'.format(current_date)) current_date_str = '{:04}-{:02}-{:02}'.format(current_date.year, current_date.month, current_date.day) case_st_archive(self._case, last_date_str=current_date_str, copy_only=False) archive_dates = [get_file_date(fname) for fname in glob.glob(os.path.join(rest_dir, '*'))] while next_datecheck <= current_date: expect(next_datecheck in archive_dates, 'Not all dates generated and/or archived: ' + '{} is missing'.format(next_datecheck)) next_datecheck += delta_day for date in archive_dates: expect(date <= current_date, 'Archived date greater than specified by last-date: ' + '{}'.format(date)) num_days = random.randint(1, min(3, days_left)) days_left -= num_days current_date += num_days * delta_day
def verify_chksum(input_data_root, rundir, filename, isdirectory): """ For file in filename perform a chksum and compare the result to that stored in the local checksumfile, if isdirectory chksum all files in the directory of form *.* """ if not chksum_hash: hashfile = os.path.join(rundir, local_chksum_file) if not os.path.isfile(hashfile): logger.warning("Failed to find or download file {}".format(hashfile)) return with open(hashfile) as fd: lines = fd.readlines() for line in lines: fchksum, fname = line.split() if fname in chksum_hash: expect(chksum_hash[fname] == fchksum, " Inconsistent hashes in chksum for file {}".format(fname)) else: chksum_hash[fname] = fchksum if isdirectory: filenames = glob.glob(os.path.join(filename,"*.*")) else: filenames = [filename] for fname in filenames: if not os.sep in fname: continue chksum = md5(os.path.join(input_data_root, fname)) if chksum_hash: if not fname in chksum_hash: logger.warning("Did not find hash for file {} in chksum file {}".format(filename, hashfile)) else: expect(chksum == chksum_hash[fname], "chksum mismatch for file {} expected {} found {}". format(os.path.join(input_data_root,fname),chksum, chksum_hash[fname]))
def __init__(self, infile=None, files=None, machine=None): """ initialize an object if a filename is provided it will be used, otherwise if a files object is provided it will be used otherwise create a files object from default values """ self.machine_node = None self.machine = None self.machines_dir = None if infile is None: if files is None: files = Files() infile = files.get_value("MACHINES_SPEC_FILE", resolved=False) infile = files.get_resolved_value(infile) self.machines_dir = os.path.dirname(infile) GenericXML.__init__(self, infile) # Append the contents of $HOME/.cime/config_machines.xml if it exists # This could cause problems if node matchs are repeated when only one is expected infile = os.path.join(os.environ.get("HOME"),".cime","config_machines.xml") logger.debug("Infile: %s" , infile) if os.path.exists(infile): GenericXML.read(self, infile) if machine is None: machine = self.probe_machine_name() expect(machine is not None, "Could not initialize machine object") self.set_machine(machine)
def build_libraries(exeroot, caseroot, cimeroot, libroot, mpilib, lid, machines_file): ############################################################################### if (mpilib == "mpi-serial"): for header_to_copy in glob.glob(os.path.join(cimeroot, "externals/mct/mpi-serial/*.h")): shutil.copy(header_to_copy, os.path.join(libroot, "include")) sharedpath = os.environ["SHAREDPATH"] shared_lib = os.path.join(sharedpath, "lib") shared_inc = os.path.join(sharedpath, "include") for shared_item in [shared_lib, shared_inc]: if (not os.path.exists(shared_item)): os.makedirs(shared_item) libs = ["mct", "gptl", "pio", "csm_share"] logs = [] for lib in libs: full_lib_path = os.path.join(sharedpath, lib) if (not os.path.exists(full_lib_path)): os.makedirs(full_lib_path) file_build = os.path.join(sharedpath, "%s.bldlog.%s" % (lib, lid)) with open(file_build, "w") as fd: fd.write("Current env:\n%s" % "\n".join([" %s = %s" % (env, os.environ[env]) for env in sorted(os.environ)])) my_file = os.path.join(os.path.dirname(machines_file), "buildlib.%s" % lib) stat = run_cmd("%s %s %s >> %s 2>&1" % (my_file, sharedpath, caseroot, file_build), from_dir=exeroot, ok_to_fail=True, verbose=True)[0] expect(stat == 0, "ERROR: buildlib.%s failed, cat %s" % (lib, file_build)) logs.append(file_build) return logs
def wait_for_tests_impl(test_paths, no_wait=False, check_throughput=False, check_memory=False, ignore_namelists=False, ignore_memleak=False): ############################################################################### results = Queue.Queue() for test_path in test_paths: t = threading.Thread(target=wait_for_test, args=(test_path, results, not no_wait, check_throughput, check_memory, ignore_namelists, ignore_memleak)) t.daemon = True t.start() while threading.active_count() > 1: time.sleep(1) test_results = {} completed_test_paths = [] while (not results.empty()): test_name, test_path, test_status = results.get() if (test_name in test_results): prior_path, prior_status = test_results[test_name] if (test_status == prior_status): logging.warning("Test name '%s' was found in both '%s' and '%s'" % (test_name, test_path, prior_path)) else: raise SystemExit("Test name '%s' was found in both '%s' and '%s' with different results" % (test_name, test_path, prior_path)) test_results[test_name] = (test_path, test_status) completed_test_paths.append(test_path) expect(set(test_paths) == set(completed_test_paths), "Missing results for test paths: %s" % (set(test_paths) - set(completed_test_paths)) ) return test_results
def set_default_value(self, vid, attributes=None): """ Set the value of an entry to the default value for that entry vid can be an xml node pointer or a string identifier of a node """ value = None if (type(vid) != type(str())): node = vid vid = node.attrib["id"] else: nodes = self.get_node("entry", {"id":vid}) if (nodes is None): return expect(len(nodes) == 1, "More than one match found for id " + vid) node = nodes[0] valnodes = self.get_node("value",root=node) if (valnodes is not None): for valnode in valnodes: for att in valnode.attributes: if (att.key in attributes): if (re.search(attributes[att.key],att.text)): value = valnode.text logger.info("id %s value %s" % (vid, valnode.text)) if (value is None): value = self.get_node("default_value", root=node) if (value is not None): node.set("value", value[0].text) return value[0].text
def case_test(case, testname=None, reset=False): if testname is None: testname = case.get_value('TESTCASE') expect(testname is not None, "testname argument not resolved") logging.warning("Running test for {}".format(testname)) _set_up_signal_handlers() try: # The following line can throw exceptions if the testname is # not found or the test constructor throws. We need to be # sure to leave TestStatus in the appropriate state if that # happens. test = find_system_test(testname, case)(case) except: caseroot = case.get_value("CASEROOT") with TestStatus(test_dir=caseroot) as ts: ts.set_status(RUN_PHASE, TEST_FAIL_STATUS, comments="failed to initialize") append_testlog(str(sys.exc_info()[1])) return False if reset: logger.info("Reset test to initial conditions and exit") # pylint: disable=protected-access test._resetup_case(RUN_PHASE) return True success = test.run() return success
def get_value(self, name, resolved=True): """ Get Value of fields in the config_machines.xml file """ expect(self.machine is not None, "Machine object has no machine defined") value = None # COMPILER and MPILIB are special, if called without arguments they get the default value from the # COMPILERS and MPILIBS lists in the file. if (name == "COMPILER"): value = self.get_default_compiler() elif (name == "MPILIB"): value = self.get_default_MPIlib() else: nodes = self.get_node(name, root=self.machine) if (nodes): node = nodes[0] expect(node is not None, "No match found for %s in machine %s" % (name, self.name)) value = node.text if (value is None): # if all else fails value = GenericXML.get_value(self, name) if (resolved): value = self.get_resolved_value(value) return value
def get_env(self, short_name): full_name = "env_%s.xml" % (short_name) for env_file in self._files: if os.path.basename(env_file.filename) == full_name: return env_file expect(False, "Could not find object for %s in case"%full_name)
def compare_xml(self, other, root=None, otherroot=None): xmldiffs = {} if root is not None: expect(otherroot is not None," inconsistant request") f1nodes = self.scan_children("entry", root=root) for node in f1nodes: vid = self.get(node, "id") logger.debug("Compare vid {}".format(vid)) f2match = other.scan_optional_child("entry", attributes={"id":vid},root=otherroot) expect(f2match is not None,"Could not find {} in Locked file".format(vid)) if node != f2match: f1val = self.get_value(vid, resolved=False) if f1val is not None: f2val = other.get_value(vid, resolved=False) if f1val != f2val: xmldiffs[vid] = [f1val, f2val] elif hasattr(self, "_components"): # pylint: disable=no-member for comp in self._components: f1val = self.get_value("{}_{}".format(vid,comp), resolved=False) if f1val is not None: f2val = other.get_value("{}_{}".format(vid,comp), resolved=False) if f1val != f2val: xmldiffs[vid] = [f1val, f2val] else: if node != f2match: f1value_nodes = self.get_children("value", root=node) for valnode in f1value_nodes: f2valnodes = other.get_children("value", root=f2match, attributes=self.attrib(valnode)) for f2valnode in f2valnodes: if self.attrib(valnode) is None and self.attrib(f2valnode) is None or \ self.attrib(f2valnode) == self.attrib(valnode): if other.get_resolved_value(self.text(f2valnode)) != self.get_resolved_value(self.text(valnode)): xmldiffs["{}:{}".format(vid, self.attrib(valnode))] = [self.text(valnode), self.text(f2valnode)] return xmldiffs
def _parse_timing_files(timing_files): """ Parse every file in list for timing information and return data dict """ data = {} for timing_file in timing_files: timing = _read_timing_file(timing_file) logger.debug('ntasks: %s' % "; ".join([str(k) + ":" + str(timing[k]['ntasks']) for k in timing.keys()])) logger.debug('cost: %s' % "; ".join([str(k) + ":" + str(timing[k]['cost']) for k in timing.keys()])) for key in timing: if key not in data: data[key] = {'cost':[], 'ntasks':[], 'nthrds':[]} if timing[key]['ntasks'] in data[key]['ntasks']: logger.warning('WARNING: duplicate timing run data in %s ' 'for %s ntasks=%d.', timing_file, key, timing[key]['ntasks']) index = data[key]['ntasks'].index(timing[key]['ntasks']) logger.warning('Existing value: cost=%s. Ignoring new value: ' 'cost=%s', data[key]['cost'][index], timing[key]['cost']) elif 'name' in data[key] and data[key]['name'] != timing[key]['name']: expect(False, "Timing files have inconsistant model components {} has {} vs {}" .format(key, data[key]['name'], timing[key]['name'])) else: data[key]['name'] = timing[key]['name'] data[key]['cost'].append(timing[key]['cost']) data[key]['ntasks'].append(timing[key]['ntasks']) data[key]['nthrds'].append(timing[key]['nthrds']) return data
def run_gmake(case, compclass, libroot, libname="", user_cppdefs=""): ############################################################################### caseroot = case.get_value("CASEROOT") casetools = case.get_value("CASETOOLS") gmake_j = case.get_value("GMAKE_J") gmake = case.get_value("GMAKE") mach = case.get_value("MACH") complib = "" if libname: complib = os.path.join(libroot, "lib%s.a" % libname) else: complib = os.path.join(libroot, "lib%s.a" % compclass) makefile = os.path.join(casetools, "Makefile") macfile = os.path.join(caseroot, "Macros.%s" % mach) if user_cppdefs: cmd = "%s complib -j %d MODEL=%s COMPLIB=%s -f %s MACFILE=%s USER_CPPDEFS=%s" \ % (gmake, gmake_j, compclass, complib, makefile, macfile, user_cppdefs ) else: cmd = "%s complib -j %d MODEL=%s COMPLIB=%s -f %s MACFILE=%s " \ % (gmake, gmake_j, compclass, complib, makefile, macfile ) rc, out, err = run_cmd(cmd, ok_to_fail=True) expect(rc == 0, "Command %s failed rc=%d\nout=%s\nerr=%s" % (cmd, rc, out, err)) logger.info("Command %s completed with output %s\nerr %s" ,cmd, out, err)
def get_resolved_value(self, raw_value): """ A value in the xml file may contain references to other xml variables or to environment variables. These are refered to in the perl style with $name and $ENV{name}. >>> obj = GenericXML() >>> os.environ["FOO"] = "BAR" >>> os.environ["BAZ"] = "BARF" >>> obj.get_resolved_value("one $ENV{FOO} two $ENV{BAZ} three") 'one BAR two BARF three' >>> obj.get_resolved_value("2 + 3 - 1") '4' >>> obj.get_resolved_value("0001-01-01") '0001-01-01' """ logger.debug("raw_value %s" % raw_value) reference_re = re.compile(r'\${?(\w+)}?') env_ref_re = re.compile(r'\$ENV\{(\w+)\}') math_re = re.compile(r'\s[+-/*]\s') item_data = raw_value if item_data is None: return None if type(item_data) is not str: return item_data for m in env_ref_re.finditer(item_data): logger.debug("look for %s in env" % item_data) env_var = m.groups()[0] expect(env_var in os.environ, "Undefined env var '%s'" % env_var) item_data = item_data.replace(m.group(), os.environ[env_var]) for m in reference_re.finditer(item_data): var = m.groups()[0] logger.debug("find: %s" % var) ref = self.get_value(var) if ref is not None: logger.debug("resolve: " + str(ref)) item_data = item_data.replace(m.group(), self.get_resolved_value(str(ref))) elif var == "CIMEROOT": cimeroot = get_cime_root() item_data = item_data.replace(m.group(), cimeroot) elif var == "SRCROOT": srcroot = os.path.join(get_cime_root(),"..") item_data = item_data.replace(m.group(), srcroot) elif var in os.environ: # this is a list of suppressed warnings (things normally expected to be resolved in env) if var not in ("USER",): logging.warn("Resolved from env: " + var) item_data = item_data.replace(m.group(), os.environ[var]) if math_re.search(item_data): try: tmp = eval(item_data) except: tmp = item_data item_data = str(tmp) return item_data
def __init__(self, infile=None): """ Initialize an object """ logger.debug("Initializing %s" , infile) self.tree = None if infile == None: # if file is not defined just return self.filename = None return if os.path.isfile(infile) and os.access(infile, os.R_OK): # If file is defined and exists, read it self.filename = infile self.read(infile) else: # if file does not exist create a root xml element # and set it's id to file logger.debug("File %s does not exists." , infile) expect("$" not in infile,"File path not fully resolved %s"%infile) self.filename = infile root = ET.Element("xml") self.root = ET.SubElement(root, "file") self.root.set("id", os.path.basename(infile)) self.tree = ET.ElementTree(root)
def check_all_input_data(self, protocol=None, address=None, input_data_root=None, data_list_dir="Buildconf", download=True, chksum=False): """ Read through all files of the form *.input_data_list in the data_list_dir directory. These files contain a list of input and boundary files needed by each model component. For each file in the list confirm that it is available in input_data_root and if not (optionally download it from a server at address using protocol. Perform a chksum of the downloaded file. """ success = False if protocol is not None and address is not None: success = self.check_input_data(protocol=protocol, address=address, download=download, input_data_root=input_data_root, data_list_dir=data_list_dir, chksum=chksum) else: if chksum: _download_checksum_file(self.get_value("RUNDIR")) success = self.check_input_data(protocol=protocol, address=address, download=False, input_data_root=input_data_root, data_list_dir=data_list_dir, chksum=chksum) if download and not success: if not chksum: _download_checksum_file(self.get_value("RUNDIR")) success = _downloadfromserver(self, input_data_root, data_list_dir) expect(not download or (download and success), "Could not find all inputdata on any server") self.stage_refcase(input_data_root=input_data_root, data_list_dir=data_list_dir) return success
def _build_usernl_files(case, model, comp): ############################################################################### """ Create user_nl_xxx files, expects cwd is caseroot """ model = model.upper() model_file = case.get_value("CONFIG_%s_FILE" % model) model_dir = os.path.dirname(model_file) expect(os.path.isdir(model_dir), "cannot find cime_config directory %s for component %s" % (model_dir, comp)) if comp == "cpl": if not os.path.exists("user_nl_cpl"): shutil.copy(os.path.join(model_dir, "user_nl_cpl"), ".") else: ninst = case.get_value("NINST_%s" % model) nlfile = "user_nl_%s" % comp model_nl = os.path.join(model_dir, nlfile) if os.path.exists(model_nl): if ninst > 1: for inst_counter in xrange(1, ninst+1): case_nlfile = "%s_%04d" % (nlfile, inst_counter) if not os.path.exists(case_nlfile): shutil.copy(model_nl, case_nlfile) else: if not os.path.exists(nlfile): shutil.copy(model_nl, nlfile)
def build_xcpl_nml(case, caseroot, compname): ############################################################################### compclasses = case.get_values("COMP_CLASSES") compclass = None for compclass in compclasses: if case.get_value("COMP_%s"%compclass) == compname: break expect(compclass is not None, "Could not identify compclass for compname %s"%compname) rundir = case.get_value("RUNDIR") ninst = case.get_value("NINST_%s" % compclass.upper()) nx = case.get_value("%s_NX" % compclass.upper()) ny = case.get_value("%s_NY" % compclass.upper()) if compname == "xrof": flood_mode = case.get_value('XROF_FLOOD_MODE') extras = [] dtype = 1 npes = 0 length = 0 if compname == "xatm": if ny == 1: dtype = 2 extras = [["24", "ncpl number of communications w/coupler per dat"], ["0.0", "simul time proxy (secs): time between cpl comms"]] elif compname == "xglc" or compname == "xice": dtype = 2 elif compname == "xlnd": dtype = 11 elif compname == "xocn": dtype = 4 elif compname == "xrof": dtype = 11 if flood_mode == "ACTIVE": extras = [[".true.", "flood flag"]] else: extras = [[".false.", "flood flag"]] for i in range(1, ninst + 1): # If only 1 file, name is 'compclass_in' # otherwise files are 'compclass_in0001', 'compclass_in0002', etc if ninst == 1: filename = os.path.join(rundir, "%s_in" % compname) else: filename = os.path.join(rundir, "%s_in_%4.4d" % (compname, i)) with open(filename, 'w') as infile: infile.write("%-20d ! i-direction global dimension\n" % nx) infile.write("%-20d ! j-direction global dimension\n" % ny) infile.write("%-20d ! decomp_type 1=1d-by-lat, 2=1d-by-lon," " 3=2d, 4=2d evensquare, 11=segmented\n" % dtype) infile.write("%-20d ! num of pes for i (type 3 only)\n" % npes) infile.write("%-20d ! length of segments (type 4 only)\n" % length) for extra in extras: infile.write("%-20s ! %s\n" % (extra[0], extra[1]))
def add_default(self, name, value=None, ignore_abs_path=None): """Add a value for the specified variable to the namelist. If the specified variable is already defined in the object, the existing value is preserved. Otherwise, the `value` argument, if provided, will be used to set the value. If no such value is found, the defaults file will be consulted. If null values are present in any of the above, the result will be a merged array of values. If no value for the variable is found via any of the above, this method will raise an exception. """ # pylint: disable=protected-access group = self._definition.get_group(name) # Use this to see if we need to raise an error when nothing is found. have_value = False # Check for existing value. current_literals = self._namelist.get_variable_value(group, name) # Check for input argument. if value is not None: have_value = True # if compression were to occur, this is where it does literals = self._to_namelist_literals(name, value) current_literals = merge_literal_lists(literals, current_literals) # Check for default value. default = self.get_default(name, allow_none=True) if default is not None: have_value = True default_literals = self._to_namelist_literals(name, default) current_literals = merge_literal_lists(default_literals, current_literals) expect(have_value, "No default value found for {}.".format(name)) # Go through file names and prepend input data root directory for # absolute pathnames. var_type, _, var_size = self._definition.split_type_string(name) if var_type == "character" and ignore_abs_path is None: var_input_pathname = self._definition.get_input_pathname(name) if var_input_pathname == 'abs': current_literals = expand_literal_list(current_literals) for i, literal in enumerate(current_literals): if literal == '': continue file_path = character_literal_to_string(literal) # NOTE - these are hard-coded here and a better way is to make these extensible if file_path == 'UNSET' or file_path == 'unset' or file_path == 'idmap': continue if file_path == 'null': continue file_path = self.set_abs_file_path(file_path) if not os.path.exists(file_path): logger.warning("File not found: {} = {}, will attempt to download in check_input_data phase".format(name, literal)) current_literals[i] = string_to_character_literal(file_path) current_literals = compress_literal_list(current_literals) # Set the new value. self._namelist.set_variable_value(group, name, current_literals, var_size)
def compare_files(gold_file, compare_file, case=None): ############################################################################### expect(os.path.exists(gold_file), "File not found: %s" % gold_file) expect(os.path.exists(compare_file), "File not found: %s" % compare_file) return compare_data(open(gold_file, "r").readlines(), open(compare_file, "r").readlines(), case)
def __init__(self): """ initialize an object >>> files = Files() >>> files.get_value('CASEFILE_HEADERS',resolved=False) '$CIMEROOT/config/config_headers.xml' """ cimeroot = get_cime_root() infile = os.path.join(cimeroot, "config", get_model(), "config_files.xml") expect(os.path.isfile(infile), "Could not find or open file %s" % infile) schema = os.path.join(cimeroot, "config", "xml_schemas", "entry_id.xsd") EntryID.__init__(self, infile, schema=schema)
def _to_namelist_literals(self, name, value): """Transform a literal list as needed for `set_value`. This is the inverse of `_to_python_value`, except that many of the changes have potentially already been performed. """ var_type, _, var_size, = self._definition.split_type_string(name) if var_size == 1 and not isinstance(value, list): value = [value] for i, scalar in enumerate(value): if scalar is None: value[i] = "" elif var_type == 'character': expect(not isinstance(scalar, list), name) value[i] = self.quote_string(scalar) return compress_literal_list(value)
def get_valid_value_string(self, node, value, vid=None, ignore_type=False): valid_values = self._get_valid_values(node) if ignore_type: expect(isinstance(value, six.string_types), "Value must be type string if ignore_type is true") str_value = value return str_value type_str = self._get_type_info(node) str_value = convert_to_string(value, type_str, vid) if valid_values is not None and not str_value.startswith('$'): expect( str_value in valid_values, "Did not find {} in valid values for {}: {}".format( value, vid, valid_values)) return str_value
def handle_conflicts(is_merge=False, auto_conf=False): ############################################################################### logging.info("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") logging.info("There are conflicts, analyzing...") logging.info("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") remaining_conflicts = handle_easy_conflicts(is_merge) if auto_conf else True if remaining_conflicts: expect(False, "There are merge conflicts. Please fix, commit, and re-run this tool with --resume") else: logging.info("All conflicts were automatically resovled, continuing") run_cmd_no_fail("git commit --no-edit") logging.info("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") logging.info("Automatic conflict resolution complete") logging.info("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
def check_case(self, skip_pnl=False): self.check_lockedfiles() if not skip_pnl: self.create_namelists() # Must be called before check_all_input_data logger.info( "Checking that inputdata is available as part of case submission") self.check_all_input_data() if self.get_value('COMP_WAV') == 'ww': # the ww3 buildnml has dependencies on inputdata so we must run it again self.create_namelists(component='WAV') expect( self.get_value("BUILD_COMPLETE"), "Build complete is " "not True please rebuild the model by calling case.build") logger.info("Check case OK")
def overwrite_existing_entries(self): # if there exist two nodes with the same id delete the first one. for node in self.get_children("entry"): vid = self.get(node, "id") samenodes = self.get_nodes_by_id(vid) if len(samenodes) > 1: expect( len(samenodes) == 2, "Too many matchs for id {} in file {}".format(vid, self.filename), ) logger.debug("Overwriting node {}".format(vid)) read_only = self.read_only if read_only: self.read_only = False self.remove_child(samenodes[0]) self.read_only = read_only
def check_code(files, num_procs=10, interactive=False): ############################################################################### """ Check all python files in the given directory Returns True if all files had no problems """ # Get list of files to check, we look to see if user-provided file argument # is a valid file, if not, we search the repo for a file with similar name. files_to_check = [] if files: repo_files = run_cmd_no_fail('git ls-files', from_dir=get_cime_root(), verbose=False).splitlines() for filearg in files: if os.path.exists(filearg): files_to_check.append(os.path.abspath(filearg)) else: found = False for repo_file in repo_files: if repo_file.endswith(filearg): found = True files_to_check.append( repo_file) # could have multiple matches if not found: logger.warning( "Could not find file matching argument '%s'" % filearg) else: # Check every python file files_to_check = get_all_checkable_files() if "scripts/lib/six.py" in files_to_check: files_to_check.remove("scripts/lib/six.py") logger.info("Not checking contributed file six.py") expect(len(files_to_check) > 0, "No matching files found") # No point in using more threads than files if len(files_to_check) < num_procs: num_procs = len(files_to_check) pool = ThreadPool(num_procs) results = pool.map(lambda x: _run_pylint(x, interactive), files_to_check) pool.close() pool.join() return dict(results)
def get_full_test_names(testargs, machine, compiler): ############################################################################### """ Return full test names in the form: TESTCASE.GRID.COMPSET.MACHINE_COMPILER.TESTMODS Testmods are optional Testargs can be categories or test names and support the NOT symbol '^' >>> get_full_test_names(["cime_tiny"], "melvin", "gnu") ['ERS.f19_g16_rx1.A.melvin_gnu', 'NCK.f19_g16_rx1.A.melvin_gnu'] >>> get_full_test_names(["cime_tiny", "PEA_P1_M.f45_g37_rx1.A"], "melvin", "gnu") ['ERS.f19_g16_rx1.A.melvin_gnu', 'NCK.f19_g16_rx1.A.melvin_gnu', 'PEA_P1_M.f45_g37_rx1.A.melvin_gnu'] >>> get_full_test_names(['ERS.f19_g16_rx1.A', 'NCK.f19_g16_rx1.A', 'PEA_P1_M.f45_g37_rx1.A'], "melvin", "gnu") ['ERS.f19_g16_rx1.A.melvin_gnu', 'NCK.f19_g16_rx1.A.melvin_gnu', 'PEA_P1_M.f45_g37_rx1.A.melvin_gnu'] >>> get_full_test_names(["cime_tiny", "^NCK.f19_g16_rx1.A"], "melvin", "gnu") ['ERS.f19_g16_rx1.A.melvin_gnu'] """ expect(machine is not None, "Must define a machine") expect(compiler is not None, "Must define a compiler") acme_test_suites = get_test_suites() tests_to_run = set() negations = set() for testarg in testargs: # remove any whitespace in name testarg = testarg.strip() if (testarg.startswith("^")): negations.add(testarg[1:]) elif (testarg in acme_test_suites): tests_to_run.update(get_test_suite(testarg, machine, compiler)) else: tests_to_run.add(CIME.utils.get_full_test_name(testarg, machine=machine, compiler=compiler)) for negation in negations: if (negation in acme_test_suites): tests_to_run -= set(get_test_suite(negation, machine, compiler)) else: fullname = CIME.utils.get_full_test_name(negation, machine=machine, compiler=compiler) if (fullname in tests_to_run): tests_to_run.remove(fullname) return list(sorted(tests_to_run))
def _interpret_value(value_str, filename): ############################################################################### comma_re = re.compile(r'\s*,\s*') dict_re = re.compile(r"^'(\S+)\s*->\s*(\S+)'") value_str = _normalize_lists(value_str) tokens = [ item.strip() for item in comma_re.split(value_str) if item.strip() != "" ] if ("->" in value_str): # dict rv = OrderedDict() for token in tokens: m = dict_re.match(token) expect( m is not None, "In file '{}', Dict entry '{}' does not match expected format". format(filename, token)) k, v = m.groups() rv[k] = _interpret_value(v, filename) return rv else: new_tokens = [] for token in tokens: if "*" in token: try: # the following ensure that the following to namelist settings trigger a match # nmlvalue = 1,1,1 versus nmlvalue = 3*1 sub_tokens = [item.strip() for item in token.split("*")] expect( len(sub_tokens) == 2, "Incorrect usage of multiplication in token '{}'". format(token)) new_tokens.extend([sub_tokens[1]] * int(sub_tokens[0])) except: # User probably did not intend to use the * operator as a namelist multiplier new_tokens.append(token) else: new_tokens.append(token) if "," in value_str or len(new_tokens) > 1: return new_tokens else: return new_tokens[0]
def run_phase(self): # pylint: disable=arguments-differ ########################################################################### self._activate_case2() should_match = (self._case.get_value("DESP_MODE") == "NOCHANGE") SystemTestsCompareTwo.run_phase(self, success_change=not should_match) # Look for expected coupler restart files logger = logging.getLogger(__name__) self._activate_case1() rundir1 = self._case.get_value("RUNDIR") self._activate_case2() rundir2 = self._case.get_value("RUNDIR") compare_ok = True multi_driver = self._case.get_value("MULTI_DRIVER") comps = self._case.get_values("COMP_CLASSES") for comp in comps: if not self._case.get_value("PAUSE_ACTIVE_{}".format(comp)): continue if comp == "CPL": if multi_driver: ninst = self._case.get_value("NINST_MAX") else: ninst = 1 else: ninst = self._case.get_value("NINST_{}".format(comp)) comp_name = self._case.get_value('COMP_{}'.format(comp)) for index in range(1, ninst + 1): if ninst == 1: rname = '*.{}.r.*'.format(comp_name) else: rname = '*.{}_{:04d}.r.*'.format(comp_name, index) restart_files_1 = glob.glob(os.path.join(rundir1, rname)) expect((len(restart_files_1) > 0), "No case1 restart files for {}".format(comp)) restart_files_2 = glob.glob(os.path.join(rundir2, rname)) expect((len(restart_files_2) > len(restart_files_1)), "No pause (restart) files found in case2 for {}".format( comp)) # Do cprnc of restart files. rfile1 = restart_files_1[len(restart_files_1) - 1] # rfile2 has to match rfile1 (same time string) parts = os.path.basename(rfile1).split(".") glob_str = "*.{}".format(".".join(parts[len(parts) - 4:])) restart_files_2 = glob.glob(os.path.join(rundir2, glob_str)) expect((len(restart_files_2) == 1), "Missing case2 restart file, {}", glob_str) rfile2 = restart_files_2[0] ok = cprnc(comp, rfile1, rfile2, self._case, rundir2)[0] logger.warning("CPRNC result for {}: {}".format( os.path.basename(rfile1), "PASS" if (ok == should_match) else "FAIL")) compare_ok = compare_ok and (should_match == ok) expect( compare_ok, "Not all restart files {}".format( "matched" if should_match else "failed to match"))
def get_workflow_jobs(self, machine, workflowid="default"): """ Return a list of jobs with the first element the name of the script and the second a dict of qualifiers for the job """ jobs = [] bnodes = [] findmore = True prepend = False while findmore: bnode = self.get_optional_child("workflow_jobs", attributes={"id":workflowid}) expect(bnode,"No workflow {} found in file {}".format(workflowid, self.filename)) if prepend: bnodes = [bnode] + bnodes else: bnodes.append(bnode) prepend = False workflow_attribs = self.attrib(bnode) if "prepend" in workflow_attribs: workflowid = workflow_attribs["prepend"] prepend = True elif "append" in workflow_attribs: workflowid = workflow_attribs["append"] else: findmore = False for bnode in bnodes: for jnode in self.get_children(root=bnode): if self.name(jnode) == "job": name = self.get(jnode, "name") jdict = {} for child in self.get_children(root=jnode): if self.name(child) == "runtime_parameters": attrib = self.attrib(child) if attrib and attrib == {'MACH' : machine}: for rtchild in self.get_children(root=child): jdict[self.name(rtchild)] = self.text(rtchild) elif not attrib: for rtchild in self.get_children(root=child): if self.name(rtchild) not in jdict: jdict[self.name(rtchild)] = self.text(rtchild) else: jdict[self.name(child)] = self.text(child) jobs.append((name, jdict)) return jobs
def _main_func(options, work_dir): ############################################################################### """Construct grids html from an XML file.""" # Initialize a variables for the html template all_grids = dict() model_version = options.version[0] # read in txtfile with the grids definitions txtfile = options.txtfile[0] expect((txtfile), 'Cannot find query_config --grids --long output text file %s' %txtfile) # read the grids file looking for aliases and load the all_grids dict with open( txtfile, 'r') as f: for line in f: if "alias:" in line: line = line.rstrip('\n') alias = line.split(':') alias.pop(0) key = alias[0].lstrip() all_grids[key] = '' else: all_grids[key] += line f.close() # load up jinja template templateLoader = jinja2.FileSystemLoader( searchpath='{0}/templates'.format(work_dir) ) templateEnv = jinja2.Environment( loader=templateLoader ) # TODO - get the cesm_version for the CIME root tmplFile = 'griddef2html.tmpl' template = templateEnv.get_template( tmplFile ) templateVars = { 'today' : _now, 'model_version' : model_version, 'all_grids' : all_grids } # render the template grid_tmpl = template.render( templateVars ) # write the output file with open( options.htmlfile[0], 'w') as html: html.write(grid_tmpl) return 0
def handle_easy_conflicts(is_merge): ############################################################################### conflicting_files = run_cmd_no_fail( "git diff --name-only --diff-filter=U").splitlines() if not conflicting_files: expect( False, "Merge appears to have failed for reasons other than merge conflicts" ) rv = [] for conflicting_file in conflicting_files: able_to_handle = handle_easy_conflict(conflicting_file, is_merge) if not able_to_handle: rv.append(conflicting_file) return rv
def wait_for_tests_impl(test_paths, no_wait=False, check_throughput=False, check_memory=False, ignore_namelists=False, ignore_memleak=False, no_run=False): ############################################################################### results = queue.Queue() for test_path in test_paths: t = threading.Thread(target=wait_for_test, args=(test_path, results, not no_wait, check_throughput, check_memory, ignore_namelists, ignore_memleak, no_run)) t.daemon = True t.start() while threading.active_count() > 1: time.sleep(1) test_results = {} completed_test_paths = [] while (not results.empty()): test_name, test_path, test_status, test_phase = results.get() if (test_name in test_results): prior_path, prior_status, _ = test_results[test_name] if (test_status == prior_status): logging.warning( "Test name '{}' was found in both '{}' and '{}'".format( test_name, test_path, prior_path)) else: raise CIMEError( "Test name '{}' was found in both '{}' and '{}' with different results" .format(test_name, test_path, prior_path)) expect(test_name is not None, "Failed to get test name for test_path: {}".format(test_path)) test_results[test_name] = (test_path, test_status, test_phase) completed_test_paths.append(test_path) expect( set(test_paths) == set(completed_test_paths), "Missing results for test paths: {}".format( set(test_paths) - set(completed_test_paths))) return test_results
def update_acme_tests(xml_file, categories, platform=None): ############################################################################### # Retrieve all supported ACME platforms, killing the third entry (MPI lib) # for the moment. supported_platforms = [p[:2] for p in find_all_supported_platforms()] # Fish all of the existing machine/compiler combos out of the XML file. if (platform is not None): platforms = [tuple(platform.split(","))] else: platforms = find_all_platforms(xml_file) # Prune the non-supported platforms from our list. for p in platforms: if p not in supported_platforms: logging.info("pruning unsupported platform %s" % repr(p)) platforms = [p for p in platforms if p in supported_platforms] manage_xml_entries = os.path.join(CIME.utils.get_cime_root(), "scripts", "manage_testlists") expect( os.path.isfile(manage_xml_entries), "Couldn't find manage_testlists, expected it to be here: '%s'" % manage_xml_entries) for category in categories: # Remove any existing acme test category from the file. if (platform is None): run_cmd_no_fail( "%s -model acme -component allactive -removetests -category %s" % (manage_xml_entries, category)) else: run_cmd_no_fail( "%s -model acme -component allactive -removetests -category %s -machine %s -compiler %s" % (manage_xml_entries, category, platforms[0][0], platforms[0][1])) # Generate a list of test entries corresponding to our suite at the top # of the file. new_test_file = generate_acme_test_entries(category, platforms) run_cmd_no_fail( "%s -model acme -component allactive -addlist -file %s -category %s" % (manage_xml_entries, new_test_file, category)) os.unlink(new_test_file) print "SUCCESS"
def run_phase(self): ########################################################################### SystemTestsCompareTwo.run_phase(self) # Look for expected coupler restart files logger = logging.getLogger(__name__) self._activate_case1() rundir1 = self._case.get_value("RUNDIR") self._activate_case2() rundir2 = self._case.get_value("RUNDIR") should_match = (self._case.get_value("DESP_MODE") == "NOCHANGE") compare_ok = True pause_comps = self._case.get_value("PAUSE_COMPONENT_LIST") expect((pause_comps != 'none'), "Pause/Resume (PRE) test has no pause components") if pause_comps == 'all': pause_comps = self._case.get_values("COMP_CLASSES") else: pause_comps = pause_comps.split(':') for comp in pause_comps: comp_name = self._case.get_value('COMP_%s' % comp.upper()) rname = '*.%s.r.*' % comp_name restart_files_1 = glob.glob(os.path.join(rundir1, rname)) expect((len(restart_files_1) > 0), "No case1 restart files for %s" % comp) restart_files_2 = glob.glob(os.path.join(rundir2, rname)) expect((len(restart_files_2) > len(restart_files_1)), "No pause (restart) files found in case2 for %s" % comp) # Do cprnc of restart files. rfile1 = restart_files_1[len(restart_files_1) - 1] # rfile2 has to match rfile1 (same time string) parts = os.path.basename(rfile1).split(".") glob_str = "*.%s" % ".".join(parts[len(parts) - 4:]) restart_files_2 = glob.glob(os.path.join(rundir2, glob_str)) expect((len(restart_files_2) == 1), "Missing case2 restart file, %s", glob_str) rfile2 = restart_files_2[0] ok, out = cprnc(comp, rfile1, rfile2, self._case, rundir2) # pylint: disable=unused-variable logger.warning("CPRNC result for %s: %s" % (os.path.basename(rfile1), "PASS" if (ok == should_match) else "FAIL")) compare_ok = compare_ok and (should_match == ok) expect( compare_ok, "Not all restart files %s" % ("matched" if should_match else "failed to match"))
def __init__(self): """ initialize an object """ expect( get_model() == "cesm", "testreport is only meant to populate the CESM test database.", ) self.root = None GenericXML.__init__( self, root_name_override="testrecord", read_only=False, infile="TestRecord.xml", )
def get_test_suite(suite, machine=None, compiler=None, skip_inherit=False, skip_tests=None): ############################################################################### """ Return a list of FULL test names for a suite. """ expect(suite in get_test_suites(), "Unknown test suite: '{}'".format(suite)) machobj = Machines(machine=machine) machine = machobj.get_machine_name() if compiler is None: compiler = machobj.get_default_compiler() expect( machobj.is_valid_compiler(compiler), "Compiler {} not valid for machine {}".format(compiler, machine), ) inherits_from, _, _, tests_raw = get_test_data(suite) tests = [] for item in tests_raw: expect( isinstance(item, str), "Bad type of test {}, expected string".format(item), ) test_mods = None test_components = item.split(".") expect(len(test_components) in [3, 4], "Bad test name {}".format(item)) if len(test_components) == 4: test_name = ".".join(test_components[:-1]) test_mods = test_components[-1] else: test_name = item if not skip_tests or not test_name in skip_tests: tests.append( CIME.utils.get_full_test_name( test_name, machine=machine, compiler=compiler, testmods_string=test_mods, )) if not skip_inherit: for inherits in inherits_from: inherited_tests = get_test_suite(inherits, machine, compiler) for inherited_test in inherited_tests: if inherited_test not in tests: tests.append(inherited_test) return tests
def get_node(self, nodename, attributes=None, root=None, xpath=None): """ Get an xml element matching nodename with optional attributes. Error unless exactly one match. """ nodes = self.get_nodes(nodename, attributes=attributes, root=root, xpath=xpath) expect( len(nodes) == 1, "Incorrect number of matches, %d, for nodename '%s' and attrs '%s' in file '%s'" % (len(nodes), nodename, attributes, self.filename)) return nodes[0]
def query_grids(files, long_output, xml=False): """ query all grids. """ config_file = files.get_value("GRIDS_SPEC_FILE") expect( os.path.isfile(config_file), "Cannot find config_file {} on disk".format(config_file), ) grids = Grids(config_file) if xml: print("{}".format(grids.get_raw_record().decode("UTF-8"))) elif long_output: grids.print_values(long_output=long_output) else: grids.print_values()
def case_lt_archive(case): ############################################################################### caseroot = case.get_value("CASEROOT") # max number of threads needed by scripts os.environ["maxthrds"] = "1" # document start append_status("lt_archive starting", caseroot=caseroot, sfile="CaseStatus") # determine status of run and short term archiving runComplete = does_file_have_string(os.path.join(caseroot, "CaseStatus"), "Run SUCCESSFUL") staComplete = does_file_have_string(os.path.join(caseroot, "CaseStatus"), "st_archiving completed") # set up envrionment vars and call the lt_archive.sh script if runComplete and staComplete: os.environ["DOUT_S_ROOT"] = case.get_value("DOUT_S_ROOT") os.environ["DOUT_L_MSROOT"] = case.get_value("DOUT_L_MSROOT") os.environ["DOUT_L_HPSS_ACCNT"] = case.get_value("DOUT_L_HPSS_ACCNT") lid = time.strftime("%y%m%d-%H%M%S") lt_archive = LTArchive(case.get_value("MACH")) lt_archive_args = lt_archive.get_lt_archive_args() if lt_archive_args is None: lt_archive_args = " " cmd = os.path.join(caseroot, "Toolslt_archive.sh") \ + lt_archive_args + "ltArchiveStatus." + lid + " 2>&1" run_cmd_no_fail(cmd, from_dir=caseroot) else: logger.warn("runComplete %s staComplete %s" % (runComplete, staComplete)) expect( False, "lt_archive: run or st_archive is not yet complete or was not successful." "Unable to perform long term archive...") # document completion append_status("lt_archive completed", caseroot=caseroot, sfile="CaseStatus") return True
def verify_chksum(input_data_root, rundir, filename, isdirectory): """ For file in filename perform a chksum and compare the result to that stored in the local checksumfile, if isdirectory chksum all files in the directory of form *.* """ hashfile = os.path.join(rundir, local_chksum_file) if not chksum_hash: if not os.path.isfile(hashfile): logger.warning( "Failed to find or download file {}".format(hashfile)) return with open(hashfile) as fd: lines = fd.readlines() for line in lines: fchksum, fname = line.split() if fname in chksum_hash: expect( chksum_hash[fname] == fchksum, " Inconsistent hashes in chksum for file {}".format( fname), ) else: chksum_hash[fname] = fchksum if isdirectory: filenames = glob.glob(os.path.join(filename, "*.*")) else: filenames = [filename] for fname in filenames: if not os.sep in fname: continue chksum = md5(os.path.join(input_data_root, fname)) if chksum_hash: if not fname in chksum_hash: logger.warning( "Did not find hash for file {} in chksum file {}".format( filename, hashfile)) else: expect( chksum == chksum_hash[fname], "chksum mismatch for file {} expected {} found {}".format( os.path.join(input_data_root, fname), chksum, chksum_hash[fname]), )
def set_batch_system(self, batch_system, machine=None): """ Sets the batch system block in the Batch object """ machine = machine if machine is not None else self.machine if self.batch_system != batch_system or self.batch_system_node is None: nodes = self.get_nodes("batch_system",{"type" : batch_system}) for node in nodes: mach = node.get("MACH") if mach is None: self.batch_system_node = node elif mach == machine: self.machine = machine self.machine_node = node expect(self.batch_system_node is not None, "No batch system '%s' found" % batch_system) return batch_system
def get_test_data(suite): ############################################################################### """ For a given suite, returns (inherit, time, share, tests) """ raw_dict = _ALL_TESTS[suite] for key in raw_dict.keys(): expect( key in ["inherit", "time", "share", "tests"], "Unexpected test key '{}'".format(key), ) return ( _get_key_data(raw_dict, "inherit", tuple), _get_key_data(raw_dict, "time", str), _get_key_data(raw_dict, "share", bool), _get_key_data(raw_dict, "tests", tuple), )
def _signal_handler(signum, _): name = "Unknown" for signame in _iter_signal_names(): if signum == getattr(signal, signame): name = signame # Terminate children proc_ids = find_proc_id(children_only=True) for proc_id in proc_ids: try: os.kill(proc_id, signal.SIGKILL) except OSError: # If the batch system killed the entire process group, these # processes might already be dying pass # Throw an exception so SystemTest infrastructure can handle this error expect(False, "Job killed due to receiving signal {:d} ({})".format(signum, name))
def __init__(self): """ initialize an object >>> files = Files() >>> files.get_value('CASEFILE_HEADERS',resolved=False) '$CIMEROOT/config/config_headers.xml' """ cimeroot = get_cime_root() infile = os.path.join(cimeroot, "config", get_model(), "config_files.xml") expect(os.path.isfile(infile), "Could not find or open file {}".format(infile)) schema = os.path.join(cimeroot, "config", "xml_schemas", "entry_id.xsd") EntryID.__init__(self, infile, schema=schema) config_files_override = os.path.join(os.path.dirname(cimeroot),".config_files.xml") # .config_file.xml at the top level may overwrite COMP_ROOT_DIR_ nodes in config_files if os.path.isfile(config_files_override): self.read(config_files_override) self.overwrite_existing_entries()
def get_compset_components(self): #If are doing a create_clone then, self._compsetname is not set yet components = [] compset = self.get_value("COMPSET") if compset is None: compset = self._compsetname expect(compset is not None, "ERROR: compset is not set") # the first element is always the date operator - skip it elements = compset.split('_')[1:] # pylint: disable=maybe-no-member for element in elements: # ignore the possible BGC or TEST modifier if element.startswith("BGC%") or element.startswith("TEST"): continue else: element_component = element.split('%')[0].lower() element_component = re.sub(r'[0-9]*', "", element_component) components.append(element_component) return components
def wait_for_tests_impl(test_paths, no_wait=False, check_throughput=False, check_memory=False, ignore_namelists=False, ignore_memleak=False): ############################################################################### results = Queue.Queue() for test_path in test_paths: t = threading.Thread(target=wait_for_test, args=(test_path, results, not no_wait, check_throughput, check_memory, ignore_namelists, ignore_memleak)) t.daemon = True t.start() while threading.active_count() > 1: time.sleep(1) test_results = {} completed_test_paths = [] while (not results.empty()): test_name, test_path, test_status = results.get() if (test_name in test_results): prior_path, prior_status = test_results[test_name] if (test_status == prior_status): logging.warning( "Test name '%s' was found in both '%s' and '%s'" % (test_name, test_path, prior_path)) else: raise SystemExit( "Test name '%s' was found in both '%s' and '%s' with different results" % (test_name, test_path, prior_path)) test_results[test_name] = (test_path, test_status) completed_test_paths.append(test_path) expect( set(test_paths) == set(completed_test_paths), "Missing results for test paths: %s" % (set(test_paths) - set(completed_test_paths))) return test_results
def get_recommended_test_time(test_full_name): ############################################################################### """ >>> get_recommended_test_time("ERS.f19_g16_rx1.A.melvin_gnu") '0:10:00' >>> get_recommended_test_time("ERP_Ln9.ne30_ne30.FC5.melvin_gun.cam-outfrq9s") '0:45:00' >>> get_recommended_test_time("PET_Ln9.ne30_ne30.FC5.sandiatoss3_intel.cam-outfrq9s") '03:00:00' >>> get_recommended_test_time("PET_Ln20.ne30_ne30.FC5.sandiatoss3_intel.cam-outfrq9s") >>> """ _, _, _, _, machine, compiler, _ = CIME.utils.parse_test_name(test_full_name) expect(machine is not None, "{} is not a full test name".format(test_full_name)) best_time = None suites = get_test_suites() for suite in suites: _, rec_time, tests_raw = _ALL_TESTS[suite] for item in tests_raw: test_mod = None if (isinstance(item, six.string_types)): test_name = item else: test_name = item[0] if (len(item) == 2): test_mod = item[1] else: test_mod_machines = [item[2]] if isinstance(item[2], six.string_types) else item[2] if (machine in test_mod_machines): test_mod = item[1] full_test = CIME.utils.get_full_test_name(test_name, machine=machine, compiler=compiler, testmod=test_mod) if full_test == test_full_name and rec_time is not None: if best_time is None or \ convert_to_seconds(rec_time) < convert_to_seconds(best_time): best_time = rec_time return best_time
def make_fake_teststatus(path, testname, status, phase): expect(phase in test_status.CORE_PHASES, "Bad phase '%s'" % phase) with test_status.TestStatus(test_dir=path, test_name=testname) as ts: for core_phase in test_status.CORE_PHASES: if core_phase == phase: ts.set_status( core_phase, status, comments=("time=42" if phase == test_status.RUN_PHASE else ""), ) break else: ts.set_status( core_phase, test_status.TEST_PASS_STATUS, comments=("time=42" if phase == test_status.RUN_PHASE else ""), )