def run(self, skip_pnl=False): """ Do NOT override this method, this method is the framework that controls the run phase. run_phase is the extension point that subclasses should use. """ success = True start_time = time.time() self._skip_pnl = skip_pnl try: self._resetup_case(RUN_PHASE) with self._test_status: self._test_status.set_status(RUN_PHASE, TEST_PEND_STATUS) self.run_phase() if self._case.get_value("GENERATE_BASELINE"): self._phase_modifying_call(GENERATE_PHASE, self._generate_baseline) if self._case.get_value("COMPARE_BASELINE"): self._phase_modifying_call(BASELINE_PHASE, self._compare_baseline) self._phase_modifying_call(MEMCOMP_PHASE, self._compare_memory) self._phase_modifying_call(THROUGHPUT_PHASE, self._compare_throughput) self._phase_modifying_call(MEMLEAK_PHASE, self._check_for_memleak) self._phase_modifying_call(STARCHIVE_PHASE, self._st_archive_case_test) except BaseException as e: # We want KeyboardInterrupts to generate FAIL status success = False if isinstance(e, CIMEError): # Don't want to print stacktrace for a model failure since that # is not a CIME/infrastructure problem. excmsg = str(e) else: excmsg = "Exception during run:\n{}\n{}".format(str(e), traceback.format_exc()) append_testlog(excmsg) raise finally: # Writing the run status should be the very last thing due to wait_for_tests time_taken = time.time() - start_time status = TEST_PASS_STATUS if success else TEST_FAIL_STATUS with self._test_status: self._test_status.set_status(RUN_PHASE, status, comments=("time={:d}".format(int(time_taken)))) if success and get_model() == "e3sm": save_test_time(self._case.get_value("BASELINE_ROOT"), self._casebaseid, time_taken) if get_model() == "cesm" and self._case.get_value("GENERATE_BASELINE"): baseline_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASEGEN_CASE")) generate_teststatus(self._caseroot, baseline_dir) # We return success if the run phase worked; memleaks, diffs will not be taken into account # with this return value. return success
def test_f_createnewcase_with_user_compset(self): cls = self.__class__ testdir = os.path.join(cls._testroot, "testcreatenewcase_with_user_compset") if os.path.exists(testdir): shutil.rmtree(testdir) cls._testdirs.append(testdir) if utils.get_model() == "cesm": if utils.get_cime_default_driver() == "nuopc": pesfile = os.path.join( utils.get_src_root(), "components", "cmeps", "cime_config", "config_pes.xml", ) else: pesfile = os.path.join( utils.get_src_root(), "components", "cpl7", "driver", "cime_config", "config_pes.xml", ) else: pesfile = os.path.join( utils.get_src_root(), "driver-mct", "cime_config", "config_pes.xml" ) args = ( "--case %s --compset 2000_SATM_XLND_SICE_SOCN_XROF_XGLC_SWAV --pesfile %s --res f19_g16 --output-root %s --handle-preexisting-dirs=r" % (testdir, pesfile, cls._testroot) ) if utils.get_model() == "cesm": args += " --run-unsupported" if self.TEST_COMPILER is not None: args += " --compiler %s" % self.TEST_COMPILER if self.TEST_MPILIB is not None: args = args + " --mpilib %s" % self.TEST_MPILIB args += f" --machine {self.MACHINE.get_machine_name()}" self.run_cmd_assert_result( "%s/create_newcase %s" % (self.SCRIPT_DIR, args), from_dir=self.SCRIPT_DIR ) self.run_cmd_assert_result("./case.setup", from_dir=testdir) self.run_cmd_assert_result("./case.build", from_dir=testdir) cls._do_teardown.append(testdir)
def bless_history(test_name, testcase_dir_for_test, baseline_name, baseline_root, compiler, report_only, force): ############################################################################### with Case(testcase_dir_for_test) as case: if get_model() == "acme": baseline_full_dir = os.path.join(baseline_root, compiler, baseline_name, case.get_value("CASEBASEID")) else: baseline_full_dir = os.path.join(baseline_root, baseline_name, case.get_value("CASEBASEID")) result, comments = compare_baseline(case, baseline_dir=baseline_full_dir, outfile_suffix=None) if result: logging.info("Diff appears to have been already resolved.") return True, None else: print comments if (not report_only and (force or raw_input("Update this diff (y/n)? ").upper() in ["Y", "YES"])): result, comments = generate_baseline( case, baseline_dir=baseline_full_dir) if not result: logging.warning("Hist file bless FAILED for test %s" % test_name) return False, "Generate baseline failed: %s" % comments else: print comments return True, None else: return True, None
def test_cime_case_test_walltime_mgmt_3(self): if utils.get_model() != "e3sm": self.skipTest( "Skipping walltime test. Depends on E3SM batch settings") test_name = "ERS_P64.f19_g16_rx1.A" casedir = self._create_test( [ "--no-setup", "--machine=blues", "--non-local", "--walltime=0:10:00", test_name, ], test_id=self._baseline_name, env_changes="unset CIME_GLOBAL_WALLTIME &&", ) result = self.run_cmd_assert_result( "./xmlquery JOB_WALLCLOCK_TIME -N --subgroup=case.test --value", from_dir=casedir, ) self.assertEqual(result, "00:10:00") result = self.run_cmd_assert_result( "./xmlquery JOB_QUEUE -N --subgroup=case.test --value", from_dir=casedir) self.assertEqual(result, "biggpu") # Not smart enough to select faster queue
def build_phase(self, sharedlib_only=False, model_only=False): # Subtle issue: case1 is already in a writeable state since it tends to be opened # with a with statement in all the API entrances in CIME. case2 was created via clone, # not a with statement, so it's not in a writeable state, so we need to use a with # statement here to put it in a writeable state. with self._case2: if self._separate_builds: self._activate_case1() self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) self._activate_case2() # Although we're doing separate builds, it still makes sense # to share the sharedlibroot area with case1 so we can reuse # pieces of the build from there. if get_model() != "e3sm": # We need to turn off this change for E3SM because it breaks # the MPAS build system self._case2.set_value("SHAREDLIBROOT", self._case1.get_value("SHAREDLIBROOT")) self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) else: self._activate_case1() self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) # pio_typename may be changed during the build if the default is not a # valid value for this build, update case2 to reflect this change for comp in self._case1.get_values("COMP_CLASSES"): comp_pio_typename = "{}_PIO_TYPENAME".format(comp) self._case2.set_value(comp_pio_typename, self._case1.get_value(comp_pio_typename)) # The following is needed when _case_two_setup has a case_setup call # despite sharing the build (e.g., to change NTHRDS) self._case2.set_value("BUILD_COMPLETE",True)
def uses_kokkos(case): ############################################################################### cam_target = case.get_value("CAM_TARGET") # atm_comp = case.get_value("COMP_ATM") # scream does not use the shared kokkoslib for now return get_model() == "e3sm" and cam_target in ("preqx_kokkos", "theta-l", "theta-l_kokkos")
def _create_caseroot_sourcemods(self): components = self.get_compset_components() for component in components: directory = os.path.join(self._caseroot,"SourceMods","src.%s"%component) if not os.path.exists(directory): os.makedirs(directory) directory = os.path.join(self._caseroot, "SourceMods", "src.share") if not os.path.exists(directory): os.makedirs(directory) directory = os.path.join(self._caseroot,"SourceMods","src.drv") if not os.path.exists(directory): os.makedirs(directory) if get_model() == "cesm": # Note: this is CESM specific, given that we are referencing cism explitly if "cism" in components: directory = os.path.join(self._caseroot, "SourceMods", "src.cism", "glimmer-cism") if not os.path.exists(directory): os.makedirs(directory) readme_file = os.path.join(directory, "README") str_to_write = """ Put source mods for the glimmer-cism library in the glimmer-cism subdirectory This includes any files that are in the glimmer-cism subdirectory of $cimeroot/../components/cism Anything else (e.g., mods to source_glc or drivers) goes in this directory, NOT in glimmer-cism/""" with open(readme_file, "w") as fd: fd.write(str_to_write)
def _get_procs_needed(self, test, phase, threads_in_flight=None, no_batch=False): ########################################################################### if phase == RUN_PHASE and (self._no_batch or no_batch): test_dir = self._get_test_dir(test) total_pes = int(run_cmd_no_fail("./xmlquery TOTALPES --value", from_dir=test_dir)) threads = eval(run_cmd_no_fail("./xmlquery NTHRDS --value", from_dir=test_dir)) max_threads = 0 for item in threads: _, comp_threads = item.split(":") comp_threads = int(comp_threads) if comp_threads > max_threads: max_threads = comp_threads max_cores = total_pes * max_threads return max_cores elif (phase == SHAREDLIB_BUILD_PHASE): if get_model() == "cesm": # Will force serialization of sharedlib builds # TODO - instead of serializing, compute all library configs needed and build # them all in parallel for _, _, running_phase in threads_in_flight.values(): if (running_phase == SHAREDLIB_BUILD_PHASE): return self._proc_pool + 1 return 1 elif (phase == MODEL_BUILD_PHASE): # Model builds now happen in parallel return self._model_build_cost else: return 1
def bless_namelists(test_name, test_dir, report_only, force, baseline_name, baseline_root): ############################################################################### # Be aware that restart test will overwrite the original namelist files # with versions of the files that should not be blessed. This forces us to # re-run create_test. # Update namelist files logger.info("Test '{}' had namelist diff".format(test_name)) if (not report_only and (force or six.moves.input("Update namelists (y/n)? ").upper() in ["Y", "YES"])): if baseline_name is None: stat, baseline_name, _ = run_cmd("./xmlquery --value BASELINE_NAME_CMP", from_dir=test_dir) if stat != 0 or not baseline_name: baseline_name = CIME.utils.get_current_branch(repo=CIME.utils.get_cime_root()) if baseline_root is None: stat, baseline_root, _ = run_cmd("./xmlquery --value BASELINE_ROOT", from_dir=test_dir) if stat != 0 or not baseline_root: return False, "Could not determine baseline root" create_test_gen_args = " -g {} ".format(baseline_name if get_model() == "cesm" else " -g -b {} ".format(baseline_name)) stat, out, _ = run_cmd("{}/create_test {} -n {} --baseline-root {} -o".format(get_scripts_root(), test_name, create_test_gen_args, baseline_root), combine_output=True) if stat != 0: return False, "Namelist regen failed: '{}'".format(out) else: return True, None else: return True, None
def build_phase(self, sharedlib_only=False, model_only=False): if self._separate_builds: self._activate_case1() self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) self._activate_case2() # Although we're doing separate builds, it still makes sense # to share the sharedlibroot area with case1 so we can reuse # pieces of the build from there. if get_model() != "e3sm": # We need to turn off this change for E3SM because it breaks # the MPAS build system self._case2.set_value("SHAREDLIBROOT", self._case1.get_value("SHAREDLIBROOT")) self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) else: self._activate_case1() self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) # pio_typename may be changed during the build if the default is not a # valid value for this build, update case2 to reflect this change for comp in self._case1.get_values("COMP_CLASSES"): comp_pio_typename = "{}_PIO_TYPENAME".format(comp) self._case2.set_value(comp_pio_typename, self._case1.get_value(comp_pio_typename)) # The following is needed when _case_two_setup has a case_setup call # despite sharing the build (e.g., to change NTHRDS) self._case2.set_value("BUILD_COMPLETE",True) self._case2.flush()
def _build_model_thread( config_dir, compclass, compname, caseroot, libroot, bldroot, incroot, file_build, thread_bad_results, smp, compiler, ): ############################################################################### logger.info("Building {} with output to {}".format(compclass, file_build)) t1 = time.time() cmd = os.path.join(caseroot, "SourceMods", "src." + compname, "buildlib") if os.path.isfile(cmd): logger.warning( "WARNING: using local buildlib script for {}".format(compname)) else: cmd = os.path.join(config_dir, "buildlib") expect(os.path.isfile(cmd), "Could not find buildlib for {}".format(compname)) compile_cmd = "COMP_CLASS={compclass} COMP_NAME={compname} {cmd} {caseroot} {libroot} {bldroot} ".format( compclass=compclass, compname=compname, cmd=cmd, caseroot=caseroot, libroot=libroot, bldroot=bldroot, ) if get_model() != "ufs": compile_cmd = "SMP={} {}".format(stringify_bool(smp), compile_cmd) if is_python_executable(cmd): logging_options = get_logging_options() if logging_options != "": compile_cmd = compile_cmd + logging_options with open(file_build, "w") as fd: stat = run_cmd(compile_cmd, from_dir=bldroot, arg_stdout=fd, arg_stderr=subprocess.STDOUT)[0] if stat != 0: thread_bad_results.append( "BUILD FAIL: {}.buildlib failed, cat {}".format( compname, file_build)) analyze_build_log(compclass, file_build, compiler) for mod_file in glob.glob(os.path.join(bldroot, "*_[Cc][Oo][Mm][Pp]_*.mod")): safe_copy(mod_file, incroot) t2 = time.time() logger.info("{} built in {:f} seconds".format(compname, (t2 - t1)))
def test_cime_case_test_custom_project(self): test_name = "ERS_P1.f19_g16_rx1.A" # have to use a machine both models know and one that doesn't put PROJECT in any key paths if utils.get_model() == "e3sm": machine = "mappy" else: machine = "melvin" compiler = "gnu" casedir = self._create_test( [ "--no-setup", "--machine={}".format(machine), "--compiler={}".format(compiler), "--project=testproj", test_name, "--mpilib=mpi-serial", ], test_id=self._baseline_name, env_changes="unset CIME_GLOBAL_WALLTIME &&", ) result = self.run_cmd_assert_result( "./xmlquery --value PROJECT --subgroup=case.test", from_dir=casedir) self.assertEqual(result, "testproj")
def compare_baseline(case, baseline_dir=None, outfile_suffix=""): """ compare the current test output to a baseline result case - The case containing the hist files to be compared against baselines baseline_dir - Optionally, specify a specific baseline dir, otherwise it will be computed from case config outfile_suffix - if non-blank, then the cprnc output file name ends with this suffix (with a '.' added before the given suffix). if None, no output file saved. returns (SUCCESS, comments) SUCCESS means all hist files matched their corresponding baseline """ rundir = case.get_value("RUNDIR") if baseline_dir is None: baselineroot = case.get_value("BASELINE_ROOT") basecmp_dir = os.path.join(baselineroot, case.get_value("BASECMP_CASE")) dirs_to_check = (baselineroot, basecmp_dir) else: basecmp_dir = baseline_dir dirs_to_check = (basecmp_dir,) for bdir in dirs_to_check: if not os.path.isdir(bdir): return False, "ERROR {} baseline directory '{}' does not exist".format(TEST_NO_BASELINES_COMMENT,bdir) success, comments = _compare_hists(case, rundir, basecmp_dir, outfile_suffix=outfile_suffix) if get_model() == "e3sm": bless_log = os.path.join(basecmp_dir, BLESS_LOG_NAME) if os.path.exists(bless_log): last_line = open(bless_log, "r").readlines()[-1] comments += "\n Most recent bless: {}".format(last_line) return success, comments
def _setup_cs_files(self): ########################################################################### try: python_libs_root = CIME.utils.get_python_libs_root() template_file = os.path.join(python_libs_root, "cs.status.template") template = open(template_file, "r").read() template = template.replace("<PATH>", os.path.join(self._cime_root,"scripts","Tools")).replace\ ("<TESTID>", self._test_id).replace\ ("<TESTROOT>", self._test_root) if not os.path.exists(self._test_root): os.makedirs(self._test_root) cs_status_file = os.path.join(self._test_root, "cs.status.{}".format(self._test_id)) with open(cs_status_file, "w") as fd: fd.write(template) os.chmod( cs_status_file, os.stat(cs_status_file).st_mode | stat.S_IXUSR | stat.S_IXGRP) template_file = os.path.join(python_libs_root, "cs.submit.template") template = open(template_file, "r").read() setup_cmd = "./case.setup" if self._no_setup else ":" build_cmd = "./case.build" if self._no_build else ":" test_cmd = "./case.submit" template = template.replace("<SETUP_CMD>", setup_cmd).\ replace("<BUILD_CMD>", build_cmd).\ replace("<RUN_CMD>", test_cmd).\ replace("<TESTID>", self._test_id) if self._no_run: cs_submit_file = os.path.join( self._test_root, "cs.submit.{}".format(self._test_id)) with open(cs_submit_file, "w") as fd: fd.write(template) os.chmod( cs_submit_file, os.stat(cs_submit_file).st_mode | stat.S_IXUSR | stat.S_IXGRP) if get_model() == "cesm": template_file = os.path.join(python_libs_root, "testreporter.template") template = open(template_file, "r").read() template = template.replace( "<PATH>", os.path.join(self._cime_root, "scripts", "Tools")) testreporter_file = os.path.join(self._test_root, "testreporter") with open(testreporter_file, "w") as fd: fd.write(template) os.chmod( testreporter_file, os.stat(testreporter_file).st_mode | stat.S_IXUSR | stat.S_IXGRP) except Exception as e: logger.warning("FAILED to set up cs files: {}".format(str(e)))
def setup(self, env_archive, components, files=None): if files is None: files = Files() components_node = env_archive.make_child("components", attributes={"version":"2.0"}) model = get_model() if 'drv' not in components: components.append('drv') if 'dart' not in components and model == 'cesm': components.append('dart') for comp in components: infile = files.get_value("ARCHIVE_SPEC_FILE", {"component":comp}) if infile is not None and os.path.isfile(infile): arch = Archive(infile=infile, files=files) specs = arch.get_optional_child(name="comp_archive_spec", attributes={"compname":comp}) else: if infile is None: logger.debug("No archive file defined for component {}".format(comp)) else: logger.debug("Archive file {} for component {} not found".format(infile,comp)) specs = self.get_optional_child(name="comp_archive_spec", attributes={"compname":comp}) if specs is None: logger.debug("No archive specs found for component {}".format(comp)) else: logger.debug("adding archive spec for {}".format(comp)) env_archive.add_child(specs, root=components_node)
def _create_caseroot_sourcemods(self): components = self.get_compset_components() for component in components: directory = os.path.join(self._caseroot, "SourceMods", "src.%s" % component) if not os.path.exists(directory): os.makedirs(directory) directory = os.path.join(self._caseroot, "SourceMods", "src.share") if not os.path.exists(directory): os.makedirs(directory) directory = os.path.join(self._caseroot, "SourceMods", "src.drv") if not os.path.exists(directory): os.makedirs(directory) if get_model() == "cesm": # Note: this is CESM specific, given that we are referencing cism explitly if "cism" in components: directory = os.path.join(self._caseroot, "SourceMods", "src.cism", "glimmer-cism") if not os.path.exists(directory): os.makedirs(directory) readme_file = os.path.join(directory, "README") str_to_write = """ Put source mods for the glimmer-cism library in the glimmer-cism subdirectory This includes any files that are in the glimmer-cism subdirectory of $cimeroot/../components/cism Anything else (e.g., mods to source_glc or drivers) goes in this directory, NOT in glimmer-cism/""" with open(readme_file, "w") as fd: fd.write(str_to_write)
def simple_test(self, testdir, expected_results, extra_args="", build_name=None): # Need these flags to test dashboard if e3sm if utils.get_model() == "e3sm" and build_name is not None: extra_args += " -b %s" % build_name expected_stat = 0 for expected_result in expected_results: if not (expected_result == "PASS" or (expected_result == "PEND" and "-n" in extra_args)): expected_stat = utils.TESTS_FAILED_ERR_CODE output = self.run_cmd_assert_result( "%s/wait_for_tests -p ACME_test */TestStatus %s" % (self.TOOLS_DIR, extra_args), from_dir=testdir, expected_stat=expected_stat, ) lines = [ line for line in output.splitlines() if (line.startswith("PASS") or line.startswith("FAIL") or line.startswith("PEND")) ] self.assertEqual(len(lines), len(expected_results)) for idx, line in enumerate(lines): testname, status = test_utils.parse_test_status(line) self.assertEqual(status, expected_results[idx]) self.assertEqual(testname, "Test_%d" % idx)
def test_cime_case_test_walltime_mgmt_8(self): if utils.get_model() != "e3sm": self.skipTest( "Skipping walltime test. Depends on E3SM batch settings") test_name = "SMS_P25600.f19_g16_rx1.A" machine, compiler = "theta", "gnu" casedir = self._create_test( [ "--no-setup", "--machine={}".format(machine), "--compiler={}".format(compiler), "--project e3sm", test_name, ], test_id=self._baseline_name, env_changes="unset CIME_GLOBAL_WALLTIME &&", ) result = self.run_cmd_assert_result( "./xmlquery JOB_WALLCLOCK_TIME --subgroup=case.test --value", from_dir=casedir, ) self.assertEqual(result, "09:00:00") result = self.run_cmd_assert_result( "./xmlquery JOB_QUEUE --subgroup=case.test --value", from_dir=casedir) self.assertEqual(result, "default")
def setup(self, env_archive, components, files=None): if files is None: files = Files() components_node = ET.Element("components") components_node.set("version", "2.0") model = get_model() if 'cpl' not in components: components.append('cpl') if 'dart' not in components and model == 'cesm': components.append('dart') for comp in components: infile = files.get_value("ARCHIVE_SPEC_FILE", {"component":comp}) if infile is not None and os.path.isfile(infile): arch = Archive(infile=infile, files=files) specs = arch.get_node("comp_archive_spec", {"compname":comp}) else: if infile is None: logger.debug("No archive file defined for component %s"%comp) else: logger.debug("Archive file %s for component %s not found"%(infile,comp)) specs = self.get_optional_node("comp_archive_spec", attributes={"compname":comp}) if specs is None: logger.debug("No archive specs found for component %s"%comp) else: logger.debug("adding archive spec for %s"%comp) components_node.append(specs) env_archive.add_child(components_node)
def __init__(self, comp_interface="mct"): """ initialize an object >>> files = Files() >>> files.get_value('CASEFILE_HEADERS',resolved=False) '$CIMEROOT/config/config_headers.xml' """ cimeroot = get_cime_root() infile = os.path.join(cimeroot, "config", get_model(), "config_files.xml") expect(os.path.isfile(infile), "Could not find or open file {}".format(infile)) schema = os.path.join(cimeroot, "config", "xml_schemas", "entry_id.xsd") EntryID.__init__(self, infile, schema=schema) config_files_override = os.path.join(os.path.dirname(cimeroot), ".config_files.xml") # variables COMP_ROOT_DIR_{} are mutable, all other variables are read only self.COMP_ROOT_DIR = {} self._comp_interface = comp_interface # .config_file.xml at the top level may overwrite COMP_ROOT_DIR_ nodes in config_files if os.path.isfile(config_files_override): self.read(config_files_override) self.overwrite_existing_entries()
def test_cime_case_test_walltime_mgmt_5(self): if utils.get_model() != "e3sm": self.skipTest( "Skipping walltime test. Depends on E3SM batch settings") test_name = "ERS_P1.f19_g16_rx1.A" casedir = self._create_test( ["--no-setup", "--machine=blues", test_name], test_id=self._baseline_name, env_changes="unset CIME_GLOBAL_WALLTIME &&", ) self.run_cmd_assert_result( "./xmlchange JOB_QUEUE=slartibartfast --subgroup=case.test", from_dir=casedir, expected_stat=1, ) self.run_cmd_assert_result( "./xmlchange JOB_QUEUE=slartibartfast --force --subgroup=case.test", from_dir=casedir, ) result = self.run_cmd_assert_result( "./xmlquery JOB_WALLCLOCK_TIME --subgroup=case.test --value", from_dir=casedir, ) self.assertEqual(result, "01:00:00") result = self.run_cmd_assert_result( "./xmlquery JOB_QUEUE --subgroup=case.test --value", from_dir=casedir) self.assertEqual(result, "slartibartfast")
def bless_namelists(test_name, report_only, force, baseline_name, baseline_root): ############################################################################### # Be aware that restart test will overwrite the original namelist files # with versions of the files that should not be blessed. This forces us to # re-run create_test. # Update namelist files logger.info("Test '{}' had namelist diff".format(test_name)) if (not report_only and (force or six.moves.input("Update namelists (y/n)? ").upper() in ["Y", "YES"])): create_test_gen_args = " -g {} ".format(baseline_name if get_model( ) == "cesm" else " -g -b {} ".format(baseline_name)) stat, out, _ = run_cmd( "{}/create_test {} -n {} --baseline-root {} -o".format( get_scripts_root(), test_name, create_test_gen_args, baseline_root), combine_output=True) if stat != 0: return False, "Namelist regen failed: '{}'".format(out) else: return True, None else: return True, None
def setup(self, env_archive, components, files=None): if files is None: files = Files() components_node = env_archive.make_child("components", attributes={"version":"2.0"}) model = get_model() if 'drv' not in components: components.append('drv') if 'dart' not in components and model == 'cesm': components.append('dart') for comp in components: infile = files.get_value("ARCHIVE_SPEC_FILE", {"component":comp}) if infile is not None and os.path.isfile(infile): arch = Archive(infile=infile, files=files) specs = arch.get_child(name="comp_archive_spec", attributes={"compname":comp}) else: if infile is None: logger.debug("No archive file defined for component {}".format(comp)) else: logger.debug("Archive file {} for component {} not found".format(infile,comp)) specs = self.get_optional_child(name="comp_archive_spec", attributes={"compname":comp}) if specs is None: logger.debug("No archive specs found for component {}".format(comp)) else: logger.debug("adding archive spec for {}".format(comp)) env_archive.add_child(specs, root=components_node)
def test_n_createnewcase_bad_compset(self): cls = self.__class__ model = utils.get_model() testdir = os.path.join(cls._testroot, "testcreatenewcase_bad_compset") if os.path.exists(testdir): shutil.rmtree(testdir) args = ( " --case %s --compset InvalidCompsetName --output-root %s --handle-preexisting-dirs=r " % (testdir, cls._testroot) ) if model == "cesm": args += " --run-unsupported" if self.TEST_COMPILER is not None: args = args + " --compiler %s" % self.TEST_COMPILER if self.TEST_MPILIB is not None: args = args + " --mpilib %s" % self.TEST_MPILIB if utils.get_cime_default_driver() == "nuopc": args += " --res f19_g17 " else: args += " --res f19_g16 " args += f" --machine {self.MACHINE.get_machine_name()}" self.run_cmd_assert_result( "./create_newcase %s" % (args), from_dir=self.SCRIPT_DIR, expected_stat=1 ) self.assertFalse(os.path.exists(testdir))
def __init__(self, infile=None): """ initialize an object """ if infile is None: infile = os.path.join(get_cime_root(), "cime_config", get_model(), "archive.xml") GenericXML.__init__(self, infile)
def _restart_fake_phase(self): # Swap out model.exe for one that emits node failures rundir = self._case.get_value("RUNDIR") exeroot = self._case.get_value("EXEROOT") driver = self._case.get_value("COMP_INTERFACE") if driver == "nuopc": logname = "drv" else: logname = "cpl" fake_exe = """#!/bin/bash fail_sentinel={0} cpl_log={1}/{4}.log.$LID model_log={1}/{2}.log.$LID touch $cpl_log touch $fail_sentinel declare -i num_fails=$(cat $fail_sentinel | wc -l) declare -i times_to_fail=${{NODEFAIL_NUM_FAILS:-3}} if ((num_fails < times_to_fail)); then echo FAKE FAIL >> $cpl_log echo FAIL >> $fail_sentinel echo '{3}' >> $model_log sleep 1 exit -1 else echo Insta pass echo SUCCESSFUL TERMINATION > $cpl_log fi """.format( self._fail_sentinel, rundir, get_model(), self._fail_str, logname ) fake_exe_file = os.path.join(exeroot, "fake.sh") with open(fake_exe_file, "w") as fd: fd.write(fake_exe) os.chmod(fake_exe_file, 0o755) prev_run_exe = self._case.get_value("run_exe") env_mach_specific = self._case.get_env("mach_specific") env_mach_specific.set_value("run_exe", fake_exe_file) self._case.flush(flushall=True) # This flag is needed by mpt to run a script under mpiexec mpilib = self._case.get_value("MPILIB") if mpilib == "mpt": os.environ["MPI_SHEPHERD"] = "true" self.run_indv(suffix=None) if mpilib == "mpt": del os.environ["MPI_SHEPHERD"] env_mach_specific = self._case.get_env("mach_specific") env_mach_specific.set_value("run_exe", prev_run_exe) self._case.flush(flushall=True)
def __init__(self): """ initialize an object """ expect(get_model() == 'cesm', "testreport is only meant to populate the CESM test database." ) self.root = None GenericXML.__init__(self)
def __init__(self): """ initialize an object """ expect(get_model() == 'cesm', "testreport is only meant to populate the CESM test database." ) self.root = None GenericXML.__init__(self, root_name_override="testrecord", read_only=False, infile="TestRecord.xml")
def __init__(self, infile=None): """ initialize an object """ if infile is None: infile = os.path.join(get_cime_root(), "cime_config", get_model(), "config_archive.xml") GenericXML.__init__(self, infile)
def test_success_recording(self): if utils.get_model() != "e3sm": self.skipTest("Skipping success recording tests. E3SM feature") fake_test1 = "faketest1" fake_test2 = "faketest2" baseline_dir = os.path.join(self._baseline_area, self._baseline_name) # Test initial state was_success, last_pass, trans_fail = provenance.get_test_success( baseline_dir, None, fake_test1, testing=True) self.assertFalse(was_success, msg="Broken initial was_success") self.assertEqual(last_pass, None, msg="Broken initial last_pass") self.assertEqual(trans_fail, None, msg="Broken initial trans_fail") # Test first result (test1 fails, test2 passes) # test_name , success, commit , expP , expTF, baseline) self._record_success(fake_test1, False, "AAA", None, "AAA", baseline_dir) self._record_success(fake_test2, True, "AAA", "AAA", None, baseline_dir) # Test second result matches first (no transition) (test1 fails, test2 passes) # test_name , success, commit , expP , expTF, baseline) self._record_success(fake_test1, False, "BBB", None, "AAA", baseline_dir) self._record_success(fake_test2, True, "BBB", "BBB", None, baseline_dir) # Test transition to new state (first real transition) (test1 passes, test2 fails) # test_name , success, commit , expP , expTF, baseline) self._record_success(fake_test1, True, "CCC", "CCC", "AAA", baseline_dir) self._record_success(fake_test2, False, "CCC", "BBB", "CCC", baseline_dir) # Test transition to new state (second real transition) (test1 fails, test2 passes) # test_name , success, commit , expP , expTF, baseline) self._record_success(fake_test1, False, "DDD", "CCC", "DDD", baseline_dir) self._record_success(fake_test2, True, "DDD", "DDD", "CCC", baseline_dir) # Test final repeat (test1 fails, test2 passes) # test_name , success, commit , expP , expTF, baseline) self._record_success(fake_test1, False, "EEE", "CCC", "DDD", baseline_dir) self._record_success(fake_test2, True, "EEE", "EEE", "CCC", baseline_dir) # Test final transition (test1 passes, test2 fails) # test_name , success, commit , expP , expTF, baseline) self._record_success(fake_test1, True, "FFF", "FFF", "DDD", baseline_dir) self._record_success(fake_test2, False, "FFF", "EEE", "FFF", baseline_dir)
def __init__(self, machine, infile=None): """ initialize an object """ if (infile is None): infile = os.path.join(get_cime_root(), "cime_config", get_model(), "machines", "config_lt_archive.xml") GenericXML.__init__(self, infile) self.machine = machine
def __init__(self): """ initialize an object >>> files = Files() >>> files.get_value('CASEFILE_HEADERS',resolved=False) '$CIMEROOT/cime_config/config_headers.xml' """ infile = os.path.join(get_cime_root(), "cime_config", get_model(), "config_files.xml") EntryID.__init__(self, infile)
def compare_namelists(case, baseline_name, baseline_root, logfile_name, compiler): ############################################################################### if get_model() == "acme": baseline_name = os.path.join(compiler, baseline_name) log_lvl = logging.getLogger().getEffectiveLevel() logging.disable(logging.CRITICAL) success = case_cmpgen_namelists(case, compare=True, compare_name=baseline_name, baseline_root=baseline_root, logfile_name=logfile_name) logging.getLogger().setLevel(log_lvl) return success
def build_phase(self, sharedlib_only=False, model_only=False): if "TESTBUILDFAIL_PASS" in os.environ: TESTRUNPASS.build_phase(self, sharedlib_only, model_only) else: if (not sharedlib_only): blddir = self._case.get_value("EXEROOT") bldlog = os.path.join(blddir, "{}.bldlog.{}".format(get_model(), get_timestamp("%y%m%d-%H%M%S"))) with open(bldlog, "w") as fd: fd.write("BUILD FAIL: Intentional fail for testing infrastructure") expect(False, "BUILD FAIL: Intentional fail for testing infrastructure")
def _create_caseroot_tools(self): cime_model = get_model() machines_dir = os.path.abspath(self.get_value("MACHDIR")) toolsdir = os.path.join(self.get_value("CIMEROOT"),"scripts","Tools") caseroot = self.get_value("CASEROOT") # setup executable files in caseroot/ exefiles = (os.path.join(toolsdir, "case.setup"), os.path.join(toolsdir, "case.build"), os.path.join(toolsdir, "case.submit"), os.path.join(toolsdir, "preview_namelists"), os.path.join(toolsdir, "testcase.setup"), os.path.join(toolsdir, "check_input_data"), os.path.join(toolsdir, "check_case"), os.path.join(toolsdir, "archive_metadata.sh"), os.path.join(toolsdir, "create_production_test"), os.path.join(toolsdir, "xmlchange"), os.path.join(toolsdir, "xmlquery")) try: for exefile in exefiles: destfile = os.path.join(caseroot,os.path.basename(exefile)) os.symlink(exefile, destfile) except Exception as e: logger.warning("FAILED to set up exefiles: %s" % str(e)) # set up utility files in caseroot/Tools/ toolfiles = (os.path.join(toolsdir, "check_lockedfiles"), os.path.join(toolsdir, "lt_archive.sh"), os.path.join(toolsdir, "st_archive"), os.path.join(toolsdir, "getTiming"), os.path.join(toolsdir, "compare_namelists.pl"), os.path.join(machines_dir,"taskmaker.pl"), os.path.join(machines_dir,"Makefile"), os.path.join(machines_dir,"mkSrcfiles"), os.path.join(machines_dir,"mkDepends")) for toolfile in toolfiles: destfile = os.path.join(caseroot,"Tools",os.path.basename(toolfile)) expect(os.path.isfile(toolfile)," File %s does not exist"%toolfile) try: os.symlink(toolfile, destfile) except Exception as e: logger.warning("FAILED to set up toolfiles: %s %s %s" % (str(e), toolfile, destfile)) # Copy any system or compiler Depends files to the case machine = self.get_value("MACH") compiler = self.get_value("COMPILER") for dep in (machine, compiler): dfile = "Depends.%s"%dep if os.path.isfile(os.path.join(machines_dir,dfile)): shutil.copyfile(os.path.join(machines_dir,dfile), os.path.join(caseroot,dfile)) dfile = "Depends.%s.%s"%(machine,compiler) if os.path.isfile(os.path.join(machines_dir,dfile)): shutil.copyfile(os.path.join(machines_dir,dfile), os.path.join(caseroot, dfile))
def simple_test(self, manual_timing=False): if self.NO_FORTRAN_RUN: self.skipTest("Skipping fortran test") timing_flag = "" if manual_timing else "--save-timing" driver = utils.get_cime_default_driver() if driver == "mct": walltime = "00:15:00" else: walltime = "00:30:00" self._create_test( [ "SMS_Ln9_P1.f19_g16_rx1.A", timing_flag, "--walltime=" + walltime ], test_id=self._baseline_name, ) statuses = glob.glob("%s/*%s/TestStatus" % (self._testroot, self._baseline_name)) self.assertEqual( len(statuses), 1, msg="Should have had exactly one match, found %s" % statuses, ) casedir = os.path.dirname(statuses[0]) with Case(casedir, read_only=True) as case: lids = utils.get_lids(case) timing_dir = case.get_value("SAVE_TIMING_DIR") casename = case.get_value("CASE") self.assertEqual(len(lids), 1, msg="Expected one LID, found %s" % lids) if manual_timing: self.run_cmd_assert_result("cd %s && %s/save_provenance postrun" % (casedir, self.TOOLS_DIR)) if utils.get_model() == "e3sm": provenance_glob = os.path.join( timing_dir, "performance_archive", getpass.getuser(), casename, lids[0] + "*", ) provenance_dirs = glob.glob(provenance_glob) self.assertEqual( len(provenance_dirs), 1, msg= "wrong number of provenance dirs, expected 1, got {}, looked for {}" .format(provenance_dirs, provenance_glob), ) self.verify_perms("".join(provenance_dirs))
def __init__(self): """ initialize an object """ expect(get_model() == 'cesm', "testreport is only meant to populate the CESM test database.") self.root = None GenericXML.__init__(self, root_name_override="testrecord", read_only=False, infile="TestRecord.xml")
def generate_teststatus(testdir, baseline_dir): """ CESM stores it's TestStatus file in baselines. Do not let exceptions escape from this function. """ if get_model() == "cesm": try: if not os.path.isdir(baseline_dir): os.makedirs(baseline_dir) safe_copy(os.path.join(testdir, TEST_STATUS_FILENAME), baseline_dir) except Exception as e: logger.warning("Could not copy {} to baselines, {}".format(os.path.join(testdir, TEST_STATUS_FILENAME), str(e)))
def __init__(self): """ initialize an object >>> files = Files() >>> files.get_value('CASEFILE_HEADERS',resolved=False) '$CIMEROOT/config/config_headers.xml' """ cimeroot = get_cime_root() infile = os.path.join(cimeroot, "config", get_model(), "config_files.xml") expect(os.path.isfile(infile), "Could not find or open file {}".format(infile)) schema = os.path.join(cimeroot, "config", "xml_schemas", "entry_id.xsd") EntryID.__init__(self, infile, schema=schema)
def setUp(self): if utils.get_model() != "e3sm": self.skipTest("Skipping Jenkins tests. E3SM feature") super().setUp() # Need to run in a subdir in order to not have CTest clash. Name it # such that it should be cleaned up by the parent tearDown self._testdir = os.path.join(self._testroot, "jenkins_test_%s" % self._baseline_name) os.makedirs(self._testdir) # Change root to avoid clashing with other jenkins_generic_jobs self._jenkins_root = os.path.join(self._testdir, "J")
def write_provenance_info(machine, test_compiler, test_mpilib, test_root): curr_commit = get_current_commit(repo=CIMEROOT) logging.info("Testing commit %s" % curr_commit) cime_model = get_model() logging.info("Using cime_model = %s" % cime_model) logging.info("Testing machine = %s" % machine.get_machine_name()) if test_compiler is not None: logging.info("Testing compiler = %s" % test_compiler) if test_mpilib is not None: logging.info("Testing mpilib = %s" % test_mpilib) logging.info("Test root: %s" % test_root) logging.info("Test driver: %s" % CIME.utils.get_cime_default_driver()) logging.info("Python version {}\n".format(sys.version))
def _restart_fake_phase(self): # Swap out model.exe for one that emits node failures rundir = self._case.get_value("RUNDIR") exeroot = self._case.get_value("EXEROOT") fake_exe = \ """#!/bin/bash fail_sentinel={0} cpl_log={1}/cpl.log.$LID model_log={1}/{2}.log.$LID touch $cpl_log touch $fail_sentinel declare -i num_fails=$(cat $fail_sentinel | wc -l) declare -i times_to_fail=${{NODEFAIL_NUM_FAILS:-3}} if ((num_fails < times_to_fail)); then echo FAKE FAIL >> $cpl_log echo FAIL >> $fail_sentinel echo '{3}' >> $model_log sleep 1 exit -1 else echo Insta pass echo SUCCESSFUL TERMINATION > $cpl_log fi """.format(self._fail_sentinel, rundir, get_model(), self._fail_str) fake_exe_file = os.path.join(exeroot, "fake.sh") with open(fake_exe_file, "w") as fd: fd.write(fake_exe) os.chmod(fake_exe_file, 0o755) prev_run_exe = self._case.get_value("run_exe") env_mach_specific = self._case.get_env("mach_specific") env_mach_specific.set_value("run_exe", fake_exe_file) self._case.flush(flushall=True) # This flag is needed by mpt to run a script under mpiexec mpilib = self._case.get_value("MPILIB") if mpilib == "mpt": os.environ["MPI_SHEPHERD"] = "true" self.run_indv(suffix=None) if mpilib == "mpt": del os.environ["MPI_SHEPHERD"] env_mach_specific = self._case.get_env("mach_specific") env_mach_specific.set_value("run_exe", prev_run_exe) self._case.flush(flushall=True)
def __init__(self, batch_system=None, machine=None, infile=None): """ initialize an object """ if infile is None: infile = os.path.join(get_cime_root(), "cime_config", get_model(), "machines", "config_batch.xml") GenericXML.__init__(self, infile) self.batch_system_node = None self.machine_node = None self.batch_system = batch_system self.machine = machine if self.batch_system is not None: self.set_batch_system(self.batch_system, machine=machine)
def compare_history(case, baseline_name, baseline_root, log_id, compiler): ############################################################################### if get_model() == "acme": baseline_full_dir = os.path.join(baseline_root, compiler, baseline_name, case.get_value("CASEBASEID")) else: baseline_full_dir = os.path.join(baseline_root, baseline_name, case.get_value("CASEBASEID")) outfile_suffix = "{}.{}".format(baseline_name, log_id) try: result, comments = compare_baseline(case, baseline_dir=baseline_full_dir, outfile_suffix=outfile_suffix) except IOError: result, comments = compare_baseline(case, baseline_dir=baseline_full_dir, outfile_suffix=None) return result, comments
def run(self, skip_pnl=False): """ Do NOT override this method, this method is the framework that controls the run phase. run_phase is the extension point that subclasses should use. """ success = True start_time = time.time() self._skip_pnl = skip_pnl try: self._resetup_case(RUN_PHASE) with self._test_status: self._test_status.set_status(RUN_PHASE, TEST_PEND_STATUS) self.run_phase() if self._case.get_value("GENERATE_BASELINE"): self._generate_baseline() if self._case.get_value("COMPARE_BASELINE"): self._compare_baseline() self._check_for_memleak() except BaseException as e: success = False msg = e.__str__() if "RUN FAIL" in msg: # Don't want to print stacktrace for a model failure since that # is not a CIME/infrastructure problem. excmsg = msg else: excmsg = "Exception during run:\n{}\n{}".format(msg, traceback.format_exc()) logger.warning(excmsg) append_testlog(excmsg) # Writing the run status should be the very last thing due to wait_for_tests time_taken = time.time() - start_time status = TEST_PASS_STATUS if success else TEST_FAIL_STATUS with self._test_status: self._test_status.set_status(RUN_PHASE, status, comments=("time={:d}".format(int(time_taken)))) if success and get_model() == "e3sm": save_test_time(self._case.get_value("BASELINE_ROOT"), self._casebaseid, time_taken) # We return success if the run phase worked; memleaks, diffs will not be taken into account # with this return value. return success
def _setup_cs_files(self): ########################################################################### try: python_libs_root = CIME.utils.get_python_libs_root() template_file = os.path.join(python_libs_root, "cs.status.template") template = open(template_file, "r").read() template = template.replace("<PATH>", os.path.join(self._cime_root,"scripts","Tools")).replace\ ("<TESTID>", self._test_id).replace\ ("<TESTROOT>", self._test_root) if not os.path.exists(self._test_root): os.makedirs(self._test_root) cs_status_file = os.path.join(self._test_root, "cs.status.{}".format(self._test_id)) with open(cs_status_file, "w") as fd: fd.write(template) os.chmod(cs_status_file, os.stat(cs_status_file).st_mode | stat.S_IXUSR | stat.S_IXGRP) template_file = os.path.join(python_libs_root, "cs.submit.template") template = open(template_file, "r").read() setup_cmd = "./case.setup" if self._no_setup else ":" build_cmd = "./case.build" if self._no_build else ":" test_cmd = "./case.submit" template = template.replace("<SETUP_CMD>", setup_cmd).\ replace("<BUILD_CMD>", build_cmd).\ replace("<RUN_CMD>", test_cmd).\ replace("<TESTID>", self._test_id) if self._no_run: cs_submit_file = os.path.join(self._test_root, "cs.submit.{}".format(self._test_id)) with open(cs_submit_file, "w") as fd: fd.write(template) os.chmod(cs_submit_file, os.stat(cs_submit_file).st_mode | stat.S_IXUSR | stat.S_IXGRP) if get_model() == "cesm": template_file = os.path.join(python_libs_root, "testreporter.template") template = open(template_file, "r").read() template = template.replace("<PATH>", os.path.join(self._cime_root, "scripts", "Tools")) testreporter_file = os.path.join(self._test_root, "testreporter") with open(testreporter_file, "w") as fd: fd.write(template) os.chmod(testreporter_file, os.stat(testreporter_file).st_mode | stat.S_IXUSR | stat.S_IXGRP) except Exception as e: logger.warning("FAILED to set up cs files: {}".format(str(e)))
def bless_namelists(test_name, report_only, force, baseline_name, baseline_root): ############################################################################### # Be aware that restart test will overwrite the original namelist files # with versions of the files that should not be blessed. This forces us to # re-run create_test. # Update namelist files print("Test '{}' had namelist diff".format(test_name)) if (not report_only and (force or six.moves.input("Update namelists (y/n)? ").upper() in ["Y", "YES"])): create_test_gen_args = " -g {} ".format(baseline_name if get_model() == "cesm" else " -g -b {} ".format(baseline_name)) stat, _, err = run_cmd("{}/create_test {} -n {} --baseline-root {} -o".format(get_scripts_root(), test_name, create_test_gen_args, baseline_root)) if stat != 0: return False, "Namelist regen failed: '{}'".format(err) else: return True, None else: return True, None
def __init__(self, batch_system=None, machine=None, infile=None): """ initialize an object """ if infile is None: infile = os.path.join(get_cime_root(), "cime_config", get_model(), "machines", "config_batch.xml") GenericXML.__init__(self, infile) self.batch_system_node = None self.machine_node = None self.batch_system = batch_system self.machine = machine #Append the contents of $HOME/.cime/config_batch.xml if it exists #This could cause problems if node matchs are repeated when only one is expected infile = os.path.join(os.environ.get("HOME"),".cime","config_batch.xml") if os.path.exists(infile): GenericXML.read(self, infile) if self.batch_system is not None: self.set_batch_system(self.batch_system, machine=machine)
def __init__(self): """ initialize an object >>> files = Files() >>> files.get_value('CASEFILE_HEADERS',resolved=False) '$CIMEROOT/config/config_headers.xml' """ cimeroot = get_cime_root() infile = os.path.join(cimeroot, "config", get_model(), "config_files.xml") expect(os.path.isfile(infile), "Could not find or open file {}".format(infile)) schema = os.path.join(cimeroot, "config", "xml_schemas", "entry_id.xsd") EntryID.__init__(self, infile, schema=schema) config_files_override = os.path.join(os.path.dirname(cimeroot),".config_files.xml") # variables COMP_ROOT_DIR_{} are mutable, all other variables are read only self.COMP_ROOT_DIR = {} # .config_file.xml at the top level may overwrite COMP_ROOT_DIR_ nodes in config_files if os.path.isfile(config_files_override): self.read(config_files_override) self.overwrite_existing_entries()
def bless_history(test_name, testcase_dir_for_test, baseline_name, baseline_root, compiler, report_only, force): ############################################################################### with Case(testcase_dir_for_test) as case: if get_model() == "acme": baseline_full_dir = os.path.join(baseline_root, compiler, baseline_name, case.get_value("CASEBASEID")) else: baseline_full_dir = os.path.join(baseline_root, baseline_name, case.get_value("CASEBASEID")) result, comments = compare_baseline(case, baseline_dir=baseline_full_dir, outfile_suffix=None) if result: return True, None else: print(comments) if (not report_only and (force or six.moves.input("Update this diff (y/n)? ").upper() in ["Y", "YES"])): result, comments = generate_baseline(case, baseline_dir=baseline_full_dir) if not result: logging.warning("Hist file bless FAILED for test {}".format(test_name)) return False, "Generate baseline failed: {}".format(comments) else: print(comments) return True, None else: return True, None
import CIME.utils from CIME.utils import expect, convert_to_seconds, parse_test_name, get_cime_root, get_model from CIME.XML.machines import Machines import six, sys, os # Expect that, if a model wants to use python-based test lists, they will have a file # config/$model/tests.py , containing a test dictionary called _TESTS sys.path.insert(0, os.path.join(get_cime_root(), "config", get_model())) _ALL_TESTS = {} try: from tests import _TESTS # pylint: disable=import-error _ALL_TESTS.update(_TESTS) except: pass # Here are the tests belonging to e3sm suites. Format is # <test>.<grid>.<compset>. # suite_name -> (inherits_from, timelimit, [test [, mods[, machines]]]) # To elaborate, if no mods are needed, a string representing the testname is all that is needed. # If testmods are needed, a 2-ple must be provided (test, mods) # If you want to restrict the test mods to certain machines, than a 3-ple is needed (test, mods, [machines]) _CIME_TESTS = { "cime_tiny" : (None, "0:10:00", ("ERS.f19_g16_rx1.A", "NCK.f19_g16_rx1.A") ), "cime_test_only_pass" : (None, "0:10:00",
def __init__(self, test_names, test_data=None, no_run=False, no_build=False, no_setup=False, no_batch=None, test_root=None, test_id=None, machine_name=None, compiler=None, baseline_root=None, baseline_cmp_name=None, baseline_gen_name=None, clean=False, namelists_only=False, project=None, parallel_jobs=None, walltime=None, proc_pool=None, use_existing=False, save_timing=False, queue=None, allow_baseline_overwrite=False, output_root=None, force_procs=None, force_threads=None, mpilib=None, input_dir=None, pesfile=None, mail_user=None, mail_type=None): ########################################################################### self._cime_root = CIME.utils.get_cime_root() self._cime_model = get_model() self._save_timing = save_timing self._queue = queue self._test_data = {} if test_data is None else test_data # Format: {test_name -> {data_name -> data}} self._mpilib = mpilib # allow override of default mpilib self._completed_tests = 0 self._input_dir = input_dir self._pesfile = pesfile self._allow_baseline_overwrite = allow_baseline_overwrite self._mail_user = mail_user self._mail_type = mail_type self._machobj = Machines(machine=machine_name) self._model_build_cost = 4 # If user is forcing procs or threads, re-write test names to reflect this. if force_procs or force_threads: test_names = _translate_test_names_for_new_pecount(test_names, force_procs, force_threads) self._no_setup = no_setup self._no_build = no_build or no_setup or namelists_only self._no_run = no_run or self._no_build self._output_root = output_root # Figure out what project to use if project is None: self._project = CIME.utils.get_project() if self._project is None: self._project = self._machobj.get_value("PROJECT") else: self._project = project # We will not use batch system if user asked for no_batch or if current # machine is not a batch machine self._no_batch = no_batch or not self._machobj.has_batch_system() expect(not (self._no_batch and self._queue is not None), "Does not make sense to request a queue without batch system") # Determine and resolve test_root if test_root is not None: self._test_root = test_root elif self._output_root is not None: self._test_root = self._output_root else: self._test_root = self._machobj.get_value("CIME_OUTPUT_ROOT") if self._project is not None: self._test_root = self._test_root.replace("$PROJECT", self._project) self._test_root = os.path.abspath(self._test_root) self._test_id = test_id if test_id is not None else CIME.utils.get_timestamp() self._compiler = self._machobj.get_default_compiler() if compiler is None else compiler self._clean = clean self._namelists_only = namelists_only self._walltime = walltime if parallel_jobs is None: self._parallel_jobs = min(len(test_names), self._machobj.get_value("MAX_MPITASKS_PER_NODE")) else: self._parallel_jobs = parallel_jobs self._baseline_cmp_name = baseline_cmp_name # Implies comparison should be done if not None self._baseline_gen_name = baseline_gen_name # Implies generation should be done if not None # Compute baseline_root self._baseline_root = baseline_root if baseline_root is not None \ else self._machobj.get_value("BASELINE_ROOT") if self._project is not None: self._baseline_root = self._baseline_root.replace("$PROJECT", self._project) self._baseline_root = os.path.abspath(self._baseline_root) if baseline_cmp_name or baseline_gen_name: if self._baseline_cmp_name: full_baseline_dir = os.path.join(self._baseline_root, self._baseline_cmp_name) expect(os.path.isdir(full_baseline_dir), "Missing baseline comparison directory {}".format(full_baseline_dir)) # the following is to assure that the existing generate directory is not overwritten if self._baseline_gen_name: full_baseline_dir = os.path.join(self._baseline_root, self._baseline_gen_name) existing_baselines = [] for test_name in test_names: test_baseline = os.path.join(full_baseline_dir, test_name) if os.path.isdir(test_baseline): existing_baselines.append(test_baseline) expect(allow_baseline_overwrite or len(existing_baselines) == 0, "Baseline directories already exists {}\n" \ "Use -o to avoid this error".format(existing_baselines)) if self._cime_model == "e3sm": _order_tests_by_runtime(test_names, self._baseline_root) # This is the only data that multiple threads will simultaneously access # Each test has it's own value and setting/retrieving items from a dict # is atomic, so this should be fine to use without mutex. # name -> (phase, status) self._tests = OrderedDict() for test_name in test_names: self._tests[test_name] = (TEST_START, TEST_PASS_STATUS) # Oversubscribe by 1/4 if proc_pool is None: pes = int(self._machobj.get_value("MAX_TASKS_PER_NODE")) self._proc_pool = int(pes * 1.25) else: self._proc_pool = int(proc_pool) self._procs_avail = self._proc_pool # Setup phases self._phases = list(PHASES) if self._no_setup: self._phases.remove(SETUP_PHASE) if self._no_build: self._phases.remove(SHAREDLIB_BUILD_PHASE) self._phases.remove(MODEL_BUILD_PHASE) if self._no_run: self._phases.remove(RUN_PHASE) if use_existing: for test in self._tests: with TestStatus(self._get_test_dir(test)) as ts: for phase, status in ts: if phase in CORE_PHASES: if status in [TEST_PEND_STATUS, TEST_FAIL_STATUS]: if status == TEST_FAIL_STATUS: # Import for potential subsequent waits ts.set_status(phase, TEST_PEND_STATUS) # We need to pick up here break else: if phase != SUBMIT_PHASE: # Somewhat subtle. Create_test considers submit/run to be the run phase, # so don't try to update test status for a passed submit phase self._update_test_status(test, phase, TEST_PEND_STATUS) self._update_test_status(test, phase, status) if phase == RUN_PHASE: logger.info("Test {} passed and will not be re-run".format(test)) logger.info("Using existing test directory {}".format(self._get_test_dir(test))) else: # None of the test directories should already exist. for test in self._tests: expect(not os.path.exists(self._get_test_dir(test)), "Cannot create new case in directory '{}', it already exists." " Pick a different test-id".format(self._get_test_dir(test))) logger.info("Creating test directory {}".format(self._get_test_dir(test)))
def build_model(build_threaded, exeroot, clm_config_opts, incroot, complist, lid, caseroot, cimeroot, compiler): ############################################################################### logs = [] thread_bad_results = [] for model, comp, nthrds, _, config_dir in complist: # aquap has a dependency on atm so we will build it after the threaded loop if comp == "aquap": logger.debug("Skip aquap ocn build here") continue # coupler handled seperately if model == "cpl": continue # special case for clm # clm 4_0 is not a shared library and must be built here # clm 4_5 and newer is a shared library and should be built in build_libraries if comp == "clm": if "clm4_0" in clm_config_opts: logger.info(" - Building clm4_0 Library ") else: continue smp = nthrds > 1 or build_threaded bldroot = os.path.join(exeroot, model, "obj") libroot = os.path.join(exeroot, "lib") file_build = os.path.join(exeroot, "%s.bldlog.%s" % (model, lid)) logger.debug("bldroot is %s" % bldroot) logger.debug("libroot is %s" % libroot) # make sure bldroot and libroot exist for build_dir in [bldroot, libroot]: if not os.path.exists(build_dir): os.makedirs(build_dir) # build the component library # thread_bad_results captures error output from thread (expected to be empty) # logs is a list of log files to be compressed and added to the case logs/bld directory t = threading.Thread(target=_build_model_thread, args=(config_dir, model, caseroot, libroot, bldroot, incroot, file_build, thread_bad_results, smp, compiler)) t.start() for mod_file in glob.glob(os.path.join(bldroot, "*_[Cc][Oo][Mm][Pp]_*.mod")): shutil.copy(mod_file, incroot) logs.append(file_build) # Wait for threads to finish while(threading.active_count() > 1): time.sleep(1) # aquap has a dependancy on atm so we build it after the threaded loop for model, comp, nthrds, _, config_dir in complist: smp = nthrds > 1 or build_threaded if comp == "aquap": logger.debug("Now build aquap ocn component") # thread_bad_results captures error output from thread (expected to be empty) # logs is a list of log files to be compressed and added to the case logs/bld directory _build_model_thread(config_dir, comp, caseroot, libroot, bldroot, incroot, file_build, thread_bad_results, smp, compiler) logs.append(file_build) expect(not thread_bad_results, "\n".join(thread_bad_results)) # # Now build the executable # cime_model = get_model() file_build = os.path.join(exeroot, "%s.bldlog.%s" % (cime_model, lid)) config_dir = os.path.join(cimeroot, "driver_cpl", "cime_config") f = open(file_build, "w") bldroot = os.path.join(exeroot, "cpl", "obj") stat = run_cmd("%s/buildexe %s %s %s" % (config_dir, caseroot, libroot, bldroot), from_dir=bldroot, verbose=True, arg_stdout=f, arg_stderr=subprocess.STDOUT)[0] f.close() analyze_build_log("%s exe"%cime_model, file_build, compiler) expect(stat == 0, "ERROR: buildexe failed, cat %s" % file_build) # Copy the just-built ${MODEL}.exe to ${MODEL}.exe.$LID shutil.copy("%s/%s.exe" % (exeroot, cime_model), "%s/%s.exe.%s" % (exeroot, cime_model, lid)) logs.append(file_build) return logs