def _create_test(self, extra_args, test_id=None, run_errors=False, env_changes=""): """ Convenience wrapper around create_test. Returns list of full paths to created cases. If multiple cases, the order of the returned list is not guaranteed to match the order of the arguments. """ # All stub model not supported in nuopc driver driver = utils.get_cime_default_driver() if driver == "nuopc" and "cime_developer" in extra_args: extra_args.append( " ^SMS_Ln3.T42_T42.S ^PRE.f19_f19.ADESP_TEST ^PRE.f19_f19.ADESP ^DAE.ww3a.ADWAV" ) test_id = ( "{}-{}".format(self._baseline_name, utils.get_timestamp()) if test_id is None else test_id ) extra_args.append("-t {}".format(test_id)) extra_args.append("--baseline-root {}".format(self._baseline_area)) if self.NO_BATCH: extra_args.append("--no-batch") if self.TEST_COMPILER and ( [extra_arg for extra_arg in extra_args if "--compiler" in extra_arg] == [] ): extra_args.append("--compiler={}".format(self.TEST_COMPILER)) if self.TEST_MPILIB and ( [extra_arg for extra_arg in extra_args if "--mpilib" in extra_arg] == [] ): extra_args.append("--mpilib={}".format(self.TEST_MPILIB)) if [extra_arg for extra_arg in extra_args if "--machine" in extra_arg] == []: extra_args.append(f"--machine {self.MACHINE.get_machine_name()}") extra_args.append("--test-root={0} --output-root={0}".format(self._testroot)) full_run = ( set(extra_args) & set(["-n", "--namelist-only", "--no-setup", "--no-build", "--no-run"]) ) == set() if full_run and not self.NO_BATCH: extra_args.append("--wait") expected_stat = 0 if not run_errors else utils.TESTS_FAILED_ERR_CODE output = self.run_cmd_assert_result( "{} {}/create_test {}".format( env_changes, self.SCRIPT_DIR, " ".join(extra_args) ), expected_stat=expected_stat, ) cases = [] for line in output.splitlines(): if "Case dir:" in line: casedir = line.split()[-1] self.assertTrue( os.path.isdir(casedir), msg="Missing casedir {}".format(casedir) ) cases.append(casedir) self.assertTrue(len(cases) > 0, "create_test made no cases") return cases[0] if len(cases) == 1 else cases
def post_build(case, logs, build_complete=False): ############################################################################### logdir = case.get_value("LOGDIR") #zip build logs to CASEROOT/logs if logdir: bldlogdir = os.path.join(logdir, "bld") if not os.path.exists(bldlogdir): os.makedirs(bldlogdir) for log in logs: logger.info("Copying build log {} to {}".format(log, bldlogdir)) with open(log, 'rb') as f_in: with gzip.open("{}.gz".format(log), 'wb') as f_out: shutil.copyfileobj(f_in, f_out) if "sharedlibroot" not in log: shutil.copy("{}.gz".format(log), os.path.join(bldlogdir, "{}.gz".format(os.path.basename(log)))) os.remove(log) if build_complete: # Set XML to indicate build complete case.set_value("BUILD_COMPLETE", True) case.set_value("BUILD_STATUS", 0) if "SMP_VALUE" in os.environ: case.set_value("SMP_BUILD", os.environ["SMP_VALUE"]) case.flush() lock_file("env_build.xml") # must ensure there's an lid lid = os.environ["LID"] if "LID" in os.environ else get_timestamp("%y%m%d-%H%M%S") save_build_provenance(case, lid=lid)
def make_new_tag(prefix, old_tag, remote="origin", commit="HEAD"): ############################################################################### new_tag = "{}{}".format(prefix, get_timestamp(timestamp_format="%Y-%m-%d")) expect(old_tag != new_tag, "New tag must have different name than old tag") run_cmd_no_fail("git tag {} {}".format(new_tag, commit), verbose=True) run_cmd_no_fail("git push {} {}".format(remote, new_tag), verbose=True) return new_tag
def make_new_tag(prefix, old_tag, remote="origin", commit="HEAD"): ############################################################################### new_tag = "{}{}".format(prefix, get_timestamp(timestamp_format="%Y-%m-%d")) expect(old_tag != new_tag, "New tag must have different name than old tag") run_cmd_no_fail("git tag {} {}".format(new_tag, commit), verbose=True) run_cmd_no_fail("git push {} {}".format(remote, new_tag), verbose=True) return new_tag
def build_phase(self, sharedlib_only=False, model_only=False): if "TESTBUILDFAIL_PASS" in os.environ: TESTRUNPASS.build_phase(self, sharedlib_only, model_only) else: if (not sharedlib_only): blddir = self._case.get_value("EXEROOT") bldlog = os.path.join(blddir, "{}.bldlog.{}".format(get_model(), get_timestamp("%y%m%d-%H%M%S"))) with open(bldlog, "w") as fd: fd.write("BUILD FAIL: Intentional fail for testing infrastructure") expect(False, "BUILD FAIL: Intentional fail for testing infrastructure")
def setUp(self): self._thread_error = None self._unset_proxy = self.setup_proxy() self._machine = self.MACHINE.get_machine_name() self._compiler = (self.MACHINE.get_default_compiler() if self.TEST_COMPILER is None else self.TEST_COMPILER) self._baseline_name = "fake_testing_only_%s" % utils.get_timestamp() self._baseline_area = os.path.join(self.TEST_ROOT, "baselines") self._testroot = self.TEST_ROOT self._hasbatch = self.MACHINE.has_batch_system() and not self.NO_BATCH self._do_teardown = not self.NO_TEARDOWN self._root_dir = os.getcwd()
def test_jenkins_generic_job(self): # Generate fresh baselines so that this test is not impacted by # unresolved diffs self.simple_test(True, "-t cime_test_only_pass -g -b %s" % self._baseline_name) self.assert_num_leftovers("cime_test_only_pass") build_name = "jenkins_generic_job_pass_%s" % utils.get_timestamp() self.simple_test( True, "-t cime_test_only_pass -b %s" % self._baseline_name, build_name=build_name, ) self.assert_num_leftovers( "cime_test_only_pass" ) # jenkins_generic_job should have automatically cleaned up leftovers from prior run self.assert_dashboard_has_build(build_name)
def test_jenkins_generic_job_realistic_dash(self): # The actual quality of the cdash results for this test can only # be inspected manually # Generate fresh baselines so that this test is not impacted by # unresolved diffs self.simple_test(False, "-t cime_test_all -g -b %s" % self._baseline_name) self.assert_num_leftovers("cime_test_all") # Should create a diff os.environ["TESTRUNDIFF_ALTERNATE"] = "True" # Should create a nml diff # Modify namelist fake_nl = """ &fake_nml fake_item = 'fake' fake = .true. /""" baseline_glob = glob.glob( os.path.join(self._baseline_area, self._baseline_name, "TESTRUNPASS*")) self.assertEqual( len(baseline_glob), 1, msg="Expected one match, got:\n%s" % "\n".join(baseline_glob), ) for baseline_dir in baseline_glob: nl_path = os.path.join(baseline_dir, "CaseDocs", "datm_in") self.assertTrue(os.path.isfile(nl_path), msg="Missing file %s" % nl_path) os.chmod(nl_path, stat.S_IRUSR | stat.S_IWUSR) with open(nl_path, "a") as nl_file: nl_file.write(fake_nl) build_name = "jenkins_generic_job_mixed_%s" % utils.get_timestamp() self.simple_test(False, "-t cime_test_all -b %s" % self._baseline_name, build_name=build_name) self.assert_num_leftovers( "cime_test_all" ) # jenkins_generic_job should have automatically cleaned up leftovers from prior run self.assert_dashboard_has_build(build_name)
def post_build(case, logs, build_complete=False, save_build_provenance=True): ############################################################################### for log in logs: gzip_existing_file(log) if build_complete: # must ensure there's an lid lid = os.environ["LID"] if "LID" in os.environ else get_timestamp("%y%m%d-%H%M%S") if save_build_provenance: save_build_provenance_sub(case, lid=lid) # Set XML to indicate build complete case.set_value("BUILD_COMPLETE", True) case.set_value("BUILD_STATUS", 0) if "SMP_VALUE" in os.environ: case.set_value("SMP_BUILD", os.environ["SMP_VALUE"]) case.flush() lock_file("env_build.xml")
def test_jenkins_generic_job_kill(self): build_name = "jenkins_generic_job_kill_%s" % utils.get_timestamp() run_thread = threading.Thread( target=self.threaded_test, args=(False, " -t cime_test_only_slow_pass -b master", build_name), ) run_thread.daemon = True run_thread.start() time.sleep(120) self.kill_subprocesses(sig=signal.SIGTERM) run_thread.join(timeout=30) self.assertFalse(run_thread.is_alive(), msg="jenkins_generic_job should have finished") self.assertTrue( self._thread_error is None, msg="Thread had failure: %s" % self._thread_error, ) self.assert_dashboard_has_build(build_name)
def test_bless_test_results(self): if self.NO_FORTRAN_RUN: self.skipTest("Skipping fortran test") # Test resubmit scenario if Machine has a batch system if self.MACHINE.has_batch_system(): test_names = [ "TESTRUNDIFFRESUBMIT_Mmpi-serial.f19_g16_rx1.A", "TESTRUNDIFF_Mmpi-serial.f19_g16_rx1.A", ] else: test_names = ["TESTRUNDIFF_P1.f19_g16_rx1.A"] # Generate some baselines for test_name in test_names: if utils.get_model() == "e3sm": genargs = ["-g", "-o", "-b", self._baseline_name, test_name] compargs = ["-c", "-b", self._baseline_name, test_name] else: genargs = [ "-g", self._baseline_name, "-o", test_name, "--baseline-root ", self._baseline_area, ] compargs = [ "-c", self._baseline_name, test_name, "--baseline-root ", self._baseline_area, ] self._create_test(genargs) # Hist compare should pass self._create_test(compargs) # Change behavior os.environ["TESTRUNDIFF_ALTERNATE"] = "True" # Hist compare should now fail test_id = "%s-%s" % (self._baseline_name, utils.get_timestamp()) self._create_test(compargs, test_id=test_id, run_errors=True) # compare_test_results should detect the fail cpr_cmd = "{}/compare_test_results --test-root {} -t {} ".format( self.TOOLS_DIR, self._testroot, test_id ) output = self.run_cmd_assert_result( cpr_cmd, expected_stat=utils.TESTS_FAILED_ERR_CODE ) # use regex expected_pattern = re.compile(r"FAIL %s[^\s]* BASELINE" % test_name) the_match = expected_pattern.search(output) self.assertNotEqual( the_match, None, msg="Cmd '%s' failed to display failed test %s in output:\n%s" % (cpr_cmd, test_name, output), ) # Bless utils.run_cmd_no_fail( "{}/bless_test_results --test-root {} --hist-only --force -t {}".format( self.TOOLS_DIR, self._testroot, test_id ) ) # Hist compare should now pass again self._create_test(compargs) self.verify_perms(self._baseline_area) if "TESTRUNDIFF_ALTERNATE" in os.environ: del os.environ["TESTRUNDIFF_ALTERNATE"]
def setUp(self): self._testroot = os.path.join(self.TEST_ROOT, "TestWaitForTests") self._timestamp = utils.get_timestamp() # basic tests self._testdir_all_pass = os.path.join( self._testroot, "scripts_regression_tests.testdir_all_pass") self._testdir_with_fail = os.path.join( self._testroot, "scripts_regression_tests.testdir_with_fail") self._testdir_unfinished = os.path.join( self._testroot, "scripts_regression_tests.testdir_unfinished") self._testdir_unfinished2 = os.path.join( self._testroot, "scripts_regression_tests.testdir_unfinished2") # live tests self._testdir_teststatus1 = os.path.join( self._testroot, "scripts_regression_tests.testdir_teststatus1") self._testdir_teststatus2 = os.path.join( self._testroot, "scripts_regression_tests.testdir_teststatus2") self._testdirs = [ self._testdir_all_pass, self._testdir_with_fail, self._testdir_unfinished, self._testdir_unfinished2, self._testdir_teststatus1, self._testdir_teststatus2, ] basic_tests = self._testdirs[:self._testdirs. index(self._testdir_teststatus1)] for testdir in self._testdirs: if os.path.exists(testdir): shutil.rmtree(testdir) os.makedirs(testdir) for r in range(10): for testdir in basic_tests: os.makedirs(os.path.join(testdir, str(r))) test_utils.make_fake_teststatus( os.path.join(testdir, str(r)), "Test_%d" % r, test_status.TEST_PASS_STATUS, test_status.RUN_PHASE, ) test_utils.make_fake_teststatus( os.path.join(self._testdir_with_fail, "5"), "Test_5", test_status.TEST_FAIL_STATUS, test_status.RUN_PHASE, ) test_utils.make_fake_teststatus( os.path.join(self._testdir_unfinished, "5"), "Test_5", test_status.TEST_PEND_STATUS, test_status.RUN_PHASE, ) test_utils.make_fake_teststatus( os.path.join(self._testdir_unfinished2, "5"), "Test_5", test_status.TEST_PASS_STATUS, test_status.SUBMIT_PHASE, ) integration_tests = self._testdirs[len(basic_tests):] for integration_test in integration_tests: os.makedirs(os.path.join(integration_test, "0")) test_utils.make_fake_teststatus( os.path.join(integration_test, "0"), "Test_0", test_status.TEST_PASS_STATUS, test_status.CORE_PHASES[0], ) # Set up proxy if possible self._unset_proxy = self.setup_proxy() self._thread_error = None
def test_rebless_namelist(self): # Generate some namelist baselines if self.NO_FORTRAN_RUN: self.skipTest("Skipping fortran test") test_to_change = "TESTRUNPASS_P1.f19_g16_rx1.A" if utils.get_model() == "e3sm": genargs = ["-g", "-o", "-b", self._baseline_name, "cime_test_only_pass"] compargs = ["-c", "-b", self._baseline_name, "cime_test_only_pass"] else: genargs = ["-g", self._baseline_name, "-o", "cime_test_only_pass"] compargs = ["-c", self._baseline_name, "cime_test_only_pass"] self._create_test(genargs) # Basic namelist compare test_id = "%s-%s" % (self._baseline_name, utils.get_timestamp()) cases = self._create_test(compargs, test_id=test_id) casedir = self.get_casedir(test_to_change, cases) # Check standalone case.cmpgen_namelists self.run_cmd_assert_result("./case.cmpgen_namelists", from_dir=casedir) # compare_test_results should pass cpr_cmd = "{}/compare_test_results --test-root {} -n -t {} ".format( self.TOOLS_DIR, self._testroot, test_id ) output = self.run_cmd_assert_result(cpr_cmd) # use regex expected_pattern = re.compile(r"PASS %s[^\s]* NLCOMP" % test_to_change) the_match = expected_pattern.search(output) msg = f"Cmd {cpr_cmd} failed to display passed test in output:\n{output}" self.assertNotEqual( the_match, None, msg=msg, ) # Modify namelist fake_nl = """ &fake_nml fake_item = 'fake' fake = .true. /""" baseline_area = self._baseline_area baseline_glob = glob.glob( os.path.join(baseline_area, self._baseline_name, "TEST*") ) self.assertEqual( len(baseline_glob), 3, msg="Expected three matches, got:\n%s" % "\n".join(baseline_glob), ) for baseline_dir in baseline_glob: nl_path = os.path.join(baseline_dir, "CaseDocs", "datm_in") self.assertTrue(os.path.isfile(nl_path), msg="Missing file %s" % nl_path) os.chmod(nl_path, stat.S_IRUSR | stat.S_IWUSR) with open(nl_path, "a") as nl_file: nl_file.write(fake_nl) # Basic namelist compare should now fail test_id = "%s-%s" % (self._baseline_name, utils.get_timestamp()) self._create_test(compargs, test_id=test_id, run_errors=True) casedir = self.get_casedir(test_to_change, cases) # Unless namelists are explicitly ignored test_id2 = "%s-%s" % (self._baseline_name, utils.get_timestamp()) self._create_test(compargs + ["--ignore-namelists"], test_id=test_id2) self.run_cmd_assert_result( "./case.cmpgen_namelists", from_dir=casedir, expected_stat=100 ) # preview namelists should work self.run_cmd_assert_result("./preview_namelists", from_dir=casedir) # This should still fail self.run_cmd_assert_result( "./case.cmpgen_namelists", from_dir=casedir, expected_stat=100 ) # compare_test_results should fail cpr_cmd = "{}/compare_test_results --test-root {} -n -t {} ".format( self.TOOLS_DIR, self._testroot, test_id ) output = self.run_cmd_assert_result( cpr_cmd, expected_stat=utils.TESTS_FAILED_ERR_CODE ) # use regex expected_pattern = re.compile(r"FAIL %s[^\s]* NLCOMP" % test_to_change) the_match = expected_pattern.search(output) self.assertNotEqual( the_match, None, msg="Cmd '%s' failed to display passed test in output:\n%s" % (cpr_cmd, output), ) # Bless new_test_id = "%s-%s" % (self._baseline_name, utils.get_timestamp()) utils.run_cmd_no_fail( "{}/bless_test_results --test-root {} -n --force -t {} --new-test-root={} --new-test-id={}".format( self.TOOLS_DIR, self._testroot, test_id, self._testroot, new_test_id ) ) # Basic namelist compare should now pass again self._create_test(compargs) self.verify_perms(self._baseline_area)
def _generate_baseline(self): """ generate a new baseline case based on the current test """ with self._test_status: # generate baseline # BEGIN: modified CIME.hist_utils.generate_baseline rundir = self._case.get_value("RUNDIR") basegen_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASEGEN_CASE")) testcase = self._case.get_value("CASE") if not os.path.isdir(basegen_dir): os.makedirs(basegen_dir) if os.path.isdir(os.path.join(basegen_dir, testcase)): expect(False, " Cowardly refusing to overwrite existing baseline directory") comments = "Generating baselines into '{}'\n".format(basegen_dir) num_gen = 0 model = 'cam' comments += " generating for model '{}'\n".format(model) hists = _get_all_hist_files(testcase, model, rundir) logger.debug("mvk_hist_files: {}".format(hists)) num_gen += len(hists) for hist in hists: basename = hist[hist.rfind(model):] baseline = os.path.join(basegen_dir, basename) if os.path.exists(baseline): os.remove(baseline) shutil.copy(hist, baseline) comments += " generating baseline '{}' from file {}\n".format(baseline, hist) newestcpllogfile = self._case.get_latest_cpl_log(coupler_log_path=self._case.get_value("LOGDIR")) if newestcpllogfile is None: logger.warning("No cpl.log file found in log directory {}".format(self._case.get_value("LOGDIR"))) else: shutil.copyfile(newestcpllogfile, os.path.join(basegen_dir, "cpl.log.gz")) expect(num_gen > 0, "Could not generate any hist files for case '{}', something is seriously wrong".format( os.path.join(rundir, testcase))) # make sure permissions are open in baseline directory for root, _, files in os.walk(basegen_dir): for name in files: try: os.chmod(os.path.join(root, name), stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH) except OSError: # We tried. Not worth hard failure here. pass if get_model() == "e3sm": bless_log = os.path.join(basegen_dir, BLESS_LOG_NAME) with open(bless_log, "a") as fd: fd.write("sha:{} date:{}\n".format(get_current_commit(repo=self._case.get_value("CIMEROOT")), get_timestamp(timestamp_format="%Y-%m-%d_%H:%M:%S"))) # END: modified CIME.hist_utils.generate_baseline append_testlog(comments) status = CIME.test_status.TEST_PASS_STATUS baseline_name = self._case.get_value("BASEGEN_CASE") self._test_status.set_status("{}".format(CIME.test_status.GENERATE_PHASE), status, comments=os.path.dirname(baseline_name)) basegen_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASEGEN_CASE")) # copy latest cpl log to baseline # drop the date so that the name is generic newestcpllogfiles = self._get_latest_cpl_logs() for cpllog in newestcpllogfiles: m = re.search(r"/(cpl.*.log).*.gz", cpllog) if m is not None: baselog = os.path.join(basegen_dir, m.group(1)) + ".gz" shutil.copyfile(cpllog, os.path.join(basegen_dir, baselog))
def case_run(case, skip_pnl=False): ############################################################################### # Set up the run, run the model, do the postrun steps run_with_submit = case.get_value("RUN_WITH_SUBMIT") expect(run_with_submit, "You are not calling the run script via the submit script. " "As a result, short-term archiving will not be called automatically." "Please submit your run using the submit script like so:" " ./case.submit Time: {}".format(get_timestamp())) # Forces user to use case.submit if they re-submit if case.get_value("TESTCASE") is None: case.set_value("RUN_WITH_SUBMIT", False) prerun_script = case.get_value("PRERUN_SCRIPT") postrun_script = case.get_value("POSTRUN_SCRIPT") data_assimilation = case.get_value("DATA_ASSIMILATION") data_assimilation_cycles = case.get_value("DATA_ASSIMILATION_CYCLES") data_assimilation_script = case.get_value("DATA_ASSIMILATION_SCRIPT") # set up the LID lid = new_lid() save_prerun_provenance(case) for cycle in range(data_assimilation_cycles): # After the first DA cycle, runs are restart runs if cycle > 0: case.set_value("CONTINUE_RUN", "TRUE") lid = new_lid() if prerun_script: case.flush() do_external(prerun_script, case.get_value("CASEROOT"), case.get_value("RUNDIR"), lid, prefix="prerun") case.read_xml() lid = run_model(case, lid, skip_pnl, da_cycle=cycle) save_logs(case, lid) # Copy log files back to caseroot if case.get_value("CHECK_TIMING") or case.get_value("SAVE_TIMING"): get_timing(case, lid) # Run the getTiming script if data_assimilation: case.flush() do_data_assimilation(data_assimilation_script, case.get_value("CASEROOT"), cycle, lid, case.get_value("RUNDIR")) case.read_xml() if postrun_script: case.flush() do_external(postrun_script, case.get_value("CASEROOT"), case.get_value("RUNDIR"), lid, prefix="postrun") case.read_xml() save_postrun_provenance(case) logger.warning("check for resubmit") resubmit_check(case) return True
def __init__(self, test_names, test_data=None, no_run=False, no_build=False, no_setup=False, no_batch=None, test_root=None, test_id=None, machine_name=None, compiler=None, baseline_root=None, baseline_cmp_name=None, baseline_gen_name=None, clean=False, namelists_only=False, project=None, parallel_jobs=None, walltime=None, proc_pool=None, use_existing=False, save_timing=False, queue=None, allow_baseline_overwrite=False, output_root=None, force_procs=None, force_threads=None, mpilib=None, input_dir=None, pesfile=None, mail_user=None, mail_type=None, allow_pnl=False, non_local=False): ########################################################################### self._cime_root = get_cime_root() self._cime_model = get_model() self._cime_driver = "mct" self._save_timing = save_timing self._queue = queue self._test_data = {} if test_data is None else test_data # Format: {test_name -> {data_name -> data}} self._mpilib = mpilib # allow override of default mpilib self._completed_tests = 0 self._input_dir = input_dir self._pesfile = pesfile self._allow_baseline_overwrite = allow_baseline_overwrite self._allow_pnl = allow_pnl self._non_local = non_local self._mail_user = mail_user self._mail_type = mail_type self._machobj = Machines(machine=machine_name) self._model_build_cost = 4 # If user is forcing procs or threads, re-write test names to reflect this. if force_procs or force_threads: test_names = _translate_test_names_for_new_pecount(test_names, force_procs, force_threads) self._no_setup = no_setup self._no_build = no_build or no_setup or namelists_only self._no_run = no_run or self._no_build self._output_root = output_root # Figure out what project to use if project is None: self._project = get_project() if self._project is None: self._project = self._machobj.get_value("PROJECT") else: self._project = project # We will not use batch system if user asked for no_batch or if current # machine is not a batch machine self._no_batch = no_batch or not self._machobj.has_batch_system() expect(not (self._no_batch and self._queue is not None), "Does not make sense to request a queue without batch system") # Determine and resolve test_root if test_root is not None: self._test_root = test_root elif self._output_root is not None: self._test_root = self._output_root else: self._test_root = self._machobj.get_value("CIME_OUTPUT_ROOT") if self._project is not None: self._test_root = self._test_root.replace("$PROJECT", self._project) self._test_root = os.path.abspath(self._test_root) self._test_id = test_id if test_id is not None else get_timestamp() self._compiler = self._machobj.get_default_compiler() if compiler is None else compiler self._clean = clean self._namelists_only = namelists_only self._walltime = walltime if parallel_jobs is None: self._parallel_jobs = min(len(test_names), self._machobj.get_value("MAX_MPITASKS_PER_NODE")) else: self._parallel_jobs = parallel_jobs self._baseline_cmp_name = baseline_cmp_name # Implies comparison should be done if not None self._baseline_gen_name = baseline_gen_name # Implies generation should be done if not None # Compute baseline_root self._baseline_root = baseline_root if baseline_root is not None \ else self._machobj.get_value("BASELINE_ROOT") if self._project is not None: self._baseline_root = self._baseline_root.replace("$PROJECT", self._project) self._baseline_root = os.path.abspath(self._baseline_root) if baseline_cmp_name or baseline_gen_name: if self._baseline_cmp_name: full_baseline_dir = os.path.join(self._baseline_root, self._baseline_cmp_name) expect(os.path.isdir(full_baseline_dir), "Missing baseline comparison directory {}".format(full_baseline_dir)) # the following is to assure that the existing generate directory is not overwritten if self._baseline_gen_name: full_baseline_dir = os.path.join(self._baseline_root, self._baseline_gen_name) existing_baselines = [] for test_name in test_names: test_baseline = os.path.join(full_baseline_dir, test_name) if os.path.isdir(test_baseline): existing_baselines.append(test_baseline) expect(allow_baseline_overwrite or len(existing_baselines) == 0, "Baseline directories already exists {}\n" \ "Use -o to avoid this error".format(existing_baselines)) if self._cime_model == "e3sm": _order_tests_by_runtime(test_names, self._baseline_root) # This is the only data that multiple threads will simultaneously access # Each test has it's own value and setting/retrieving items from a dict # is atomic, so this should be fine to use without mutex. # name -> (phase, status) self._tests = OrderedDict() for test_name in test_names: self._tests[test_name] = (TEST_START, TEST_PASS_STATUS) # Oversubscribe by 1/4 if proc_pool is None: pes = int(self._machobj.get_value("MAX_TASKS_PER_NODE")) self._proc_pool = int(pes * 1.25) else: self._proc_pool = int(proc_pool) self._procs_avail = self._proc_pool # Setup phases self._phases = list(PHASES) if self._no_setup: self._phases.remove(SETUP_PHASE) if self._no_build: self._phases.remove(SHAREDLIB_BUILD_PHASE) self._phases.remove(MODEL_BUILD_PHASE) if self._no_run: self._phases.remove(RUN_PHASE) if use_existing: for test in self._tests: with TestStatus(self._get_test_dir(test)) as ts: for phase, status in ts: if phase in CORE_PHASES: if status in [TEST_PEND_STATUS, TEST_FAIL_STATUS]: if status == TEST_FAIL_STATUS: # Import for potential subsequent waits ts.set_status(phase, TEST_PEND_STATUS) # We need to pick up here break else: if phase != SUBMIT_PHASE: # Somewhat subtle. Create_test considers submit/run to be the run phase, # so don't try to update test status for a passed submit phase self._update_test_status(test, phase, TEST_PEND_STATUS) self._update_test_status(test, phase, status) if phase == RUN_PHASE: logger.info("Test {} passed and will not be re-run".format(test)) logger.info("Using existing test directory {}".format(self._get_test_dir(test))) else: # None of the test directories should already exist. for test in self._tests: expect(not os.path.exists(self._get_test_dir(test)), "Cannot create new case in directory '{}', it already exists." " Pick a different test-id".format(self._get_test_dir(test))) logger.info("Creating test directory {}".format(self._get_test_dir(test)))
def test_c_use_existing(self): tests = get_tests.get_full_test_names( [ "TESTBUILDFAIL_P1.f19_g16_rx1.A", "TESTRUNFAIL_P1.f19_g16_rx1.A", "TESTRUNPASS_P1.f19_g16_rx1.A", ], self._machine, self._compiler, ) test_id = "%s-%s" % (self._baseline_name, utils.get_timestamp()) ct = test_scheduler.TestScheduler( tests, test_id=test_id, no_batch=self.NO_BATCH, test_root=self._testroot, output_root=self._testroot, compiler=self._compiler, mpilib=self.TEST_MPILIB, machine_name=self.MACHINE.get_machine_name(), ) build_fail_test = [item for item in tests if "TESTBUILDFAIL" in item][0] run_fail_test = [item for item in tests if "TESTRUNFAIL" in item][0] pass_test = [item for item in tests if "TESTRUNPASS" in item][0] log_lvl = logging.getLogger().getEffectiveLevel() logging.disable(logging.CRITICAL) try: ct.run_tests() finally: logging.getLogger().setLevel(log_lvl) test_statuses = glob.glob("%s/*%s/TestStatus" % (self._testroot, test_id)) self.assertEqual(len(tests), len(test_statuses)) self._wait_for_tests(test_id, expect_works=False) for x in test_statuses: casedir = os.path.dirname(x) ts = test_status.TestStatus(test_dir=casedir) test_name = ts.get_name() if test_name == build_fail_test: self.assert_test_status( test_name, ts, test_status.MODEL_BUILD_PHASE, test_status.TEST_FAIL_STATUS, ) with test_status.TestStatus(test_dir=casedir) as ts: ts.set_status(test_status.MODEL_BUILD_PHASE, test_status.TEST_PEND_STATUS) elif test_name == run_fail_test: self.assert_test_status(test_name, ts, test_status.RUN_PHASE, test_status.TEST_FAIL_STATUS) with test_status.TestStatus(test_dir=casedir) as ts: ts.set_status(test_status.SUBMIT_PHASE, test_status.TEST_PEND_STATUS) else: self.assertTrue(test_name == pass_test) self.assert_test_status( test_name, ts, test_status.MODEL_BUILD_PHASE, test_status.TEST_PASS_STATUS, ) self.assert_test_status( test_name, ts, test_status.SUBMIT_PHASE, test_status.TEST_PASS_STATUS, ) self.assert_test_status(test_name, ts, test_status.RUN_PHASE, test_status.TEST_PASS_STATUS) os.environ["TESTBUILDFAIL_PASS"] = "******" os.environ["TESTRUNFAIL_PASS"] = "******" ct2 = test_scheduler.TestScheduler( tests, test_id=test_id, no_batch=self.NO_BATCH, use_existing=True, test_root=self._testroot, output_root=self._testroot, compiler=self._compiler, mpilib=self.TEST_MPILIB, machine_name=self.MACHINE.get_machine_name(), ) log_lvl = logging.getLogger().getEffectiveLevel() logging.disable(logging.CRITICAL) try: ct2.run_tests() finally: logging.getLogger().setLevel(log_lvl) self._wait_for_tests(test_id) for x in test_statuses: ts = test_status.TestStatus(test_dir=os.path.dirname(x)) test_name = ts.get_name() self.assert_test_status( test_name, ts, test_status.MODEL_BUILD_PHASE, test_status.TEST_PASS_STATUS, ) self.assert_test_status(test_name, ts, test_status.SUBMIT_PHASE, test_status.TEST_PASS_STATUS) self.assert_test_status(test_name, ts, test_status.RUN_PHASE, test_status.TEST_PASS_STATUS) del os.environ["TESTBUILDFAIL_PASS"] del os.environ["TESTRUNFAIL_PASS"] # test that passed tests are not re-run ct2 = test_scheduler.TestScheduler( tests, test_id=test_id, no_batch=self.NO_BATCH, use_existing=True, test_root=self._testroot, output_root=self._testroot, compiler=self._compiler, mpilib=self.TEST_MPILIB, machine_name=self.MACHINE.get_machine_name(), ) log_lvl = logging.getLogger().getEffectiveLevel() logging.disable(logging.CRITICAL) try: ct2.run_tests() finally: logging.getLogger().setLevel(log_lvl) self._wait_for_tests(test_id) for x in test_statuses: ts = test_status.TestStatus(test_dir=os.path.dirname(x)) test_name = ts.get_name() self.assert_test_status( test_name, ts, test_status.MODEL_BUILD_PHASE, test_status.TEST_PASS_STATUS, ) self.assert_test_status(test_name, ts, test_status.SUBMIT_PHASE, test_status.TEST_PASS_STATUS) self.assert_test_status(test_name, ts, test_status.RUN_PHASE, test_status.TEST_PASS_STATUS)
def _case_build_impl(caseroot, case, sharedlib_only, model_only, buildlist, save_build_provenance): ############################################################################### t1 = time.time() expect(not (sharedlib_only and model_only), "Contradiction: both sharedlib_only and model_only") logger.info("Building case in directory {}".format(caseroot)) logger.info("sharedlib_only is {}".format(sharedlib_only)) logger.info("model_only is {}".format(model_only)) expect(os.path.isdir(caseroot), "'{}' is not a valid directory".format(caseroot)) os.chdir(caseroot) expect(os.path.exists(get_batch_script_for_job(case.get_primary_job())), "ERROR: must invoke case.setup script before calling build script ") cimeroot = case.get_value("CIMEROOT") comp_classes = case.get_values("COMP_CLASSES") case.check_lockedfiles(skip="env_batch") # Retrieve relevant case data # This environment variable gets set for cesm Make and # needs to be unset before building again. if "MODEL" in os.environ: del os.environ["MODEL"] build_threaded = case.get_build_threaded() casetools = case.get_value("CASETOOLS") exeroot = os.path.abspath(case.get_value("EXEROOT")) incroot = os.path.abspath(case.get_value("INCROOT")) libroot = os.path.abspath(case.get_value("LIBROOT")) sharedlibroot = os.path.abspath(case.get_value("SHAREDLIBROOT")) multi_driver = case.get_value("MULTI_DRIVER") complist = [] ninst = 1 for comp_class in comp_classes: if comp_class == "CPL": config_dir = None if multi_driver: ninst = case.get_value("NINST_MAX") else: config_dir = os.path.dirname(case.get_value("CONFIG_{}_FILE".format(comp_class))) if multi_driver: ninst = 1 else: ninst = case.get_value("NINST_{}".format(comp_class)) comp = case.get_value("COMP_{}".format(comp_class)) thrds = case.get_value("NTHRDS_{}".format(comp_class)) expect(ninst is not None,"Failed to get ninst for comp_class {}".format(comp_class)) complist.append((comp_class.lower(), comp, thrds, ninst, config_dir )) os.environ["COMP_{}".format(comp_class)] = comp ocn_submodel = case.get_value("OCN_SUBMODEL") profile_papi_enable = case.get_value("PROFILE_PAPI_ENABLE") compiler = case.get_value("COMPILER") comp_interface = case.get_value("COMP_INTERFACE") mpilib = case.get_value("MPILIB") use_esmf_lib = case.get_value("USE_ESMF_LIB") debug = case.get_value("DEBUG") ninst_build = case.get_value("NINST_BUILD") smp_value = case.get_value("SMP_VALUE") clm_use_petsc = case.get_value("CLM_USE_PETSC") cism_use_trilinos = case.get_value("CISM_USE_TRILINOS") mali_use_albany = case.get_value("MALI_USE_ALBANY") use_moab = case.get_value("USE_MOAB") clm_config_opts = case.get_value("CLM_CONFIG_OPTS") cam_config_opts = case.get_value("CAM_CONFIG_OPTS") pio_config_opts = case.get_value("PIO_CONFIG_OPTS") ninst_value = case.get_value("NINST_VALUE") mach = case.get_value("MACH") os_ = case.get_value("OS") # Load some params into env os.environ["CIMEROOT"] = cimeroot os.environ["CASETOOLS"] = casetools os.environ["EXEROOT"] = exeroot os.environ["INCROOT"] = incroot os.environ["LIBROOT"] = libroot os.environ["SHAREDLIBROOT"] = sharedlibroot os.environ["CASEROOT"] = caseroot os.environ["COMPILER"] = compiler os.environ["COMP_INTERFACE"] = comp_interface os.environ["NINST_VALUE"] = str(ninst_value) os.environ["BUILD_THREADED"] = stringify_bool(build_threaded) os.environ["MACH"] = mach os.environ["USE_ESMF_LIB"] = stringify_bool(use_esmf_lib) os.environ["MPILIB"] = mpilib os.environ["DEBUG"] = stringify_bool(debug) os.environ["OS"] = os_ os.environ["CLM_CONFIG_OPTS"] = clm_config_opts if clm_config_opts is not None else "" os.environ["CAM_CONFIG_OPTS"] = cam_config_opts if cam_config_opts is not None else "" os.environ["PIO_CONFIG_OPTS"] = pio_config_opts if pio_config_opts is not None else "" os.environ["OCN_SUBMODEL"] = ocn_submodel if ocn_submodel is not None else "" os.environ["PROFILE_PAPI_ENABLE"] = stringify_bool(profile_papi_enable) os.environ["CLM_USE_PETSC"] = stringify_bool(clm_use_petsc) os.environ["CISM_USE_TRILINOS"] = stringify_bool(cism_use_trilinos) os.environ["MALI_USE_ALBANY"] = stringify_bool(mali_use_albany) os.environ["USE_MOAB"] = stringify_bool(use_moab) if get_model() == "e3sm" and mach == "titan" and compiler == "pgiacc": case.set_value("CAM_TARGET", "preqx_acc") # This is a timestamp for the build , not the same as the testid, # and this case may not be a test anyway. For a production # experiment there may be many builds of the same case. lid = get_timestamp("%y%m%d-%H%M%S") os.environ["LID"] = lid # Set the overall USE_PETSC variable to TRUE if any of the # *_USE_PETSC variables are TRUE. # For now, there is just the one CLM_USE_PETSC variable, but in # the future there may be others -- so USE_PETSC will be true if # ANY of those are true. use_petsc = clm_use_petsc case.set_value("USE_PETSC", use_petsc) os.environ["USE_PETSC"] = stringify_bool(use_petsc) # Set the overall USE_TRILINOS variable to TRUE if any of the # *_USE_TRILINOS variables are TRUE. # For now, there is just the one CISM_USE_TRILINOS variable, but in # the future there may be others -- so USE_TRILINOS will be true if # ANY of those are true. use_trilinos = False if cism_use_trilinos is None else cism_use_trilinos case.set_value("USE_TRILINOS", use_trilinos) os.environ["USE_TRILINOS"] = stringify_bool(use_trilinos) # Set the overall USE_ALBANY variable to TRUE if any of the # *_USE_ALBANY variables are TRUE. # For now, there is just the one MALI_USE_ALBANY variable, but in # the future there may be others -- so USE_ALBANY will be true if # ANY of those are true. use_albany = stringify_bool(mali_use_albany) case.set_value("USE_ALBANY", use_albany) os.environ["USE_ALBANY"] = use_albany # Load modules case.load_env() sharedpath = _build_checks(case, build_threaded, comp_interface, use_esmf_lib, debug, compiler, mpilib, complist, ninst_build, smp_value, model_only, buildlist) t2 = time.time() logs = [] if not model_only: logs = _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid, compiler, buildlist, comp_interface) if not sharedlib_only: os.environ["INSTALL_SHAREDPATH"] = os.path.join(exeroot, sharedpath) # for MPAS makefile generators logs.extend(_build_model(build_threaded, exeroot, clm_config_opts, incroot, complist, lid, caseroot, cimeroot, compiler, buildlist, comp_interface)) if not buildlist: # in case component build scripts updated the xml files, update the case object case.read_xml() # Note, doing buildlists will never result in the system thinking the build is complete post_build(case, logs, build_complete=not (buildlist or sharedlib_only), save_build_provenance=save_build_provenance) t3 = time.time() if not sharedlib_only: logger.info("Time spent not building: {:f} sec".format(t2 - t1)) logger.info("Time spent building: {:f} sec".format(t3 - t2)) logger.info("MODEL BUILD HAS FINISHED SUCCESSFULLY") return True
############################################################################### expect(os.path.isfile(da_script), "Data Assimilation script {} not found".format(da_script)) filename = "da.log.{}".format(lid) outfile = os.path.join(rundir, filename) run_sub_or_cmd(da_script, [caseroot, cycle], os.path.basename(da_script), [caseroot, cycle], logfile=outfile,combine_output=True) ############################################################################### def case_run(case, skip_pnl=False): ############################################################################### # Set up the run, run the model, do the postrun steps run_with_submit = case.get_value("RUN_WITH_SUBMIT") expect(run_with_submit, "You are not calling the run script via the submit script. " "As a result, short-term archiving will not be called automatically." "Please submit your run using the submit script like so:" " ./case.submit Time: {}".format(get_timestamp())) # Forces user to use case.submit if they re-submit if case.get_value("TESTCASE") is None: case.set_value("RUN_WITH_SUBMIT", False) prerun_script = case.get_value("PRERUN_SCRIPT") postrun_script = case.get_value("POSTRUN_SCRIPT") data_assimilation = case.get_value("DATA_ASSIMILATION") data_assimilation_cycles = case.get_value("DATA_ASSIMILATION_CYCLES") data_assimilation_script = case.get_value("DATA_ASSIMILATION_SCRIPT") # set up the LID lid = new_lid()
def test_b_full(self): tests = get_tests.get_full_test_names(["cime_test_only"], self._machine, self._compiler) test_id = "%s-%s" % (self._baseline_name, utils.get_timestamp()) ct = test_scheduler.TestScheduler( tests, test_id=test_id, no_batch=self.NO_BATCH, test_root=self._testroot, output_root=self._testroot, compiler=self._compiler, mpilib=self.TEST_MPILIB, machine_name=self.MACHINE.get_machine_name(), ) build_fail_test = [item for item in tests if "TESTBUILDFAIL_" in item][0] build_fail_exc_test = [ item for item in tests if "TESTBUILDFAILEXC" in item ][0] run_fail_test = [item for item in tests if "TESTRUNFAIL_" in item][0] run_fail_exc_test = [ item for item in tests if "TESTRUNFAILEXC" in item ][0] pass_test = [item for item in tests if "TESTRUNPASS" in item][0] test_diff_test = [item for item in tests if "TESTTESTDIFF" in item][0] mem_fail_test = [item for item in tests if "TESTMEMLEAKFAIL" in item][0] mem_pass_test = [item for item in tests if "TESTMEMLEAKPASS" in item][0] st_arch_fail_test = [ item for item in tests if "TESTRUNSTARCFAIL" in item ][0] log_lvl = logging.getLogger().getEffectiveLevel() logging.disable(logging.CRITICAL) try: ct.run_tests() finally: logging.getLogger().setLevel(log_lvl) self._wait_for_tests(test_id, expect_works=False) test_statuses = glob.glob("%s/*%s/TestStatus" % (self._testroot, test_id)) self.assertEqual(len(tests), len(test_statuses)) for x in test_statuses: ts = test_status.TestStatus(test_dir=os.path.dirname(x)) test_name = ts.get_name() log_files = glob.glob("%s/%s*%s/TestStatus.log" % (self._testroot, test_name, test_id)) self.assertEqual( len(log_files), 1, "Expected exactly one test_status.TestStatus.log file, found %d" % len(log_files), ) log_file = log_files[0] if test_name == build_fail_test: self.assert_test_status( test_name, ts, test_status.MODEL_BUILD_PHASE, test_status.TEST_FAIL_STATUS, ) data = open(log_file, "r").read() self.assertTrue( "Intentional fail for testing infrastructure" in data, "Broken test did not report build error:\n%s" % data, ) elif test_name == build_fail_exc_test: data = open(log_file, "r").read() self.assert_test_status( test_name, ts, test_status.SHAREDLIB_BUILD_PHASE, test_status.TEST_FAIL_STATUS, ) self.assertTrue( "Exception from init" in data, "Broken test did not report build error:\n%s" % data, ) elif test_name == run_fail_test: self.assert_test_status(test_name, ts, test_status.RUN_PHASE, test_status.TEST_FAIL_STATUS) elif test_name == run_fail_exc_test: self.assert_test_status(test_name, ts, test_status.RUN_PHASE, test_status.TEST_FAIL_STATUS) data = open(log_file, "r").read() self.assertTrue( "Exception from run_phase" in data, "Broken test did not report run error:\n%s" % data, ) elif test_name == mem_fail_test: self.assert_test_status( test_name, ts, test_status.MEMLEAK_PHASE, test_status.TEST_FAIL_STATUS, ) self.assert_test_status(test_name, ts, test_status.RUN_PHASE, test_status.TEST_PASS_STATUS) elif test_name == test_diff_test: self.assert_test_status(test_name, ts, "COMPARE_base_rest", test_status.TEST_FAIL_STATUS) self.assert_test_status(test_name, ts, test_status.RUN_PHASE, test_status.TEST_PASS_STATUS) elif test_name == st_arch_fail_test: self.assert_test_status(test_name, ts, test_status.RUN_PHASE, test_status.TEST_PASS_STATUS) self.assert_test_status( test_name, ts, test_status.STARCHIVE_PHASE, test_status.TEST_FAIL_STATUS, ) else: self.assertTrue(test_name in [pass_test, mem_pass_test]) self.assert_test_status(test_name, ts, test_status.RUN_PHASE, test_status.TEST_PASS_STATUS) if test_name == mem_pass_test: self.assert_test_status( test_name, ts, test_status.MEMLEAK_PHASE, test_status.TEST_PASS_STATUS, )
def generate_baseline(case, baseline_dir=None, allow_baseline_overwrite=False): """ copy the current test output to baseline result case - The case containing the hist files to be copied into baselines baseline_dir - Optionally, specify a specific baseline dir, otherwise it will be computed from case config allow_baseline_overwrite must be true to generate baselines to an existing directory. returns (SUCCESS, comments) """ rundir = case.get_value("RUNDIR") if baseline_dir is None: baselineroot = case.get_value("BASELINE_ROOT") basegen_dir = os.path.join(baselineroot, case.get_value("BASEGEN_CASE")) else: basegen_dir = baseline_dir testcase = case.get_value("CASE") if not os.path.isdir(basegen_dir): os.makedirs(basegen_dir) if (os.path.isdir(os.path.join(basegen_dir,testcase)) and not allow_baseline_overwrite): expect(False, " Cowardly refusing to overwrite existing baseline directory") comments = "Generating baselines into '{}'\n".format(basegen_dir) num_gen = 0 for model in _iter_model_file_substrs(case): comments += " generating for model '{}'\n".format(model) hists = _get_latest_hist_files(testcase, model, rundir) logger.debug("latest_files: {}".format(hists)) num_gen += len(hists) for hist in hists: basename = hist[hist.rfind(model):] baseline = os.path.join(basegen_dir, basename) if os.path.exists(baseline): os.remove(baseline) shutil.copy(hist, baseline) comments += " generating baseline '{}' from file {}\n".format(baseline, hist) # copy latest cpl log to baseline # drop the date so that the name is generic newestcpllogfile = case.get_latest_cpl_log(coupler_log_path=case.get_value("LOGDIR")) if newestcpllogfile is None: logger.warning("No cpl.log file found in log directory {}".format(case.get_value("LOGDIR"))) else: shutil.copyfile(newestcpllogfile, os.path.join(basegen_dir, "cpl.log.gz")) expect(num_gen > 0, "Could not generate any hist files for case '{}', something is seriously wrong".format(os.path.join(rundir, testcase))) #make sure permissions are open in baseline directory for root, _, files in os.walk(basegen_dir): for name in files: try: os.chmod(os.path.join(root,name), stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH) except OSError: # We tried. Not worth hard failure here. pass if get_model() == "e3sm": bless_log = os.path.join(basegen_dir, BLESS_LOG_NAME) with open(bless_log, "a") as fd: fd.write("sha:{} date:{}\n".format(get_current_commit(repo=case.get_value("CIMEROOT")), get_timestamp(timestamp_format="%Y-%m-%d_%H:%M:%S"))) return True, comments
clm_use_petsc = case.get_value("CLM_USE_PETSC") cism_use_trilinos = case.get_value("CISM_USE_TRILINOS") mali_use_albany = case.get_value("MALI_USE_ALBANY") mach = case.get_value("MACH") # Load some params into env os.environ["BUILD_THREADED"] = stringify_bool(build_threaded) cime_model = get_model() if cime_model == "e3sm" and mach == "titan" and compiler == "pgiacc": case.set_value("CAM_TARGET", "preqx_acc") # This is a timestamp for the build , not the same as the testid, # and this case may not be a test anyway. For a production # experiment there may be many builds of the same case. lid = get_timestamp("%y%m%d-%H%M%S") os.environ["LID"] = lid # Set the overall USE_PETSC variable to TRUE if any of the # *_USE_PETSC variables are TRUE. # For now, there is just the one CLM_USE_PETSC variable, but in # the future there may be others -- so USE_PETSC will be true if # ANY of those are true. use_petsc = clm_use_petsc case.set_value("USE_PETSC", use_petsc) # Set the overall USE_TRILINOS variable to TRUE if any of the # *_USE_TRILINOS variables are TRUE. # For now, there is just the one CISM_USE_TRILINOS variable, but in # the future there may be others -- so USE_TRILINOS will be true if
def _generate_baseline(self): """ generate a new baseline case based on the current test """ with self._test_status: # generate baseline # BEGIN: modified CIME.hist_utils.generate_baseline rundir = self._case.get_value("RUNDIR") basegen_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASEGEN_CASE")) testcase = self._case.get_value("CASE") if not os.path.isdir(basegen_dir): os.makedirs(basegen_dir) if os.path.isdir(os.path.join(basegen_dir, testcase)): expect( False, " Cowardly refusing to overwrite existing baseline directory" ) comments = "Generating baselines into '{}'\n".format(basegen_dir) num_gen = 0 model = 'cam' comments += " generating for model '{}'\n".format(model) hists = _get_all_hist_files(testcase, model, rundir) logger.debug("mvk_hist_files: {}".format(hists)) num_gen += len(hists) for hist in hists: basename = hist[hist.rfind(model):] baseline = os.path.join(basegen_dir, basename) if os.path.exists(baseline): os.remove(baseline) shutil.copy(hist, baseline) comments += " generating baseline '{}' from file {}\n".format( baseline, hist) newestcpllogfile = self._case.get_latest_cpl_log( coupler_log_path=self._case.get_value("LOGDIR")) if newestcpllogfile is None: logger.warning( "No cpl.log file found in log directory {}".format( self._case.get_value("LOGDIR"))) else: shutil.copyfile(newestcpllogfile, os.path.join(basegen_dir, "cpl.log.gz")) expect( num_gen > 0, "Could not generate any hist files for case '{}', something is seriously wrong" .format(os.path.join(rundir, testcase))) # make sure permissions are open in baseline directory for root, _, files in os.walk(basegen_dir): for name in files: try: os.chmod( os.path.join(root, name), stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH) except OSError: # We tried. Not worth hard failure here. pass if get_model() == "e3sm": bless_log = os.path.join(basegen_dir, BLESS_LOG_NAME) with open(bless_log, "a") as fd: fd.write("sha:{} date:{}\n".format( get_current_commit( repo=self._case.get_value("CIMEROOT")), get_timestamp(timestamp_format="%Y-%m-%d_%H:%M:%S"))) # END: modified CIME.hist_utils.generate_baseline append_testlog(comments) status = CIME.test_status.TEST_PASS_STATUS baseline_name = self._case.get_value("BASEGEN_CASE") self._test_status.set_status( "{}".format(CIME.test_status.GENERATE_PHASE), status, comments=os.path.dirname(baseline_name)) basegen_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASEGEN_CASE")) # copy latest cpl log to baseline # drop the date so that the name is generic newestcpllogfiles = self._get_latest_cpl_logs() for cpllog in newestcpllogfiles: m = re.search(r"/(cpl.*.log).*.gz", cpllog) if m is not None: baselog = os.path.join(basegen_dir, m.group(1)) + ".gz" shutil.copyfile(cpllog, os.path.join(basegen_dir, baselog))
coupler_log_path=case.get_value("RUNDIR")) if newestcpllogfile is None: logger.warning("No cpl.log file found in directory {}".format( case.get_value("RUNDIR"))) else: safe_copy(newestcpllogfile, os.path.join(basegen_dir, "cpl.log.gz")) expect( num_gen > 0, "Could not generate any hist files for case '{}', something is seriously wrong" .format(os.path.join(rundir, testcase))) #make sure permissions are open in baseline directory for root, _, files in os.walk(basegen_dir): for name in files: try: os.chmod( os.path.join(root, name), stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH) except OSError: # We tried. Not worth hard failure here. pass if get_model() == "e3sm": bless_log = os.path.join(basegen_dir, BLESS_LOG_NAME) with open(bless_log, "a") as fd: fd.write("sha:{} date:{}\n".format( get_current_commit(repo=case.get_value("CIMEROOT")), get_timestamp(timestamp_format="%Y-%m-%d_%H:%M:%S"))) return True, comments
def _generate_baseline_impl(case, baseline_dir=None, allow_baseline_overwrite=False): """ copy the current test output to baseline result case - The case containing the hist files to be copied into baselines baseline_dir - Optionally, specify a specific baseline dir, otherwise it will be computed from case config allow_baseline_overwrite must be true to generate baselines to an existing directory. returns (SUCCESS, comments) """ rundir = case.get_value("RUNDIR") ref_case = case.get_value("RUN_REFCASE") if baseline_dir is None: baselineroot = case.get_value("BASELINE_ROOT") basegen_dir = os.path.join(baselineroot, case.get_value("BASEGEN_CASE")) else: basegen_dir = baseline_dir testcase = case.get_value("CASE") archive = case.get_env('archive') if not os.path.isdir(basegen_dir): os.makedirs(basegen_dir) if (os.path.isdir(os.path.join(basegen_dir,testcase)) and not allow_baseline_overwrite): expect(False, " Cowardly refusing to overwrite existing baseline directory") comments = "Generating baselines into '{}'\n".format(basegen_dir) num_gen = 0 for model in _iter_model_file_substrs(case): comments += " generating for model '{}'\n".format(model) if model == 'cpl': file_extensions = archive.get_hist_file_extensions(archive.get_entry('drv')) else: file_extensions = archive.get_hist_file_extensions(archive.get_entry(model)) hists = _get_latest_hist_files(model, rundir, file_extensions, ref_case=ref_case) logger.debug("latest_files: {}".format(hists)) num_gen += len(hists) for hist in hists: basename = hist[hist.rfind(model):] baseline = os.path.join(basegen_dir, basename) if os.path.exists(baseline): os.remove(baseline) safe_copy(hist, baseline, preserve_meta=False) comments += " generating baseline '{}' from file {}\n".format(baseline, hist) # copy latest cpl log to baseline # drop the date so that the name is generic if case.get_value("COMP_INTERFACE") == "nuopc": cplname = "med" else: cplname = "cpl" newestcpllogfile = case.get_latest_cpl_log(coupler_log_path=case.get_value("RUNDIR"), cplname=cplname) if newestcpllogfile is None: logger.warning("No {}.log file found in directory {}".format(cplname,case.get_value("RUNDIR"))) else: safe_copy(newestcpllogfile, os.path.join(basegen_dir, "{}.log.gz".format(cplname)), preserve_meta=False) testname = case.get_value("TESTCASE") expect(num_gen > 0 or testname == "PFS", "Could not generate any hist files for case '{}', something is seriously wrong".format(os.path.join(rundir, testcase))) if get_model() == "e3sm": bless_log = os.path.join(basegen_dir, BLESS_LOG_NAME) with open(bless_log, "a") as fd: fd.write("sha:{} date:{}\n".format(get_current_commit(repo=case.get_value("CIMEROOT")), get_timestamp(timestamp_format="%Y-%m-%d_%H:%M:%S"))) return True, comments