def save_test_success(baseline_root, src_root, test, succeeded, force_commit_test=None): """ Update success data accordingly based on succeeded flag """ if baseline_root is not None: try: with SharedArea(): success_path, prev_results = _read_success_data(baseline_root, test) the_dir = os.path.dirname(success_path) if not os.path.exists(the_dir): os.makedirs(the_dir) prev_succeeded = _is_test_working(prev_results, src_root, testing=(force_commit_test is not None)) # if no transition occurred then no update is needed if succeeded or succeeded != prev_succeeded or (prev_results[0] is None and succeeded) or (prev_results[1] is None and not succeeded): new_results = list(prev_results) my_commit = force_commit_test if force_commit_test else get_current_commit(repo=src_root) if succeeded: new_results[0] = my_commit # we passed else: new_results[1] = my_commit # we transitioned to a failing state str_results = ["None" if item is None else item for item in new_results] with open(success_path, "w") as fd: fd.write("{}\n".format(" ".join(str_results))) except Exception: # We NEVER want a failure here to kill the run logger.warning("Failed to store test success: {}".format(sys.exc_info()[1]))
def write_provenance_info(machine, test_compiler, test_mpilib, test_root): curr_commit = get_current_commit(repo=CIMEROOT) logging.info("Testing commit %s" % curr_commit) cime_model = get_model() logging.info("Using cime_model = %s" % cime_model) logging.info("Testing machine = %s" % machine.get_machine_name()) if test_compiler is not None: logging.info("Testing compiler = %s" % test_compiler) if test_mpilib is not None: logging.info("Testing mpilib = %s" % test_mpilib) logging.info("Test root: %s" % test_root) logging.info("Test driver: %s" % CIME.utils.get_cime_default_driver()) logging.info("Python version {}\n".format(sys.version))
def _record_git_provenance(srcroot, exeroot, lid): """Records git provenance Records git status, diff and logs for main repo and all submodules. """ # Save git describe describe_prov = os.path.join(exeroot, "GIT_DESCRIBE.{}".format(lid)) desc = utils.get_current_commit(tag=True, repo=srcroot) with open(describe_prov, "w") as fd: fd.write(desc) gitroot = _find_git_root(srcroot) # Save HEAD headfile = os.path.join(gitroot, "logs", "HEAD") headfile_prov = os.path.join(exeroot, "GIT_LOGS_HEAD.{}".format(lid)) if os.path.exists(headfile_prov): os.remove(headfile_prov) if os.path.exists(headfile): utils.safe_copy(headfile, headfile_prov, preserve_meta=False) # Save git submodule status submodule_prov = os.path.join(exeroot, "GIT_SUBMODULE_STATUS.{}".format(lid)) subm_status = utils.get_current_submodule_status(recursive=True, repo=srcroot) with open(submodule_prov, "w") as fd: fd.write(subm_status) # Git Status status_prov = os.path.join(exeroot, "GIT_STATUS.{}".format(lid)) _run_git_cmd_recursively("status", srcroot, status_prov) # Git Diff diff_prov = os.path.join(exeroot, "GIT_DIFF.{}".format(lid)) _run_git_cmd_recursively("diff", srcroot, diff_prov) # Git Log log_prov = os.path.join(exeroot, "GIT_LOG.{}".format(lid)) cmd = "log --first-parent --pretty=oneline -n 5" _run_git_cmd_recursively(cmd, srcroot, log_prov) # Git remote remote_prov = os.path.join(exeroot, "GIT_REMOTE.{}".format(lid)) _run_git_cmd_recursively("remote -v", srcroot, remote_prov) # Git config config_src = os.path.join(gitroot, "config") config_prov = os.path.join(exeroot, "GIT_CONFIG.{}".format(lid)) utils.safe_copy(config_src, config_prov, preserve_meta=False)
def set_model_version(self, model): version = "unknown" if model == "cesm": srcroot = self.get_value("SRCROOT") changelog = os.path.join(srcroot,"ChangeLog") expect(os.path.isfile(changelog), " No CESM ChangeLog file found") for line in open(changelog, "r"): m = re.search("Tag name: (cesm.*)$", line) if m is not None: version = m.group(1) break elif model == "acme": cimeroot = self.get_value("CIMEROOT") version = get_current_commit(True, cimeroot) self.set_value("MODEL_VERSION", version)
def _save_build_provenance_e3sm(case, lid): cimeroot = case.get_value("CIMEROOT") exeroot = case.get_value("EXEROOT") caseroot = case.get_value("CASEROOT") # Save git describe describe_prov = os.path.join(exeroot, "GIT_DESCRIBE.{}".format(lid)) desc = get_current_commit(tag=True, repo=cimeroot) with open(describe_prov, "w") as fd: fd.write(desc) # Save HEAD headfile = os.path.join(cimeroot, ".git", "logs", "HEAD") headfile_prov = os.path.join(exeroot, "GIT_LOGS_HEAD.{}".format(lid)) if os.path.exists(headfile_prov): os.remove(headfile_prov) if os.path.exists(headfile): copy_umask(headfile, headfile_prov) # Save SourceMods sourcemods = os.path.join(caseroot, "SourceMods") sourcemods_prov = os.path.join(exeroot, "SourceMods.{}.tar.gz".format(lid)) if os.path.exists(sourcemods_prov): os.remove(sourcemods_prov) if os.path.isdir(sourcemods): with tarfile.open(sourcemods_prov, "w:gz") as tfd: tfd.add(sourcemods, arcname="SourceMods") # Save build env env_prov = os.path.join(exeroot, "build_environment.{}.txt".format(lid)) if os.path.exists(env_prov): os.remove(env_prov) env_module = case.get_env("mach_specific") env_module.save_all_env_info(env_prov) # For all the just-created post-build provenance files, symlink a generic name # to them to indicate that these are the most recent or active. for item in ["GIT_DESCRIBE", "GIT_LOGS_HEAD", "SourceMods", "build_environment"]: globstr = "{}/{}.{}*".format(exeroot, item, lid) matches = glob.glob(globstr) expect(len(matches) < 2, "Multiple matches for glob {} should not have happened".format(globstr)) if matches: the_match = matches[0] generic_name = the_match.replace(".{}".format(lid), "") if os.path.exists(generic_name): os.remove(generic_name) os.symlink(the_match, generic_name)
def set_model_version(self, model): version = "unknown" srcroot = self.get_value("SRCROOT") if model == "cesm": changelog = os.path.join(srcroot, "ChangeLog") if os.path.isfile(changelog): for line in open(changelog, "r"): m = re.search("Tag name: (cesm.*)$", line) if m is not None: version = m.group(1) break elif model == "acme": version = get_current_commit(True, srcroot) self.set_value("MODEL_VERSION", version) if version != "unknown": logger.info("%s model version found: %s" % (model, version)) else: logger.warn("WARNING: No %s Model version found." % (model))
def set_model_version(self, model): version = "unknown" srcroot = self.get_value("SRCROOT") if model == "cesm": changelog = os.path.join(srcroot,"ChangeLog") if os.path.isfile(changelog): for line in open(changelog, "r"): m = re.search("Tag name: (cesm.*)$", line) if m is not None: version = m.group(1) break elif model == "acme": version = get_current_commit(True, srcroot) self.set_value("MODEL_VERSION", version) if version != "unknown": logger.info("%s model version found: %s"%(model, version)) else: logger.warn("WARNING: No %s Model version found."%(model))
coupler_log_path=case.get_value("RUNDIR")) if newestcpllogfile is None: logger.warning("No cpl.log file found in directory {}".format( case.get_value("RUNDIR"))) else: safe_copy(newestcpllogfile, os.path.join(basegen_dir, "cpl.log.gz")) expect( num_gen > 0, "Could not generate any hist files for case '{}', something is seriously wrong" .format(os.path.join(rundir, testcase))) #make sure permissions are open in baseline directory for root, _, files in os.walk(basegen_dir): for name in files: try: os.chmod( os.path.join(root, name), stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH) except OSError: # We tried. Not worth hard failure here. pass if get_model() == "e3sm": bless_log = os.path.join(basegen_dir, BLESS_LOG_NAME) with open(bless_log, "a") as fd: fd.write("sha:{} date:{}\n".format( get_current_commit(repo=case.get_value("CIMEROOT")), get_timestamp(timestamp_format="%Y-%m-%d_%H:%M:%S"))) return True, comments
def _save_prerun_timing_e3sm(case, lid): project = case.get_value("PROJECT", subgroup=case.get_primary_job()) if not case.is_save_timing_dir_project(project): return timing_dir = case.get_value("SAVE_TIMING_DIR") if timing_dir is None or not os.path.isdir(timing_dir): logger.warning("SAVE_TIMING_DIR {} is not valid. E3SM requires a valid SAVE_TIMING_DIR to archive timing data.".format(timing_dir)) return logger.info("Archiving timing data and associated provenance in {}.".format(timing_dir)) rundir = case.get_value("RUNDIR") blddir = case.get_value("EXEROOT") caseroot = case.get_value("CASEROOT") cimeroot = case.get_value("CIMEROOT") base_case = case.get_value("CASE") full_timing_dir = os.path.join(timing_dir, "performance_archive", getpass.getuser(), base_case, lid) if os.path.exists(full_timing_dir): logger.warning("{} already exists. Skipping archive of timing data and associated provenance.".format(full_timing_dir)) return try: os.makedirs(full_timing_dir) except OSError: logger.warning("{} cannot be created. Skipping archive of timing data and associated provenance.".format(full_timing_dir)) return mach = case.get_value("MACH") compiler = case.get_value("COMPILER") # For some batch machines save queue info job_id = _get_batch_job_id_for_syslog(case) if job_id is not None: if mach == "mira": for cmd, filename in [("qstat -f", "qstatf"), ("qstat -lf %s" % job_id, "qstatf_jobid")]: filename = "%s.%s" % (filename, lid) run_cmd_no_fail(cmd, arg_stdout=filename, from_dir=full_timing_dir) gzip_existing_file(os.path.join(full_timing_dir, filename)) elif mach == "theta": for cmd, filename in [("qstat -l --header JobID:JobName:User:Project:WallTime:QueuedTime:Score:RunTime:TimeRemaining:Nodes:State:Location:Mode:Command:Args:Procs:Queue:StartTime:attrs:Geometry", "qstatf"), ("qstat -lf %s" % job_id, "qstatf_jobid"), ("xtnodestat", "xtnodestat"), ("xtprocadmin", "xtprocadmin")]: filename = "%s.%s" % (filename, lid) run_cmd_no_fail(cmd, arg_stdout=filename, from_dir=full_timing_dir) gzip_existing_file(os.path.join(full_timing_dir, filename)) elif mach in ["edison", "cori-haswell", "cori-knl"]: for cmd, filename in [("sinfo -a -l", "sinfol"), ("sqs -f %s" % job_id, "sqsf_jobid"), # ("sqs -f", "sqsf"), ("squeue -o '%.10i %.15P %.20j %.10u %.7a %.2t %.6D %.8C %.10M %.10l %.20S %.20V'", "squeuef"), ("squeue -t R -o '%.10i %R'", "squeues")]: filename = "%s.%s" % (filename, lid) run_cmd_no_fail(cmd, arg_stdout=filename, from_dir=full_timing_dir) gzip_existing_file(os.path.join(full_timing_dir, filename)) elif mach == "titan": for cmd, filename in [("qstat -f %s >" % job_id, "qstatf_jobid"), ("xtnodestat >", "xtnodestat"), # ("qstat -f >", "qstatf"), # ("xtdb2proc -f", "xtdb2proc"), ("showq >", "showq")]: full_cmd = cmd + " " + filename run_cmd_no_fail(full_cmd + "." + lid, from_dir=full_timing_dir) gzip_existing_file(os.path.join(full_timing_dir, filename + "." + lid)) # mdiag_reduce = os.path.join(full_timing_dir, "mdiag_reduce." + lid) # run_cmd_no_fail("./mdiag_reduce.csh", arg_stdout=mdiag_reduce, from_dir=os.path.join(caseroot, "Tools")) # gzip_existing_file(mdiag_reduce) elif mach == "anvil": for cmd, filename in [("qstat -f -1 acme >", "qstatf"), ("qstat -f %s >" % job_id, "qstatf_jobid"), ("qstat -r acme >", "qstatr")]: full_cmd = cmd + " " + filename run_cmd_no_fail(full_cmd + "." + lid, from_dir=full_timing_dir) gzip_existing_file(os.path.join(full_timing_dir, filename + "." + lid)) # copy/tar SourceModes source_mods_dir = os.path.join(caseroot, "SourceMods") if os.path.isdir(source_mods_dir): with tarfile.open(os.path.join(full_timing_dir, "SourceMods.{}.tar.gz".format(lid)), "w:gz") as tfd: tfd.add(source_mods_dir, arcname="SourceMods") # Save various case configuration items case_docs = os.path.join(full_timing_dir, "CaseDocs.{}".format(lid)) os.mkdir(case_docs) globs_to_copy = [ "CaseDocs/*", "*.run", ".*.run", "*.xml", "user_nl_*", "*env_mach_specific*", "Macros*", "README.case", "Depends.{}".format(mach), "Depends.{}".format(compiler), "Depends.{}.{}".format(mach, compiler), "software_environment.txt" ] for glob_to_copy in globs_to_copy: for item in glob.glob(os.path.join(caseroot, glob_to_copy)): copy_umask(item, os.path.join(case_docs, "{}.{}".format(os.path.basename(item).lstrip("."), lid))) # Copy some items from build provenance blddir_globs_to_copy = [ "GIT_LOGS_HEAD", "build_environment.txt" ] for blddir_glob_to_copy in blddir_globs_to_copy: for item in glob.glob(os.path.join(blddir, blddir_glob_to_copy)): copy_umask(item, os.path.join(full_timing_dir, os.path.basename(item) + "." + lid)) # Save state of repo from_repo = cimeroot if os.path.exists(os.path.join(cimeroot, ".git")) else os.path.dirname(cimeroot) desc = get_current_commit(tag=True, repo=from_repo) with open(os.path.join(full_timing_dir, "GIT_DESCRIBE.{}".format(lid)), "w") as fd: fd.write(desc) # What this block does is mysterious to me (JGF) if job_id is not None: # Kill mach_syslog from previous run if one exists syslog_jobid_path = os.path.join(rundir, "syslog_jobid.{}".format(job_id)) if os.path.exists(syslog_jobid_path): try: with open(syslog_jobid_path, "r") as fd: syslog_jobid = int(fd.read().strip()) os.kill(syslog_jobid, signal.SIGTERM) except (ValueError, OSError) as e: logger.warning("Failed to kill syslog: {}".format(e)) finally: os.remove(syslog_jobid_path) # If requested, spawn a mach_syslog process to monitor job progress sample_interval = case.get_value("SYSLOG_N") if sample_interval > 0: archive_checkpoints = os.path.join(full_timing_dir, "checkpoints.{}".format(lid)) os.mkdir(archive_checkpoints) touch("{}/e3sm.log.{}".format(rundir, lid)) syslog_jobid = run_cmd_no_fail("./mach_syslog {:d} {} {} {} {}/timing/checkpoints {} >& /dev/null & echo $!".format(sample_interval, job_id, lid, rundir, rundir, archive_checkpoints), from_dir=os.path.join(caseroot, "Tools")) with open(os.path.join(rundir, "syslog_jobid.{}".format(job_id)), "w") as fd: fd.write("{}\n".format(syslog_jobid))
newestcpllogfile = case.get_latest_cpl_log(coupler_log_path=case.get_value("RUNDIR"), cplname=cplname) if newestcpllogfile is None: logger.warning("No {}.log file found in directory {}".format(cplname,case.get_value("RUNDIR"))) else: safe_copy(newestcpllogfile, os.path.join(basegen_dir, "{}.log.gz".format(cplname)), preserve_meta=False) testname = case.get_value("TESTCASE") testopts = parse_test_name(case.get_value("CASEBASEID"))[1] testopts = [] if testopts is None else testopts expect(num_gen > 0 or (testname in ["PFS", "TSC"] or "B" in testopts), "Could not generate any hist files for case '{}', something is seriously wrong".format(os.path.join(rundir, testcase))) if get_model() == "e3sm": bless_log = os.path.join(basegen_dir, BLESS_LOG_NAME) with open(bless_log, "a") as fd: fd.write("sha:{} date:{}\n".format(get_current_commit(repo=case.get_value("CIMEROOT")), get_timestamp(timestamp_format="%Y-%m-%d_%H:%M:%S"))) return True, comments def generate_baseline(case, baseline_dir=None, allow_baseline_overwrite=False): with SharedArea(): return _generate_baseline_impl(case, baseline_dir=baseline_dir, allow_baseline_overwrite=allow_baseline_overwrite) def get_ts_synopsis(comments): r""" Reduce case diff comments down to a single line synopsis so that we can put something in the TestStatus file. It's expected that the comments provided to this function came from compare_baseline, not compare_tests. >>> get_ts_synopsis('')
def _generate_baseline(self): """ generate a new baseline case based on the current test """ with self._test_status: # generate baseline # BEGIN: modified CIME.hist_utils.generate_baseline rundir = self._case.get_value("RUNDIR") basegen_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASEGEN_CASE")) testcase = self._case.get_value("CASE") if not os.path.isdir(basegen_dir): os.makedirs(basegen_dir) if os.path.isdir(os.path.join(basegen_dir, testcase)): expect( False, " Cowardly refusing to overwrite existing baseline directory" ) comments = "Generating baselines into '{}'\n".format(basegen_dir) num_gen = 0 model = 'cam' comments += " generating for model '{}'\n".format(model) hists = _get_all_hist_files(testcase, model, rundir) logger.debug("mvk_hist_files: {}".format(hists)) num_gen += len(hists) for hist in hists: basename = hist[hist.rfind(model):] baseline = os.path.join(basegen_dir, basename) if os.path.exists(baseline): os.remove(baseline) shutil.copy(hist, baseline) comments += " generating baseline '{}' from file {}\n".format( baseline, hist) newestcpllogfile = self._case.get_latest_cpl_log( coupler_log_path=self._case.get_value("LOGDIR")) if newestcpllogfile is None: logger.warning( "No cpl.log file found in log directory {}".format( self._case.get_value("LOGDIR"))) else: shutil.copyfile(newestcpllogfile, os.path.join(basegen_dir, "cpl.log.gz")) expect( num_gen > 0, "Could not generate any hist files for case '{}', something is seriously wrong" .format(os.path.join(rundir, testcase))) # make sure permissions are open in baseline directory for root, _, files in os.walk(basegen_dir): for name in files: try: os.chmod( os.path.join(root, name), stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH) except OSError: # We tried. Not worth hard failure here. pass if get_model() == "e3sm": bless_log = os.path.join(basegen_dir, BLESS_LOG_NAME) with open(bless_log, "a") as fd: fd.write("sha:{} date:{}\n".format( get_current_commit( repo=self._case.get_value("CIMEROOT")), get_timestamp(timestamp_format="%Y-%m-%d_%H:%M:%S"))) # END: modified CIME.hist_utils.generate_baseline append_testlog(comments) status = CIME.test_status.TEST_PASS_STATUS baseline_name = self._case.get_value("BASEGEN_CASE") self._test_status.set_status( "{}".format(CIME.test_status.GENERATE_PHASE), status, comments=os.path.dirname(baseline_name)) basegen_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASEGEN_CASE")) # copy latest cpl log to baseline # drop the date so that the name is generic newestcpllogfiles = self._get_latest_cpl_logs() for cpllog in newestcpllogfiles: m = re.search(r"/(cpl.*.log).*.gz", cpllog) if m is not None: baselog = os.path.join(basegen_dir, m.group(1)) + ".gz" shutil.copyfile(cpllog, os.path.join(basegen_dir, baselog))
"w:gz") as tfd: tfd.add(source_mods_dir, arcname="SourceMods") # Save various case configuration items case_docs = os.path.join(full_timing_dir, "CaseDocs.{}".format(lid)) os.mkdir(case_docs) _copy_caseroot_files(mach, compiler, caseroot, case_docs, lid) _copy_blddir_files(blddir, full_timing_dir, lid) _copy_rundir_files(rundir, full_timing_dir, lid) # Save state of repo from_repo = (srcroot if os.path.exists(os.path.join(srcroot, ".git")) else os.path.dirname(srcroot)) desc = utils.get_current_commit(tag=True, repo=from_repo) with open(os.path.join(full_timing_dir, "GIT_DESCRIBE.{}".format(lid)), "w") as fd: fd.write(desc) # What this block does is mysterious to me (JGF) if job_id is not None: _record_syslog(case, lid, job_id, caseroot, rundir, full_timing_dir) def _record_queue_info(mach, job_id, lid, full_timing_dir): if mach == "theta": _record_anl_theta_queue(job_id, lid, full_timing_dir) elif mach in ["cori-haswell", "cori-knl"]: _record_nersc_queue(job_id, lid, full_timing_dir) elif mach in ["anvil", "chrysalis", "compy"]:
testname = case.get_value("TESTCASE") testopts = parse_test_name(case.get_value("CASEBASEID"))[1] testopts = [] if testopts is None else testopts expect( num_gen > 0 or (testname in NO_HIST_TESTS or "B" in testopts), "Could not generate any hist files for case '{}', something is seriously wrong".format( os.path.join(rundir, testcase) ), ) if get_model() == "e3sm": bless_log = os.path.join(basegen_dir, BLESS_LOG_NAME) with open(bless_log, "a", encoding="utf-8") as fd: fd.write( "sha:{} date:{}\n".format( get_current_commit(repo=case.get_value("SRCROOT")), get_timestamp(timestamp_format="%Y-%m-%d_%H:%M:%S"), ) ) return True, comments def generate_baseline(case, baseline_dir=None, allow_baseline_overwrite=False): with SharedArea(): return _generate_baseline_impl( case, baseline_dir=baseline_dir, allow_baseline_overwrite=allow_baseline_overwrite, )
def _generate_baseline(self): """ generate a new baseline case based on the current test """ with self._test_status: # generate baseline # BEGIN: modified CIME.hist_utils.generate_baseline rundir = self._case.get_value("RUNDIR") basegen_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASEGEN_CASE")) testcase = self._case.get_value("CASE") if not os.path.isdir(basegen_dir): os.makedirs(basegen_dir) if os.path.isdir(os.path.join(basegen_dir, testcase)): expect(False, " Cowardly refusing to overwrite existing baseline directory") comments = "Generating baselines into '{}'\n".format(basegen_dir) num_gen = 0 model = 'cam' comments += " generating for model '{}'\n".format(model) hists = _get_all_hist_files(testcase, model, rundir) logger.debug("mvk_hist_files: {}".format(hists)) num_gen += len(hists) for hist in hists: basename = hist[hist.rfind(model):] baseline = os.path.join(basegen_dir, basename) if os.path.exists(baseline): os.remove(baseline) shutil.copy(hist, baseline) comments += " generating baseline '{}' from file {}\n".format(baseline, hist) newestcpllogfile = self._case.get_latest_cpl_log(coupler_log_path=self._case.get_value("LOGDIR")) if newestcpllogfile is None: logger.warning("No cpl.log file found in log directory {}".format(self._case.get_value("LOGDIR"))) else: shutil.copyfile(newestcpllogfile, os.path.join(basegen_dir, "cpl.log.gz")) expect(num_gen > 0, "Could not generate any hist files for case '{}', something is seriously wrong".format( os.path.join(rundir, testcase))) # make sure permissions are open in baseline directory for root, _, files in os.walk(basegen_dir): for name in files: try: os.chmod(os.path.join(root, name), stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH) except OSError: # We tried. Not worth hard failure here. pass if get_model() == "e3sm": bless_log = os.path.join(basegen_dir, BLESS_LOG_NAME) with open(bless_log, "a") as fd: fd.write("sha:{} date:{}\n".format(get_current_commit(repo=self._case.get_value("CIMEROOT")), get_timestamp(timestamp_format="%Y-%m-%d_%H:%M:%S"))) # END: modified CIME.hist_utils.generate_baseline append_testlog(comments) status = CIME.test_status.TEST_PASS_STATUS baseline_name = self._case.get_value("BASEGEN_CASE") self._test_status.set_status("{}".format(CIME.test_status.GENERATE_PHASE), status, comments=os.path.dirname(baseline_name)) basegen_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASEGEN_CASE")) # copy latest cpl log to baseline # drop the date so that the name is generic newestcpllogfiles = self._get_latest_cpl_logs() for cpllog in newestcpllogfiles: m = re.search(r"/(cpl.*.log).*.gz", cpllog) if m is not None: baselog = os.path.join(basegen_dir, m.group(1)) + ".gz" shutil.copyfile(cpllog, os.path.join(basegen_dir, baselog))
remote_prov = os.path.join(exeroot, "GIT_REMOTE.{}".format(lid)) _run_git_cmd_recursively("remote -v", srcroot, remote_prov) # Git config config_src = os.path.join(srcroot, ".git", "config") config_prov = os.path.join(exeroot, "GIT_CONFIG.{}".format(lid)) safe_copy(config_src, config_prov, preserve_meta=False) def _save_build_provenance_e3sm(case, lid): srcroot = case.get_value("SRCROOT") exeroot = case.get_value("EXEROOT") caseroot = case.get_value("CASEROOT") # Save git describe describe_prov = os.path.join(exeroot, "GIT_DESCRIBE.{}".format(lid)) desc = get_current_commit(tag=True, repo=srcroot) with open(describe_prov, "w") as fd: fd.write(desc) # Save HEAD headfile = os.path.join(srcroot, ".git", "logs", "HEAD") headfile_prov = os.path.join(exeroot, "GIT_LOGS_HEAD.{}".format(lid)) if os.path.exists(headfile_prov): os.remove(headfile_prov) if os.path.exists(headfile): safe_copy(headfile, headfile_prov, preserve_meta=False) # Save git submodule status submodule_prov = os.path.join(exeroot, "GIT_SUBMODULE_STATUS.{}".format(lid)) subm_status = get_current_submodule_status(recursive=True, repo=srcroot) with open(submodule_prov, "w") as fd:
def generate_baseline(case, baseline_dir=None, allow_baseline_overwrite=False): """ copy the current test output to baseline result case - The case containing the hist files to be copied into baselines baseline_dir - Optionally, specify a specific baseline dir, otherwise it will be computed from case config allow_baseline_overwrite must be true to generate baselines to an existing directory. returns (SUCCESS, comments) """ rundir = case.get_value("RUNDIR") if baseline_dir is None: baselineroot = case.get_value("BASELINE_ROOT") basegen_dir = os.path.join(baselineroot, case.get_value("BASEGEN_CASE")) else: basegen_dir = baseline_dir testcase = case.get_value("CASE") if not os.path.isdir(basegen_dir): os.makedirs(basegen_dir) if (os.path.isdir(os.path.join(basegen_dir,testcase)) and not allow_baseline_overwrite): expect(False, " Cowardly refusing to overwrite existing baseline directory") comments = "Generating baselines into '{}'\n".format(basegen_dir) num_gen = 0 for model in _iter_model_file_substrs(case): comments += " generating for model '{}'\n".format(model) hists = _get_latest_hist_files(testcase, model, rundir) logger.debug("latest_files: {}".format(hists)) num_gen += len(hists) for hist in hists: basename = hist[hist.rfind(model):] baseline = os.path.join(basegen_dir, basename) if os.path.exists(baseline): os.remove(baseline) shutil.copy(hist, baseline) comments += " generating baseline '{}' from file {}\n".format(baseline, hist) # copy latest cpl log to baseline # drop the date so that the name is generic newestcpllogfile = case.get_latest_cpl_log(coupler_log_path=case.get_value("LOGDIR")) if newestcpllogfile is None: logger.warning("No cpl.log file found in log directory {}".format(case.get_value("LOGDIR"))) else: shutil.copyfile(newestcpllogfile, os.path.join(basegen_dir, "cpl.log.gz")) expect(num_gen > 0, "Could not generate any hist files for case '{}', something is seriously wrong".format(os.path.join(rundir, testcase))) #make sure permissions are open in baseline directory for root, _, files in os.walk(basegen_dir): for name in files: try: os.chmod(os.path.join(root,name), stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH) except OSError: # We tried. Not worth hard failure here. pass if get_model() == "e3sm": bless_log = os.path.join(basegen_dir, BLESS_LOG_NAME) with open(bless_log, "a") as fd: fd.write("sha:{} date:{}\n".format(get_current_commit(repo=case.get_value("CIMEROOT")), get_timestamp(timestamp_format="%Y-%m-%d_%H:%M:%S"))) return True, comments
def _generate_baseline_impl(case, baseline_dir=None, allow_baseline_overwrite=False): """ copy the current test output to baseline result case - The case containing the hist files to be copied into baselines baseline_dir - Optionally, specify a specific baseline dir, otherwise it will be computed from case config allow_baseline_overwrite must be true to generate baselines to an existing directory. returns (SUCCESS, comments) """ rundir = case.get_value("RUNDIR") ref_case = case.get_value("RUN_REFCASE") if baseline_dir is None: baselineroot = case.get_value("BASELINE_ROOT") basegen_dir = os.path.join(baselineroot, case.get_value("BASEGEN_CASE")) else: basegen_dir = baseline_dir testcase = case.get_value("CASE") archive = case.get_env('archive') if not os.path.isdir(basegen_dir): os.makedirs(basegen_dir) if (os.path.isdir(os.path.join(basegen_dir,testcase)) and not allow_baseline_overwrite): expect(False, " Cowardly refusing to overwrite existing baseline directory") comments = "Generating baselines into '{}'\n".format(basegen_dir) num_gen = 0 for model in _iter_model_file_substrs(case): comments += " generating for model '{}'\n".format(model) if model == 'cpl': file_extensions = archive.get_hist_file_extensions(archive.get_entry('drv')) else: file_extensions = archive.get_hist_file_extensions(archive.get_entry(model)) hists = _get_latest_hist_files(model, rundir, file_extensions, ref_case=ref_case) logger.debug("latest_files: {}".format(hists)) num_gen += len(hists) for hist in hists: basename = hist[hist.rfind(model):] baseline = os.path.join(basegen_dir, basename) if os.path.exists(baseline): os.remove(baseline) safe_copy(hist, baseline, preserve_meta=False) comments += " generating baseline '{}' from file {}\n".format(baseline, hist) # copy latest cpl log to baseline # drop the date so that the name is generic if case.get_value("COMP_INTERFACE") == "nuopc": cplname = "med" else: cplname = "cpl" newestcpllogfile = case.get_latest_cpl_log(coupler_log_path=case.get_value("RUNDIR"), cplname=cplname) if newestcpllogfile is None: logger.warning("No {}.log file found in directory {}".format(cplname,case.get_value("RUNDIR"))) else: safe_copy(newestcpllogfile, os.path.join(basegen_dir, "{}.log.gz".format(cplname)), preserve_meta=False) testname = case.get_value("TESTCASE") expect(num_gen > 0 or testname == "PFS", "Could not generate any hist files for case '{}', something is seriously wrong".format(os.path.join(rundir, testcase))) if get_model() == "e3sm": bless_log = os.path.join(basegen_dir, BLESS_LOG_NAME) with open(bless_log, "a") as fd: fd.write("sha:{} date:{}\n".format(get_current_commit(repo=case.get_value("CIMEROOT")), get_timestamp(timestamp_format="%Y-%m-%d_%H:%M:%S"))) return True, comments