def _download_if_in_repo(server, input_data_root, rel_path, isdirectory=False): """ Return True if successfully downloaded """ if not server.fileexists(rel_path): return False full_path = os.path.join(input_data_root, rel_path) logging.info("Trying to download file: '{}' to path '{}'".format( rel_path, full_path)) # Make sure local path exists, create if it does not if isdirectory or full_path.endswith(os.sep): if not os.path.exists(full_path): logger.info("Creating directory {}".format(full_path)) os.makedirs(full_path) isdirectory = True elif not os.path.exists(os.path.dirname(full_path)): os.makedirs(os.path.dirname(full_path)) # Use umask to make sure files are group read/writable. As long as parent directories # have +s, then everything should work. with SharedArea(): if isdirectory: return server.getdirectory(rel_path, full_path) else: return server.getfile(rel_path, full_path)
def save_test_success(baseline_root, src_root, test, succeeded, force_commit_test=None): """ Update success data accordingly based on succeeded flag """ if baseline_root is not None: try: with SharedArea(): success_path, prev_results = _read_success_data(baseline_root, test) the_dir = os.path.dirname(success_path) if not os.path.exists(the_dir): os.makedirs(the_dir) prev_succeeded = _is_test_working(prev_results, src_root, testing=(force_commit_test is not None)) # if no transition occurred then no update is needed if succeeded or succeeded != prev_succeeded or (prev_results[0] is None and succeeded) or (prev_results[1] is None and not succeeded): new_results = list(prev_results) my_commit = force_commit_test if force_commit_test else get_current_commit(repo=src_root) if succeeded: new_results[0] = my_commit # we passed else: new_results[1] = my_commit # we transitioned to a failing state str_results = ["None" if item is None else item for item in new_results] with open(success_path, "w") as fd: fd.write("{}\n".format(" ".join(str_results))) except Exception: # We NEVER want a failure here to kill the run logger.warning("Failed to store test success: {}".format(sys.exc_info()[1]))
def download_if_in_repo(svn_loc, input_data_root, rel_path): """ Return True if successfully downloaded """ rel_path = rel_path.strip('/') full_url = os.path.join(svn_loc, rel_path) full_path = os.path.join(input_data_root, rel_path) logging.info("Trying to download file: '%s' to path '%s'" % (full_url, full_path)) # Make sure local path exists, create if it does not if (not os.path.exists(os.path.dirname(full_path))): os.makedirs(os.path.dirname(full_path)) stat, out, err = run_cmd( "svn --non-interactive --trust-server-cert ls %s" % full_url) if (stat != 0): logging.warning( "FAIL: SVN repo '%s' does not have file '%s'\nReason:%s\n%s\n" % (svn_loc, full_url, out, err)) return False else: # Use umask to make sure files are group read/writable. As long as parent directories # have +s, then everything should work. with SharedArea(): stat, output, errput = \ run_cmd("svn --non-interactive --trust-server-cert export %s %s" % (full_url, full_path)) if (stat != 0): logging.warning( "svn export failed with output: %s and errput %s\n" % (output, errput)) return False else: logging.info("SUCCESS\n") return True
def _download_if_in_repo(server, input_data_root, rel_path, isdirectory=False): """ Return True if successfully downloaded server is an object handle of type CIME.Servers input_data_root is the local path to inputdata (DIN_LOC_ROOT) rel_path is the path to the file or directory relative to input_data_root user is the user name of the person running the script isdirectory indicates that this is a directory download rather than a single file """ if not (rel_path or server.fileexists(rel_path)): return False full_path = os.path.join(input_data_root, rel_path) logging.info("Trying to download file: '{}' to path '{}' using {} protocol.".format(rel_path, full_path, type(server).__name__)) # Make sure local path exists, create if it does not if isdirectory or full_path.endswith(os.sep): if not os.path.exists(full_path): logger.info("Creating directory {}".format(full_path)) os.makedirs(full_path+".tmp") isdirectory = True elif not os.path.exists(os.path.dirname(full_path)): os.makedirs(os.path.dirname(full_path)) # Use umask to make sure files are group read/writable. As long as parent directories # have +s, then everything should work. with SharedArea(): if isdirectory: success = server.getdirectory(rel_path, full_path+".tmp") # this is intended to prevent a race condition in which # one case attempts to use a refdir before another one has # completed the download os.rename(full_path+".tmp",full_path) else: success = server.getfile(rel_path, full_path) return success
def _download_checksum_file(rundir): """ Download the checksum files from each server and merge them into rundir. """ inputdata = Inputdata() protocol = "svn" # download and merge all available chksum files. while protocol is not None: protocol, address, user, passwd, chksum_file = inputdata.get_next_server( ) if protocol not in vars(CIME.Servers): logger.warning("Client protocol {} not enabled".format(protocol)) continue logger.info("Using protocol {} with user {} and passwd {}".format( protocol, user, passwd)) if protocol == "svn": server = CIME.Servers.SVN(address, user, passwd) elif protocol == "gftp": server = CIME.Servers.GridFTP(address, user, passwd) elif protocol == "ftp": server = CIME.Servers.FTP(address, user, passwd) elif protocol == "wget": server = CIME.Servers.WGET(address, user, passwd) else: expect(False, "Unsupported inputdata protocol: {}".format(protocol)) if not chksum_file: continue success = False rel_path = chksum_file full_path = os.path.join(rundir, local_chksum_file) new_file = full_path + '.raw' protocol = type(server).__name__ logging.info( "Trying to download file: '{}' to path '{}' using {} protocol.". format(rel_path, new_file, protocol)) tmpfile = None if os.path.isfile(full_path): tmpfile = full_path + ".tmp" os.rename(full_path, tmpfile) # Use umask to make sure files are group read/writable. As long as parent directories # have +s, then everything should work. with SharedArea(): success = server.getfile(rel_path, new_file) if success: _reformat_chksum_file(full_path, new_file) if tmpfile: _merge_chksum_files(full_path, tmpfile) chksum_hash.clear() else: if tmpfile and os.path.isfile(tmpfile): os.rename(tmpfile, full_path) logger.warning("Could not automatically download file " + full_path + " Restoring existing version.") else: logger.warning( "Could not automatically download file {}".format( full_path))
def run_phase(self): rundir = self._case.get_value("RUNDIR") exeroot = self._case.get_value("EXEROOT") baseline = self._case.get_value("BASELINE_ROOT") compare = self._case.get_value("COMPARE_BASELINE") generate = self._case.get_value("GENERATE_BASELINE") basegen = self._case.get_value("BASEGEN_CASE") gmake = self._case.get_value("GMAKE") log = os.path.join(rundir, "homme.log") if os.path.exists(log): os.remove(log) if generate: full_baseline_dir = os.path.join(baseline, basegen, "tests", "baseline") stat = run_cmd("{} -j 4 baseline".format(gmake), arg_stdout=log, combine_output=True, from_dir=exeroot)[0] if stat == 0: if os.path.isdir(full_baseline_dir): shutil.rmtree(full_baseline_dir) with SharedArea(): dir_util.copy_tree(os.path.join(exeroot, "tests", "baseline"), full_baseline_dir, preserve_mode=False) elif compare: stat = run_cmd("{} -j 4 check".format(gmake), arg_stdout=log, combine_output=True, from_dir=exeroot)[0] else: stat = run_cmd("{} -j 4 baseline".format(gmake), arg_stdout=log, combine_output=True, from_dir=exeroot)[0] if stat == 0: stat = run_cmd("{} -j 4 check".format(gmake), arg_stdout=log, combine_output=True, from_dir=exeroot)[0] # Add homme.log output to TestStatus.log so that it can # appear on the dashboard. Otherwise, the TestStatus.log # is pretty useless for this test. append_testlog(open(log, "r").read()) expect(stat == 0, "RUN FAIL for HOMME")
def generate_teststatus(testdir, baseline_dir): """ CESM stores it's TestStatus file in baselines. Do not let exceptions escape from this function. """ if get_model() == "cesm": try: with SharedArea(): if not os.path.isdir(baseline_dir): os.makedirs(baseline_dir) safe_copy(os.path.join(testdir, TEST_STATUS_FILENAME), baseline_dir, preserve_meta=False) except Exception as e: logger.warning("Could not copy {} to baselines, {}".format(os.path.join(testdir, TEST_STATUS_FILENAME), str(e)))
def save_test_time(baseline_root, test, time_seconds, commit): if baseline_root is not None: try: with SharedArea(): the_dir = os.path.join(baseline_root, _WALLTIME_BASELINE_NAME, test) if not os.path.exists(the_dir): os.makedirs(the_dir) the_path = os.path.join(the_dir, _WALLTIME_FILE_NAME) with open(the_path, "a") as fd: fd.write("{} {}\n".format(int(time_seconds), commit)) except Exception: # We NEVER want a failure here to kill the run logger.warning("Failed to store test time: {}".format(sys.exc_info()[1]))
def download_if_in_repo(svn_loc, input_data_root, rel_path): """ Return True if successfully downloaded """ rel_path = rel_path.strip('/') full_url = os.path.join(svn_loc, rel_path) err = run_cmd( "svn --non-interactive --trust-server-cert ls {}".format(svn_loc))[0] if err != 0: logging.warning(""" Could not connect to svn repo '{0}' This is most likely either a credential, proxy, or network issue . To check connection and store your credential run 'svn ls {0}' and permanently store your password""" .format(svn_loc)) return False full_path = os.path.join(input_data_root, rel_path) logging.info("Trying to download file: '{}' to path '{}'".format( full_url, full_path)) # Make sure local path exists, create if it does not if (not os.path.exists(os.path.dirname(full_path))): os.makedirs(os.path.dirname(full_path)) stat, out, err = run_cmd( "svn --non-interactive --trust-server-cert ls {}".format(full_url)) if (stat != 0): logging.warning( "FAIL: SVN repo '{}' does not have file '{}'\nReason:{}\n{}\n". format(svn_loc, full_url, out, err)) return False else: # Use umask to make sure files are group read/writable. As long as parent directories # have +s, then everything should work. with SharedArea(): stat, output, errput = \ run_cmd("svn --non-interactive --trust-server-cert export {} {}".format(full_url, full_path)) if (stat != 0): logging.warning( "svn export failed with output: {} and errput {}\n".format( output, errput)) return False else: logging.info("SUCCESS\n") return True
def _generate_baseline(self): super(TSC, self)._generate_baseline() with SharedArea(): basegen_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASEGEN_CASE")) rundir = self._case.get_value("RUNDIR") ref_case = self._case.get_value("RUN_REFCASE") model = 'cam' hists = _get_all_hist_files(model, rundir, [r'h\d*.*\.nc\.DT\d*'], ref_case=ref_case) logger.debug("TSC additional baseline files: {}".format(hists)) for hist in hists: basename = hist[hist.rfind(model):] baseline = os.path.join(basegen_dir, basename) if os.path.exists(baseline): os.remove(baseline) safe_copy(hist, baseline, preserve_meta=False)
{manic}{args} (2) If you don't need provenance information, rebuild with --skip-provenance-check """.format(out=indent_string(out, 4), err=indent_string(err, 4), srcroot=srcroot, manic=manic, args=args) expect(stat==0,errmsg) caseroot = case.get_value("CASEROOT") with open(os.path.join(caseroot, "CaseStatus"), "a") as fd: if version is not None and version != "unknown": fd.write("CESM version is {}\n".format(version)) if out is not None: fd.write("{}\n".format(out)) def save_build_provenance(case, lid=None): with SharedArea(): model = case.get_value("MODEL") lid = os.environ["LID"] if lid is None else lid if model == "e3sm": _save_build_provenance_e3sm(case, lid) elif model == "cesm": _save_build_provenance_cesm(case, lid) def _save_prerun_timing_e3sm(case, lid): project = case.get_value("PROJECT", subgroup=case.get_primary_job()) if not case.is_save_timing_dir_project(project): return timing_dir = case.get_value("SAVE_TIMING_DIR") if timing_dir is None or not os.path.isdir(timing_dir):