def _download_if_in_repo(server, input_data_root, rel_path, isdirectory=False):
"""
Return True if successfully downloaded
"""
if not server.fileexists(rel_path):
return False
full_path = os.path.join(input_data_root, rel_path)
logging.info("Trying to download file: '{}' to path '{}'".format(
rel_path, full_path))
# Make sure local path exists, create if it does not
if isdirectory or full_path.endswith(os.sep):
if not os.path.exists(full_path):
logger.info("Creating directory {}".format(full_path))
os.makedirs(full_path)
isdirectory = True
elif not os.path.exists(os.path.dirname(full_path)):
os.makedirs(os.path.dirname(full_path))
# Use umask to make sure files are group read/writable. As long as parent directories
# have +s, then everything should work.
with SharedArea():
if isdirectory:
return server.getdirectory(rel_path, full_path)
else:
return server.getfile(rel_path, full_path)
def save_test_success(baseline_root, src_root, test, succeeded, force_commit_test=None):
"""
Update success data accordingly based on succeeded flag
"""
if baseline_root is not None:
try:
with SharedArea():
success_path, prev_results = _read_success_data(baseline_root, test)
the_dir = os.path.dirname(success_path)
if not os.path.exists(the_dir):
os.makedirs(the_dir)
prev_succeeded = _is_test_working(prev_results, src_root, testing=(force_commit_test is not None))
# if no transition occurred then no update is needed
if succeeded or succeeded != prev_succeeded or (prev_results[0] is None and succeeded) or (prev_results[1] is None and not succeeded):
new_results = list(prev_results)
my_commit = force_commit_test if force_commit_test else get_current_commit(repo=src_root)
if succeeded:
new_results[0] = my_commit # we passed
else:
new_results[1] = my_commit # we transitioned to a failing state
str_results = ["None" if item is None else item for item in new_results]
with open(success_path, "w") as fd:
fd.write("{}\n".format(" ".join(str_results)))
except Exception:
# We NEVER want a failure here to kill the run
logger.warning("Failed to store test success: {}".format(sys.exc_info()[1]))
def download_if_in_repo(svn_loc, input_data_root, rel_path):
"""
Return True if successfully downloaded
"""
rel_path = rel_path.strip('/')
full_url = os.path.join(svn_loc, rel_path)
full_path = os.path.join(input_data_root, rel_path)
logging.info("Trying to download file: '%s' to path '%s'" %
(full_url, full_path))
# Make sure local path exists, create if it does not
if (not os.path.exists(os.path.dirname(full_path))):
os.makedirs(os.path.dirname(full_path))
stat, out, err = run_cmd(
"svn --non-interactive --trust-server-cert ls %s" % full_url)
if (stat != 0):
logging.warning(
"FAIL: SVN repo '%s' does not have file '%s'\nReason:%s\n%s\n" %
(svn_loc, full_url, out, err))
return False
else:
# Use umask to make sure files are group read/writable. As long as parent directories
# have +s, then everything should work.
with SharedArea():
stat, output, errput = \
run_cmd("svn --non-interactive --trust-server-cert export %s %s" % (full_url, full_path))
if (stat != 0):
logging.warning(
"svn export failed with output: %s and errput %s\n" %
(output, errput))
return False
else:
logging.info("SUCCESS\n")
return True
def _download_if_in_repo(server, input_data_root, rel_path, isdirectory=False):
"""
Return True if successfully downloaded
server is an object handle of type CIME.Servers
input_data_root is the local path to inputdata (DIN_LOC_ROOT)
rel_path is the path to the file or directory relative to input_data_root
user is the user name of the person running the script
isdirectory indicates that this is a directory download rather than a single file
"""
if not (rel_path or server.fileexists(rel_path)):
return False
full_path = os.path.join(input_data_root, rel_path)
logging.info("Trying to download file: '{}' to path '{}' using {} protocol.".format(rel_path, full_path, type(server).__name__))
# Make sure local path exists, create if it does not
if isdirectory or full_path.endswith(os.sep):
if not os.path.exists(full_path):
logger.info("Creating directory {}".format(full_path))
os.makedirs(full_path+".tmp")
isdirectory = True
elif not os.path.exists(os.path.dirname(full_path)):
os.makedirs(os.path.dirname(full_path))
# Use umask to make sure files are group read/writable. As long as parent directories
# have +s, then everything should work.
with SharedArea():
if isdirectory:
success = server.getdirectory(rel_path, full_path+".tmp")
# this is intended to prevent a race condition in which
# one case attempts to use a refdir before another one has
# completed the download
os.rename(full_path+".tmp",full_path)
else:
success = server.getfile(rel_path, full_path)
return success
def _download_checksum_file(rundir):
"""
Download the checksum files from each server and merge them into rundir.
"""
inputdata = Inputdata()
protocol = "svn"
# download and merge all available chksum files.
while protocol is not None:
protocol, address, user, passwd, chksum_file = inputdata.get_next_server(
)
if protocol not in vars(CIME.Servers):
logger.warning("Client protocol {} not enabled".format(protocol))
continue
logger.info("Using protocol {} with user {} and passwd {}".format(
protocol, user, passwd))
if protocol == "svn":
server = CIME.Servers.SVN(address, user, passwd)
elif protocol == "gftp":
server = CIME.Servers.GridFTP(address, user, passwd)
elif protocol == "ftp":
server = CIME.Servers.FTP(address, user, passwd)
elif protocol == "wget":
server = CIME.Servers.WGET(address, user, passwd)
else:
expect(False,
"Unsupported inputdata protocol: {}".format(protocol))
if not chksum_file:
continue
success = False
rel_path = chksum_file
full_path = os.path.join(rundir, local_chksum_file)
new_file = full_path + '.raw'
protocol = type(server).__name__
logging.info(
"Trying to download file: '{}' to path '{}' using {} protocol.".
format(rel_path, new_file, protocol))
tmpfile = None
if os.path.isfile(full_path):
tmpfile = full_path + ".tmp"
os.rename(full_path, tmpfile)
# Use umask to make sure files are group read/writable. As long as parent directories
# have +s, then everything should work.
with SharedArea():
success = server.getfile(rel_path, new_file)
if success:
_reformat_chksum_file(full_path, new_file)
if tmpfile:
_merge_chksum_files(full_path, tmpfile)
chksum_hash.clear()
else:
if tmpfile and os.path.isfile(tmpfile):
os.rename(tmpfile, full_path)
logger.warning("Could not automatically download file " +
full_path + " Restoring existing version.")
else:
logger.warning(
"Could not automatically download file {}".format(
full_path))
def run_phase(self):
rundir = self._case.get_value("RUNDIR")
exeroot = self._case.get_value("EXEROOT")
baseline = self._case.get_value("BASELINE_ROOT")
compare = self._case.get_value("COMPARE_BASELINE")
generate = self._case.get_value("GENERATE_BASELINE")
basegen = self._case.get_value("BASEGEN_CASE")
gmake = self._case.get_value("GMAKE")
log = os.path.join(rundir, "homme.log")
if os.path.exists(log):
os.remove(log)
if generate:
full_baseline_dir = os.path.join(baseline, basegen, "tests",
"baseline")
stat = run_cmd("{} -j 4 baseline".format(gmake),
arg_stdout=log,
combine_output=True,
from_dir=exeroot)[0]
if stat == 0:
if os.path.isdir(full_baseline_dir):
shutil.rmtree(full_baseline_dir)
with SharedArea():
dir_util.copy_tree(os.path.join(exeroot, "tests",
"baseline"),
full_baseline_dir,
preserve_mode=False)
elif compare:
stat = run_cmd("{} -j 4 check".format(gmake),
arg_stdout=log,
combine_output=True,
from_dir=exeroot)[0]
else:
stat = run_cmd("{} -j 4 baseline".format(gmake),
arg_stdout=log,
combine_output=True,
from_dir=exeroot)[0]
if stat == 0:
stat = run_cmd("{} -j 4 check".format(gmake),
arg_stdout=log,
combine_output=True,
from_dir=exeroot)[0]
# Add homme.log output to TestStatus.log so that it can
# appear on the dashboard. Otherwise, the TestStatus.log
# is pretty useless for this test.
append_testlog(open(log, "r").read())
expect(stat == 0, "RUN FAIL for HOMME")
def generate_teststatus(testdir, baseline_dir):
"""
CESM stores it's TestStatus file in baselines. Do not let exceptions
escape from this function.
"""
if get_model() == "cesm":
try:
with SharedArea():
if not os.path.isdir(baseline_dir):
os.makedirs(baseline_dir)
safe_copy(os.path.join(testdir, TEST_STATUS_FILENAME), baseline_dir, preserve_meta=False)
except Exception as e:
logger.warning("Could not copy {} to baselines, {}".format(os.path.join(testdir, TEST_STATUS_FILENAME), str(e)))
def save_test_time(baseline_root, test, time_seconds, commit):
if baseline_root is not None:
try:
with SharedArea():
the_dir = os.path.join(baseline_root, _WALLTIME_BASELINE_NAME, test)
if not os.path.exists(the_dir):
os.makedirs(the_dir)
the_path = os.path.join(the_dir, _WALLTIME_FILE_NAME)
with open(the_path, "a") as fd:
fd.write("{} {}\n".format(int(time_seconds), commit))
except Exception:
# We NEVER want a failure here to kill the run
logger.warning("Failed to store test time: {}".format(sys.exc_info()[1]))
def download_if_in_repo(svn_loc, input_data_root, rel_path):
"""
Return True if successfully downloaded
"""
rel_path = rel_path.strip('/')
full_url = os.path.join(svn_loc, rel_path)
err = run_cmd(
"svn --non-interactive --trust-server-cert ls {}".format(svn_loc))[0]
if err != 0:
logging.warning("""
Could not connect to svn repo '{0}'
This is most likely either a credential, proxy, or network issue .
To check connection and store your credential run 'svn ls {0}' and permanently store your password"""
.format(svn_loc))
return False
full_path = os.path.join(input_data_root, rel_path)
logging.info("Trying to download file: '{}' to path '{}'".format(
full_url, full_path))
# Make sure local path exists, create if it does not
if (not os.path.exists(os.path.dirname(full_path))):
os.makedirs(os.path.dirname(full_path))
stat, out, err = run_cmd(
"svn --non-interactive --trust-server-cert ls {}".format(full_url))
if (stat != 0):
logging.warning(
"FAIL: SVN repo '{}' does not have file '{}'\nReason:{}\n{}\n".
format(svn_loc, full_url, out, err))
return False
else:
# Use umask to make sure files are group read/writable. As long as parent directories
# have +s, then everything should work.
with SharedArea():
stat, output, errput = \
run_cmd("svn --non-interactive --trust-server-cert export {} {}".format(full_url, full_path))
if (stat != 0):
logging.warning(
"svn export failed with output: {} and errput {}\n".format(
output, errput))
return False
else:
logging.info("SUCCESS\n")
return True
def _generate_baseline(self):
super(TSC, self)._generate_baseline()
with SharedArea():
basegen_dir = os.path.join(self._case.get_value("BASELINE_ROOT"),
self._case.get_value("BASEGEN_CASE"))
rundir = self._case.get_value("RUNDIR")
ref_case = self._case.get_value("RUN_REFCASE")
model = 'cam'
hists = _get_all_hist_files(model,
rundir, [r'h\d*.*\.nc\.DT\d*'],
ref_case=ref_case)
logger.debug("TSC additional baseline files: {}".format(hists))
for hist in hists:
basename = hist[hist.rfind(model):]
baseline = os.path.join(basegen_dir, basename)
if os.path.exists(baseline):
os.remove(baseline)
safe_copy(hist, baseline, preserve_meta=False)
{manic}{args}
(2) If you don't need provenance information, rebuild with --skip-provenance-check
""".format(out=indent_string(out, 4), err=indent_string(err, 4),
srcroot=srcroot, manic=manic, args=args)
expect(stat==0,errmsg)
caseroot = case.get_value("CASEROOT")
with open(os.path.join(caseroot, "CaseStatus"), "a") as fd:
if version is not None and version != "unknown":
fd.write("CESM version is {}\n".format(version))
if out is not None:
fd.write("{}\n".format(out))
def save_build_provenance(case, lid=None):
with SharedArea():
model = case.get_value("MODEL")
lid = os.environ["LID"] if lid is None else lid
if model == "e3sm":
_save_build_provenance_e3sm(case, lid)
elif model == "cesm":
_save_build_provenance_cesm(case, lid)
def _save_prerun_timing_e3sm(case, lid):
project = case.get_value("PROJECT", subgroup=case.get_primary_job())
if not case.is_save_timing_dir_project(project):
return
timing_dir = case.get_value("SAVE_TIMING_DIR")
if timing_dir is None or not os.path.isdir(timing_dir):
