def git_clone_repository(config):
logger.info("Clone user-specified git repository...")
user_config = utils.config_section_map(config, "USERCONFIG")
temp = user_config["scratch_dir"]
repo = user_config["repo_url"]
branch = user_config["repo_branch"]
clone = temp + "/builds/nu-wrf_repo"
if not branch:
cmd = ["git", "clone", "-q", repo, clone]
else:
cmd = ["git", "clone", "-q", "-b", branch, repo, clone]
if os.path.exists(clone):
return
results_dir = temp + "/results/"
logger.info("Cloning %s into %s", repo, clone)
cwd = os.getcwd()
proc = sp.Popen(cmd)
proc.wait()
os.chdir(clone)
cmd = "git log --pretty=format:'%h - %an, %ar : %s' --since=1.day"
os.system(cmd + ">" + results_dir + "gitLog")
os.chdir(cwd)
def add_batch_directives(
config, fp, build, run_dir, npes=8, is_type=0, wall_time="2:00:00", qos="allnccs"
):
user_config = utils.config_section_map(config, "USERCONFIG")
if "yes" in user_config["use_batch"]:
sponsor_id = user_config["sponsor_id"]
out_name = run_dir + "/" + build + ".out"
err_name = run_dir + "/" + build + ".err"
str_npes = str(npes)
int_npes = int(npes)
# Batch directives are only used if
# 0) use_batch='yes' AND
# 1) it's a build script or (is_type == 0)
# 2) it's a run script or (is_type == 1)
# 3) it's a component script (is_type == 2) AND setup_runs ='yes'
if "discover" in platform.node():
case_3 = is_type == 2 and "yes" in user_config["setup_runs"]
if is_type == 0 or is_type == 1 or case_3:
fp.write("#SBATCH -J " + build + "\n")
fp.write("#SBATCH -o " + out_name + "\n")
fp.write("#SBATCH -e " + err_name + "\n")
fp.write("#SBATCH --account=" + sponsor_id + "\n")
fp.write("#SBATCH --ntasks=" + str_npes + "\n")
fp.write("#SBATCH --constraint=hasw" + "\n")
fp.write("#SBATCH --qos=" + qos + "\n")
if 'debug' in qos:
fp.write("#SBATCH --time=1:00:00" + "\n")
else:
fp.write("#SBATCH --time=" + wall_time + "\n")
else: # Must be Pleiades
if int_npes > 24:
if int_npes % 24 == 0:
nodes = int_npes / 24
else:
nodes = (int_npes / 24) + 1
else:
nodes = 1
fp.write(
"#PBS -lselect="
+ str(nodes)
+ ":ncpus="
+ str_npes
+ ":mpiprocs="
+ str_npes
+ ":model=has\n"
)
fp.write("#PBS -q devel\n")
fp.write("#PBS -o " + out_name + "\n")
fp.write("#PBS -e " + err_name + "\n")
fp.write("#PBS -W group_list=" + sponsor_id + "\n")
wall_time = "02:00:00"
if "scm" in build:
wall_time = "00:10:00"
elif "arw_" in build:
wall_time = "00:30:00"
fp.write("#PBS -l walltime=" + wall_time + "\n")
def create_build_script(config, comp_config, build, build_dir, ideal_build, comp_mpi):
""" Create a build script file and return a command to run it."""
user_config = utils.config_section_map(config, "USERCONFIG")
filename = build_dir + "/" + build + ".bash"
logger.debug("Creating build script in: " + build_dir)
with open(filename, "w") as fp:
fp.write("#!/bin/bash" + "\n")
add_batch_directives(config, fp, build, build_dir, is_type=0)
fp.write("cd " + build_dir + "\n")
build_arg = "lis,wrf,wps,ldt,geos2wrf,sst2wrf,lisWrfDomain"
if ideal_build: # Only one ideal case is tested
build_arg = "lis,ideal_scm_lis_xy,wps,ldt,lis4scm"
else:
if "chem" in build:
build_arg = "lis,chem,wps,ldt,utils"
elif "kpp" in build:
build_arg = "lis,kpp,wps,prep_chem_sources,gocart2wrf"
logger.debug(build + " build components: " + build_arg)
comp_config_section = utils.config_section_map(comp_config, "COMPCONFIG")
comp_mpi_vendor = "intel-intelmpi"
for mod in comp_config_section["modulelist"].split(","):
if re.search(comp_mpi, mod):
comp_mpi_vendor = mod
libdir_tag = os.path.join(user_config["nuwrflib_dir"], comp_mpi_vendor)
fp.write("export LIBDIR_TAG=" + libdir_tag + "\n")
fp.write("export COMPILER_VENDOR=" + comp_mpi_vendor.split("-")[0] + "\n")
fp.write("export MPI_VENDOR=" + comp_mpi_vendor.split("-")[1] + "\n")
fp.write("./build.sh " + build_arg + "\n")
fp.write("if [ ! -f ./.build_failed ]; then touch .build_ok; fi" + "\n")
if "yes" in user_config["use_batch"]:
script_command = "cd " + build_dir + ";sbatch " + build + ".bash"
else:
if "Darwin" in platform.system():
script_command = "chmod +x " + filename + ";" + filename
else:
script_command = "chmod +x " + filename + ";ssh discover-sp3 " + filename
return script_command
def setup_testing_environment(config, comp_config, task_list):
logger.info("Setup testing environment...")
user_config = utils.config_section_map(config, "USERCONFIG")
if "yes" in user_config["clean_scratch"]:
utils.clean_scratch(config)
if not user_config["model_dir"]:
if user_config["repo_type"] == "git":
git_clone_repository(config)
else:
svn_checkout_repository(config)
create_test_dir_structure(config, comp_config, task_list)
def create_test_dir_structure(config, comp_config, test_list):
logger.info("Create directories for each NU-WRF build...")
user_config = utils.config_section_map(config, "USERCONFIG")
builds_dir = user_config["scratch_dir"] + "/builds"
results_dir = user_config["scratch_dir"] + "/results"
time_stamp = test_list[1].get_opt("time_stamp")
build_tasks = test_list[0]
for item in build_tasks:
compiler = item[0]
build = item[1]
logger.debug("Create " + build + " build task using " + compiler + " settings")
available = get_compilers(comp_config)
# Sanity check
if compiler.split("-")[0] in available:
build_dir = builds_dir + "/" + compiler + "." + time_stamp + "/" + build
if user_config["model_dir"]:
build_dir_link = builds_dir + "/" + compiler + "." + time_stamp + "/"
utils.mkdir_p(build_dir_link)
repo_dir = user_config["model_dir"]
logger.debug("Creating symlink " + repo_dir + " -> " + build_dir)
os.symlink(repo_dir, build_dir)
else:
logger.debug("Creating build dir " + build_dir)
repo_dir = builds_dir + "/nu-wrf_repo"
copytree(
repo_dir,
build_dir,
symlinks=True,
ignore=ignore_patterns(
"GSDSU",
"MET",
"UPP",
"lvt",
"docs",
"." + user_config["repo_type"],
),
)
else:
logger.error("Unable to create directory for " + build + " build")
# Create directories to hold NU-WRF results for each run
iter_runs = iter(test_list)
next(iter_runs)
for t in iter_runs:
compiler = t.get_opt("compiler")
run_dir = results_dir + "/" + compiler + "." + time_stamp + "/" + t.name
logger.debug("Creating results dir " + run_dir)
utils.mkdir_p(run_dir)
def svn_checkout_repository(config):
logger.info("Checkout user-specified svn repository...")
user_config = utils.config_section_map(config, "USERCONFIG")
repo = user_config["repo_url"]
svn_repo = user_config["scratch_dir"] + "/builds/nu-wrf_repo"
if os.path.exists(svn_repo):
return
cmd = ["svn", "--quiet", "checkout", repo, svn_repo]
logger.info("Checking out %s into %s", repo, svn_repo)
cwd = os.getcwd()
proc = sp.Popen(cmd)
proc.wait()
os.chdir(cwd)
def reg_runs(config, comp_config, test_list, build_results):
user_config = utils.config_section_map(config, "USERCONFIG")
run_commands = create_run_commands(config, comp_config, test_list, build_results)
try:
if "yes" in user_config["setup_runs"]:
logger.info("Setup is DONE.")
sys.exit()
except KeyError as e:
logger.error(str(e) + " option does not exist in your config file.")
sys.exit()
if run_commands:
logger.info("Running test cases...")
if "DEBUG_REG" in os.environ:
logger.info(run_commands)
else:
pool.run_commands(run_commands, user_config["use_batch"])
def set_libdir_environment(config, my_build):
# Use COMPILER_VENDOR, MPI_VENDOR and LIBDIR_TAG values set in environment
# If not set then grab from the config file. If not set, abort.
lib_config = u.config_section_map(config, "libconfig")
if os.environ.get("LIBDIR_TAG") is None:
try:
libdir = lib_config["libdir_tag"]
except KeyError as err:
print("Error: {} is not defined.".format(err))
else:
libdir = os.environ.get("LIBDIR_TAG")
if os.environ.get("COMPILER_VENDOR") is None:
try:
compiler_vendor = lib_config["compiler_vendor"]
except KeyError as err:
print("Error: {} is not defined.".format(err))
else:
compiler_vendor = os.environ.get("COMPILER_VENDOR")
if os.environ.get("MPI_VENDOR") is None:
try:
mpi_vendor = lib_config["mpi_vendor"]
except KeyError as err:
print("Error: {} is not defined.".format(err))
else:
mpi_vendor = os.environ.get("MPI_VENDOR")
# Set these so that values can by used in other config sections
config.set("libconfig", "libdir_tag", libdir)
config.set("libconfig", "compiler_vendor", compiler_vendor)
config.set("libconfig", "mpi_vendor", mpi_vendor)
logger.debug("LIBDIR_TAG: " + libdir)
logger.debug("COMPILER_VENDOR: " + compiler_vendor)
logger.debug("MPI_VENDOR: " + mpi_vendor)
my_build.env_vars["BASELIBS"] = libdir
my_build.env_vars["LIBDIR_TAG"] = libdir
my_build.env_vars["COMPILER_VENDOR"] = compiler_vendor
my_build.env_vars["MPI_VENDOR"] = mpi_vendor
return compiler_vendor, mpi_vendor
def create_build_commands(config, comp_config, test_list):
user_config = utils.config_section_map(config, "USERCONFIG")
builds_dir = user_config["scratch_dir"] + "/builds"
time_stamp = test_list[1].get_opt("time_stamp")
build_commands = []
# Create build commands for each NU-WRF build
for item in test_list[0]:
comp_mpi = item[0]
build = item[1]
available = get_compilers(comp_config)
if comp_mpi.split("-")[0] in available:
build_dir = builds_dir + "/" + comp_mpi + "." + time_stamp + "/" + build
ideal_build = True if "scm" in build else False
script_command = create_build_script(
config, comp_config, build, build_dir, ideal_build, comp_mpi
)
if script_command:
build_commands.append(script_command)
return build_commands
def create_run_commands(config, comp_config, test_list, build_results):
user_config = utils.config_section_map(config, "USERCONFIG")
run_commands = []
# Skip first item which contains build tasks.
iter_runs = iter(test_list)
next(iter_runs)
for t in iter_runs:
if "compile_only" in t.get_opt("verification"):
continue
build = t.name.split("_", 1)[0]
if "wrflis" in t.name:
if "scm" in t.name:
build = "scm"
else:
build = "wrf"
run_command = None
if user_config["model_dir"]:
compilers = t.get_opt("compiler").split(",")
for compiler in compilers:
run_command = create_run_script(config, comp_config, t, compiler)
else:
for build_set, result in build_results.items():
build_ok = "b+" in result
is_the_build = build == build_set[0]
is_the_compiler = t.get_opt("compiler") in build_set
built = build_ok and is_the_build and is_the_compiler
if built or "DEBUG_REG" in os.environ:
run_command = create_run_script(
config, comp_config, t, build_set[1]
)
if run_command:
run_commands.append(run_command)
return run_commands
def get_compilers(config):
comp_config = utils.config_section_map(config, "COMPCONFIG")
comp_list = comp_config["compilers"].split(",")
compilers = []
[compilers.append(c.strip()) for c in comp_list]
return compilers
def send_test_report(config, compconfig, end_time):
# Create a report and notify via email
user_config = utils.config_section_map(config, "USERCONFIG")
mail_to = user_config["mail_to"]
branch = user_config["repo_branch"]
if not branch:
branch = "none specified"
results_dir = user_config["scratch_dir"] + "/results/"
builds_dir = user_config["scratch_dir"] + "/builds/"
build_type = user_config["build_type"]
message = user_config["message"]
compilers = get_compilers(compconfig)
use_html = user_config["use_html"]
test_report = results_dir + "/" + "test_report.txt"
with open(test_report, "w") as fp:
if "yes" in use_html:
fp.write("<html><pre>\n")
if "DEBUG_REG" in os.environ:
fp.write(message + " - DRY RUN ONLY! \n")
else:
fp.write(message + " \n")
if user_config["model_dir"]:
fp.write("NU-WRF build directory: " + user_config["model_dir"] + "\n")
fp.write("-" * 80 + "\n")
else:
fp.write("Repository: " + user_config["repo_url"] + "\n")
fp.write("-" * 80 + "\n")
fp.write("Branch: " + branch)
fp.write(" -- Build type: " + build_type + "\n")
fp.write("-" * 80 + "\n")
fp.write(
"{0:^40}{1:^20}{2:^10}{3:^10}\n".format(
"BUILD and/or RUN NAME".center(40, " "),
"COMPILER".center(20, " "),
"RESULT".center(10, " "),
"BASELINE",
).center(10, " ")
)
fp.write("-" * 80 + "\n")
sp.call(
"find "
+ builds_dir
+ " -name \*.diff -exec cat {} \; >"
+ builds_dir
+ "/"
+ "alldiffs",
shell=True,
)
sp.call(
"cat "
+ builds_dir
+ "/"
+ "alldiffs | sort -k 1,1 >"
+ builds_dir
+ "/"
+ "sorteddiffs",
shell=True,
)
with open(builds_dir + "/" + "sorteddiffs", "r") as inf:
fp.write(inf.read())
os.remove(builds_dir + "/" + "alldiffs")
os.remove(builds_dir + "/" + "sorteddiffs")
sp.call(
"find "
+ results_dir
+ " -name \*.diff -exec cat {} \; >"
+ results_dir
+ "/"
+ "alldiffs",
shell=True,
)
sp.call(
"cat "
+ results_dir
+ "/"
+ "alldiffs | sort -k 1,1 >"
+ results_dir
+ "/"
+ "sorteddiffs",
shell=True,
)
with open(results_dir + "/" + "sorteddiffs", "r") as inf:
fp.write(inf.read())
os.remove(results_dir + "/" + "alldiffs")
os.remove(results_dir + "/" + "sorteddiffs")
fp.write("-" * 80 + "\n")
hhmmss = time.strftime("%H:%M:%S", time.gmtime(end_time))
fp.write("Time taken = %s \n" % hhmmss)
fp.write("-" * 80 + "\n")
fp.write("Legend:\n")
fp.write("-" * 7 + "\n")
fp.write("+ : task success\n")
fp.write("F : task failure\n")
fp.write("C : created baseline\n")
fp.write("b : _build_ task\n")
fp.write("r : _run_ task\n")
fp.write("v : _verification_ task\n")
fp.write("- : Not available\n")
fp.write("Notes:\n")
fp.write("-" * 6 + "\n")
comp_config = utils.config_section_map(compconfig, "COMPCONFIG")
comp_versions = comp_config["compiler_versions"].split(",")
i = 0
for comp in compilers:
fp.write(comp + " compiler version: " + comp_versions[i] + "\n")
i += 1
fp.write("Results in: " + results_dir + "\n")
fp.write("-" * 80 + "\n")
if "DEBUG_REG" not in os.environ and not user_config["model_dir"]:
if user_config["repo_type"] == "git":
fp.write("Commits from last day:\n")
with open(results_dir + "/gitLog", "r") as inf:
fp.write(inf.read())
fp.write("\n")
fp.write("-" * 80 + "\n")
if "yes" in use_html:
fp.write("</pre><html>\n")
subject = '"[NU-WRF-regression]" '
cmd = "/usr/bin/mail -s " + subject + mail_to + " < " + test_report
if "yes" in use_html:
pref = 'mutt -e "set content_type=text/html" -s '
cmd = pref + subject + mail_to + " < " + test_report
sp.call(cmd, shell=True)
def reg_finalize(config, comp_config, end_time):
user_config = utils.config_section_map(config, "USERCONFIG")
# Notify results via email
if user_config["mail_to"]:
send_test_report(config, comp_config, end_time)
return None
def create_common_script(
config, comp_config, comp, run_dir, build_dir, data_dir, exe_dir=None
):
user_config = utils.config_section_map(config, "USERCONFIG")
comp_config_section = utils.config_section_map(comp_config, "COMPCONFIG")
with open(run_dir + "/common.reg", "w") as fp:
# Module environment
comp_mpi_vendor = "intel-sgimpt"
if "yes" in user_config["use_modules"]:
if "Darwin" in platform.system():
# CC: This is not portable...just my MAC so far
fp.write(". /opt/local/share/Modules/3.2.10/init/bash" + "\n")
fp.write("module purge" + "\n")
else:
fp.write("umask 022" + "\n")
fp.write(". /usr/share/modules/init/bash" + "\n")
fp.write("module purge" + "\n")
fp.write("unset LD_LIBRARY_PATH" + "\n")
for mod in comp_config_section["modulelist"].split(","):
if re.search(comp, mod):
comp_mpi_vendor = mod
for mm in comp_config_section[mod].split(","):
cmd = "module load " + mm + "\n"
fp.write(cmd)
libdir_tag = os.path.join(user_config["nuwrflib_dir"], comp_mpi_vendor)
fp.write("NUWRFLIB=" + libdir_tag + "\n")
if "Linux" in platform.system():
fp.write("export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib64" + "\n")
fp.write(
"export LD_LIBRARY_PATH=$NUWRFLIB/fortrangis/lib:$LD_LIBRARY_PATH"
+ "\n"
)
fp.write(
"export LD_LIBRARY_PATH=$NUWRFLIB/gdal/lib:$LD_LIBRARY_PATH" + "\n"
)
fp.write(
"export LD_LIBRARY_PATH=$NUWRFLIB/jasper/lib:$LD_LIBRARY_PATH" + "\n"
)
fp.write(
"export LD_LIBRARY_PATH=$NUWRFLIB/netcdf4/lib:$LD_LIBRARY_PATH" + "\n"
)
fp.write(
"export LD_LIBRARY_PATH=$NUWRFLIB/esmf/lib:$LD_LIBRARY_PATH" + "\n"
)
fp.write(
"export LD_LIBRARY_PATH=$NUWRFLIB/hdf5/lib:$LD_LIBRARY_PATH" + "\n"
)
fp.write(
"export LD_LIBRARY_PATH=$NUWRFLIB/hdf4/lib:$LD_LIBRARY_PATH" + "\n"
)
fp.write(
"export PATH=$NUWRFLIB/netcdf4/bin:$PATH" + "\n"
)
fp.write("LISDIR=" + user_config["lis_project_dir"] + "\n")
fp.write("DATADIR=" + data_dir + "\n")
fp.write("RUNDIR=" + run_dir + "\n")
fp.write("NUWRFDIR=" + build_dir + "\n")
if exe_dir:
fp.write("EXEDIR=" + exe_dir + "\n")
else:
fp.write("EXEDIR=" + build_dir + "\n")
def create_run_script(config, comp_config, run, comp):
""" This function creates a run script file and returns a command to executed."""
user_config = utils.config_section_map(config, "USERCONFIG")
logger.debug("Creating run script for test: " + run.name)
time_stamp = run.get_opt("time_stamp")
results_dir = run.get_opt("scratch_dir") + "/results/" + comp + "." + time_stamp
run_dir = results_dir + "/" + run.name
build_dir = run.get_opt("scratch_dir") + "/builds/" + comp + "." + time_stamp
test_case = run.name.split("_", 1)
build = test_case[0]
if "wrflis_" in run.name:
if "scm" in run.name:
build = "scm"
else:
build = "wrf"
if "wrf_" in run.name:
build = "wrf"
if "chem_" in run.name or "kpp_" in run.name:
if "chem_" in run.name:
build = "chem"
else:
build = "kpp"
model_dir = run.get_opt("model_dir")
if model_dir:
build_dir = model_dir
else:
build_dir = build_dir + "/" + build
exe_dir = None
if user_config["exe_dir"]:
exe_dir = user_config["exe_dir"]
data_dir = run.get_opt("data_dir") + "/" + "/".join(test_case)
# This is hardwired relative to NUWRFDIR
test_dir = build_dir + "/testcases/" + "/".join(test_case)
filename = results_dir + "/" + run.name + "/" + run.name + ".bash"
with open(filename, "w") as fp:
fp.write("#!/bin/bash" + "\n")
npes_list = [int(x) for x in run.get_opt("npes")]
add_batch_directives(
config,
fp,
test_case[1],
run_dir,
npes=max(npes_list),
is_type=1,
wall_time=run.get_opt("wall_time"),
qos=run.get_opt("qos"),
)
fp.write("cd " + run_dir + "\n")
fp.write("# source common file for modules and paths \n")
fp.write("source ./common.reg || exit 1 \n")
fp.write("if [ -d " + data_dir + " ]; then \n")
fp.write("ln -sf " + data_dir + "\n")
fp.write("fi \n")
fp.write(
"python "
+ run.get_opt("scripts_dir")
+ "/"
+ "regression.py "
+ run.name
+ "\n"
)
fp.write(" " + "\n")
fp.close()
create_common_script(
config, comp_config, comp, run_dir, build_dir, data_dir, exe_dir
)
create_component_script(
config, run, comp, test_dir, run_dir
)
create_run_config(results_dir, run)
if "yes" in run.get_opt("use_batch"):
run_command = "sbatch " + filename
else:
run_command = "chmod +x " + filename + "; " + filename
return run_command
def reg_builds(config, comp_config, test_list, time_stamp):
user_config = utils.config_section_map(config, "USERCONFIG")
builds_dir = user_config["scratch_dir"] + "/builds"
# Accumulate build results
build_results = dict()
if not user_config["model_dir"]:
build_commands = create_build_commands(config, comp_config, test_list)
if build_commands:
logger.info("Compiling...")
if "DEBUG_REG" in os.environ:
logger.info(build_commands)
else:
if "Darwin" in platform.system():
pool.run_commands(build_commands, "no")
else:
pool.run_commands(build_commands, user_config["use_batch"])
for item in test_list[0]:
compiler = item[0]
build = item[1]
available = get_compilers(comp_config)
report_file = (
builds_dir + "/" + compiler + "." + time_stamp + "/builds.diff"
)
if "DEBUG_REG" in os.environ:
build_results[(build, compiler)] = "b+"
logger.info("Build " + build + " was successful.")
results = [
build,
compiler.center(10, " "),
"b+".center(10, " "),
"-".center(10, " "),
]
else:
# Sanity check
if compiler.split("-")[0] in available:
build_dir = (
builds_dir + "/" + compiler + "." + time_stamp + "/" + build
)
results = [
build,
compiler.center(10, " "),
"-".center(10, " "),
"-".center(10, " "),
]
if os.path.exists(build_dir + "/.build_ok"):
build_results[(build, compiler)] = "b+"
logger.info("Build " + build + " was successful.")
results = [
build,
compiler.center(10, " "),
"b+".center(10, " "),
"-".center(10, " "),
]
else:
build_results[(build, compiler)] = "bF"
logger.info("Build " + build + " FAILED.")
results = [
build,
compiler.center(10, " "),
"bF".center(10, " "),
"-".center(10, " "),
]
with open(report_file, "a") as fp:
utils.write_results(results, fp)
return build_results