def run_case(self, base_case_root, run_type, run_length, overwrite=False):
user_mods_dirs = [
os.path.join(self.cesmroot, "cime_config", "usermods_dirs", "NEON",
self.name)
]
expect(os.path.isdir(base_case_root),
"Error base case does not exist in {}".format(base_case_root))
case_root = os.path.abspath(
os.path.join(base_case_root, "..", self.name + "." + run_type))
if os.path.isdir(case_root) and overwrite:
logger.info("---- removing the existing case -------")
shutil.rmtree(case_root)
if not os.path.isdir(case_root):
# read_only = False should not be required here
with Case(base_case_root, read_only=False) as basecase:
logger.info(
"---- cloning the base case in {}".format(case_root))
basecase.create_clone(case_root,
keepexe=True,
user_mods_dirs=user_mods_dirs)
with Case(case_root, read_only=False) as case:
case.set_value("STOP_OPTION", "nyears")
case.set_value("STOP_N", run_length)
case.set_value("REST_N", 100)
case.set_value("CONTINUE_RUN", False)
if run_type == "ad":
case.set_value("CLM_FORCE_COLDSTART", "on")
case.set_value("CLM_ACCELERATED_SPINUP", "on")
case.set_value("RUN_REFDATE", "0018-01-01")
case.set_value("RUN_STARTDATE", "0018-01-01")
def build_xcpl_nml(argv, compclass):
###############################################################################
caseroot = parse_input(argv)
compname = "x" + compclass
caseroot = parse_input(argv)
case = Case(caseroot)
rundir = case.get_value("RUNDIR")
ninst = case.get_value("NINST_%s" % compclass.upper())
nx = case.get_value("%s_NX" % compclass.upper())
ny = case.get_value("%s_NY" % compclass.upper())
extras = []
dtype = 1
npes = 0
length = 0
if compname == "xatm":
if ny == 1:
dtype = 2
extras = [["24",
"ncpl number of communications w/coupler per dat"],
["0.0",
"simul time proxy (secs): time between cpl comms"]]
elif compname == "xglc" or compname == "xice":
dtype = 2
elif compname == "xlnd":
dtype = 11
elif compname == "xocn":
dtype = 4
elif compname == "xrof":
dtype = 11
flood_mode = Case('XROF_FLOOD_MODE')
if flood_mode == "ACTIVE":
extras = [[".true.", "flood flag"]]
else:
extras = [[".false.", "flood flag"]]
for i in range(1, ninst + 1):
# If only 1 file, name is 'compclass_in'
# otherwise files are 'compclass_in0001', 'compclass_in0002', etc
if ninst == 1:
filename = os.path.join(rundir, "%s_in" % compname)
else:
filename = os.path.join(rundir, "%s_in%4.4d" % (compname, i))
with open(filename, 'w') as infile:
infile.write("%-20d ! i-direction global dimension\n" % nx)
infile.write("%-20d ! j-direction global dimension\n" % ny)
infile.write("%-20d ! decomp_type 1=1d-by-lat, 2=1d-by-lon,"
" 3=2d, 4=2d evensquare, 11=segmented\n" % dtype)
infile.write("%-20d ! num of pes for i (type 3 only)\n" % npes)
infile.write("%-20d ! length of segments (type 4 only)\n"
% length)
for extra in extras:
infile.write("%-20s ! %s\n" % (extra[0], extra[1]))
def build_data_nml(argv, compclass):
###############################################################################
# This function is just a wrapper for the one below, to avoid having to
# indent everything for the "with" block.
caseroot = parse_input(argv)
with Case(caseroot) as case:
_build_data_nml(case, caseroot, compclass)
def build_base_case(baseroot, basecasename, res, compset, overwrite,
user_mods_dir):
print(">>>>> BUILDING BASE CASE...")
caseroot = os.path.join(baseroot,basecasename+'.'+basecase_startval)
if overwrite and os.path.isdir(caseroot):
shutil.rmtree(caseroot)
with Case(caseroot, read_only=False) as case:
if not os.path.isdir(caseroot):
case.create(os.path.basename(caseroot), cesmroot, compset, res,
machine_name="cheyenne", driver="mct",
run_unsupported=True, answer="r",walltime="01:00:00",
user_mods_dir=user_mods_dir, project=project)
# make sure that changing the casename will not affect these variables
case.set_value("EXEROOT",case.get_value("EXEROOT", resolved=True))
case.set_value("RUNDIR",case.get_value("RUNDIR",resolved=True)+".00")
case.set_value("RUN_TYPE","startup")
case.set_value("GET_REFCASE",False)
rundir = case.get_value("RUNDIR")
caseroot = case.get_value("CASEROOT")
print(">> base case_setup...")
case.case_setup()
print(">> base case_build...")
build.case_build(caseroot, case=case)
return caseroot
def test_submit(self, read_xml, get_value, init, _submit): # pylint: disable=unused-argument
with tempfile.TemporaryDirectory() as tempdir:
get_value.side_effect = [
tempdir,
tempdir,
True,
"baseid",
None,
True,
]
make_valid_case(tempdir)
with Case(tempdir) as case:
case.submit(chksum=True)
_submit.assert_called_with(
case,
job=None,
no_batch=False,
prereq=None,
allow_fail=False,
resubmit=False,
resubmit_immediate=False,
skip_pnl=False,
mail_user=None,
mail_type=None,
batch_args=None,
workflow=True,
chksum=True,
)
def bless_history(test_name, testcase_dir_for_test, baseline_name,
baseline_root, compiler, report_only, force):
###############################################################################
with Case(testcase_dir_for_test) as case:
if get_model() == "acme":
baseline_full_dir = os.path.join(baseline_root, compiler,
baseline_name,
case.get_value("CASEBASEID"))
else:
baseline_full_dir = os.path.join(baseline_root, baseline_name,
case.get_value("CASEBASEID"))
result, comments = compare_baseline(case,
baseline_dir=baseline_full_dir,
outfile_suffix=None)
if result:
logging.info("Diff appears to have been already resolved.")
return True, None
else:
print comments
if (not report_only and
(force or raw_input("Update this diff (y/n)? ").upper()
in ["Y", "YES"])):
result, comments = generate_baseline(
case, baseline_dir=baseline_full_dir)
if not result:
logging.warning("Hist file bless FAILED for test %s" %
test_name)
return False, "Generate baseline failed: %s" % comments
else:
print comments
return True, None
else:
return True, None
def test_cmd_arg(self, get_value, flush, init): # pylint: disable=unused-argument
Case._force_read_only = False # pylint: disable=protected-access
mocked_open = mock.mock_open()
with self.tempdir as tempdir, mock.patch("CIME.case.case.open",
mocked_open):
with Case(tempdir) as case:
get_value.side_effect = [
tempdir,
"/src",
]
# We didn't need to make tempdir look like a valid case for the Case
# constructor because we mock that constructor, but we *do* need to make
# it look like a valid case for record_cmd.
make_valid_case(tempdir)
case.record_cmd(["/some/custom/command", "arg1"])
expected = [
"/some/custom/command arg1\n\n",
]
handle = mocked_open()
handle.writelines.assert_called_with(expected)
def bless_history(test_name, test_dir, baseline_name, baseline_root, report_only, force):
###############################################################################
with Case(test_dir) as case:
real_user = case.get_value("REALUSER")
with EnvironmentContext(USER=real_user):
if baseline_name is None:
baseline_name = case.get_value("BASELINE_NAME_CMP")
if not baseline_name:
baseline_name = CIME.utils.get_current_branch(repo=CIME.utils.get_cime_root())
if baseline_root is None:
baseline_root = case.get_value("BASELINE_ROOT")
baseline_full_dir = os.path.join(baseline_root, baseline_name, case.get_value("CASEBASEID"))
cmp_result, cmp_comments = compare_baseline(case, baseline_dir=baseline_full_dir, outfile_suffix=None)
if cmp_result:
logger.info("Diff appears to have been already resolved.")
return True, None
else:
logger.info(cmp_comments)
if (not report_only and
(force or six.moves.input("Update this diff (y/n)? ").upper() in ["Y", "YES"])):
gen_result, gen_comments = generate_baseline(case, baseline_dir=baseline_full_dir)
if not gen_result:
logger.warning("Hist file bless FAILED for test {}".format(test_name))
return False, "Generate baseline failed: {}".format(gen_comments)
else:
logger.info(gen_comments)
return True, None
else:
return True, None
def parse_input(argv):
###############################################################################
parser = argparse.ArgumentParser()
setup_standard_logging_options(parser)
parser.add_argument("caseroot", default=os.getcwd(),
help="Case directory")
parser.add_argument("libroot",
help="root for creating the library")
parser.add_argument("bldroot",
help="root for building library")
args = parse_args_and_handle_standard_logging_options(argv, parser)
# Some compilers have trouble with long include paths, setting
# EXEROOT to the relative path from bldroot solves the problem
# doing it in the environment means we don't need to change all of
# the component buildlib scripts
with Case(args.caseroot) as case:
os.environ["EXEROOT"] = os.path.relpath(case.get_value("EXEROOT"), args.bldroot)
return args.caseroot, args.libroot, args.bldroot
def test_create(self, get_user, getfqdn, configure, create_caseroot, # pylint: disable=unused-argument
apply_user_mods, set_lookup_value, lock_file, strftime, # pylint: disable=unused-argument
read_xml): # pylint: disable=unused-argument
with self.tempdir as tempdir:
caseroot = os.path.join(tempdir, "test1")
with Case(caseroot, read_only=False) as case:
os.environ["CIME_MODEL"] = "cesm"
srcroot = os.path.abspath(os.path.join(
os.path.dirname(__file__), "../../../../../"))
case.create("test1", srcroot, "A", "f19_g16_rx1",
machine_name="ubuntu-latest")
# Check that they're all called
configure.assert_called_with("A", "f19_g16_rx1",
machine_name="ubuntu-latest",
project=None, pecount=None,
compiler=None, mpilib=None,
pesfile=None, gridfile=None,
multi_driver=False, ninst=1,
test=False, walltime=None,
queue=None, output_root=None,
run_unsupported=False, answer=None,
input_dir=None, driver=None,
workflowid="default",
non_local=False,
extra_machines_dir=None,
case_group=None)
create_caseroot.assert_called()
apply_user_mods.assert_called()
lock_file.assert_called()
set_lookup_value.assert_called_with("CASE_HASH",
"134a939f62115fb44bf08a46bfb2bd13426833b5c8848cf7c4884af7af05b91a")
def test_sub_relative(self, strftime, get_value, flush, init): # pylint: disable=unused-argument
Case._force_read_only = False # pylint: disable=protected-access
mocked_open = mock.mock_open()
with self.tempdir as tempdir, mock.patch("CIME.case.case.open",
mocked_open):
with Case(tempdir) as case:
get_value.side_effect = [
tempdir,
"/src"
]
case.record_cmd(init=True)
expected = [
"#!/bin/bash\n\n",
"set -e\n\n",
"# Created 00:00:00\n\n",
"CASEDIR=\"{}\"\n\n".format(tempdir),
"/src/scripts/create_newcase\n\n",
"cd \"${CASEDIR}\"\n\n",
]
handle = mocked_open()
handle.writelines.assert_called_with(expected)
def set_ref_case(self, case):
rundir = case.get_value("RUNDIR")
case_root = case.get_value("CASEROOT")
if case_root.endswith(".postad"):
ref_case_root = case_root.replace(".postad", ".ad")
root = ".ad"
else:
ref_case_root = case_root.replace(".transient", ".postad")
root = ".postad"
expect(
os.path.isdir(ref_case_root),
"ERROR: spinup must be completed first, could not find directory {}"
.format(ref_case_root))
with Case(ref_case_root) as refcase:
refrundir = refcase.get_value("RUNDIR")
case.set_value("RUN_REFDIR", refrundir)
case.set_value("RUN_REFCASE", os.path.basename(ref_case_root))
for reffile in glob.iglob(refrundir +
"/{}{}.*.nc".format(self.name, root)):
m = re.search("(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile)
if m:
refdate = m.group(1)
symlink_force(reffile,
os.path.join(rundir, os.path.basename(reffile)))
for rpfile in glob.iglob(refrundir + "/rpointer*"):
safe_copy(rpfile, rundir)
if not os.path.isdir(os.path.join(
rundir, "inputdata")) and os.path.isdir(
os.path.join(refrundir, "inputdata")):
symlink_force(os.path.join(refrundir, "inputdata"),
os.path.join(rundir, "inputdata"))
case.set_value("RUN_REFDATE", refdate)
if case_root.endswith(".postad"):
case.set_value("RUN_STARTDATE", refdate)
def bless_history(test_name, testcase_dir_for_test, baseline_name,
baseline_root, report_only, force):
###############################################################################
with Case(testcase_dir_for_test) as case:
baseline_full_dir = os.path.join(baseline_root, baseline_name,
case.get_value("CASEBASEID"))
result, comments = compare_baseline(case,
baseline_dir=baseline_full_dir)
if result:
return True, None
else:
print comments
if (not report_only and
(force or raw_input("Update this diff (y/n)? ").upper()
in ["Y", "YES"])):
result, comments = generate_baseline(
case, baseline_dir=baseline_full_dir)
if not result:
logging.warning("Hist file bless FAILED for test %s" %
test_name)
return False, "Generate baseline failed: %s" % comments
else:
print comments
return True, None
else:
return True, None
def _case_from_existing_caseroot(self, caseroot):
"""
Returns a Case object from an existing caseroot directory
Args:
caseroot (str): path to existing caseroot
"""
return Case(case_root=caseroot, read_only=False)
def build_data_lib(argv, compclass):
###############################################################################
caseroot, libroot, bldroot = parse_input(argv)
case = Case(caseroot)
cimeroot = case.get_value("CIMEROOT")
# Write directory list (Filepath)
compname = "d" + compclass
with open('Filepath', 'w') as out:
out.write(os.path.join(caseroot, "SourceMods", "src.%s" %compname) + "\n")
out.write(os.path.join(cimeroot, "components", "data_comps", compname) + "\n")
# Build the component
run_gmake(case, compclass, libroot)
def clone_base_case(caseroot, ensemble, user_mods_dir, overwrite, paramdict):
print(">>>>> CLONING BASE CASE...")
startval = ensemble_startval
nint = len(ensemble_startval)
cloneroot = caseroot
for i in range(int(startval), int(startval)+ensemble):
member_string = '{{0:0{0:d}d}}'.format(nint).format(i)
print("member_string="+member_string)
if ensemble > 1:
caseroot = caseroot[:-nint] + member_string
if overwrite and os.path.isdir(caseroot):
shutil.rmtree(caseroot)
if not os.path.isdir(caseroot):
with Case(cloneroot, read_only=False) as clone:
clone.create_clone(caseroot, keepexe=True,
user_mods_dir=user_mods_dir)
with Case(caseroot, read_only=False) as case:
per_run_case_updates(case, user_mods_dir, member_string, nint, paramdict)
def build_xcpl_nml(argv, compclass):
###############################################################################
caseroot = parse_input(argv)
compname = "x" + compclass
caseroot = parse_input(argv)
with Case(caseroot) as case:
rundir = case.get_value("RUNDIR")
ninst = case.get_value("NINST_%s" % compclass.upper())
nx = case.get_value("%s_NX" % compclass.upper())
ny = case.get_value("%s_NY" % compclass.upper())
if compname == "xrof":
flood_mode = case.get_value('XROF_FLOOD_MODE')
extras = []
dtype = 1
npes = 0
length = 0
if compname == "xatm":
if ny == 1:
dtype = 2
extras = [["24", "ncpl number of communications w/coupler per dat"],
["0.0", "simul time proxy (secs): time between cpl comms"]]
elif compname == "xglc" or compname == "xice":
dtype = 2
elif compname == "xlnd":
dtype = 11
elif compname == "xocn":
dtype = 4
elif compname == "xrof":
dtype = 11
if flood_mode == "ACTIVE":
extras = [[".true.", "flood flag"]]
else:
extras = [[".false.", "flood flag"]]
for i in range(1, ninst + 1):
# If only 1 file, name is 'compclass_in'
# otherwise files are 'compclass_in0001', 'compclass_in0002', etc
if ninst == 1:
filename = os.path.join(rundir, "%s_in" % compname)
else:
filename = os.path.join(rundir, "%s_in_%4.4d" % (compname, i))
with open(filename, 'w') as infile:
infile.write("%-20d ! i-direction global dimension\n" % nx)
infile.write("%-20d ! j-direction global dimension\n" % ny)
infile.write("%-20d ! decomp_type 1=1d-by-lat, 2=1d-by-lon,"
" 3=2d, 4=2d evensquare, 11=segmented\n" % dtype)
infile.write("%-20d ! num of pes for i (type 3 only)\n" % npes)
infile.write("%-20d ! length of segments (type 4 only)\n" % length)
for extra in extras:
infile.write("%-20s ! %s\n" % (extra[0], extra[1]))
def __init__(self, casename):
super(CaseWindow, self).__init__()
self.setupUi(self)
self.setMouseTracking(1)
with Case(casename) as case:
self.populate_case_tab(case)
self.populate_setup_tab(case)
self.populate_build_tab(case)
self.populate_run_tab(case)
def compare_history(testcase_dir_for_test, baseline_name, baseline_root):
###############################################################################
with Case(testcase_dir_for_test) as case:
baseline_full_dir = os.path.join(baseline_root, baseline_name,
case.get_value("CASEBASEID"))
result, comments = compare_baseline(case,
baseline_dir=baseline_full_dir)
if result:
return True, None
else:
logging.info(comments)
return False, "Diff'd"
def create_new_case(self):
srcroot = os.path.dirname(os.path.abspath(get_cime_root()))
casename = self.CreateNewcaseArgs["case"]
with Case(os.path.join(self._casedir, casename),
read_only=False) as case:
case.create(casename,
srcroot,
self.CreateNewcaseArgs["compset"],
self.CreateNewcaseArgs["res"],
machine_name=self.CreateNewcaseArgs["machine"],
compiler=self.CreateNewcaseArgs["compiler"],
mpilib=self.CreateNewcaseArgs["mpilib"])
def compare_history(testcase_dir_for_test, baseline_name, baseline_root,
log_id):
###############################################################################
with Case(testcase_dir_for_test) as case:
baseline_full_dir = os.path.join(baseline_root, baseline_name,
case.get_value("CASEBASEID"))
outfile_suffix = "%s.%s" % (baseline_name, log_id)
result, comments = compare_baseline(case,
baseline_dir=baseline_full_dir,
outfile_suffix=outfile_suffix)
return result, comments
def download_input_data(rundir):
"""Implementation of the download_input_data command
Args:
rundir: str - path to directory containing input_data_list files
"""
_create_lilac_input_data_list(rundir)
case = Case(os.path.realpath(os.path.join(rundir, os.pardir, 'case')))
case.check_all_input_data(data_list_dir=rundir,
download=True,
chksum=False)
os.remove(os.path.join(rundir, 'lilac.input_data_list'))
def clone_base_case(date, caseroot, ensemble, sdrestdir, user_mods_dir,
overwrite):
startval = "01"
nint = len(startval)
cloneroot = caseroot
for i in range(int(startval), int(startval) + ensemble):
member_string = '{{0:0{0:d}d}}'.format(nint).format(i)
if ensemble > 1:
caseroot = caseroot[:-nint] + member_string
if overwrite and os.path.isdir(caseroot):
shutil.rmtree(caseroot)
if not os.path.isdir(caseroot):
with Case(cloneroot, read_only=False) as clone:
clone.create_clone(caseroot,
keepexe=True,
user_mods_dir=user_mods_dir)
with Case(caseroot, read_only=True) as case:
# rundir is initially 00 reset to current member
rundir = case.get_value("RUNDIR")
rundir = rundir[:-nint] + member_string
case.set_value("RUNDIR", rundir)
per_run_case_updates(case, date, sdrestdir, user_mods_dir, rundir)
def cleanup_queue(test_root, test_id):
###############################################################################
"""
Delete all jobs left in the queue
"""
for teststatus_file in glob.iglob("{}/*{}*/TestStatus".format(test_root, test_id)):
case_dir = os.path.dirname(teststatus_file)
with Case(case_dir, read_only=True) as case:
jobmap = case.get_job_info()
jobkills = []
for jobname, jobid in jobmap.items():
logging.warning("Found leftover batch job {} ({}) that need to be deleted".format(jobid, jobname))
jobkills.append(jobid)
case.cancel_batch_jobs(jobkills)
def test_error(self, strftime, get_value, flush, init): # pylint: disable=unused-argument
Case._force_read_only = False # pylint: disable=protected-access
with self.tempdir as tempdir, mock.patch("CIME.case.case.open",
mock.mock_open()) as m:
m.side_effect = PermissionError()
with Case(tempdir) as case:
get_value.side_effect = [tempdir, "/src"]
# We didn't need to make tempdir look like a valid case for the Case
# constructor because we mock that constructor, but we *do* need to make
# it look like a valid case for record_cmd.
make_valid_case(tempdir)
case.record_cmd()
def build_data_lib(argv, compclass):
###############################################################################
caseroot, libroot, _ = parse_input(argv)
with Case(caseroot) as case:
cimeroot = case.get_value("CIMEROOT")
# Write directory list (Filepath)
compname = "d" + compclass
with open('Filepath', 'w') as out:
out.write(os.path.join(caseroot, "SourceMods", "src.{}\n".format(compname)) + "\n")
out.write(os.path.join(cimeroot, "src", "components", "data_comps", compname) + "\n")
# Build the component
run_gmake(case, compclass, libroot)
def test_copy(self, getuser, getfqdn, configure, create_caseroot, # pylint: disable=unused-argument
apply_user_mods, set_lookup_value, lock_file, strftime, # pylint: disable=unused-argument
read_xml): # pylint: disable=unused-argument
expected_first_hash = "134a939f62115fb44bf08a46bfb2bd13426833b5c8848cf7c4884af7af05b91a"
expected_second_hash = "3561339a49daab999e3c4ea2f03a9c6acc33296a5bc35f1bfb82e7b5e10bdf38"
with self.tempdir as tempdir:
caseroot = os.path.join(tempdir, "test1")
with Case(caseroot, read_only=False) as case:
os.environ["CIME_MODEL"] = "cesm"
srcroot = os.path.abspath(os.path.join(
os.path.dirname(__file__), "../../../../../"))
case.create("test1", srcroot, "A", "f19_g16_rx1",
machine_name="ubuntu-latest")
# Check that they're all called
configure.assert_called_with("A", "f19_g16_rx1",
machine_name="ubuntu-latest",
project=None, pecount=None,
compiler=None, mpilib=None,
pesfile=None, gridfile=None,
multi_driver=False, ninst=1,
test=False, walltime=None,
queue=None, output_root=None,
run_unsupported=False, answer=None,
input_dir=None, driver=None,
workflowid="default",
non_local=False,
extra_machines_dir=None,
case_group=None)
create_caseroot.assert_called()
apply_user_mods.assert_called()
lock_file.assert_called()
set_lookup_value.assert_called_with("CASE_HASH",
expected_first_hash)
strftime.return_value = "10:00:00"
with mock.patch("CIME.case.case.Case.set_value") as set_value, \
mock.patch("sys.argv", ["/src/create_clone"]):
case.copy("test2", "{}_2".format(tempdir))
set_value.assert_called_with("CASE_HASH",
expected_second_hash)
def build_stub_lib(argv, compclass):
###############################################################################
caseroot, libroot, _ = parse_input(argv)
with Case(caseroot) as case:
cimeroot = case.get_value("CIMEROOT")
# Write directory list
compname = "s" + compclass
with open('Filepath', 'w') as out:
out.write(os.path.join(caseroot, "SourceMods", "src.{}", compname) + "\n")
out.write(os.path.join(cimeroot, "src", "components", "stub_comps", "xshare") + "\n")
out.write(os.path.join(cimeroot, "src", "components", "stub_comps",compname, "cpl") + "\n")
# Build the component
run_gmake(case, compclass, libroot)
def test_new_hash(self, getuser, getfqdn, strftime, read_xml): # pylint: disable=unused-argument
with self.tempdir as tempdir:
with Case(tempdir) as case:
expected = "134a939f62115fb44bf08a46bfb2bd13426833b5c8848cf7c4884af7af05b91a"
# Check idempotency
for _ in range(2):
value = case.new_hash()
self.assertTrue(value == expected,
"{} != {}".format(value, expected))
expected = "bb59f1c473ac07e9dd30bfab153c0530a777f89280b716cf42e6fe2f49811a6e"
value = case.new_hash()
self.assertTrue(value == expected,
"{} != {}".format(value, expected))
def _main_func(description):
casename, _ = parse_command_line(sys.argv, description)
caseroot = os.path.abspath(casename)
# forcing_dir = "/glade/collections/nmme/ncgd0022/jcaron/CFSv2/forcing_files/"
forcing_dir = "/glade/scratch/ssfcst/forcing_files/"
forcing_files = {
"Precip": glob.glob(forcing_dir + "*Prec*.nc"),
"Solar": glob.glob(forcing_dir + "*Solar*.nc"),
"TPQW": glob.glob(forcing_dir + "*TPQW*.nc")
}
with Case(caseroot, read_only=False) as case:
rundir = case.get_value("RUNDIR")
for ftype in ["Solar", "Precip", "TPQW"]:
forcing_files[ftype] = sorted(forcing_files[ftype], reverse=True)
streamfile = os.path.join(rundir,
"datm.streams.txt.CLMCRUNCEP." + ftype)
newfile = os.path.join(caseroot,
"user_" + os.path.basename(streamfile))
#print("New forcing_files from {} to {}".format(forcing_files["Solar"][0],forcing_files["Solar"][-1]))
with open(streamfile) as fin, open(newfile, "w") as fout:
input_lines = fin.readlines()
skip_line = False
for line in input_lines:
if "</filePath>" in line:
skip_line = False
if skip_line:
continue
if "domain.lnd" in line:
line = "domain.ncepCFSv2.c2019.0.2d.nc\n"
if "clmforc.cruncep" in line or "clmforc.NCEPCFSv2" in line:
while forcing_files[ftype]:
newline = os.path.basename(
forcing_files[ftype].pop())
fout.write(newline + "\n")
continue
fout.write(line)
if "<filePath>" in line:
skip_line = True
fout.write("/glade/scratch/ssfcst/forcing_files/\n")
def build_base_case(date,
baseroot,
basecasename,
basemonth,
res,
compset,
overwrite,
sdrestdir,
user_mods_dir,
pecount=None):
caseroot = os.path.join(baseroot,
basecasename + ".{:02d}".format(basemonth) + ".00")
if overwrite and os.path.isdir(caseroot):
shutil.rmtree(caseroot)
with Case(caseroot, read_only=False) as case:
if not os.path.isdir(caseroot):
case.create(os.path.basename(caseroot),
cesmroot,
compset,
res,
run_unsupported=True,
answer="r",
walltime="04:00:00",
user_mods_dir=user_mods_dir,
pecount=pecount,
project="NCGD0042")
# make sure that changing the casename will not affect these variables
case.set_value("EXEROOT", case.get_value("EXEROOT", resolved=True))
case.set_value("RUNDIR",
case.get_value("RUNDIR", resolved=True) + ".00")
case.set_value("RUN_TYPE", "hybrid")
case.set_value("GET_REFCASE", False)
case.set_value("RUN_REFDIR", sdrestdir)
if basecasename == "70Lwaccm6":
case.set_value("RUN_REFCASE",
"b.e21.BWHIST.SD.{}.002.nudgedOcn".format(res))
case.set_value("OCN_TRACER_MODULES", "")
case.set_value("NTHRDS", 1)
else:
def test_cmd_arg(self, get_value, flush, init): # pylint: disable=unused-argument
Case._force_read_only = False # pylint: disable=protected-access
mocked_open = mock.mock_open()
with self.tempdir as tempdir, mock.patch("CIME.case.case.open",
mocked_open):
with Case(tempdir) as case:
get_value.side_effect = [
tempdir,
"/src",
]
case.record_cmd(["/some/custom/command", "arg1"])
expected = [
"/some/custom/command arg1\n\n",
]
handle = mocked_open()
handle.writelines.assert_called_with(expected)
def preview_namelists(dryrun=False, case=None, casedir=None):
if (case is None):
if (casedir is None):
case = Case()
else:
case = Case(case_root=casedir)
# Get data from XML
exeroot = case.get_value("EXEROOT")
libroot = case.get_value("LIBROOT")
incroot = case.get_value("INCROOT")
rundir = case.get_value("RUNDIR")
caseroot = case.get_value("CASEROOT")
cimeroot = case.get_value("CIMEROOT")
casebuild = case.get_value("CASEBUILD")
testcase = case.get_value("TESTCASE")
compiler = case.get_value("COMPILER")
mach = case.get_value("MACH")
debug = case.get_value("DEBUG")
mpilib = case.get_value("MPILIB")
logging.debug("LID is: '%s'" % os.getenv("LID", ""))
logging.debug("caseroot is: '%s'" % caseroot)
dryrun = True if (testcase == "SBN") else dryrun
models = ["atm", "lnd", "ice", "ocn", "glc", "wav", "rof", "cpl"]
docdir = os.path.join(caseroot, "CaseDocs")
if (dryrun):
# Only create rundir
try:
os.makedirs(rundir)
except OSError:
logging.warning("Not able to create $RUNDIR, trying a subdirectory of $CASEROOT")
rundir = os.path.join(caseroot, rundir)
try:
os.makedirs(rundir)
logging.info("Success! Setting RUNDIR=%s" % rundir)
case.set_value("RUNDIR", rundir)
except OSError:
expect(False, "Could not create rundir")
else:
# Load modules
env_module = EnvModule(mach, compiler, cimeroot, caseroot, mpilib, debug)
env_module.load_env_for_case()
# Make necessary directories
dirs_to_make = [os.path.join(exeroot, model, "obj") for model in models]
dirs_to_make.extend([exeroot, libroot, incroot, rundir, docdir])
for dir_to_make in dirs_to_make:
if (not os.path.isdir(dir_to_make)):
try:
logging.debug("Making dir '%s'" % dir_to_make)
os.makedirs(dir_to_make)
except OSError as e:
expect(False, "Could not make directory '%s', error: %s" % (dir_to_make, e))
# Create namelists
for model in models:
model_str = "drv" if model == "cpl" else model
config_file = case.get_value("CONFIG_%s_FILE" % model_str.upper())
config_dir = os.path.dirname(config_file)
cmd = os.path.join(config_dir, "buildnml")
if (logging.getLogger().level == logging.DEBUG):
run_cmd("PREVIEW_NML=1 %s %s" % (cmd, caseroot))
else:
run_cmd("%s %s" % (cmd, caseroot))
# Save namelists to docdir
if (not os.path.isdir(docdir)):
os.makedirs(docdir)
try:
with open(os.path.join(docdir, "README"), "w") as fd:
fd.write(" CESM Resolved Namelist Files\n For documentation only DO NOT MODIFY\n")
except (OSError, IOError) as e:
expect(False, "Failed to write %s/README: %s" % (docdir, e))
for cpglob in ["*_in_[0-9]*", "*modelio*", "*_in",
"*streams*txt*", "*stxt", "*maps.rc", "*cism.config*"]:
for file_to_copy in glob.glob(os.path.join(rundir, cpglob)):
logging.debug("Copy file from '%s' to '%s'" % (file_to_copy, docdir))
shutil.copy2(file_to_copy, docdir)
# Copy over chemistry mechanism docs if they exist
if (os.path.isdir(os.path.join(casebuild, "camconf"))):
for file_to_copy in glob.glob(os.path.join(casebuild, "camconf", "*chem_mech*")):
shutil.copy2(file_to_copy, docdir)
def case_build(caseroot, case=None, testmode=False, sharedlib_only=False, model_only=False):
###############################################################################
t1 = time.time()
expect(not (sharedlib_only and model_only),
"Contradiction: both sharedlib_only and model_only")
logger.info("sharedlib_only is %s" % sharedlib_only)
logger.info("model_only is %s" % model_only)
expect(os.path.isdir(caseroot), "'%s' is not a valid directory" % caseroot)
os.chdir(caseroot)
expect(os.path.exists("case.run"),
"ERROR: must invoke case.setup script before calling build script ")
case = Case() if case is None else case
testcase = case.get_value("TESTCASE")
cimeroot = case.get_value("CIMEROOT")
expect(not (testcase is not None and
os.path.exists("%s/scripts/Testing/Testcases/%s_build.csh" %
(cimeroot, testcase)) and not testmode),
"%s build must be invoked via case.testbuild script" % testcase)
if not sharedlib_only:
check_all_input_data(case)
run_cmd("./Tools/check_lockedfiles --caseroot %s" % caseroot)
# Retrieve relevant case data
build_threaded = case.get_value("BUILD_THREADED")
casetools = case.get_value("CASETOOLS")
exeroot = case.get_value("EXEROOT")
incroot = case.get_value("INCROOT")
libroot = case.get_value("LIBROOT")
sharedlibroot = case.get_value("SHAREDLIBROOT")
comp_atm = case.get_value("COMP_ATM")
comp_lnd = case.get_value("COMP_LND")
comp_ice = case.get_value("COMP_ICE")
comp_ocn = case.get_value("COMP_OCN")
comp_glc = case.get_value("COMP_GLC")
comp_wav = case.get_value("COMP_WAV")
comp_rof = case.get_value("COMP_ROF")
compiler = case.get_value("COMPILER")
comp_interface = case.get_value("COMP_INTERFACE")
mpilib = case.get_value("MPILIB")
use_esmf_lib = case.get_value("USE_ESMF_LIB")
debug = case.get_value("DEBUG")
ninst_build = case.get_value("NINST_BUILD")
smp_value = case.get_value("SMP_VALUE")
clm_use_petsc = case.get_value("CLM_USE_PETSC")
cism_use_trilinos = case.get_value("CISM_USE_TRILINOS")
mpasli_use_albany = case.get_value("MPASLI_USE_ALBANY")
clm_config_opts = case.get_value("CLM_CONFIG_OPTS")
cam_config_opts = case.get_value("CAM_CONFIG_OPTS")
pio_config_opts = case.get_value("PIO_CONFIG_OPTS")
ninst_value = case.get_value("NINST_VALUE")
mach = case.get_value("MACH")
os_ = case.get_value("OS")
comp_cpl = case.get_value("COMP_CPL")
machines_file = case.get_value("MACHINES_SPEC_FILE")
ocn_submodel = case.get_value("OCN_SUBMODEL")
profile_papi_enable = case.get_value("PROFILE_PAPI_ENABLE")
nthrds_cpl = int(case.get_value("NTHRDS_CPL"))
nthrds_atm = int(case.get_value("NTHRDS_ATM"))
nthrds_lnd = int(case.get_value("NTHRDS_LND"))
nthrds_ice = int(case.get_value("NTHRDS_ICE"))
nthrds_ocn = int(case.get_value("NTHRDS_OCN"))
nthrds_glc = int(case.get_value("NTHRDS_GLC"))
nthrds_wav = int(case.get_value("NTHRDS_WAV"))
nthrds_rof = int(case.get_value("NTHRDS_ROF"))
# Load some params into env
os.environ["CIMEROOT"] = cimeroot
os.environ["CASETOOLS"] = casetools
os.environ["EXEROOT"] = exeroot
os.environ["INCROOT"] = incroot
os.environ["LIBROOT"] = libroot
os.environ["SHAREDLIBROOT"] = sharedlibroot
os.environ["CASEROOT"] = caseroot
os.environ["COMPILER"] = compiler
os.environ["COMP_INTERFACE"] = comp_interface
os.environ["NINST_VALUE"] = ninst_value
os.environ["BUILD_THREADED"] = build_threaded
os.environ["MACH"] = mach
os.environ["USE_ESMF_LIB"] = use_esmf_lib
os.environ["MPILIB"] = mpilib
os.environ["DEBUG"] = debug
os.environ["OS"] = os_
os.environ["COMP_CPL"] = comp_cpl
os.environ["COMP_ATM"] = comp_atm
os.environ["COMP_LND"] = comp_lnd
os.environ["COMP_ICE"] = comp_ice
os.environ["COMP_OCN"] = comp_ocn
os.environ["COMP_GLC"] = comp_glc
os.environ["COMP_WAV"] = comp_wav
os.environ["COMP_ROF"] = comp_rof
os.environ["CLM_CONFIG_OPTS"] = clm_config_opts if clm_config_opts is not None else ""
os.environ["CAM_CONFIG_OPTS"] = cam_config_opts if cam_config_opts is not None else ""
os.environ["PIO_CONFIG_OPTS"] = pio_config_opts if pio_config_opts is not None else ""
os.environ["OCN_SUBMODEL"] = ocn_submodel if ocn_submodel is not None else ""
os.environ["PROFILE_PAPI_ENABLE"] = profile_papi_enable if profile_papi_enable is not None else ""
os.environ["CLM_USE_PETSC"] = clm_use_petsc if clm_use_petsc is not None else ""
os.environ["CISM_USE_TRILINOS"] = cism_use_trilinos if cism_use_trilinos is not None else ""
os.environ["MPASLI_USE_ALBANY"] = mpasli_use_albany if mpasli_use_albany is not None else ""
# This is a timestamp for the build , not the same as the testid,
# and this case may not be a test anyway. For a production
# experiment there may be many builds of the same case.
lid = run_cmd("date +%y%m%d-%H%M%S")
os.environ["LID"] = lid
# Set the overall USE_PETSC variable to TRUE if any of the
# XXX_USE_PETSC variables are TRUE.
# For now, there is just the one CLM_USE_PETSC variable, but in
# the future there may be others -- so USE_PETSC will be true if
# ANY of those are true.
use_petsc = "TRUE" if clm_use_petsc == "TRUE" else "FALSE"
case.set_value("USE_PETSC", use_petsc)
os.environ["USE_PETSC"] = use_petsc
# Set the overall USE_TRILINOS variable to TRUE if any of the
# XXX_USE_TRILINOS variables are TRUE.
# For now, there is just the one CISM_USE_TRILINOS variable, but in
# the future there may be others -- so USE_TRILINOS will be true if
# ANY of those are true.
use_trilinos = "TRUE" if cism_use_trilinos == "TRUE" else "FALSE"
case.set_value("USE_TRILINOS", use_trilinos)
os.environ["USE_TRILINOS"] = use_trilinos
# Set the overall USE_ALBANY variable to TRUE if any of the
# XXX_USE_ALBANY variables are TRUE.
# For now, there is just the one MPASLI_USE_ALBANY variable, but in
# the future there may be others -- so USE_ALBANY will be true if
# ANY of those are true.
use_albany = "TRUE" if mpasli_use_albany == "TRUE" else "FALSE"
case.set_value("USE_ALBANY", use_albany)
os.environ["USE_ALBANY"] = use_albany
# Load modules
env_module = EnvModule(mach, compiler, cimeroot, caseroot, mpilib, debug)
env_module.load_env_for_case()
# Need to flush case xml to disk before calling preview_namelists
case.flush()
if not sharedlib_only:
preview_namelists(case=case)
build_checks(case, build_threaded, comp_interface, use_esmf_lib, debug, compiler, mpilib,
sharedlibroot, nthrds_cpl, nthrds_atm, nthrds_lnd, nthrds_ice, nthrds_ocn,
nthrds_glc, nthrds_wav, nthrds_rof, ninst_build, smp_value)
t2 = time.time()
logs = []
if not model_only:
logs = build_libraries(exeroot, caseroot, cimeroot, libroot, mpilib, lid, machines_file)
if sharedlib_only and comp_lnd == "clm" and not "clm4_0" in clm_config_opts:
logging.info(" - Building clm4_5/clm5_0 Library ")
esmfdir = "esmf" if use_esmf_lib == "TRUE" else "noesmf"
sharedpath = os.environ["SHAREDPATH"]
bldroot = os.path.join(sharedpath, comp_interface, esmfdir)
objdir = os.path.join(bldroot, "lnd", "obj")
libdir = os.path.join(bldroot, "lib")
file_build = os.path.join(exeroot, "lnd.bldlog.%s" % lid)
config_lnd_dir = os.path.dirname(case.get_value("CONFIG_LND_FILE"))
results = []
for ndir in [bldroot, objdir, libdir]:
if(not os.path.isdir(ndir)):
os.makedirs(ndir)
_build_model_thread(config_lnd_dir, caseroot, bldroot, "clm", file_build,
exeroot, "lnd", "clm", objdir, incroot, results)
if not sharedlib_only:
logs.extend(build_model(case, build_threaded, exeroot, clm_config_opts, incroot,
comp_atm, comp_lnd, comp_ice, comp_ocn, comp_glc, comp_wav, comp_rof,
nthrds_atm, nthrds_lnd, nthrds_ice, nthrds_ocn, nthrds_glc, nthrds_wav, nthrds_rof,
lid, caseroot, cimeroot, use_esmf_lib, comp_interface))
if not sharedlib_only:
post_build(case, logs)
t3 = time.time()
logger.info("Time spent not building: %f sec" % (t2 - t1))
logger.info("Time spent building: %f sec" % (t3 - t2))
def case_build(caseroot, case=None, sharedlib_only=False, model_only=False):
###############################################################################
t1 = time.time()
expect(not (sharedlib_only and model_only),
"Contradiction: both sharedlib_only and model_only")
logger.info("sharedlib_only is %s" % sharedlib_only)
logger.info("model_only is %s" % model_only)
expect(os.path.isdir(caseroot), "'%s' is not a valid directory" % caseroot)
os.chdir(caseroot)
expect(os.path.exists("case.run"),
"ERROR: must invoke case.setup script before calling build script ")
case = Case() if case is None else case
cimeroot = case.get_value("CIMEROOT")
comp_classes = case.get_value("COMP_CLASSES").split(',')
if not sharedlib_only:
check_all_input_data(case)
run_cmd("./Tools/check_lockedfiles --caseroot %s" % caseroot)
# Retrieve relevant case data
# This environment variable gets set for cesm Make and
# needs to be unset before building again.
if "MODEL" in os.environ.keys():
del os.environ["MODEL"]
build_threaded = case.get_value("BUILD_THREADED")
casetools = case.get_value("CASETOOLS")
exeroot = case.get_value("EXEROOT")
incroot = case.get_value("INCROOT")
libroot = case.get_value("LIBROOT")
sharedlibroot = case.get_value("SHAREDLIBROOT")
complist = []
for comp_class in comp_classes:
if comp_class == "DRV":
comp_class = "CPL"
ninst = 1
config_dir = None
else:
ninst = case.get_value("NINST_%s"%comp_class)
config_dir = os.path.dirname(case.get_value("CONFIG_%s_FILE"%comp_class))
comp = case.get_value("COMP_%s"%comp_class)
thrds = case.get_value("NTHRDS_%s"%comp_class)
complist.append((comp_class.lower(), comp, thrds, ninst, config_dir ))
os.environ["COMP_%s"%comp_class] = comp
machines_file = case.get_value("MACHINES_SPEC_FILE")
ocn_submodel = case.get_value("OCN_SUBMODEL")
profile_papi_enable = case.get_value("PROFILE_PAPI_ENABLE")
compiler = case.get_value("COMPILER")
comp_interface = case.get_value("COMP_INTERFACE")
mpilib = case.get_value("MPILIB")
use_esmf_lib = case.get_value("USE_ESMF_LIB")
debug = case.get_value("DEBUG")
ninst_build = case.get_value("NINST_BUILD")
smp_value = case.get_value("SMP_VALUE")
clm_use_petsc = case.get_value("CLM_USE_PETSC")
cism_use_trilinos = case.get_value("CISM_USE_TRILINOS")
mpasli_use_albany = case.get_value("MPASLI_USE_ALBANY")
clm_config_opts = case.get_value("CLM_CONFIG_OPTS")
cam_config_opts = case.get_value("CAM_CONFIG_OPTS")
pio_config_opts = case.get_value("PIO_CONFIG_OPTS")
ninst_value = case.get_value("NINST_VALUE")
mach = case.get_value("MACH")
os_ = case.get_value("OS")
# Load some params into env
os.environ["CIMEROOT"] = cimeroot
os.environ["CASETOOLS"] = casetools
os.environ["EXEROOT"] = exeroot
os.environ["INCROOT"] = incroot
os.environ["LIBROOT"] = libroot
os.environ["SHAREDLIBROOT"] = sharedlibroot
os.environ["CASEROOT"] = caseroot
os.environ["COMPILER"] = compiler
os.environ["COMP_INTERFACE"] = comp_interface
os.environ["NINST_VALUE"] = str(ninst_value)
os.environ["BUILD_THREADED"] = stringify_bool(build_threaded)
os.environ["MACH"] = mach
os.environ["USE_ESMF_LIB"] = stringify_bool(use_esmf_lib)
os.environ["MPILIB"] = mpilib
os.environ["DEBUG"] = stringify_bool(debug)
os.environ["OS"] = os_
os.environ["CLM_CONFIG_OPTS"] = clm_config_opts if clm_config_opts is not None else ""
os.environ["CAM_CONFIG_OPTS"] = cam_config_opts if cam_config_opts is not None else ""
os.environ["PIO_CONFIG_OPTS"] = pio_config_opts if pio_config_opts is not None else ""
os.environ["OCN_SUBMODEL"] = ocn_submodel if ocn_submodel is not None else ""
os.environ["PROFILE_PAPI_ENABLE"] = stringify_bool(profile_papi_enable) if profile_papi_enable is not None else ""
os.environ["CLM_USE_PETSC"] = stringify_bool(clm_use_petsc) if clm_use_petsc is not None else ""
os.environ["CISM_USE_TRILINOS"] = stringify_bool(cism_use_trilinos) if cism_use_trilinos is not None else ""
os.environ["MPASLI_USE_ALBANY"] = stringify_bool(mpasli_use_albany) if mpasli_use_albany is not None else ""
# This is a timestamp for the build , not the same as the testid,
# and this case may not be a test anyway. For a production
# experiment there may be many builds of the same case.
lid = run_cmd("date +%y%m%d-%H%M%S")
os.environ["LID"] = lid
# Set the overall USE_PETSC variable to TRUE if any of the
# XXX_USE_PETSC variables are TRUE.
# For now, there is just the one CLM_USE_PETSC variable, but in
# the future there may be others -- so USE_PETSC will be true if
# ANY of those are true.
use_petsc = clm_use_petsc
case.set_value("USE_PETSC", use_petsc)
os.environ["USE_PETSC"] = stringify_bool(use_petsc)
# Set the overall USE_TRILINOS variable to TRUE if any of the
# XXX_USE_TRILINOS variables are TRUE.
# For now, there is just the one CISM_USE_TRILINOS variable, but in
# the future there may be others -- so USE_TRILINOS will be true if
# ANY of those are true.
use_trilinos = False if cism_use_trilinos is None else cism_use_trilinos
case.set_value("USE_TRILINOS", use_trilinos)
os.environ["USE_TRILINOS"] = stringify_bool(use_trilinos)
# Set the overall USE_ALBANY variable to TRUE if any of the
# XXX_USE_ALBANY variables are TRUE.
# For now, there is just the one MPASLI_USE_ALBANY variable, but in
# the future there may be others -- so USE_ALBANY will be true if
# ANY of those are true.
use_albany = mpasli_use_albany
case.set_value("USE_ALBANY", use_albany)
os.environ["USE_ALBANY"] = stringify_bool(use_albany)
# Load modules
env_module = case._get_env("mach_specific")
env_module.load_env_for_case(compiler=case.get_value("COMPILER"),
debug=case.get_value("DEBUG"),
mpilib=case.get_value("MPILIB"))
# Need to flush case xml to disk before calling preview_namelists
case.flush()
if not sharedlib_only:
preview_namelists(case)
build_checks(case, build_threaded, comp_interface, use_esmf_lib, debug, compiler, mpilib,
sharedlibroot, complist, ninst_build, smp_value)
t2 = time.time()
logs = []
if not model_only:
logs = build_libraries(case, exeroot, caseroot, cimeroot, libroot, mpilib, lid,
machines_file)
if not sharedlib_only:
logs.extend(build_model(case, build_threaded, exeroot, clm_config_opts, incroot, complist,
lid, caseroot, cimeroot, use_esmf_lib, comp_interface))
if not sharedlib_only:
post_build(case, logs)
t3 = time.time()
logger.info("Time spent not building: %f sec" % (t2 - t1))
logger.info("Time spent building: %f sec" % (t3 - t2))
def build_data_nml(argv, compclass):
###############################################################################
caseroot = parse_input(argv)
case = Case(caseroot)
cimeroot = case.get_value("CIMEROOT")
rundir = case.get_value("RUNDIR")
ninst = case.get_value("NINST_%s" % compclass.upper())
compname = "d" + compclass
confdir = os.path.join(caseroot,"Buildconf",compname + "conf")
if not os.path.isdir(confdir):
os.makedirs(confdir)
inst_string = ""
inst_counter = 1
while (inst_counter <= ninst):
# determine instance string
inst_string = ""
if ninst > 1:
inst_string = '_' + '%04d' % inst_counter
# If multi-instance case does not have restart file, use
# single-case restart for each instance
rpointer = "rpointer." + compname
if (os.path.isfile(os.path.join(rundir,rpointer)) and
(not os.path.isfile(os.path.join(rundir,rpointer+inst_string)))):
shutil.copy(os.path.isfile(os.path.join(rundir,rpointer),
os.path.join(rundir,rpointer+inst_string)))
# create namelist output infile using user_nl_file as input
user_nl_file = os.path.join(caseroot, "user_nl_" + compname + inst_string)
expect(os.path.isfile(user_nl_file),
"Missing required user_nl_file %s " %(user_nl_file))
namelist_infile = os.path.join(confdir, "cesm_namelist")
create_namelist_infile(case, user_nl_file, namelist_infile)
# call build-namelist
user_xml_dir = os.path.join(caseroot, "SourceMods", "src." + compname)
inst_string_label = inst_string
if not inst_string_label:
inst_string_label = "\"\""
bldnamelist = os.path.join(cimeroot, "components", "data_comps", compname, "bld", "build-namelist")
cmd = "%s -caseroot %s -cimeroot %s -inst_string %s -infile %s -user_xml_dir %s" \
% (bldnamelist, caseroot, cimeroot, inst_string_label, namelist_infile, user_xml_dir)
rc, out, err = run_cmd(cmd, from_dir=confdir, ok_to_fail=True)
expect(rc==0,"Command %s failed rc=%d\nout=%s\nerr=%s"%(cmd,rc,out,err))
# copy namelist files and stream text files, to rundir
if os.path.isdir(rundir):
filename = compname + "_in"
file_src = os.path.join(confdir, filename)
file_dest = os.path.join(rundir, filename)
if inst_string:
file_dest += inst_string
shutil.copy(file_src,file_dest)
filename = compname + "_" + compclass + "_in"
file_src = os.path.join(confdir, filename)
file_dest = os.path.join(rundir, filename)
if inst_string:
file_dest += inst_string
shutil.copy(file_src,file_dest)
for txtfile in glob.glob(os.path.join(confdir, "*txt*")):
shutil.copy(txtfile, rundir)
# increment instance counter
inst_counter = inst_counter + 1
def check_input_data(case=None, svn_loc=None, input_data_root=None, data_list_dir="Buildconf", download=False):
"""
Return True if no files missing
"""
# Fill in defaults as needed
case = Case() if case is None else case
svn_loc = SVN_LOCS[get_model()] if svn_loc is None else svn_loc
input_data_root = case.get_value("DIN_LOC_ROOT") if input_data_root is None else input_data_root
expect(os.path.isdir(input_data_root), "Invalid input_data_root directory: '%s'" % input_data_root)
expect(os.path.isdir(data_list_dir), "Invalid data_list_dir directory: '%s'" % data_list_dir)
data_list_files = find_files(data_list_dir, "*.input_data_list")
expect(data_list_files, "No .input_data_list files found in dir '%s'" % data_list_dir)
no_files_missing = True
for data_list_file in data_list_files:
logging.info("Loading input file: '%s'" % data_list_file)
with open(data_list_file, "r") as fd:
lines = fd.readlines()
for line in lines:
line = line.strip()
if (line and not line.startswith("#")):
tokens = line.split('=')
description, full_path = tokens[0].strip(), tokens[1].strip()
if(full_path):
# expand xml variables
full_path = case.get_resolved_value(full_path)
rel_path = full_path.replace(input_data_root, "")
# There are some special values of rel_path that
# we need to ignore - some of the component models
# set things like 'NULL' or 'same_as_TS' -
# basically if rel_path does not contain '/' (a
# directory tree) you can assume it's a special
# value and ignore it (perhaps with a warning)
if ("/" in rel_path and not os.path.exists(full_path)):
model = os.path.basename(data_list_file).split('.')[0]
logging.warning("Model %s missing file %s = '%s'" % (model,description,full_path))
if (download):
success = download_if_in_repo(svn_loc, input_data_root, rel_path)
if (not success):
# If ACME, try CESM repo as backup
if (get_model() == "acme" and svn_loc != SVN_LOCS["cesm"]):
success = download_if_in_repo(SVN_LOCS["cesm"], input_data_root, rel_path)
if (not success):
no_files_missing = False
else:
no_files_missing = False
else:
no_files_missing = False
else:
logging.info("Already had input file: '%s'" % full_path)
else:
model = os.path.basename(data_list_file).split('.')[0]
logging.warning("Model %s no file specified for %s"%(model,description))
return no_files_missing
def _xml_phase(self, test):
###########################################################################
test_case = CIME.utils.parse_test_name(test)[0]
# Create, fill and write an envtest object
test_dir = self._get_test_dir(test)
envtest = EnvTest(test_dir)
# Determine list of component classes that this coupler/driver knows how
# to deal with. This list follows the same order as compset longnames follow.
files = Files()
drv_config_file = files.get_value("CONFIG_DRV_FILE")
drv_comp = Component(drv_config_file)
component_classes = drv_comp.get_valid_model_components()
envtest.add_elements_by_group(drv_comp, {}, "env_test.xml")
envtest.set_value("TESTCASE", test_case)
envtest.set_value("TEST_TESTID", self._test_id)
envtest.set_value("CASEBASEID", test)
test_argv = "-testname %s -testroot %s" % (test, self._test_root)
if self._generate:
test_argv += " -generate %s" % self._baseline_gen_name
envtest.set_value("BASELINE_NAME_GEN", self._baseline_gen_name)
envtest.set_value("BASEGEN_CASE", os.path.join(self._baseline_gen_name, test))
if self._compare:
test_argv += " -compare %s" % self._baseline_cmp_name
envtest.set_value("BASELINE_NAME_CMP", self._baseline_cmp_name)
envtest.set_value("BASECMP_CASE", os.path.join(self._baseline_cmp_name, test))
envtest.set_value("TEST_ARGV", test_argv)
envtest.set_value("CLEANUP", self._clean)
if self._generate or self._compare:
envtest.set_value("BASELINE_ROOT", self._baseline_root)
envtest.set_value("GENERATE_BASELINE", self._generate)
envtest.set_value("COMPARE_BASELINE", self._compare)
envtest.set_value("CCSM_CPRNC", self._machobj.get_value("CCSM_CPRNC", resolved=False))
"""
Add the test instructions from config_test to env_test in the case
"""
config_test = Tests()
testnode = config_test.get_test_node(test_case)
envtest.add_test(testnode)
# Determine the test_case from the test name
test_case, case_opts = CIME.utils.parse_test_name(test)[:2]
# Determine case_opts from the test_case
if case_opts is not None:
logger.debug("case_opts are %s " %case_opts)
for opt in case_opts:
logger.debug("case_opt is %s" %opt)
if opt == 'D':
envtest.set_test_parameter("DEBUG", "TRUE")
logger.debug (" DEBUG set to TRUE")
elif opt == 'E':
envtest.set_test_parameter("USE_ESMF_LIB", "TRUE")
envtest.set_test_parameter("COMP_INTERFACE", "ESMF")
logger.debug (" USE_ESMF_LIB set to TRUE")
logger.debug (" COMP_INTERFACE set to ESMF")
elif opt == 'CG':
envtest.set_test_parameter("CALENDAR", "GREGORIAN")
logger.debug (" CALENDAR set to %s" %opt)
elif opt.startswith('L'):
match = re.match('L([A-Za-z])([0-9]*)', opt)
stop_option = {"y":"nyears", "m":"nmonths", "d":"ndays", "h":"nhours",
"s":"nseconds", "n":"nsteps"}
opt = match.group(1)
envtest.set_test_parameter("STOP_OPTION",stop_option[opt])
opti = match.group(2)
envtest.set_test_parameter("STOP_N", opti)
logger.debug (" STOP_OPTION set to %s" %stop_option[opt])
logger.debug (" STOP_N set to %s" %opti)
elif opt.startswith('M'):
# M option handled by create newcase
continue
elif opt.startswith('P'):
match1 = re.match('P([0-9]+)', opt)
match2 = re.match('P([0-9]+)x([0-9]+)', opt)
match3 = re.match('P[SMLX][12]?', opt)
opti_tasks = None
if match1:
opti_tasks = match1.group(1)
for component_class in component_classes:
if component_class == "DRV":
component_class = "CPL"
string = "NTASKS_" + component_class
envtest.set_test_parameter(string, opti_tasks)
string = "NTHRDS_" + component_class
envtest.set_test_parameter(string, str(1))
string = "ROOTPE_" + component_class
envtest.set_test_parameter(string, str(0))
opti_thrds = 1
elif match2:
opti_tasks = match2.group(1)
opti_thrds = match2.group(2)
for component_class in component_classes:
if component_class == "DRV":
component_class = "CPL"
string = "NTASKS_" + component_class
envtest.set_test_parameter(string, opti_tasks)
string = "NTHRDS_" + component_class
envtest.set_test_parameter(string, opti_thrds)
string = "ROOTPE_" + component_class
envtest.set_test_parameter(string, str(0))
elif match3:
# handled by create_newcase
continue
if not match3:
expect(opti_tasks is not None, "No match found for PE option %s"%opt)
logger.debug (" NTASKS_xxx set to %s" %opti_tasks)
logger.debug (" NTHRDS_xxx set to %s" %opti_thrds)
logger.debug (" ROOTPE_xxx set to %s 0")
elif opt.startswith('N'):
# handled in create_newcase
continue
elif opt.startswith('IOP'):
logger.warn("IOP test option not yet implemented")
else:
expect(False, "Could not parse option '%s' " %opt)
envtest.write()
lockedfiles = os.path.join(test_dir, "LockedFiles")
try:
os.stat(lockedfiles)
except:
os.mkdir(lockedfiles)
shutil.copy(os.path.join(test_dir,"env_run.xml"),
os.path.join(lockedfiles, "env_run.orig.xml"))
case = Case(test_dir)
case.set_value("SHAREDLIBROOT",
os.path.join(self._test_root,
"sharedlibroot.%s"%self._test_id))
envtest.set_initial_values(case)
return True