def _cs_status_xfail_arg():
"""Returns a string giving the argument to cs_status that will point to CTSM's
expected fails xml file
"""
ctsm_root = path_to_ctsm_root()
xfail_path = os.path.join(ctsm_root, 'cime_config', 'testdefs', 'ExpectedTestFails.xml')
return "--expected-fails-file {}".format(xfail_path)
def __init__(self, name, start_year, end_year, start_month, end_month):
self.name = name
self.start_year = int(start_year)
self.end_year = int(end_year)
self.start_month = int(start_month)
self.end_month = int(end_month)
self.cesmroot = path_to_ctsm_root()
def _record_git_status(testroot, dry_run):
"""Record git status and related information to stdout and a file"""
output = ''
ctsm_root = path_to_ctsm_root()
current_hash = subprocess.check_output(['git', 'show', '--no-patch', '--oneline', 'HEAD'],
cwd=ctsm_root,
universal_newlines=True)
output += "Current hash: {}".format(current_hash)
git_status = subprocess.check_output(['git', '-c', 'color.ui=always',
'status', '--short', '--branch'],
cwd=ctsm_root,
universal_newlines=True)
output += git_status
if git_status.count('\n') == 1:
# Only line in git status is the branch info
output += "(clean sandbox)\n"
manic = os.path.join('manage_externals', 'checkout_externals')
manage_externals_status = subprocess.check_output([manic, '--status', '--verbose'],
cwd=ctsm_root,
universal_newlines=True)
output += 72*'-' + '\n' + 'manage_externals status:' + '\n'
output += manage_externals_status
output += 72*'-' + '\n'
print(output)
if not dry_run:
git_status_filepath = os.path.join(testroot, 'SRCROOT_GIT_STATUS')
with open(git_status_filepath, 'w') as git_status_file:
git_status_file.write("SRCROOT: {}\n".format(ctsm_root))
git_status_file.write(output)
def _stage_runtime_inputs(build_dir, no_pnetcdf):
"""Stage CTSM and LILAC runtime inputs
Args:
build_dir (str): path to build directory
no_pnetcdf (bool): if True, use netcdf rather than pnetcdf
"""
os.makedirs(os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME))
inputdata_dir = _xmlquery('DIN_LOC_ROOT', build_dir)
fill_template_file(path_to_template=os.path.join(_PATH_TO_TEMPLATES,
'ctsm_template.cfg'),
path_to_final=os.path.join(build_dir,
_RUNTIME_INPUTS_DIRNAME,
'ctsm.cfg'),
substitutions={'INPUTDATA': inputdata_dir})
fill_template_file(path_to_template=os.path.join(_PATH_TO_TEMPLATES,
'lilac_in_template'),
path_to_final=os.path.join(build_dir,
_RUNTIME_INPUTS_DIRNAME,
'lilac_in'),
substitutions={'INPUTDATA': inputdata_dir})
pio_stride = _xmlquery('MAX_MPITASKS_PER_NODE', build_dir)
if no_pnetcdf:
pio_typename = 'netcdf'
# pio_rearranger = 1 is generally more efficient with netcdf (see
# https://github.com/ESMCI/cime/pull/3732#discussion_r508954806 and the following
# discussion)
pio_rearranger = 1
else:
pio_typename = 'pnetcdf'
pio_rearranger = 2
fill_template_file(path_to_template=os.path.join(
_PATH_TO_TEMPLATES, 'lnd_modelio_template.nml'),
path_to_final=os.path.join(build_dir,
_RUNTIME_INPUTS_DIRNAME,
'lnd_modelio.nml'),
substitutions={
'PIO_REARRANGER': pio_rearranger,
'PIO_STRIDE': pio_stride,
'PIO_TYPENAME': pio_typename
})
shutil.copyfile(src=os.path.join(path_to_ctsm_root(), 'cime_config',
'usermods_dirs', 'lilac', 'user_nl_ctsm'),
dst=os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME,
'user_nl_ctsm'))
make_link(
_PATH_TO_MAKE_RUNTIME_INPUTS,
os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME,
'make_runtime_inputs'))
make_link(
_PATH_TO_DOWNLOAD_INPUT_DATA,
os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME,
'download_input_data'))
def get_ctsm_git_sha():
"""
Returns Git short SHA for the ctsm directory.
"""
return subprocess.check_output(
['git', '-C',
path_to_ctsm_root(), 'rev-parse', '--short',
'HEAD']).strip().decode()
def main():
"""
Calls functions that subset surface, landuse, domain, and/or DATM files for a region or a
single point.
"""
# --------------------------------- #
# add logging flags from ctsm_logging
setup_logging_pre_config()
parser = get_parser()
args = parser.parse_args()
# --------------------------------- #
# print help and exit when no option is chosen
if args.run_type != "point" and args.run_type != "region":
err_msg = textwrap.dedent('''\
\n ------------------------------------
\n Must supply a positional argument: 'point' or 'region'.
''')
raise parser.error(err_msg)
if not any([
args.create_surfdata, args.create_domain, args.create_landuse,
args.create_datm
]):
err_msg = textwrap.dedent('''\
\n ------------------------------------
\n Must supply one of:
\n --create-surface \n --create-landuse \n --create-datm \n --create-domain \n
''')
raise parser.error(err_msg)
# --------------------------------- #
# process logging args (i.e. debug and verbose)
process_logging_args(args)
# --------------------------------- #
# parse defaults file
cesmroot = path_to_ctsm_root()
defaults = configparser.ConfigParser()
defaults.read(
os.path.join(cesmroot, "tools/site_and_regional", DEFAULTS_FILE))
# --------------------------------- #
myname = getuser()
pwd = os.getcwd()
logger.info("User = %s", myname)
logger.info("Current directory = %s", pwd)
# --------------------------------- #
# create files and folders necessary and return dictionary of file/folder locations
file_dict = setup_files(args, defaults, cesmroot)
if args.run_type == "point":
subset_point(args, file_dict)
elif args.run_type == "region":
subset_region(args, file_dict)
def main(description):
cesmroot = path_to_ctsm_root()
# Get the list of supported neon sites from usermods
valid_neon_sites = glob.glob(
os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON",
"[!d]*"))
valid_neon_sites = sorted([v.split("/")[-1] for v in valid_neon_sites])
(
site_list,
output_root,
run_type,
overwrite,
run_length,
base_case_root,
run_from_postad,
setup_only,
no_batch,
rerun,
) = get_parser(sys.argv, description, valid_neon_sites)
if output_root:
logger.debug("output_root : " + output_root)
if not os.path.exists(output_root):
os.makedirs(output_root)
# -- check neon listing file for available data:
available_list = check_neon_listing(valid_neon_sites)
# =================================
# -- all neon sites can be cloned from one generic case
# -- so no need to define a base_case for every site.
res = "CLM_USRDAT"
compset = "I1PtClm51Bgc"
# -- Looping over neon sites
for neon_site in available_list:
if neon_site.name in site_list:
if run_from_postad:
neon_site.finidat = None
if not base_case_root:
base_case_root = neon_site.build_base_case(
cesmroot, output_root, res, compset, overwrite, setup_only)
logger.info("-----------------------------------")
logger.info("Running CTSM for neon site : {}".format(
neon_site.name))
neon_site.run_case(
base_case_root,
run_type,
run_length,
overwrite,
setup_only,
no_batch,
rerun,
)
def setUp(self):
"""
Obtain path to the existing:
- modify_template.cfg file
- /testinputs directory and fsurdat_in, located in /testinputs
Make /_tempdir for use by these tests.
Obtain path and names for the files being created in /_tempdir:
- modify_fsurdat.cfg
- fsurdat_out.nc
"""
self._cfg_template_path = os.path.join(path_to_ctsm_root(),
'tools/modify_fsurdat/modify_template.cfg')
testinputs_path = os.path.join(path_to_ctsm_root(),
'python/ctsm/test/testinputs')
self._fsurdat_in = os.path.join(testinputs_path,
'surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214.nc')
self._tempdir = tempfile.mkdtemp()
self._cfg_file_path = os.path.join(self._tempdir, 'modify_fsurdat.cfg')
self._fsurdat_out = os.path.join(self._tempdir, 'fsurdat_out.nc')
def _stage_runtime_inputs(build_dir, no_pnetcdf):
"""Stage CTSM and LILAC runtime inputs
Args:
build_dir (str): path to build directory
no_pnetcdf (bool): if True, use netcdf rather than pnetcdf
"""
os.makedirs(os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME))
inputdata_dir = _xmlquery('DIN_LOC_ROOT', build_dir)
fill_template_file(path_to_template=os.path.join(_PATH_TO_TEMPLATES,
'ctsm_template.cfg'),
path_to_final=os.path.join(build_dir,
_RUNTIME_INPUTS_DIRNAME,
'ctsm.cfg'),
substitutions={'INPUTDATA': inputdata_dir})
fill_template_file(path_to_template=os.path.join(_PATH_TO_TEMPLATES,
'lilac_in_template'),
path_to_final=os.path.join(build_dir,
_RUNTIME_INPUTS_DIRNAME,
'lilac_in'),
substitutions={'INPUTDATA': inputdata_dir})
pio_stride = _xmlquery('MAX_MPITASKS_PER_NODE', build_dir)
if no_pnetcdf:
pio_typename = 'netcdf'
else:
pio_typename = 'pnetcdf'
fill_template_file(path_to_template=os.path.join(
_PATH_TO_TEMPLATES, 'lnd_modelio_template.nml'),
path_to_final=os.path.join(build_dir,
_RUNTIME_INPUTS_DIRNAME,
'lnd_modelio.nml'),
substitutions={
'PIO_STRIDE': pio_stride,
'PIO_TYPENAME': pio_typename
})
shutil.copyfile(src=os.path.join(path_to_ctsm_root(), 'cime_config',
'usermods_dirs', 'lilac', 'user_nl_ctsm'),
dst=os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME,
'user_nl_ctsm'))
make_link(
_PATH_TO_MAKE_RUNTIME_INPUTS,
os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME,
'make_runtime_inputs'))
make_link(
_PATH_TO_DOWNLOAD_INPUT_DATA,
os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME,
'download_input_data'))
def _record_git_status(testroot, retry, dry_run):
"""Record git status and related information to stdout and a file"""
output = ''
ctsm_root = path_to_ctsm_root()
output += "create_test --retry: {}\n\n".format(retry)
current_hash = subprocess.check_output([
'git', 'show', '--no-patch', '--format=format:%h (%an, %ad) %s\n',
'HEAD'
],
cwd=ctsm_root,
universal_newlines=True)
output += "Current hash: {}".format(current_hash)
git_status = subprocess.check_output(
['git', '-c', 'color.ui=always', 'status', '--short', '--branch'],
cwd=ctsm_root,
universal_newlines=True)
output += git_status
if git_status.count('\n') == 1:
# Only line in git status is the branch info
output += "(clean sandbox)\n"
manic = os.path.join('manage_externals', 'checkout_externals')
manage_externals_status = subprocess.check_output(
[manic, '--status', '--verbose'],
cwd=ctsm_root,
universal_newlines=True)
output += 72 * '-' + '\n' + 'manage_externals status:' + '\n'
output += manage_externals_status
output += 72 * '-' + '\n'
print(output)
if not dry_run:
git_status_filepath = os.path.join(testroot, 'SRCROOT_GIT_STATUS')
if os.path.exists(git_status_filepath):
# If we're reusing an existing directory, it could happen that
# SRCROOT_GIT_STATUS already exists. It's still helpful to record the current
# SRCROOT_GIT_STATUS information, but we don't want to clobber the old. So
# make a new file with a date/time-stamp.
now = datetime.now()
now_str = now.strftime("%m%d-%H%M%S")
git_status_filepath = git_status_filepath + '_' + now_str
with open(git_status_filepath, 'w') as git_status_file:
git_status_file.write(' '.join(sys.argv) + '\n\n')
git_status_file.write("SRCROOT: {}\n".format(ctsm_root))
git_status_file.write(output)
def get_ctsm_git_describe():
"""
Function for giving the recent tag of the CTSM repository
Args:
Raises:
Returns:
label (str) : ouput of running 'git describe' for the CTSM repository
"""
label = (
subprocess.check_output(["git", "-C", path_to_ctsm_root(), "describe"])
.strip()
.decode()
)
return label
def get_ctsm_git_long_hash():
"""
Returns Git long SHA for the CTSM repository.
Args:
Raises:
Returns:
sha (str) : git long hash for ctsm repository
"""
sha = (
subprocess.check_output(["git", "-C", path_to_ctsm_root(), "rev-parse", "HEAD"])
.strip()
.decode()
)
return sha
logger = logging.getLogger(__name__)
# ========================================================================
# Define some constants
# ========================================================================
# this matches the machine name in config_machines_template.xml
_MACH_NAME = 'ctsm_build'
# these are arbitrary, since we only use the case for its build, not any of the runtime
# settings; they just need to be valid
_COMPSET = 'I2000Ctsm50NwpSpAsRs'
_RES = 'f10_f10_mg37'
_PATH_TO_TEMPLATES = os.path.join(path_to_ctsm_root(), 'lilac',
'bld_templates')
_PATH_TO_MAKE_RUNTIME_INPUTS = os.path.join(path_to_ctsm_root(), 'lilac',
'make_runtime_inputs')
_PATH_TO_DOWNLOAD_INPUT_DATA = os.path.join(path_to_ctsm_root(), 'lilac',
'download_input_data')
_MACHINE_CONFIG_DIRNAME = 'machine_configuration'
_INPUTDATA_DIRNAME = 'inputdata'
_RUNTIME_INPUTS_DIRNAME = 'runtime_inputs'
_GPTL_NANOTIMERS_CPPDEFS = '-DHAVE_NANOTIME -DBIT64 -DHAVE_VPRINTF -DHAVE_BACKTRACE -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY' # pylint: disable=line-too-long
# ========================================================================
def buildnml(cime_path, rundir):
###############################################################################
"""Build the ctsm namelist
"""
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
ctsm_cfg_path = os.path.join(rundir, 'ctsm.cfg')
# read the config file
config = ConfigParser()
config.read(ctsm_cfg_path)
lnd_domain_file = get_config_value(config, 'buildnml_input', 'lnd_domain_file', ctsm_cfg_path)
fsurdat = get_config_value(config, 'buildnml_input', 'fsurdat', ctsm_cfg_path)
finidat = get_config_value(config, 'buildnml_input', 'finidat', ctsm_cfg_path)
ctsm_phys = get_config_value(config, 'buildnml_input', 'ctsm_phys', ctsm_cfg_path,
allowed_values=['clm4_5', 'clm5_0', 'clm5_1'])
configuration = get_config_value(config, 'buildnml_input', 'configuration', ctsm_cfg_path,
allowed_values=['nwp', 'clm'])
structure = get_config_value(config, 'buildnml_input', 'structure', ctsm_cfg_path,
allowed_values=['fast', 'standard'])
bgc_mode = get_config_value(config, 'buildnml_input', 'bgc_mode', ctsm_cfg_path,
allowed_values=['sp', 'bgc', 'cn', 'fates'])
crop = get_config_value(config, 'buildnml_input', 'crop', ctsm_cfg_path,
allowed_values=['off', 'on'])
vichydro = get_config_value(config, 'buildnml_input', 'vichydro', ctsm_cfg_path,
allowed_values=['off', 'on'])
bldnml_opts = determine_bldnml_opts(bgc_mode=bgc_mode,
crop=crop,
vichydro=vichydro)
co2_ppmv = get_config_value(config, 'buildnml_input', 'co2_ppmv', ctsm_cfg_path)
use_case = get_config_value(config, 'buildnml_input', 'use_case', ctsm_cfg_path)
lnd_tuning_mode = get_config_value(config, 'buildnml_input', 'lnd_tuning_mode', ctsm_cfg_path)
spinup = get_config_value(config, 'buildnml_input', 'spinup', ctsm_cfg_path,
allowed_values=['off', 'on'])
inputdata_path = get_config_value(config, 'buildnml_input', 'inputdata_path', ctsm_cfg_path)
# Parse the user_nl_ctsm file
infile = os.path.join(rundir, '.namelist')
create_namelist_infile(case=CaseFake(),
user_nl_file=os.path.join(rundir, 'user_nl_ctsm'),
namelist_infile=infile)
# create config_cache.xml file
# Note that build-namelist utilizes the contents of the config_cache.xml file in
# the namelist_defaults.xml file to obtain namelist variables
config_cache = os.path.join(rundir, "config_cache.xml")
config_cache_text = _CONFIG_CACHE_TEMPLATE.format(clm_phys=ctsm_phys)
with open(config_cache, 'w') as tempfile:
tempfile.write(config_cache_text)
# create temporary env_lilac.xml
env_lilac = os.path.join(rundir, "env_lilac.xml")
env_lilac_text = _ENV_LILAC_TEMPLATE.format()
with open(env_lilac, 'w') as tempfile:
tempfile.write(env_lilac_text)
# remove any existing clm.input_data_list file
inputdatalist_path = os.path.join(rundir, "ctsm.input_data_list")
if os.path.exists(inputdatalist_path):
os.remove(inputdatalist_path)
# determine if fsurdat and/or finidat should appear in the -namelist option
extra_namelist_opts = ''
if fsurdat != _UNSET:
# NOTE(wjs, 2020-06-30) With the current logic, fsurdat should never be _UNSET,
# but it's possible that this will change in the future.
extra_namelist_opts = extra_namelist_opts + " fsurdat = '{}' ".format(fsurdat)
if finidat != _UNSET:
extra_namelist_opts = extra_namelist_opts + " finidat = '{}' ".format(finidat)
# call build-namelist
cmd = os.path.abspath(os.path.join(path_to_ctsm_root(), "bld", "build-namelist"))
command = [cmd,
'-cimeroot', cime_path,
'-infile', infile,
'-csmdata', inputdata_path,
'-inputdata', inputdatalist_path,
# Hard-code start_ymd of year-2000. This is used to set the run type (for
# which a setting of 2000 gives 'startup', which is what we want) and pick
# the initial conditions file (which is pretty much irrelevant when running
# with lilac).
'-namelist', '&clm_inparm start_ymd=20000101 {} /'.format(extra_namelist_opts),
'-use_case', use_case,
# For now, we assume ignore_ic_year, not ignore_ic_date
'-ignore_ic_year',
# -clm_start_type seems unimportant (see discussion in
# https://github.com/ESCOMP/CTSM/issues/876)
'-clm_start_type', 'default',
'-configuration', configuration,
'-structure', structure,
'-lnd_frac', lnd_domain_file,
'-glc_nec', str(10),
'-co2_ppmv', co2_ppmv,
'-co2_type', 'constant',
'-clm_accelerated_spinup', spinup,
'-lnd_tuning_mode', lnd_tuning_mode,
# Eventually make -no-megan dynamic (see
# https://github.com/ESCOMP/CTSM/issues/926)
'-no-megan',
'-config', os.path.join(rundir, "config_cache.xml"),
'-envxml_dir', rundir]
# NOTE(wjs, 2020-06-16) Note that we do NOT use the -mask argument; it's possible that
# we should be using it in some circumstances (I haven't looked into how it's used).
command.extend(['-res', 'lilac',
'-clm_usr_name', 'lilac'])
command.extend(bldnml_opts.split())
subprocess.check_call(command,
universal_newlines=True)
# remove temporary files in rundir
os.remove(os.path.join(rundir, "config_cache.xml"))
os.remove(os.path.join(rundir, "env_lilac.xml"))
os.remove(os.path.join(rundir, "drv_flds_in"))
os.remove(infile)
