def _cs_status_xfail_arg(): """Returns a string giving the argument to cs_status that will point to CTSM's expected fails xml file """ ctsm_root = path_to_ctsm_root() xfail_path = os.path.join(ctsm_root, 'cime_config', 'testdefs', 'ExpectedTestFails.xml') return "--expected-fails-file {}".format(xfail_path)
def __init__(self, name, start_year, end_year, start_month, end_month): self.name = name self.start_year = int(start_year) self.end_year = int(end_year) self.start_month = int(start_month) self.end_month = int(end_month) self.cesmroot = path_to_ctsm_root()
def _record_git_status(testroot, dry_run): """Record git status and related information to stdout and a file""" output = '' ctsm_root = path_to_ctsm_root() current_hash = subprocess.check_output(['git', 'show', '--no-patch', '--oneline', 'HEAD'], cwd=ctsm_root, universal_newlines=True) output += "Current hash: {}".format(current_hash) git_status = subprocess.check_output(['git', '-c', 'color.ui=always', 'status', '--short', '--branch'], cwd=ctsm_root, universal_newlines=True) output += git_status if git_status.count('\n') == 1: # Only line in git status is the branch info output += "(clean sandbox)\n" manic = os.path.join('manage_externals', 'checkout_externals') manage_externals_status = subprocess.check_output([manic, '--status', '--verbose'], cwd=ctsm_root, universal_newlines=True) output += 72*'-' + '\n' + 'manage_externals status:' + '\n' output += manage_externals_status output += 72*'-' + '\n' print(output) if not dry_run: git_status_filepath = os.path.join(testroot, 'SRCROOT_GIT_STATUS') with open(git_status_filepath, 'w') as git_status_file: git_status_file.write("SRCROOT: {}\n".format(ctsm_root)) git_status_file.write(output)
def _stage_runtime_inputs(build_dir, no_pnetcdf): """Stage CTSM and LILAC runtime inputs Args: build_dir (str): path to build directory no_pnetcdf (bool): if True, use netcdf rather than pnetcdf """ os.makedirs(os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME)) inputdata_dir = _xmlquery('DIN_LOC_ROOT', build_dir) fill_template_file(path_to_template=os.path.join(_PATH_TO_TEMPLATES, 'ctsm_template.cfg'), path_to_final=os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, 'ctsm.cfg'), substitutions={'INPUTDATA': inputdata_dir}) fill_template_file(path_to_template=os.path.join(_PATH_TO_TEMPLATES, 'lilac_in_template'), path_to_final=os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, 'lilac_in'), substitutions={'INPUTDATA': inputdata_dir}) pio_stride = _xmlquery('MAX_MPITASKS_PER_NODE', build_dir) if no_pnetcdf: pio_typename = 'netcdf' # pio_rearranger = 1 is generally more efficient with netcdf (see # https://github.com/ESMCI/cime/pull/3732#discussion_r508954806 and the following # discussion) pio_rearranger = 1 else: pio_typename = 'pnetcdf' pio_rearranger = 2 fill_template_file(path_to_template=os.path.join( _PATH_TO_TEMPLATES, 'lnd_modelio_template.nml'), path_to_final=os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, 'lnd_modelio.nml'), substitutions={ 'PIO_REARRANGER': pio_rearranger, 'PIO_STRIDE': pio_stride, 'PIO_TYPENAME': pio_typename }) shutil.copyfile(src=os.path.join(path_to_ctsm_root(), 'cime_config', 'usermods_dirs', 'lilac', 'user_nl_ctsm'), dst=os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, 'user_nl_ctsm')) make_link( _PATH_TO_MAKE_RUNTIME_INPUTS, os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, 'make_runtime_inputs')) make_link( _PATH_TO_DOWNLOAD_INPUT_DATA, os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, 'download_input_data'))
def get_ctsm_git_sha(): """ Returns Git short SHA for the ctsm directory. """ return subprocess.check_output( ['git', '-C', path_to_ctsm_root(), 'rev-parse', '--short', 'HEAD']).strip().decode()
def main(): """ Calls functions that subset surface, landuse, domain, and/or DATM files for a region or a single point. """ # --------------------------------- # # add logging flags from ctsm_logging setup_logging_pre_config() parser = get_parser() args = parser.parse_args() # --------------------------------- # # print help and exit when no option is chosen if args.run_type != "point" and args.run_type != "region": err_msg = textwrap.dedent('''\ \n ------------------------------------ \n Must supply a positional argument: 'point' or 'region'. ''') raise parser.error(err_msg) if not any([ args.create_surfdata, args.create_domain, args.create_landuse, args.create_datm ]): err_msg = textwrap.dedent('''\ \n ------------------------------------ \n Must supply one of: \n --create-surface \n --create-landuse \n --create-datm \n --create-domain \n ''') raise parser.error(err_msg) # --------------------------------- # # process logging args (i.e. debug and verbose) process_logging_args(args) # --------------------------------- # # parse defaults file cesmroot = path_to_ctsm_root() defaults = configparser.ConfigParser() defaults.read( os.path.join(cesmroot, "tools/site_and_regional", DEFAULTS_FILE)) # --------------------------------- # myname = getuser() pwd = os.getcwd() logger.info("User = %s", myname) logger.info("Current directory = %s", pwd) # --------------------------------- # # create files and folders necessary and return dictionary of file/folder locations file_dict = setup_files(args, defaults, cesmroot) if args.run_type == "point": subset_point(args, file_dict) elif args.run_type == "region": subset_region(args, file_dict)
def main(description): cesmroot = path_to_ctsm_root() # Get the list of supported neon sites from usermods valid_neon_sites = glob.glob( os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", "[!d]*")) valid_neon_sites = sorted([v.split("/")[-1] for v in valid_neon_sites]) ( site_list, output_root, run_type, overwrite, run_length, base_case_root, run_from_postad, setup_only, no_batch, rerun, ) = get_parser(sys.argv, description, valid_neon_sites) if output_root: logger.debug("output_root : " + output_root) if not os.path.exists(output_root): os.makedirs(output_root) # -- check neon listing file for available data: available_list = check_neon_listing(valid_neon_sites) # ================================= # -- all neon sites can be cloned from one generic case # -- so no need to define a base_case for every site. res = "CLM_USRDAT" compset = "I1PtClm51Bgc" # -- Looping over neon sites for neon_site in available_list: if neon_site.name in site_list: if run_from_postad: neon_site.finidat = None if not base_case_root: base_case_root = neon_site.build_base_case( cesmroot, output_root, res, compset, overwrite, setup_only) logger.info("-----------------------------------") logger.info("Running CTSM for neon site : {}".format( neon_site.name)) neon_site.run_case( base_case_root, run_type, run_length, overwrite, setup_only, no_batch, rerun, )
def setUp(self): """ Obtain path to the existing: - modify_template.cfg file - /testinputs directory and fsurdat_in, located in /testinputs Make /_tempdir for use by these tests. Obtain path and names for the files being created in /_tempdir: - modify_fsurdat.cfg - fsurdat_out.nc """ self._cfg_template_path = os.path.join(path_to_ctsm_root(), 'tools/modify_fsurdat/modify_template.cfg') testinputs_path = os.path.join(path_to_ctsm_root(), 'python/ctsm/test/testinputs') self._fsurdat_in = os.path.join(testinputs_path, 'surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214.nc') self._tempdir = tempfile.mkdtemp() self._cfg_file_path = os.path.join(self._tempdir, 'modify_fsurdat.cfg') self._fsurdat_out = os.path.join(self._tempdir, 'fsurdat_out.nc')
def _stage_runtime_inputs(build_dir, no_pnetcdf): """Stage CTSM and LILAC runtime inputs Args: build_dir (str): path to build directory no_pnetcdf (bool): if True, use netcdf rather than pnetcdf """ os.makedirs(os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME)) inputdata_dir = _xmlquery('DIN_LOC_ROOT', build_dir) fill_template_file(path_to_template=os.path.join(_PATH_TO_TEMPLATES, 'ctsm_template.cfg'), path_to_final=os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, 'ctsm.cfg'), substitutions={'INPUTDATA': inputdata_dir}) fill_template_file(path_to_template=os.path.join(_PATH_TO_TEMPLATES, 'lilac_in_template'), path_to_final=os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, 'lilac_in'), substitutions={'INPUTDATA': inputdata_dir}) pio_stride = _xmlquery('MAX_MPITASKS_PER_NODE', build_dir) if no_pnetcdf: pio_typename = 'netcdf' else: pio_typename = 'pnetcdf' fill_template_file(path_to_template=os.path.join( _PATH_TO_TEMPLATES, 'lnd_modelio_template.nml'), path_to_final=os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, 'lnd_modelio.nml'), substitutions={ 'PIO_STRIDE': pio_stride, 'PIO_TYPENAME': pio_typename }) shutil.copyfile(src=os.path.join(path_to_ctsm_root(), 'cime_config', 'usermods_dirs', 'lilac', 'user_nl_ctsm'), dst=os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, 'user_nl_ctsm')) make_link( _PATH_TO_MAKE_RUNTIME_INPUTS, os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, 'make_runtime_inputs')) make_link( _PATH_TO_DOWNLOAD_INPUT_DATA, os.path.join(build_dir, _RUNTIME_INPUTS_DIRNAME, 'download_input_data'))
def _record_git_status(testroot, retry, dry_run): """Record git status and related information to stdout and a file""" output = '' ctsm_root = path_to_ctsm_root() output += "create_test --retry: {}\n\n".format(retry) current_hash = subprocess.check_output([ 'git', 'show', '--no-patch', '--format=format:%h (%an, %ad) %s\n', 'HEAD' ], cwd=ctsm_root, universal_newlines=True) output += "Current hash: {}".format(current_hash) git_status = subprocess.check_output( ['git', '-c', 'color.ui=always', 'status', '--short', '--branch'], cwd=ctsm_root, universal_newlines=True) output += git_status if git_status.count('\n') == 1: # Only line in git status is the branch info output += "(clean sandbox)\n" manic = os.path.join('manage_externals', 'checkout_externals') manage_externals_status = subprocess.check_output( [manic, '--status', '--verbose'], cwd=ctsm_root, universal_newlines=True) output += 72 * '-' + '\n' + 'manage_externals status:' + '\n' output += manage_externals_status output += 72 * '-' + '\n' print(output) if not dry_run: git_status_filepath = os.path.join(testroot, 'SRCROOT_GIT_STATUS') if os.path.exists(git_status_filepath): # If we're reusing an existing directory, it could happen that # SRCROOT_GIT_STATUS already exists. It's still helpful to record the current # SRCROOT_GIT_STATUS information, but we don't want to clobber the old. So # make a new file with a date/time-stamp. now = datetime.now() now_str = now.strftime("%m%d-%H%M%S") git_status_filepath = git_status_filepath + '_' + now_str with open(git_status_filepath, 'w') as git_status_file: git_status_file.write(' '.join(sys.argv) + '\n\n') git_status_file.write("SRCROOT: {}\n".format(ctsm_root)) git_status_file.write(output)
def get_ctsm_git_describe(): """ Function for giving the recent tag of the CTSM repository Args: Raises: Returns: label (str) : ouput of running 'git describe' for the CTSM repository """ label = ( subprocess.check_output(["git", "-C", path_to_ctsm_root(), "describe"]) .strip() .decode() ) return label
def get_ctsm_git_long_hash(): """ Returns Git long SHA for the CTSM repository. Args: Raises: Returns: sha (str) : git long hash for ctsm repository """ sha = ( subprocess.check_output(["git", "-C", path_to_ctsm_root(), "rev-parse", "HEAD"]) .strip() .decode() ) return sha
logger = logging.getLogger(__name__) # ======================================================================== # Define some constants # ======================================================================== # this matches the machine name in config_machines_template.xml _MACH_NAME = 'ctsm_build' # these are arbitrary, since we only use the case for its build, not any of the runtime # settings; they just need to be valid _COMPSET = 'I2000Ctsm50NwpSpAsRs' _RES = 'f10_f10_mg37' _PATH_TO_TEMPLATES = os.path.join(path_to_ctsm_root(), 'lilac', 'bld_templates') _PATH_TO_MAKE_RUNTIME_INPUTS = os.path.join(path_to_ctsm_root(), 'lilac', 'make_runtime_inputs') _PATH_TO_DOWNLOAD_INPUT_DATA = os.path.join(path_to_ctsm_root(), 'lilac', 'download_input_data') _MACHINE_CONFIG_DIRNAME = 'machine_configuration' _INPUTDATA_DIRNAME = 'inputdata' _RUNTIME_INPUTS_DIRNAME = 'runtime_inputs' _GPTL_NANOTIMERS_CPPDEFS = '-DHAVE_NANOTIME -DBIT64 -DHAVE_VPRINTF -DHAVE_BACKTRACE -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY' # pylint: disable=line-too-long # ========================================================================
def buildnml(cime_path, rundir): ############################################################################### """Build the ctsm namelist """ # pylint: disable=too-many-locals # pylint: disable=too-many-statements ctsm_cfg_path = os.path.join(rundir, 'ctsm.cfg') # read the config file config = ConfigParser() config.read(ctsm_cfg_path) lnd_domain_file = get_config_value(config, 'buildnml_input', 'lnd_domain_file', ctsm_cfg_path) fsurdat = get_config_value(config, 'buildnml_input', 'fsurdat', ctsm_cfg_path) finidat = get_config_value(config, 'buildnml_input', 'finidat', ctsm_cfg_path) ctsm_phys = get_config_value(config, 'buildnml_input', 'ctsm_phys', ctsm_cfg_path, allowed_values=['clm4_5', 'clm5_0', 'clm5_1']) configuration = get_config_value(config, 'buildnml_input', 'configuration', ctsm_cfg_path, allowed_values=['nwp', 'clm']) structure = get_config_value(config, 'buildnml_input', 'structure', ctsm_cfg_path, allowed_values=['fast', 'standard']) bgc_mode = get_config_value(config, 'buildnml_input', 'bgc_mode', ctsm_cfg_path, allowed_values=['sp', 'bgc', 'cn', 'fates']) crop = get_config_value(config, 'buildnml_input', 'crop', ctsm_cfg_path, allowed_values=['off', 'on']) vichydro = get_config_value(config, 'buildnml_input', 'vichydro', ctsm_cfg_path, allowed_values=['off', 'on']) bldnml_opts = determine_bldnml_opts(bgc_mode=bgc_mode, crop=crop, vichydro=vichydro) co2_ppmv = get_config_value(config, 'buildnml_input', 'co2_ppmv', ctsm_cfg_path) use_case = get_config_value(config, 'buildnml_input', 'use_case', ctsm_cfg_path) lnd_tuning_mode = get_config_value(config, 'buildnml_input', 'lnd_tuning_mode', ctsm_cfg_path) spinup = get_config_value(config, 'buildnml_input', 'spinup', ctsm_cfg_path, allowed_values=['off', 'on']) inputdata_path = get_config_value(config, 'buildnml_input', 'inputdata_path', ctsm_cfg_path) # Parse the user_nl_ctsm file infile = os.path.join(rundir, '.namelist') create_namelist_infile(case=CaseFake(), user_nl_file=os.path.join(rundir, 'user_nl_ctsm'), namelist_infile=infile) # create config_cache.xml file # Note that build-namelist utilizes the contents of the config_cache.xml file in # the namelist_defaults.xml file to obtain namelist variables config_cache = os.path.join(rundir, "config_cache.xml") config_cache_text = _CONFIG_CACHE_TEMPLATE.format(clm_phys=ctsm_phys) with open(config_cache, 'w') as tempfile: tempfile.write(config_cache_text) # create temporary env_lilac.xml env_lilac = os.path.join(rundir, "env_lilac.xml") env_lilac_text = _ENV_LILAC_TEMPLATE.format() with open(env_lilac, 'w') as tempfile: tempfile.write(env_lilac_text) # remove any existing clm.input_data_list file inputdatalist_path = os.path.join(rundir, "ctsm.input_data_list") if os.path.exists(inputdatalist_path): os.remove(inputdatalist_path) # determine if fsurdat and/or finidat should appear in the -namelist option extra_namelist_opts = '' if fsurdat != _UNSET: # NOTE(wjs, 2020-06-30) With the current logic, fsurdat should never be _UNSET, # but it's possible that this will change in the future. extra_namelist_opts = extra_namelist_opts + " fsurdat = '{}' ".format(fsurdat) if finidat != _UNSET: extra_namelist_opts = extra_namelist_opts + " finidat = '{}' ".format(finidat) # call build-namelist cmd = os.path.abspath(os.path.join(path_to_ctsm_root(), "bld", "build-namelist")) command = [cmd, '-cimeroot', cime_path, '-infile', infile, '-csmdata', inputdata_path, '-inputdata', inputdatalist_path, # Hard-code start_ymd of year-2000. This is used to set the run type (for # which a setting of 2000 gives 'startup', which is what we want) and pick # the initial conditions file (which is pretty much irrelevant when running # with lilac). '-namelist', '&clm_inparm start_ymd=20000101 {} /'.format(extra_namelist_opts), '-use_case', use_case, # For now, we assume ignore_ic_year, not ignore_ic_date '-ignore_ic_year', # -clm_start_type seems unimportant (see discussion in # https://github.com/ESCOMP/CTSM/issues/876) '-clm_start_type', 'default', '-configuration', configuration, '-structure', structure, '-lnd_frac', lnd_domain_file, '-glc_nec', str(10), '-co2_ppmv', co2_ppmv, '-co2_type', 'constant', '-clm_accelerated_spinup', spinup, '-lnd_tuning_mode', lnd_tuning_mode, # Eventually make -no-megan dynamic (see # https://github.com/ESCOMP/CTSM/issues/926) '-no-megan', '-config', os.path.join(rundir, "config_cache.xml"), '-envxml_dir', rundir] # NOTE(wjs, 2020-06-16) Note that we do NOT use the -mask argument; it's possible that # we should be using it in some circumstances (I haven't looked into how it's used). command.extend(['-res', 'lilac', '-clm_usr_name', 'lilac']) command.extend(bldnml_opts.split()) subprocess.check_call(command, universal_newlines=True) # remove temporary files in rundir os.remove(os.path.join(rundir, "config_cache.xml")) os.remove(os.path.join(rundir, "env_lilac.xml")) os.remove(os.path.join(rundir, "drv_flds_in")) os.remove(infile)