def write_to_netcdf(self, xr_ds, nc_fname):
"""
Writes a netcdf file if
- the file does not exist.
or
- overwrite flag is chosen.
Args:
xr_ds : Xarray Dataset
The xarray dataset that we are write out to netcdf file.
nc_fname : str
Netcdf file name
Raises:
Error and aborts the code if the file exists and --overwrite is not used.
"""
if not os.path.exists(nc_fname) or self.overwrite:
# mode 'w' overwrites file
xr_ds.to_netcdf(path=nc_fname, mode="w", format="NETCDF3_64BIT")
else:
err_msg = (
"File "
+ nc_fname
+ " already exists."
+ "\n Either remove the file or use "
+ "--overwrite to overwrite the existing files."
)
abort(err_msg)
def parse_command_line():
###############################################################################
"""Parse the command line, return object holding arguments"""
description = """
Script to create runtime inputs when running CTSM via LILAC
"""
parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter,
description=description)
parser.add_argument("--rundir", type=str, default=os.getcwd(),
help="Full path of the run directory (containing ctsm.cfg & user_nl_ctsm)")
add_logging_args(parser)
arguments = parser.parse_args()
# Perform some error checking on arguments
if not os.path.isdir(arguments.rundir):
abort("rundir {} does not exist".format(arguments.rundir))
return arguments
def run_cmd_output_on_error(cmd, errmsg, cwd=None):
"""Run the given command; suppress output but print it if there is an error
If there is an error running the command, print the output from the command and abort
with the given errmsg.
Args:
cmd: list of strings - command and its arguments
errmsg: string - error message to print if the command returns an error code
cwd: string or None - path from which the command should be run
"""
try:
_ = subprocess.check_output(cmd,
stderr=subprocess.STDOUT,
universal_newlines=True,
cwd=cwd)
except subprocess.CalledProcessError as error:
print('ERROR while running:')
print(' '.join(cmd))
if cwd is not None:
print('From {}'.format(cwd))
print('')
print(error.output)
print('')
abort(errmsg)
except:
print('ERROR trying to run:')
print(' '.join(cmd))
if cwd is not None:
print('From {}'.format(cwd))
raise
def get_config_value(config, section, item, file_path, allowed_values=None):
"""Get a given item from a given section of the config object
Give a helpful error message if we can't find the given section or item
Note that the file_path argument is only used for the sake of the error message
If allowed_values is present, it should be a list of strings giving allowed values
"""
try:
val = config.get(section, item)
except NoSectionError:
abort("ERROR: Config file {} must contain section '{}'".format(file_path, section))
except NoOptionError:
abort("ERROR: Config file {} must contain item '{}' in section '{}'".format(
file_path, item, section))
if val == _PLACEHOLDER:
abort("Error: {} needs to be specified in config file {}".format(item, file_path))
if allowed_values is not None:
if val not in allowed_values:
abort("Error: {} is not an allowed value for {} in config file {}\n"
"Allowed values: {}".format(val, item, file_path, allowed_values))
return val
def _create_build_dir(build_dir, existing_inputdata):
"""Create the given build directory and any necessary sub-directories
Args:
build_dir (str): path to build directory; this directory shouldn't exist yet!
existing_inputdata (bool): whether the inputdata directory already exists on this machine
"""
if os.path.exists(build_dir):
abort(
'When running without --rebuild, the build directory must not exist yet\n'
'(<{}> already exists)'.format(build_dir))
os.makedirs(build_dir)
if not existing_inputdata:
os.makedirs(os.path.join(build_dir, _INPUTDATA_DIRNAME))
def _handle_config_value(var, default, item, is_list, convert_to_type,
can_be_unset, allowed_values):
"""
Description
-----------
Assign the default value or the user-specified one to var.
Convert from default type (str) to reqested type (int or float).
If is_list is True, then default should be a list
"""
if var == _CONFIG_UNSET:
if can_be_unset:
return default # default may be None
abort("Must set a value for .cfg file variable: {}".format(item))
# convert string to list of strings; if there is just one element,
# we will get a list of size one, which we will convert back to a
# scalar later if needed
var = var.split()
if convert_to_type is bool:
try:
var = [_convert_to_bool(v) for v in var]
except ValueError:
abort("Non-boolean value found for .cfg file variable: {}".format(
item))
elif convert_to_type is not None:
try:
var = [convert_to_type(v) for v in var]
except ValueError:
abort("Wrong type for .cfg file variable: {}".format(item))
if allowed_values is not None:
for val in var:
if val not in allowed_values:
print("val = ", val, " in var not in allowed_values")
errmsg = ("{} is not an allowed value for {} in .cfg file. "
"Check allowed_values".format(val, item))
abort(errmsg)
if not is_list:
if len(var) > 1:
abort(
"More than 1 element found for .cfg file variable: {}".format(
item))
var = var[0]
return var
def set_lai_sai_hgts(self, dom_nat_pft, var, val):
"""
Description
-----------
If user has specified lai, sai, hgt_top, hgt_bot, replace these with
values selected by the user for dom_nat_pft. Else do nothing.
"""
if dom_nat_pft == 0: # bare soil: var must equal 0
val = [0] * 12
if len(val) != 12:
errmsg = 'Error: Variable should have exactly 12 ' \
'entries in the configure file: ' + var
abort(errmsg)
for mon in self.file.time - 1: # loop over 12 months
# set 4D variable to value for dom_nat_pft
self.setvar_lev2(var, val[int(mon)], lev1_dim=dom_nat_pft,
lev2_dim=mon)
def determine_bldnml_opts(bgc_mode, crop, vichydro):
###############################################################################
"""Return a string giving bldnml options, given some other inputs"""
bldnml_opts = ''
bldnml_opts += ' -bgc {}'.format(bgc_mode)
if bgc_mode == 'fates':
# BUG(wjs, 2020-06-12, ESCOMP/CTSM#115) For now, FATES is incompatible with MEGAN
bldnml_opts += ' -no-megan'
if crop == 'on':
if bgc_mode not in ['bgc', 'cn']:
abort("Error: setting crop to 'on' is only compatible with bgc_mode of 'bgc' or 'cn'")
bldnml_opts += ' -crop'
if vichydro == 'on':
if bgc_mode != 'sp':
abort("Error: setting vichydro to 'on' is only compatible with bgc_mode of 'sp'")
bldnml_opts += ' -vichydro'
return bldnml_opts
def create_1d_coord(filename, lon_varname, lat_varname, x_dim, y_dim):
"""
Create 1d coordinate variables for a netcdf file to enable sel() method
Parameters
----------
filename (str) : name of the netcdf file
lon_varname (str) : variable name that has 2d lon
lat_varname (str) : variable name that has 2d lat
x_dim (str) : dimension name in X -- lon
y_dim (str): dimension name in Y -- lat
Raises
------
None
Returns
-------
f_out (xarray Dataset): Xarray Dataset with 1-d coords
"""
if os.path.exists(filename):
logger.debug("Open file: %s", filename)
f_in = xr.open_dataset(filename)
else:
err_msg = "File not found : " + filename
abort(err_msg)
# create 1d coordinate variables to enable sel() method
lon0 = np.asarray(f_in[lon_varname][0, :])
lat0 = np.asarray(f_in[lat_varname][:, 0])
lon = xr.DataArray(lon0, name="lon", dims=x_dim, coords={x_dim: lon0})
lat = xr.DataArray(lat0, name="lat", dims=y_dim, coords={y_dim: lat0})
f_out = f_in.assign({"lon": lon, "lat": lat})
f_out.reset_coords([lon_varname, lat_varname])
f_in.close()
return f_out
def lon_range_0_to_360(lon_in):
"""
Description
-----------
Restrict longitude to 0 to 360 when given as -180 to 180.
"""
if -180 <= lon_in < 0:
lon_out = lon_in % 360
logger.info(
"Resetting longitude from %s to %s to keep in the range "
" 0 to 360",
str(lon_in),
str(lon_out),
)
elif 0 <= lon_in <= 360 or lon_in is None:
lon_out = lon_in
else:
errmsg = "lon_in needs to be in the range 0 to 360"
abort(errmsg)
return lon_out
def rebuild_ctsm(build_dir):
"""Re-run the build in an existing directory
Args:
build_dir (str): path to build directory
"""
if not os.path.exists(build_dir):
abort(
'When running with --rebuild, the build directory must already exist\n'
'(<{}> does not exist)'.format(build_dir))
case_dir = _get_case_dir(build_dir)
if not os.path.exists(case_dir):
abort(
'It appears there was a problem setting up the initial build in\n'
'<{}>\n'
'You should start over with a fresh build directory.'.format(
build_dir))
try:
subprocess.check_call(
[os.path.join(case_dir, 'case.build'), '--clean-depends', 'lnd'],
cwd=case_dir)
except subprocess.CalledProcessError:
abort(
'ERROR resetting build for CTSM in order to rebuild - see above for details'
)
_build_case(build_dir)
def _get_not_rectangle(lon_1, lon_2, lat_1, lat_2, longxy, latixy):
"""
Description
-----------
"""
# ensure that lon ranges 0-360 in case user entered -180 to 180
lon_1 = lon_range_0_to_360(lon_1)
lon_2 = lon_range_0_to_360(lon_2)
# determine the rectangle(s)
# TODO This is not really "nearest" for the edges but isel didn't work
rectangle_1 = (longxy >= lon_1)
rectangle_2 = (longxy <= lon_2)
eps = np.finfo(np.float32).eps # to avoid roundoff issue
rectangle_3 = (latixy >= (lat_1 - eps))
rectangle_4 = (latixy <= (lat_2 + eps))
if lon_1 <= lon_2:
# rectangles overlap
union_1 = np.logical_and(rectangle_1, rectangle_2)
else:
# rectangles don't overlap: stradling the 0-degree meridian
union_1 = np.logical_or(rectangle_1, rectangle_2)
if lat_1 < -90 or lat_1 > 90 or lat_2 < -90 or lat_2 > 90:
errmsg = 'lat_1 and lat_2 need to be in the range -90 to 90'
abort(errmsg)
elif lat_1 <= lat_2:
# rectangles overlap
union_2 = np.logical_and(rectangle_3, rectangle_4)
else:
# rectangles don't overlap: one in the north, one in the south
union_2 = np.logical_or(rectangle_3, rectangle_4)
# union rectangles overlap
rectangle = np.logical_and(union_1, union_2)
not_rectangle = np.logical_not(rectangle)
return not_rectangle
def write_output(self, fsurdat_in, fsurdat_out):
"""
Description
-----------
Write output file
Arguments
---------
fsurdat_in:
(str) Command line entry of input surface dataset
fsurdat_out:
(str) Command line entry of output surface dataset
"""
# update attributes
# TODO Better as dictionary?
title = 'Modified fsurdat file'
summary = 'Modified fsurdat file'
contact = 'N/A'
data_script = os.path.abspath(
__file__) + " -- " + get_ctsm_git_short_hash()
description = 'Modified this file: ' + fsurdat_in
update_metadata(self.file,
title=title,
summary=summary,
contact=contact,
data_script=data_script,
description=description)
# abort if output file already exists
file_exists = os.path.exists(fsurdat_out)
if file_exists:
errmsg = 'Output file already exists: ' + fsurdat_out
abort(errmsg)
# mode 'w' overwrites file if it exists
self.file.to_netcdf(path=fsurdat_out, mode='w', format="NETCDF3_64BIT")
logger.info('Successfully created fsurdat_out: %s', fsurdat_out)
self.file.close()
def _build_case(build_dir):
"""Build the CTSM library and its dependencies
Args:
build_dir (str): path to build directory
"""
# We want user to see output from the build command, so we use subprocess.check_call
# rather than run_cmd_output_on_error.
# See comment in _create_case for why we use case_dir in both the path to the command
# and in the cwd argument to check_call.
case_dir = _get_case_dir(build_dir)
try:
subprocess.check_call(
[os.path.join(case_dir, 'case.build'), '--sharedlib-only'],
cwd=case_dir)
except subprocess.CalledProcessError:
abort(
'ERROR building CTSM or its dependencies - see above for details')
make_link(os.path.join(case_dir, 'bld', 'ctsm.mk'),
os.path.join(build_dir, 'ctsm.mk'))
def download_file(url, fname):
"""
Function to download a file.
Args:
url (str):
url of the file for downloading
fname (str) :
file name to save the downloaded file.
Raises:
Error :
When the file is not available on the server (status_code:404)
Error:
When download fails for any reason.
"""
try:
response = requests.get(url)
# pylint: disable=broad-except
except Exception as err:
logger.warning("The server could not fulfill the request.")
logger.warning("Something went wrong in downloading: %s", fname)
err_msg = "Couldn't download file " + fname + "-- Error code:" + err
abort(err_msg)
with open(fname, "wb") as this_f:
this_f.write(response.content)
# -- Check if download status_code
if response.status_code == 200:
logger.info("Download finished successfully for : %s", fname)
elif response.status_code == 404:
logger.warning("This file is not available on the server: %s", fname)
err_msg = "Couldn't download file " + fname + "-- Error code: " + "404"
abort(err_msg)
def get_config_value(
config,
section,
item,
file_path,
allowed_values=None,
default=None,
is_list=False,
convert_to_type=None,
can_be_unset=False,
):
"""Get a given item from a given section of the config object
Give a helpful error message if we can't find the given section or item
Note that the file_path argument is only used for the sake of the error message
If allowed_values is present, it should be a list of strings giving allowed values
The function _handle_config_value determines what to do if we read:
- a list or
- a str that needs to be converted to int / float / bool
- _CONFIG_UNSET: anything with the value "UNSET" will become "None"
"""
try:
val = config.get(section, item)
except configparser.NoSectionError:
abort("ERROR: Config file {} must contain section '{}'".format(
file_path, section))
except configparser.NoOptionError:
abort("ERROR: Config file {} must contain item '{}' in section '{}'".
format(file_path, item, section))
if val == _CONFIG_PLACEHOLDER:
abort("Error: {} needs to be specified in config file {}".format(
item, file_path))
val = _handle_config_value(
var=val,
default=default,
item=item,
is_list=is_list,
convert_to_type=convert_to_type,
can_be_unset=can_be_unset,
allowed_values=allowed_values,
)
return val
def get_resolved_value(value):
"""Make sure get_resolved_value doesn't get called
(since we don't have a real case object to resolve values with)
"""
abort("Cannot resolve value with a '$' variable: {}".format(value))
