def main(parameters=[]):
parser = CoreParser()
if not parameters:
parameters = get_parameters(parser)
# each case id (aka, variable) has a set of parameters specified.
# print out the parameters for each variable for trouble shootting
#for p in parameters:
# attrs = vars(p)
# print (', '.join("%s: %s" % item for item in attrs.items()))
if not os.path.exists(parameters[0].results_dir):
os.makedirs(parameters[0].results_dir, 0o755)
if not parameters[0].no_viewer: # Only save provenance for full runs.
save_provenance(parameters[0].results_dir, parser)
if container.is_container():
print('Running e3sm_diags in a container.')
# Modify the parmeters so that it runs in
# the container as if it usually runs.
for p in parameters:
container.containerize_parameter(p)
if parameters[0].multiprocessing:
parameters = cdp.cdp_run.multiprocess(run_diag,
parameters,
context='fork')
elif parameters[0].distributed:
parameters = cdp.cdp_run.distribute(run_diag, parameters)
else:
parameters = cdp.cdp_run.serial(run_diag, parameters)
parameters = _collapse_results(parameters)
if container.is_container():
for p in parameters:
container.decontainerize_parameter(p)
if not parameters:
print(
'There was not a single valid diagnostics run, no viewer created.')
else:
# If you get `AttributeError: 'NoneType' object has no attribute 'no_viewer'` on this line
# then `run_diag` likely returns `None`.
if parameters[0].no_viewer:
print(
'Viewer not created because the no_viewer parameter is True.')
else:
path = os.path.join(parameters[0].results_dir, 'viewer')
if not os.path.exists(path):
os.makedirs(path)
index_path = create_viewer(path, parameters)
print('Viewer HTML generated at {}'.format(index_path))
def main(parameters=[]):
parser = CoreParser()
if not parameters:
parameters = get_parameters(parser)
expected_parameters = create_parameter_dict(parameters)
# each case id (aka, variable) has a set of parameters specified.
# print out the parameters for each variable for trouble shootting
# for p in parameters:
# attrs = vars(p)
# print (', '.join("%s: %s" % item for item in attrs.items()))
if not os.path.exists(parameters[0].results_dir):
os.makedirs(parameters[0].results_dir, 0o755)
if not parameters[0].no_viewer: # Only save provenance for full runs.
save_provenance(parameters[0].results_dir, parser)
if container.is_container():
print("Running e3sm_diags in a container.")
# Modify the parmeters so that it runs in
# the container as if it usually runs.
for p in parameters:
container.containerize_parameter(p)
if parameters[0].multiprocessing:
parameters = cdp.cdp_run.multiprocess(run_diag,
parameters,
context="fork")
elif parameters[0].distributed:
parameters = cdp.cdp_run.distribute(run_diag, parameters)
else:
parameters = cdp.cdp_run.serial(run_diag, parameters)
parameters = _collapse_results(parameters)
if container.is_container():
for p in parameters:
container.decontainerize_parameter(p)
if not parameters:
print(
"There was not a single valid diagnostics run, no viewer created.")
else:
# If you get `AttributeError: 'NoneType' object has no attribute 'no_viewer'` on this line
# then `run_diag` likely returns `None`.
if parameters[0].no_viewer:
print(
"Viewer not created because the no_viewer parameter is True.")
else:
path = os.path.join(parameters[0].results_dir, "viewer")
if not os.path.exists(path):
os.makedirs(path)
index_path = create_viewer(path, parameters)
print("Viewer HTML generated at {}".format(index_path))
actual_parameters = create_parameter_dict(parameters)
if parameters[0].fail_on_incomplete and (actual_parameters !=
expected_parameters):
d: Dict[type, Tuple[int, int]] = dict()
# Loop through all expected parameter types.
for t in expected_parameters.keys():
d[t] = (actual_parameters[t], expected_parameters[t])
message = "Not all parameters completed successfully. Check output above for errors/exceptions. The following dictionary maps parameter types to their actual and expected numbers: {}".format(
d)
raise Exception(message)
def get_parameters(parser=CoreParser()):
"""
Get the parameters from the parser.
"""
# A separate parser to just get the args used.
# The reason it's a separate object than `parser`
# is so we can parse the known args.
parser_for_args = CoreParser()
# The unknown args are _.
# These are any set-specific args that aren't needed
# for now, we just want to know what args are used.
args, _ = parser_for_args.parse_known_args()
# Below is the legacy way to run this software, pre v2.0.0.
# There weren't any arguments defined.
if not any(getattr(args, arg) for arg in vars(args)):
parser.print_help()
sys.exit()
# For when a user runs the software with commands like:
# e3sm_diags lat_lon [the other parameters]
# This use-case is usually ran when the provenance
# command is copied and pasted from the viewers.
if args.set_name in SET_TO_PARSER:
parser = SET_TO_PARSER[args.set_name]()
parameters = parser.get_parameters(cmd_default_vars=False,
argparse_vals_only=False)
# The below two clauses are for the legacy way to
# run this software, pre v2.0.0.
# Ex: e3sm_diags -p params.py -d diags.cfg
elif args.parameters and not args.other_parameters: # -p only
original_parameter = parser.get_orig_parameters(
argparse_vals_only=False)
# Load the default cfg files.
run_type = getattr(original_parameter, "run_type", "model_vs_obs")
default_diags_paths = [
get_default_diags_path(set_name, run_type)
for set_name in CoreParameter().sets
]
other_parameters = parser.get_other_parameters(
files_to_open=default_diags_paths, argparse_vals_only=False)
parameters = parser.get_parameters(
orig_parameters=original_parameter,
other_parameters=other_parameters,
cmd_default_vars=False,
argparse_vals_only=False,
)
else:
parameters = parser.get_parameters(cmd_default_vars=False,
argparse_vals_only=False)
parser.check_values_of_params(parameters)
if not parameters:
msg = "No parameters were able to be created. Please check your .py "
msg += "file, and any .cfg files or command line args you're using."
raise RuntimeError(msg)
return parameters
def __init__(self):
self.sets_to_run = CoreParameter().sets
self.parser = CoreParser()
class Run():
"""
Used to run diagnostics.
A class is needed because we often need to store some
state regarding what we need to run, like the sets selected.
"""
def __init__(self):
self.sets_to_run = CoreParameter().sets
self.parser = CoreParser()
def run_diags(self, parameters):
"""
Based on sets_to_run, run the diags with the list of parameters.
"""
final_params = self.get_final_parameters(parameters)
if not final_params:
msg = 'No parameters we able to be extracted.'
msg += ' Please check the parameters you defined.'
raise RuntimeError(msg)
main(final_params)
def get_final_parameters(self, parameters):
"""
Based on sets_to_run and the list of parameters,
get the final list of paremeters to run the diags on.
"""
if not parameters or not isinstance(parameters, list):
msg = 'You must pass in a list of parameter objects.'
raise RuntimeError(msg)
# For each of the passed in parameters, we can only have one of
# each type.
types = set([p.__class__ for p in parameters])
if len(types) != len(parameters):
msg = 'You passed in two or more parameters of the same type.'
raise RuntimeError(msg)
self._add_parent_attrs_to_children(parameters)
final_params = []
for set_name in self.sets_to_run:
other_params = self._get_other_diags(parameters[0].run_type)
# For each of the set_names, get the corresponding parameter.
param = self._get_instance_of_param_class(
SET_TO_PARAMETERS[set_name], parameters)
# Since each parameter will have lots of default values, we want to remove them.
# Otherwise when calling get_parameters(), these default values
# will take precedence over values defined in other_params.
self._remove_attrs_with_default_values(param)
param.sets = [set_name]
params = self.parser.get_parameters(orig_parameters=param,
other_parameters=other_params,
cmd_default_vars=False,
argparse_vals_only=False)
# Makes sure that any parameters that are selectors
# will be in param.
self._add_attrs_with_default_values(param)
# The select() call in get_parameters() was made for the original
# command-line way of using CDP.
# We just call it manually with the parameter object param.
params = self.parser.select(param, params)
final_params.extend(params)
self.parser.check_values_of_params(final_params)
return final_params
def _add_parent_attrs_to_children(self, parameters):
"""
For any parameter class that's inherited from another, copy
the attributes of the parent to the child.
Ex: If the user wants to run set-specific parameters for
'zonal_mean_2d', they'd pass in a ZonalMean2dParameter
and a CoreParameter.
But most of the important parameters are in CoreParameter,
so copy them over to the ZonalMean2dParameter.
"""
def get_parent(param):
"""
From parameters, get any object that's a parent
type to param.
Ex: CoreParameter object is a parent of AreaMeanTimeSeriesParameter object
"""
try:
parent_class = param.__class__.__mro__[1]
parent = self._get_instance_of_param_class(
parent_class, parameters)
except RuntimeError:
parent = None
return parent
for i in range(len(parameters)):
parent = get_parent(parameters[i])
# Make sure that the new object is actually a parent.
if not parent or type(parent) == type(parameters[i]):
continue
# Otherwise, add the the parent's attributes.
# Since we're modifying this parent object (by
# removing the default values before addition)
# make a deepcopy first.
parent = copy.deepcopy(parent)
#find attributes that are not defaults
nondefault_param_parent = self._find_attrs_with_nondefault_values(
parent)
nondefault_param_child = self._find_attrs_with_nondefault_values(
parameters[i])
self._remove_attrs_with_default_values(parent)
#Simply copy over all attribute from parent to children
#parameters[i] += parent
for attr in dir(parent):
if not attr.startswith('_') and not hasattr(
parameters[i], attr):
# This attr of parent is a user-defined one and does not
# already exist in the parameters[i] parameter object.
attr_value = getattr(parent, attr)
setattr(parameters[i], attr, attr_value)
print(list(set(nondefault_param_parent) - \
set(nondefault_param_child)))
for attr in list(set(nondefault_param_parent) - \
set(nondefault_param_child)):
#'seasons' is a corner case that don't need to get in to none-core sets, Ex. area mean time series
if attr != 'seasons':
attr_value = getattr(parent, attr)
setattr(parameters[i], attr, attr_value)
#for i in range(len(parameters)):
# attrs = vars(parameters[i])
# print('all parameters', ','.join("%s: %s" % item for item in attrs.items()))
def _add_attrs_with_default_values(self, param):
"""
In the param, add any missing parameters
with their default value.
"""
new_instance = param.__class__()
for attr in dir(new_instance):
# Ignore any of the hidden attributes.
if attr.startswith('_'):
continue
if not hasattr(param, attr):
val = getattr(new_instance, attr)
setattr(param, attr, val)
def _remove_attrs_with_default_values(self, param):
"""
In the param, remove any parameters that
have their default value.
"""
new_instance = param.__class__()
for attr in dir(param):
# Ignore any of the hidden attributes.
if attr.startswith('_'):
continue
if hasattr(new_instance, attr) and \
getattr(new_instance, attr) == getattr(param, attr):
delattr(param, attr)
def _find_attrs_with_nondefault_values(self, param):
"""
In the param, find any parameters that
have nondefault value.
"""
nondefault_attr = []
new_instance = param.__class__()
for attr in dir(param):
# Ignore any of the hidden attributes.
if attr.startswith('_'):
continue
if hasattr(new_instance, attr) and \
getattr(new_instance, attr) != getattr(param, attr): # This is only valid when the attr values are lists not numpy array
nondefault_attr.append(attr)
return nondefault_attr
def _get_instance_of_param_class(self, cls, parameters):
"""
In the list of parameters, get the class for
the parameter object corresponding to cls.
"""
# Get the Method Resolution Order (MRO) for this class.
# So get the list of the classes in the inheritance ordering.
# Ex: For the 'zonal_mean_2d' set, ZonalMean2dParameter is
# the parameter for it.
# But if a user doesn't want to modify the set-specific
# parameters for 'zonal_mean_2d', they can just pass in
# a single CoreParameter object to run_diags().
# Using this, we handle this use-case.
class_types = cls.__mro__
for cls_type in class_types:
for p in parameters:
if type(p) == cls_type:
return p
msg = "There's weren\'t any class of types {} in your parameters."
raise RuntimeError(msg.format(class_types))
def _get_other_diags(self, run_type):
"""
If the user has ran the script with a -d, get the diags for that.
If not, load the default diags based on sets_to_run and run_type.
"""
args = self.parser.view_args()
# If the user has passed in args with -d.
if args.other_parameters:
params = self.parser.get_other_parameters(argparse_vals_only=False)
else:
default_diags_paths = [
get_default_diags_path(set_name, run_type, False)
for set_name in self.sets_to_run
]
params = self.parser.get_other_parameters(
files_to_open=default_diags_paths, argparse_vals_only=False)
# For each of the params, add in the default values
# using the parameter classes in SET_TO_PARAMETERS.
for i in range(len(params)):
params[i] = SET_TO_PARAMETERS[params[i].sets[0]]() + params[i]
return params
"""
# parser = argparse.ArgumentParser()
# parser.add_argument("path")
# path = parser.parse_args().path
# path = DUMMY_FILE_PATH
path = parser.parse_args().path
print("Using the file: {}".format(path))
if not os.path.exists(path):
msg = "The file ({}) does not exist.".format(path)
raise RuntimeError(msg)
with cdms2.open(path) as f:
return f.variables.keys()
parser = CoreParser()
def list_of_vars_in_e3sm_diags():
"""
Get a list of all of the variables used in e3sm_diags.
Open all of the *.cfg files located in acme_diags/acme_diags/driver/default_diags/
and get all of the 'variables' parameters.
"""
# Get all of the 'variables' parameter from each file.
vars_used = []
try:
print("something")
parameters = get_parameters(parser)
print("USING USER ARGUMENTS")
"""
#parser = argparse.ArgumentParser()
#parser.add_argument("path")
#path = parser.parse_args().path
# path = DUMMY_FILE_PATH
path = parser.parse_args().path
print('Using the file: {}'.format(path))
if not os.path.exists(path):
msg = 'The file ({}) does not exist.'.format(path)
raise RuntimeError(msg)
with cdms2.open(path) as f:
return f.variables.keys()
parser = CoreParser()
parser.add_argument('path', default=DUMMY_FILE_PATH, nargs='?')
def list_of_vars_in_e3sm_diags():
"""
Get a list of all of the variables used in e3sm_diags.
Open all of the *.cfg files located in acme_diags/acme_diags/driver/default_diags/
and get all of the 'variables' parameters.
"""
# Get all of the 'variables' parameter from each file.
vars_used = []
#print('hi')
#args = parser.parse_args()
#print(dir(args))