def create_metadata(parameter):
"""
From a set of parameters, extract the metadata.
"""
metadata = collections.OrderedDict()
msg = 'Use this command to recreate this image:'
metadata[msg] = ''
cmd = 'e3sm_diags --no_viewer '
from acme_diags.acme_parser import ACMEParser
parser = ACMEParser()
args = parser.view_args()
supported_cmd_args = list(args.__dict__.keys())
if 'other_parameters' in supported_cmd_args:
supported_cmd_args.remove('other_parameters')
if 'parameters' in supported_cmd_args:
supported_cmd_args.remove('parameters')
if container.is_container():
container.decontainerize_parameter(parameter)
for param_name in parameter.__dict__:
param = parameter.__dict__[param_name]
# we don't want to include blank values
if not param:
continue
if param_name in supported_cmd_args:
if isinstance(param, list) or isinstance(param, tuple):
# ex: --diff_levels -7, -6, -5, -4
cmd += "--{} ".format(param_name)
for p in param:
if isinstance(p, str) and p.isdigit():
cmd += " {} ".format(str(p))
else:
cmd += " '{}' ".format(str(p))
elif isinstance(param, bool):
# ex: --multiprocessing
# note there's no value after the parameter, it's just a flag
if param: # command is True, so add --command to set it to True
cmd += "--{} ".format(param_name)
elif isinstance(param, str) and param.isdigit():
cmd += "--{} {} ".format(param_name, param)
else:
cmd += "--{} '{}' ".format(param_name, param)
metadata[msg] = cmd
return metadata
def get_parameters(parser=ACMEParser()):
"""
Get the parameters from the parser.
"""
args = parser.view_args()
# There weren't any arguments defined.
if not any(getattr(args, arg) for arg in vars(args)):
parser.print_help()
sys.exit()
if args.parameters and not args.other_parameters: # -p only
original_parameter = parser.get_orig_parameters(argparse_vals_only=False)
# Load the default cfg files.
run_type = getattr(original_parameter, 'run_type', 'model_vs_obs')
default_diags_paths = [_get_default_diags(set_name, run_type) for set_name in SET_NAMES]
other_parameters = parser.get_other_parameters(files_to_open=default_diags_paths, argparse_vals_only=False)
parameters = parser.get_parameters(orig_parameters=original_parameter,
other_parameters=other_parameters, cmd_default_vars=False, argparse_vals_only=False)
else:
parameters = parser.get_parameters(cmd_default_vars=False, argparse_vals_only=False)
parser.check_values_of_params(parameters)
return parameters
def get_parameters(parser=ACMEParser()):
"""
Get the parameters from the parser.
"""
args = parser.view_args()
# There weren't any arguments defined.
if not any(getattr(args, arg) for arg in vars(args)):
parser.print_help()
sys.exit()
if args.parameters and not args.other_parameters: # -p only
original_parameter = parser.get_orig_parameters(argparse_vals_only=False)
# Load the default cfg files.
run_type = getattr(original_parameter, 'run_type', 'model_vs_obs')
default_diags_paths = [_get_default_diags(set_name, run_type) for set_name in utils.general.SET_NAMES]
other_parameters = parser.get_other_parameters(files_to_open=default_diags_paths, argparse_vals_only=False)
parameters = parser.get_parameters(orig_parameters=original_parameter,
other_parameters=other_parameters, cmd_default_vars=False, argparse_vals_only=False)
else:
parameters = parser.get_parameters(cmd_default_vars=False, argparse_vals_only=False)
parser.check_values_of_params(parameters)
if not parameters:
msg = 'No parameters were able to be created. Please check your .py '
msg += 'file, and any .cfg files or command line args you\'re using.'
raise RuntimeError(msg)
return parameters
def list_of_vars_in_e3sm_diags():
"""
Get a list of all of the variables used in e3sm_diags.
Open all of the *.cfg files located in acme_diags/acme_diags/driver/default_diags/
and get all of the 'variables' parameters.
"""
# Looks for these files in their installed location.
pth = os.path.join(acme_diags.INSTALL_PATH)
# The first '*' is the folder of the set, the second is the actual file.
# Ex: {acme_diags.INSTALL_PATH}/lat_lon/lat_lon_model_vs_obs.cfg
file_paths = [p for p in glob.glob(pth + '*/*.cfg')]
# Get all of the 'variables' parameter from each file.
vars_used = []
parser = ACMEParser()
parser.add_argument('path') # Needed so the filename can be passed in.
parameters = parser.get_other_parameters(files_to_open=file_paths,
check_values=False)
for p in parameters:
vars_used.extend(p.variables)
# We convert to a set because we only want one of each variable.
return set(vars_used)
def main():
parser = ACMEParser()
parameters = get_parameters(parser)
if not os.path.exists(parameters[0].results_dir):
os.makedirs(parameters[0].results_dir, 0o775)
if not parameters[0].no_viewer: # Only save provenance for full runs.
save_provenance(parameters[0].results_dir, parser)
if container.is_container():
print('Running e3sm_diags in a container.')
# Parameters will decontainerized by the viewer later.
# That's to make sure the command shown in the viewer works with or without the viewer.
for p in parameters:
container.containerize_parameter(p)
if parameters[0].multiprocessing:
parameters = cdp.cdp_run.multiprocess(run_diag, parameters)
elif parameters[0].distributed:
parameters = cdp.cdp_run.distribute(run_diag, parameters)
else:
parameters = cdp.cdp_run.serial(run_diag, parameters)
parameters = _collapse_results(parameters)
if not parameters:
print('There was not a single valid diagnostics run, no viewer created.')
else:
if parameters[0].no_viewer:
print('Viewer not created because the no_viewer parameter is True.')
else:
pth = os.path.join(parameters[0].results_dir, 'viewer')
if not os.path.exists(pth):
os.makedirs(pth)
create_viewer(pth, parameters, parameters[0].output_format[0])
path = os.path.join(parameters[0].results_dir, 'viewer')
print('Viewer HTML generated at {}/index.html'.format(path))
def main():
parser = ACMEParser()
args = parser.view_args()
if args.parameters and not args.other_parameters: # -p only
cmdline_parameter = parser.get_cmdline_parameters()
# If just a -p with no command line parameters, check the py for errors
# Otherwise don't check for errors, the command line args might have some missing ones
check_values = True if not cmdline_parameter else False
original_parameter = parser.get_orig_parameters(check_values)
# Special case, until selector parameter is added.
if hasattr(cmdline_parameter, 'sets'):
original_parameter.sets = cmdline_parameter.sets
elif not hasattr(original_parameter, 'sets'):
original_parameter.sets = SET_NAMES
# load the default cfg files
default_diags_paths = []
for set_num in original_parameter.sets:
run_type = 'model_vs_obs'
dataset = ''
if hasattr(original_parameter, 'dataset'):
dataset = original_parameter.dataset
if hasattr(original_parameter, 'run_type'):
run_type = original_parameter.run_type
default_diags_paths.append(
_get_default_diags(set_num, dataset, run_type))
other_parameters = parser.get_other_parameters(
files_to_open=default_diags_paths)
# Don't put the sets from the Python parameters to each of the parameters.
# Ex. if sets=[5, 7] in the Python parameters, don't change sets in the
# default jsons like lat_lon_ACME_default.json
vars_to_ignore = ['sets']
parameters = parser.get_parameters(
cmdline_parameters=cmdline_parameter,
orig_parameters=original_parameter,
other_parameters=other_parameters,
vars_to_ignore=vars_to_ignore,
cmd_default_vars=False)
elif not args.parameters and args.other_parameters: # -d only
cmdline_parameter = parser.get_cmdline_parameters()
other_parameters = parser.get_other_parameters()
parameters = parser.get_parameters(
cmdline_parameters=cmdline_parameter,
other_parameters=other_parameters,
cmd_default_vars=False)
elif args.parameters and args.other_parameters: # -p and -d
parameters = parser.get_parameters(cmd_default_vars=False)
else: # command line args without -p or -d
parameters = parser.get_parameters(cmd_default_vars=False)
dt = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
for p in parameters:
# needed for distributed running
# chown of all generated files to the user who ran the diags
p.user = getpass.getuser()
if not hasattr(p, 'results_dir'):
p.results_dir = '{}-{}'.format('e3sm_diags_results', dt)
if not os.path.exists(parameters[0].results_dir):
os.makedirs(parameters[0].results_dir, 0o775)
if parameters[0].multiprocessing:
parameters = cdp.cdp_run.multiprocess(run_diag, parameters)
elif parameters[0].distributed:
parameters = cdp.cdp_run.distribute(run_diag, parameters)
else:
parameters = cdp.cdp_run.serial(run_diag, parameters)
parameters = _collaspse_results(parameters)
if not parameters:
print(
'There was not a single valid diagnostics run, no viewer created.')
else:
provenance(parameters[0].results_dir)
if parameters[0].no_viewer:
print(
'Viewer not created because the no_viewer parameter is True.')
else:
pth = os.path.join(parameters[0].results_dir, 'viewer')
if not os.path.exists(pth):
os.makedirs(pth)
create_viewer(pth, parameters, parameters[0].output_format[0])
for single_run in json_data[key]:
p = ACMEParameter()
for attr_name in single_run:
setattr(p, attr_name, single_run[attr_name])
# Add attributes of original_parameter to p
for var in original_parameter.__dict__:
if var not in vars_to_ignore:
p.__dict__[var] = original_parameter.__dict__[var]
p.check_values()
parameters.append(p)
return parameters
if __name__ == '__main__':
parser = ACMEParser()
original_parameter = parser.get_parameter(default_vars=False)
if not hasattr(original_parameter, 'results_dir'):
import datetime
dt = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
original_parameter.results_dir = '{}-{}'.format(
'acme_diags_results', dt)
# if user wants all of the default diags for the sets (ex: sets=[5, 7]), then
# don't overwrite the sets keyword in the default json files.
ignore_vars = [] if hasattr(original_parameter,
'custom_diags') else ['sets']
parameters = make_parameters(original_parameter,
vars_to_ignore=ignore_vars)
for parameter in parameters:
module = importlib.import_module(mod_str)
'''
except ImportError as e:
print('Import error in {}: {}'.format(mod_str, e))
print('Set {} is not supported yet. Please give us time.'.format(set_name))
continue
'''
print('Starting to run ACME Diagnostics.')
single_result = module.run_diag(parameters)
results.append(single_result)
return results
if __name__ == '__main__':
parser = ACMEParser()
args = parser.view_args()
if args.parameters and not args.other_parameters: # -p only
original_parameter = parser.get_orig_parameters(default_vars=False)
if not hasattr(original_parameter, 'sets'):
original_parameter.sets = [
'zonal_mean_xy', 'zonal_mean_2d', 'lat_lon', 'polar',
'cosp_histogram'
]
# load the default jsons
default_jsons_paths = []
for set_num in original_parameter.sets:
Given a path to an nc file, return all of the variables in it.
"""
#parser = argparse.ArgumentParser()
#parser.add_argument("path")
#path = parser.parse_args().path
# path = DUMMY_FILE_PATH
path = parser.parse_args().path
print('Using the file: {}'.format(path))
if not os.path.exists(path):
msg = 'The file ({}) does not exist.'.format(path)
raise RuntimeError(msg)
with cdms2.open(path) as f:
return f.variables.keys()
parser = ACMEParser()
parser.add_argument('path', default=DUMMY_FILE_PATH, nargs='?')
def list_of_vars_in_e3sm_diags():
"""
Get a list of all of the variables used in e3sm_diags.
Open all of the *.cfg files located in acme_diags/acme_diags/driver/default_diags/
and get all of the 'variables' parameters.
"""
# Get all of the 'variables' parameter from each file.
vars_used = []
#print('hi')
#args = parser.parse_args()
#print(dir(args))
try: