def main(parameters=[]):
parser = CoreParser()
if not parameters:
parameters = get_parameters(parser)
# each case id (aka, variable) has a set of parameters specified.
# print out the parameters for each variable for trouble shootting
#for p in parameters:
# attrs = vars(p)
# print (', '.join("%s: %s" % item for item in attrs.items()))
if not os.path.exists(parameters[0].results_dir):
os.makedirs(parameters[0].results_dir, 0o755)
if not parameters[0].no_viewer: # Only save provenance for full runs.
save_provenance(parameters[0].results_dir, parser)
if container.is_container():
print('Running e3sm_diags in a container.')
# Modify the parmeters so that it runs in
# the container as if it usually runs.
for p in parameters:
container.containerize_parameter(p)
if parameters[0].multiprocessing:
parameters = cdp.cdp_run.multiprocess(run_diag,
parameters,
context='fork')
elif parameters[0].distributed:
parameters = cdp.cdp_run.distribute(run_diag, parameters)
else:
parameters = cdp.cdp_run.serial(run_diag, parameters)
parameters = _collapse_results(parameters)
if container.is_container():
for p in parameters:
container.decontainerize_parameter(p)
if not parameters:
print(
'There was not a single valid diagnostics run, no viewer created.')
else:
# If you get `AttributeError: 'NoneType' object has no attribute 'no_viewer'` on this line
# then `run_diag` likely returns `None`.
if parameters[0].no_viewer:
print(
'Viewer not created because the no_viewer parameter is True.')
else:
path = os.path.join(parameters[0].results_dir, 'viewer')
if not os.path.exists(path):
os.makedirs(path)
index_path = create_viewer(path, parameters)
print('Viewer HTML generated at {}'.format(index_path))
def create_metadata(parameter):
"""
From a set of parameters, extract the metadata.
"""
metadata = collections.OrderedDict()
msg = 'Use this command to recreate this image:'
metadata[msg] = ''
cmd = 'e3sm_diags --no_viewer '
from acme_diags.acme_parser import ACMEParser
parser = ACMEParser()
args = parser.view_args()
supported_cmd_args = list(args.__dict__.keys())
if 'other_parameters' in supported_cmd_args:
supported_cmd_args.remove('other_parameters')
if 'parameters' in supported_cmd_args:
supported_cmd_args.remove('parameters')
if container.is_container():
container.decontainerize_parameter(parameter)
for param_name in parameter.__dict__:
param = parameter.__dict__[param_name]
# we don't want to include blank values
if not param:
continue
if param_name in supported_cmd_args:
if isinstance(param, list) or isinstance(param, tuple):
# ex: --diff_levels -7, -6, -5, -4
cmd += "--{} ".format(param_name)
for p in param:
if isinstance(p, str) and p.isdigit():
cmd += " {} ".format(str(p))
else:
cmd += " '{}' ".format(str(p))
elif isinstance(param, bool):
# ex: --multiprocessing
# note there's no value after the parameter, it's just a flag
if param: # command is True, so add --command to set it to True
cmd += "--{} ".format(param_name)
elif isinstance(param, str) and param.isdigit():
cmd += "--{} {} ".format(param_name, param)
else:
cmd += "--{} '{}' ".format(param_name, param)
metadata[msg] = cmd
return metadata
def get_output_dir(set_num, parameter, ignore_container=False):
"""
Get the directory of where to save the outputs for a run.
If ignore_container is True and the software is being ran in a container,
get the path that the user passed in.
"""
results_dir = parameter.results_dir
if ignore_container and container.is_container():
results_dir = parameter.orig_results_dir
pth = os.path.join(results_dir, '{}'.format(set_num), parameter.case_id)
if not os.path.exists(pth):
# When running diags in parallel, sometimes another process will create the dir.
try:
os.makedirs(pth, 0o775)
except OSError as e:
if e.errno != os.errno.EEXIST:
raise
return pth
def _save_parameter_files(results_dir, parser):
"""
Save the command line arguments used, and any py or cfg files.
"""
cmd_used = " ".join(sys.argv)
fnm = os.path.join(results_dir, "cmd_used.txt")
with open(fnm, "w") as f:
if container.is_container():
f.write("# e3sm_diags was ran in a container.\n")
f.write(cmd_used)
print("Saved command used to: {}".format(fnm))
args = parser.view_args()
if hasattr(args, "parameters") and args.parameters:
fnm = args.parameters
if not os.path.isfile(fnm):
print("File does not exist: {}".format(fnm))
else:
with open(fnm, "r") as f:
contents = "".join(f.readlines())
# Remove any path, just keep the filename.
new_fnm = fnm.split("/")[-1]
new_fnm = os.path.join(results_dir, new_fnm)
with open(new_fnm, "w") as f:
f.write(contents)
print("Saved py file to: {}".format(new_fnm))
if hasattr(args, "other_parameters") and args.other_parameters:
fnm = args.other_parameters[0]
if not os.path.isfile(fnm):
print("File does not exist: {}".format(fnm))
else:
with open(fnm, "r") as f:
contents = "".join(f.readlines())
# Remove any path, just keep the filename.
new_fnm = fnm.split("/")[-1]
new_fnm = os.path.join(results_dir, new_fnm)
with open(new_fnm, "w") as f:
f.write(contents)
print("Saved cfg file to: {}".format(new_fnm))
def main():
parser = ACMEParser()
parameters = get_parameters(parser)
if not os.path.exists(parameters[0].results_dir):
os.makedirs(parameters[0].results_dir, 0o775)
if not parameters[0].no_viewer: # Only save provenance for full runs.
save_provenance(parameters[0].results_dir, parser)
if container.is_container():
print('Running e3sm_diags in a container.')
# Parameters will decontainerized by the viewer later.
# That's to make sure the command shown in the viewer works with or without the viewer.
for p in parameters:
container.containerize_parameter(p)
if parameters[0].multiprocessing:
parameters = cdp.cdp_run.multiprocess(run_diag, parameters)
elif parameters[0].distributed:
parameters = cdp.cdp_run.distribute(run_diag, parameters)
else:
parameters = cdp.cdp_run.serial(run_diag, parameters)
parameters = _collapse_results(parameters)
if not parameters:
print('There was not a single valid diagnostics run, no viewer created.')
else:
if parameters[0].no_viewer:
print('Viewer not created because the no_viewer parameter is True.')
else:
pth = os.path.join(parameters[0].results_dir, 'viewer')
if not os.path.exists(pth):
os.makedirs(pth)
create_viewer(pth, parameters, parameters[0].output_format[0])
path = os.path.join(parameters[0].results_dir, 'viewer')
print('Viewer HTML generated at {}/index.html'.format(path))
def main(parameters=[]):
parser = CoreParser()
if not parameters:
parameters = get_parameters(parser)
expected_parameters = create_parameter_dict(parameters)
# each case id (aka, variable) has a set of parameters specified.
# print out the parameters for each variable for trouble shootting
# for p in parameters:
# attrs = vars(p)
# print (', '.join("%s: %s" % item for item in attrs.items()))
if not os.path.exists(parameters[0].results_dir):
os.makedirs(parameters[0].results_dir, 0o755)
if not parameters[0].no_viewer: # Only save provenance for full runs.
save_provenance(parameters[0].results_dir, parser)
if container.is_container():
print("Running e3sm_diags in a container.")
# Modify the parmeters so that it runs in
# the container as if it usually runs.
for p in parameters:
container.containerize_parameter(p)
if parameters[0].multiprocessing:
parameters = cdp.cdp_run.multiprocess(run_diag,
parameters,
context="fork")
elif parameters[0].distributed:
parameters = cdp.cdp_run.distribute(run_diag, parameters)
else:
parameters = cdp.cdp_run.serial(run_diag, parameters)
parameters = _collapse_results(parameters)
if container.is_container():
for p in parameters:
container.decontainerize_parameter(p)
if not parameters:
print(
"There was not a single valid diagnostics run, no viewer created.")
else:
# If you get `AttributeError: 'NoneType' object has no attribute 'no_viewer'` on this line
# then `run_diag` likely returns `None`.
if parameters[0].no_viewer:
print(
"Viewer not created because the no_viewer parameter is True.")
else:
path = os.path.join(parameters[0].results_dir, "viewer")
if not os.path.exists(path):
os.makedirs(path)
index_path = create_viewer(path, parameters)
print("Viewer HTML generated at {}".format(index_path))
actual_parameters = create_parameter_dict(parameters)
if parameters[0].fail_on_incomplete and (actual_parameters !=
expected_parameters):
d: Dict[type, Tuple[int, int]] = dict()
# Loop through all expected parameter types.
for t in expected_parameters.keys():
d[t] = (actual_parameters[t], expected_parameters[t])
message = "Not all parameters completed successfully. Check output above for errors/exceptions. The following dictionary maps parameter types to their actual and expected numbers: {}".format(
d)
raise Exception(message)
