def _get_default_diags(set_num, dataset):
"""Returns the path from the json corresponding to set_num"""
set_num = get_set_name(set_num)
folder = '{}'.format(set_num)
fnm = '{}_{}.json'.format(set_num, dataset)
pth = os.path.join(sys.prefix, 'share', 'acme_diags', folder, fnm)
print('Using {} for {}'.format(pth, set_num))
if not os.path.exists(pth):
raise RuntimeError(
"Plotting via set '{}' not supported, file {} not installed".
format(set_num, fnm))
return pth
def _get_plot_fcn(backend, set_num):
"""Get the actual plot() function based on the backend and set_num."""
try:
if backend in ['matplotlib', 'mpl']:
backend = 'cartopy'
set_num = get_set_name(set_num)
mod_str = 'acme_diags.plot.{}.{}_plot'.format(backend, set_num)
module = importlib.import_module(mod_str)
return module.plot
except ImportError as e:
print(e)
print('Plotting for set {} with {} is not supported'.format(set_num, backend))
def _get_default_diags(set_name, run_type):
"""
Returns the path for the default diags for plotset set_name.
These are different depending on the run_type.
"""
set_name = get_set_name(set_name)
folder = '{}'.format(set_name)
fnm = '{}_{}.cfg'.format(set_name, run_type)
pth = os.path.join(acme_diags.INSTALL_PATH, folder, fnm)
print('Using {} for {}.'.format(pth, set_name))
if not os.path.exists(pth):
raise RuntimeError(
"Plotting via set '{}' not supported, file {} not installed".format(set_name, fnm))
return pth
def _get_default_diags(set_num, dataset, run_type):
"""Returns the path for the default diags corresponding to set_num"""
set_num = get_set_name(set_num)
if dataset and run_type in ['model_vs_obs', 'obs_vs_model'
]: # either 'ACME' or 'AMWG', use the jsons
fnm = '{}_{}.json'.format(set_num, dataset)
else: # use the cfgs
fnm = '{}_{}.cfg'.format(set_num, run_type)
folder = '{}'.format(set_num)
pth = os.path.join(acme_diags.INSTALL_PATH, folder, fnm)
print('Using {} for {}'.format(pth, set_num))
if not os.path.exists(pth):
raise RuntimeError(
"Plotting via set '{}' not supported, file {} not installed".
format(set_num, fnm))
return pth
def _add_pages_and_top_row(viewer, parameters):
"""Add the page and columns of the page"""
set_to_seasons = collections.OrderedDict() # dict of {set: [seasons]}
for p in parameters:
for set_num in p.sets:
set_num = get_set_name(set_num)
for ssn in p.seasons:
if set_num not in set_to_seasons:
set_to_seasons[set_num] = []
set_to_seasons[set_num].append(ssn)
for set_num, seasons in list(set_to_seasons.items()):
ROW_INFO[set_num] = collections.OrderedDict()
col_labels = ['Description']
for s in ['ANN', 'DJF', 'MAM', 'JJA', 'SON']:
if s in seasons:
col_labels.append(s)
viewer.add_page("{}".format(_better_page_name(set_num)), col_labels)
def run_diag(parameters):
"""For a single set of parameters, run the corresponding diags."""
results = []
for pset in parameters.sets:
set_name = get_set_name(pset)
parameters.current_set = set_name
mod_str = 'acme_diags.driver.{}_driver'.format(set_name)
try:
module = importlib.import_module(mod_str)
print('Starting to run E3SM Diagnostics.')
single_result = module.run_diag(parameters)
print('')
results.append(single_result)
except Exception as e:
print('Error in {}'.format(mod_str))
traceback.print_exc()
if parameters.debug:
sys.exit()
return results
def run_diag(parameters):
"""For a single set of parameters, run the corresponding diags."""
results = []
for pset in parameters.sets:
set_name = get_set_name(pset)
parameters.current_set = set_name
mod_str = 'acme_diags.driver.{}_driver'.format(set_name)
#try:
module = importlib.import_module(mod_str)
'''
except ImportError as e:
print('Import error in {}: {}'.format(mod_str, e))
print('Set {} is not supported yet. Please give us time.'.format(set_name))
continue
'''
print('Starting to run ACME Diagnostics.')
single_result = module.run_diag(parameters)
results.append(single_result)
return results
def create_viewer(root_dir, parameters, ext):
"""Based of the parameters, find the files with
extension ext and create the viewer in root_dir."""
viewer = OutputViewer(path=root_dir, index_name='E3SM Diagnostics')
_add_pages_and_top_row(viewer, parameters)
for parameter in parameters:
results_dir = parameter.results_dir
for set_num in parameter.sets:
set_num = get_set_name(set_num)
# Filenames are like:
# ref_name-variable-season-region
# or
# ref_name-variable-plev'mb'-season-region
ref_name = parameter.ref_name
for var in parameter.variables:
for season in parameter.seasons:
for region in parameter.regions:
# because some parameters have plevs, there might be
# more than one row_name, fnm pair
row_name_and_fnm = []
if parameter.plevs == []: # 2d variables
if parameter.run_type == 'model_vs_model':
row_name = '{} {}'.format(var, region)
else:
row_name = '{} {} {}'.format(
var, region, ref_name)
fnm = '{}-{}-{}-{}'.format(ref_name, var, season,
region)
row_name_and_fnm.append((row_name, fnm))
else: # 3d variables
for plev in parameter.plevs:
if parameter.run_type == 'model_vs_model':
row_name = '{}-{} {}'.format(
var,
str(int(plev)) + 'mb', region)
else:
row_name = '{}-{} {} {}'.format(
var,
str(int(plev)) + 'mb', region,
ref_name)
fnm = '{}-{}-{}-{}-{}'.format(
ref_name, var, int(plev), season, region)
row_name_and_fnm.append((row_name, fnm))
if set_num in ['lat_lon', '5']:
metrics_path = os.path.join(
results_dir, '{}'.format(set_num),
parameter.case_id, fnm)
if os.path.exists(metrics_path + '.json'):
_add_to_lat_lon_metrics_table(
metrics_path, season, row_name)
else:
print(('JSON does not exist: {}'.format(
metrics_path + '.json')))
continue
for row_name, fnm in row_name_and_fnm:
if parameter.case_id not in ROW_INFO[set_num]:
ROW_INFO[set_num][
parameter.
case_id] = collections.OrderedDict()
if row_name not in ROW_INFO[set_num][
parameter.case_id]:
ROW_INFO[set_num][parameter.case_id][
row_name] = collections.OrderedDict()
ROW_INFO[set_num][parameter.case_id][row_name][
'descr'] = _get_description(
var, parameter)
# each season has a image_path and metadata linked to it, thus we use a dict
ROW_INFO[set_num][
parameter.case_id][row_name][season] = {}
# format fnm to support relative paths
ROW_INFO[set_num][parameter.case_id][row_name][
season]['image_path'] = os.path.join(
'..', '{}'.format(set_num),
parameter.case_id, fnm)
# If ran in a container, create_metadata() will modify *_data_path and results_dir to their original value.
ROW_INFO[set_num][parameter.case_id][row_name][
season]['metadata'] = create_metadata(
parameter)
# add all of the files in from the case_id/ folder in ANN, DJF, MAM, JJA,
# SON order
for set_num in ROW_INFO:
viewer.set_page("{}".format(_better_page_name(set_num)))
for group in ROW_INFO[set_num]:
try:
viewer.set_group(group)
except RuntimeError:
viewer.add_group(group)
for row_name in ROW_INFO[set_num][group]:
try:
viewer.set_row(row_name)
except RuntimeError:
# row of row_name wasn't in the viewer, so add it
viewer.add_row(row_name)
# the description, currently the var
viewer.add_col(ROW_INFO[set_num][group][row_name]['descr'])
# [1:] is to ignore 'Description' col
for col_season in viewer.page.columns[1:]:
if col_season not in ROW_INFO[set_num][group][row_name]:
viewer.add_col('-----', is_file=True, title='-----')
else:
metadata = ROW_INFO[set_num][group][row_name][
col_season]['metadata']
fnm = ROW_INFO[set_num][group][row_name][col_season][
'image_path']
formatted_files = []
if parameters[0].save_netcdf:
nc_files = [
fnm + nc_ext for nc_ext in
['_test.nc', '_ref.nc', '_diff.nc']
]
formatted_files = [{
'url': f,
'title': f
} for f in nc_files]
viewer.add_col(fnm + '.' + ext,
is_file=True,
title=col_season,
other_files=formatted_files,
meta=metadata)
generate_lat_lon_metrics_table(viewer, root_dir, parameters)
generate_lat_lon_taylor_diag(viewer, root_dir, parameters)
viewer.generate_viewer(prompt_user=False)
_extras(root_dir, parameters)
def create_viewer(root_dir, parameters, ext):
"""Based of the parameters, find the files with
extension ext and create the viewer in root_dir."""
viewer = OutputViewer(path=root_dir, index_name='ACME Diagnostics')
_add_pages_and_top_row(viewer, parameters)
for parameter in parameters:
for set_num in parameter.sets:
set_num = get_set_name(set_num)
# Add all of the files with extension ext from the case_id/ folder'
pth = get_output_dir(set_num, parameter) # ex: myresults/lat_lon
# Filenames are like:
# ref_name-variable-season-region
# or
# ref_name-variable-plev'mb'-season-region
ref_name = parameter.ref_name
for var in parameter.variables:
for season in parameter.seasons:
for region in parameter.regions:
row_name_and_fnm = [
] # because some parameters have plevs, there might be more than one row_name, fnm pair
if parameter.plevs == []: # 2d variables
row_name = '{} {}'.format(var, region)
fnm = '{}-{}-{}-{}'.format(ref_name, var, season,
region)
row_name_and_fnm.append((row_name, fnm))
else: # 3d variables
for plev in parameter.plevs:
row_name = '{} {} {}'.format(
var,
str(int(plev)) + ' mb', region)
fnm = '{}-{}-{}-{}-{}'.format(
ref_name, var, int(plev), season, region)
row_name_and_fnm.append((row_name, fnm))
for row_name, fnm in row_name_and_fnm:
if parameter.case_id not in ROW_INFO[set_num]:
ROW_INFO[set_num][
parameter.
case_id] = collections.OrderedDict()
if row_name not in ROW_INFO[set_num][
parameter.case_id]:
ROW_INFO[set_num][parameter.case_id][
row_name] = collections.OrderedDict()
ROW_INFO[set_num][parameter.case_id][row_name][
'descr'] = _get_description(
var, parameter)
# format fnm to support relative paths
ROW_INFO[set_num][parameter.case_id][row_name][
season] = os.path.join('..',
'{}'.format(set_num),
parameter.case_id, fnm)
# add all of the files in from the case_id/ folder in ANN, DJF, MAM, JJA, SON order
for set_num in ROW_INFO:
viewer.set_page("{}".format(_better_page_name(set_num)))
for group in ROW_INFO[set_num]:
try:
viewer.set_group(group)
except RuntimeError:
viewer.add_group(group)
for row_name in ROW_INFO[set_num][group]:
try:
viewer.set_row(row_name)
except RuntimeError:
# row of row_name wasn't in the viewer, so add it
viewer.add_row(row_name)
viewer.add_col(
ROW_INFO[set_num][group][row_name]
['descr']) # the description, currently the var
for col_season in viewer.page.columns[
1:]: # [1:] is to ignore 'Description' col
fnm = ROW_INFO[set_num][group][row_name][col_season]
formatted_files = []
if parameters[0].save_netcdf:
nc_files = [
fnm + nc_ext
for nc_ext in ['_test.nc', '_ref.nc', '_diff.nc']
]
formatted_files = [{
'url': f,
'title': f
} for f in nc_files]
viewer.add_col(fnm + '.' + ext,
is_file=True,
title=col_season,
other_files=formatted_files)
viewer.generate_viewer(prompt_user=False)
_extras(root_dir, parameters)