def setUp(self):
self.runner = Run()
self.core_param = CoreParameter()
# These parameters don't need to map to real files since we're
# just testing if the parameters are created correctly.
self.core_param.reference_data_path = '/this/can/be/whatever/for/'
self.core_param.reference_data_path += 'testing/obs_for_acme_diags/'
self.core_param.test_data_path = '/this/can//also/be/whatever/for/testing/'
self.core_param.test_data_path += 'test_model_data_for_acme_diags/climatology/'
self.core_param.test_name = 'SomeTimeStamp.beta?.SomeModelThing.ne30_ne30.somemachine'
self.core_param.results_dir = 'results'
# Define data paths for obs
input_prefix = "/global/cfs/cdirs/e3sm/acme_diags"
obs_climo = "/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/climatology"
obs_ts = "/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/time-series"
# Define data paths for test model
test_prefix = "/global/cfs/cdirs/e3sm/zhang40/postprocessing_for_e3sm_diags"
case = "20180215.DECKv1b_H1.ne30_oEC.edison"
casename = case.split(".")[1] + "." + case.split(".")[2]
climo_path = os.path.join(test_prefix, "climo/" + case + "/1980-2014/rgr")
ts_path = os.path.join(test_prefix, "monthly_ts/" + case + "/1980-2014/rgr")
dc_climo_path = os.path.join(test_prefix, "diurnal_climo/" + case + "/1980-2014/rgr")
# Define parameters for core sets: 'lat_lon','zonal_mean_xy', 'zonal_mean_2d', 'polar', 'cosp_histogram', 'meridional_mean_2d'
param = CoreParameter()
param.reference_data_path = obs_climo
param.test_data_path = climo_path
param.test_name = case
param.test_short_name = casename
# Define results path.
prefix = "/global/cfs/cdirs/e3sm/www/zhang40/tutorials/"
param.results_dir = os.path.join(prefix, "run_v230_allsets")
param.multiprocessing = True
param.num_workers = 30
param.seasons = ["ANN", "JJA"]
# Additional parameters:
# Short version of test model name being printed as plot titles
def get_parameters(parser=CoreParser()):
"""
Get the parameters from the parser.
"""
# A separate parser to just get the args used.
# The reason it's a separate object than `parser`
# is so we can parse the known args.
parser_for_args = CoreParser()
# The unknown args are _.
# These are any set-specific args that aren't needed
# for now, we just want to know what args are used.
args, _ = parser_for_args.parse_known_args()
# Below is the legacy way to run this software, pre v2.0.0.
# There weren't any arguments defined.
if not any(getattr(args, arg) for arg in vars(args)):
parser.print_help()
sys.exit()
# For when a user runs the software with commands like:
# e3sm_diags lat_lon [the other parameters]
# This use-case is usually ran when the provenance
# command is copied and pasted from the viewers.
if args.set_name in SET_TO_PARSER:
parser = SET_TO_PARSER[args.set_name]()
parameters = parser.get_parameters(cmd_default_vars=False,
argparse_vals_only=False)
# The below two clauses are for the legacy way to
# run this software, pre v2.0.0.
# Ex: e3sm_diags -p params.py -d diags.cfg
elif args.parameters and not args.other_parameters: # -p only
original_parameter = parser.get_orig_parameters(
argparse_vals_only=False)
# Load the default cfg files.
run_type = getattr(original_parameter, "run_type", "model_vs_obs")
default_diags_paths = [
get_default_diags_path(set_name, run_type)
for set_name in CoreParameter().sets
]
other_parameters = parser.get_other_parameters(
files_to_open=default_diags_paths, argparse_vals_only=False)
parameters = parser.get_parameters(
orig_parameters=original_parameter,
other_parameters=other_parameters,
cmd_default_vars=False,
argparse_vals_only=False,
)
else:
parameters = parser.get_parameters(cmd_default_vars=False,
argparse_vals_only=False)
parser.check_values_of_params(parameters)
if not parameters:
msg = "No parameters were able to be created. Please check your .py "
msg += "file, and any .cfg files or command line args you're using."
raise RuntimeError(msg)
return parameters
import os from acme_diags.parameter.core_parameter import CoreParameter from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter from acme_diags.run import runner param = CoreParameter() #For compy #machine_path_prefix = '/compyfs/e3sm_diags_data/' #For cori #machine_path_prefix = '/global/project/projectdirs/acme/acme_diags' #for acme1 machine_path_prefix = '/p/user_pub/e3sm/e3sm_diags_data/' #param.reference_data_path = os.path.join(machine_path_prefix, 'obs_for_e3sm_diags/time-series/') #param.test_data_path = os.path.join(machine_path_prefix, 'test_model_data_for_acme_diags/time-series/CESM1-CAM5_cmip/') param.reference_data_path = '/p/user_pub/e3sm/zhang40/e3sm_cmip6_xmls/v1_water_cycle/historical/' param.test_data_path = '/p/user_pub/e3sm/zhang40/e3sm_cmip6_xmls/v1_water_cycle/historical/r1i1p1f1' param.test_name = 'e3sm_v1_hist_r1' #For compy #prefix = '/compyfs/www/zhan429/doc_examples/' #For cori #prefix = '/global/project/projectdirs/acme/www/zhang40/' #For acme1 prefix = '/var/www/acme/acme-diags/zhang40/tests/' param.results_dir = os.path.join(prefix, 'area_mean_hist_r1-5_land') param.multiprocessing = True param.num_workers = 25 # We're passing in this new object as well, in
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.run import runner
param = CoreParameter()
# Location of the ref data.
param.reference_data_path = (
"/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/climatology/")
# Name of the ref obs data, used to find the climo files.
param.ref_name = "ceres_ebaf_toa_v2.8"
# Location of the test data.
param.test_data_path = (
"/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/climatology/")
# Name of the test obs data, used to find the climo files.
param.test_name = "ceres_ebaf_toa_v4.0"
# Name of the folder where the results are stored.
# Change `prefix` to use your directory.
prefix = "/global/cfs/cdirs/e3sm/www/<your directory>/examples"
param.results_dir = os.path.join(prefix, "ex7_obs_vs_obs")
# What plotsets to run the diags on.
param.sets = ["lat_lon"]
# Below are more optional arguments.
# What seasons to run the diags on.
# If not defined, diags is ran on ['ANN', 'DJF', 'MAM', 'JJA', 'SON'].
param.seasons = ["ANN"]
import os from acme_diags.parameter.core_parameter import CoreParameter from acme_diags.run import runner param = CoreParameter() # Location of the ref data. param.reference_data_path = '/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/climatology/' # Name of the ref obs data, used to find the climo files. param.ref_name = 'ceres_ebaf_toa_v2.8' # Location of the test data. param.test_data_path = '/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/climatology/' # Name of the test obs data, used to find the climo files. param.test_name = 'ceres_ebaf_toa_v4.0' # Name of the folder where the results are stored. # Change `prefix` to use your directory. #prefix = '/global/cfs/cdirs/e3sm/www/<your directory>/examples' param.results_dir = os.path.join(prefix, 'ex7_obs_vs_obs') # What plotsets to run the diags on. param.sets = ['lat_lon'] # Below are more optional arguments. # What seasons to run the diags on. # If not defined, diags is ran on ['ANN', 'DJF', 'MAM', 'JJA', 'SON']. param.seasons = ['ANN'] # 'mpl' is to create matplotlib plots, 'vcs' is for vcs plots. param.backend = 'mpl'
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.run import runner
param = CoreParameter()
# Location of the data.
param.test_data_path = "/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/time-series/CESM1-CAM5_cmip/"
param.reference_data_path = (
"/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/time-series/"
)
# Set this parameter to True.
# By default, e3sm_diags expects the test data to be climo data.
param.test_timeseries_input = True
# Years to slice the test data, base this off the years in the filenames.
param.test_start_yr = "2003"
param.test_end_yr = "2004"
# Set this parameter to True.
# By default, e3sm_diags expects the ref data to be climo data.
param.ref_timeseries_input = True
# Years to slice the ref data, base this off the years in the filenames.
param.ref_start_yr = "2003"
param.ref_end_yr = "2004"
# When running with time-series data, you don't need to specify the name of the data.
# But you should, otherwise nothing is displayed when the test/ref name is needed.
param.short_test_name = "CESM1-CAM5-historical"
# param.short_ref_name = ''
import os from acme_diags.parameter.core_parameter import CoreParameter from acme_diags.run import runner param = CoreParameter() # Location of the data. param.test_data_path = '/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/time-series/E3SM_v1' param.reference_data_path = '/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/time-series/E3SM_v1' # Set this parameter to True. # By default, e3sm_diags expects the test data to be climo data. param.test_timeseries_input = True # Years to slice the test data, base this off the years in the filenames. param.test_start_yr = '2011' param.test_end_yr = '2013' # Set this parameter to True. # By default, e3sm_diags expects the ref data to be climo data. param.ref_timeseries_input = True # Years to slice the ref data, base this off the years in the filenames. param.ref_start_yr = '1850' param.ref_end_yr = '1852' # When running with time-series data, you don't need to specify the name of the data. # But you should, otherwise nothing is displayed when the test/ref name is needed. param.short_test_name = 'historical_H1' param.short_ref_name = 'historical_H1' # This parameter modifies the software to accommodate model vs model runs. # The default setting for run_type is 'model_vs_obs'. param.run_type = 'model_vs_model'
import os from acme_diags.parameter.core_parameter import CoreParameter from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter from acme_diags.run import runner param = CoreParameter() #For compy #machine_path_prefix = '/compyfs/e3sm_diags_data/' #For cori #machine_path_prefix = '/global/project/projectdirs/acme/acme_diags' #for acme1 machine_path_prefix = '/p/user_pub/e3sm/e3sm_diags_data/' #param.reference_data_path = os.path.join(machine_path_prefix, 'obs_for_e3sm_diags/time-series/') #param.test_data_path = os.path.join(machine_path_prefix, 'test_model_data_for_acme_diags/time-series/CESM1-CAM5_cmip/') param.reference_data_path = '/p/user_pub/e3sm/zhang40/e3sm_cmip6_xmls/v1_water_cycle/abrupt-4xCO2/' param.test_data_path = '/p/user_pub/e3sm/zhang40/e3sm_cmip6_xmls/v1_water_cycle/piControl/r1i1p1f1' param.test_name = 'e3sm_v1_piControl' #For compy #prefix = '/compyfs/www/zhan429/doc_examples/' #For cori #prefix = '/global/project/projectdirs/acme/www/zhang40/' #For acme1 #prefix = '/var/www/acme/acme-diags/zhang40/tests/' #param.results_dir = os.path.join(prefix, 'area_mean_picontrol_r1') param.multiprocessing = True param.num_workers = 25 # We're passing in this new object as well, in
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter
from acme_diags.run import runner
param = CoreParameter()
#For compy
#machine_path_prefix = '/compyfs/e3sm_diags_data/'
#For cori
#machine_path_prefix = '/global/project/projectdirs/acme/acme_diags'
#for acme1
machine_path_prefix = '/p/user_pub/e3sm/e3sm_diags_data/'
param.reference_data_path = os.path.join(machine_path_prefix,
'obs_for_e3sm_diags/time-series/')
#param.test_data_path = os.path.join(machine_path_prefix, 'test_model_data_for_acme_diags/time-series/CESM1-CAM5_cmip/')
param.test_data_path = '/p/user_pub/e3sm/zhang40/e3sm_cmip6_xmls/v1_water_cycle/historical/r1i1p1f1'
param.test_name = 'e3sm_v1_hist_r1'
#For compy
#prefix = '/compyfs/www/zhan429/doc_examples/'
#For cori
#prefix = '/global/project/projectdirs/acme/www/zhang40/'
#For acme1
prefix = '/var/www/acme/acme-diags/zhang40/tests/'
param.results_dir = os.path.join(prefix, 'area_mean_with_no_obs_hist_r1')
param.multiprocessing = True
param.num_workers = 50
# We're passing in this new object as well, in
import os from acme_diags.parameter.core_parameter import CoreParameter from acme_diags.run import runner param = CoreParameter() # Location of the ref data. param.reference_data_path = '/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/climatology/' # Name of the ref model data, used to find the climo files. param.ref_name = '20161118.beta0.F1850COSP.ne30_ne30.edison' # An optional, shorter name to be used instead of the ref_name. param.short_ref_name = 'Ref: beta0.F1850COSP_ne30' # Location of the test data. param.test_data_path = '/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/climatology' # Name of the test model data, used to find the climo files. param.test_name = '20161118.beta0.FC5COSP.ne30_ne30.edison' # An optional, shorter name to be used instead of the test_name. param.short_test_name = 'Test: beta0_FC5COSP_ne30' # What plotsets to run the diags on. param.sets = ['lat_lon'] # Name of the folder where the results are stored. # Change `prefix` to use your directory. #prefix = '/global/cfs/cdirs/e3sm/www/<your directory>/examples' param.results_dir = os.path.join(prefix, 'ex4_model_to_model') # This parameter modifies the software to accommodate model vs model runs. # The default setting for run_type is 'model_vs_obs'. param.run_type = 'model_vs_model' # Below are more optional arguments.
import os from acme_diags.parameter.core_parameter import CoreParameter from acme_diags.run import runner param = CoreParameter() # Location of the data. param.test_data_path = '/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/time-series/CESM1-CAM5_cmip/' param.reference_data_path = '/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/time-series/' # Set this parameter to True. # By default, e3sm_diags expects the test data to be climo data. param.test_timeseries_input = True # Years to slice the test data, base this off the years in the filenames. param.test_start_yr = '2003' param.test_end_yr = '2004' # Set this parameter to True. # By default, e3sm_diags expects the ref data to be climo data. param.ref_timeseries_input = True # Years to slice the ref data, base this off the years in the filenames. param.ref_start_yr = '2003' param.ref_end_yr = '2004' # When running with time-series data, you don't need to specify the name of the data. # But you should, otherwise nothing is displayed when the test/ref name is needed. param.short_test_name = 'CESM1-CAM5-historical' #param.short_ref_name = '' # This parameter modifies the software to accommodate model vs model runs. # The default setting for run_type is 'model_vs_obs'.
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter
from acme_diags.run import runner
param = CoreParameter()
param.reference_data_path = '/compyfs/e3sm_diags_data/obs_for_e3sm_diags/time-series/'
param.test_data_path = '/compyfs/e3sm_diags_data/test_model_data_for_acme_diags/time-series/E3SM_v1/'
param.test_name = 'e3sm_v1'
param.seasons = [
"ANN"
] #seasons shouldn't be a parameter for area_mean_time_series, need to investigate while uncomment this line won't work
#param.seasons = [] #seasons shouldn't be a parameter for area_mean_time_series, need to investigate while uncomment this line won't work
param.test_timeseries_input = True
param.test_start_yr = '1998'
param.test_end_yr = '2004'
param.ref_timeseries_input = True
param.ref_start_yr = '1998'
param.ref_end_yr = '2004'
param.multiprocessing = True
param.num_workers = 40
prefix = '/compyfs/www/zhan429/tests'
param.results_dir = os.path.join(prefix, 'multiple_sets_2ts')
# Set specific parameters for new sets
ts_param = AreaMeanTimeSeriesParameter()
ts_param.reference_data_path = '/compyfs/e3sm_diags_data/obs_for_e3sm_diags/time-series/'
ts_param.test_data_path = '/compyfs/e3sm_diags_data/test_model_data_for_acme_diags/time-series/E3SM_v1/'
import os from acme_diags.parameter.core_parameter import CoreParameter from acme_diags.run import runner param = CoreParameter() #param.reference_data_path = '/p/user_pub/e3sm/zhang40/analysis_data_e3sm_diags/GPCP_OAFLux/climatology' param.reference_data_path = '/p/user_pub/e3sm/e3sm_diags_data/obs_for_e3sm_diags/climatology' param.test_data_path = '/p/user_pub/e3sm/e3sm_diags_data/test_model_data_for_acme_diags/climatology' param.test_name = '20161118.beta0.FC5COSP.ne30_ne30.edison' #param.seasons = ["ANN"] param.multiprocessing = True param.num_workers = 40 prefix = '/var/www/acme/acme-diags/zhang40/tests/' param.results_dir = os.path.join(prefix, 'all_vars_core_v2') # ###Set specific parameters for new sets #ts_param = AreaMeanTimeSeriesParameter() #ts_param.reference_data_path = '/compyfs/e3sm_diags_data/obs_for_e3sm_diags/time-series/' #ts_param.test_data_path = '/compyfs/e3sm_diags_data/test_model_data_for_acme_diags/time-series/E3SM_v1/' #ts_param.test_name = 'e3sm_v1' #ts_param.start_yr = '2002' #ts_param.end_yr = '2008' runner.sets_to_run = ['lat_lon','zonal_mean_xy', 'zonal_mean_2d', 'polar', 'cosp_histogram', 'meridional_mean_2d']#, 'area_mean_time_series'] #runner.run_diags([param, ts_param]) #runner.sets_to_run = ['area_mean_time_series'] #runner.run_diags([param, ts_param]) #runner.sets_to_run = ['lat_lon','zonal_mean_xy']
def __init__(self):
self.sets_to_run = CoreParameter().sets
self.parser = CoreParser()
def setUp(self):
self.parameter = CoreParameter()
def test_complete_run(self):
# Anvil
# Run `source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified.sh`
test_data_prefix = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data'
ref_data_prefix = '/lcrc/group/e3sm/public_html/diagnostics/observations/Atm'
html_prefix = '.'
param = CoreParameter()
param.reference_data_path = os.path.join(ref_data_prefix,
'climatology')
param.test_data_path = os.path.join(test_data_prefix, 'climatology/')
param.test_name = '20161118.beta0.FC5COSP.ne30_ne30.edison'
param.seasons = [
"ANN", "JJA"
] # Default setting: seasons = ["ANN", "DJF", "MAM", "JJA", "SON"]
param.results_dir = os.path.join(html_prefix, 'tutorial_2020_all_sets')
param.multiprocessing = True
param.num_workers = 30
# Additional parameters:
#param.short_test_name = 'beta0.FC5COSP.ne30'
#param.run_type = 'model_vs_model'
#param.diff_title = 'Difference'
#param.output_format = ['png']
#param.output_format_subplot = ['pdf']
#param.save_netcdf = True
# Set specific parameters for new sets
enso_param = EnsoDiagsParameter()
enso_param.reference_data_path = os.path.join(ref_data_prefix,
'time-series/')
enso_param.test_data_path = os.path.join(test_data_prefix,
'time-series/E3SM_v1/')
enso_param.test_name = 'e3sm_v1'
enso_param.start_yr = '1990'
enso_param.end_yr = '1999'
qbo_param = QboParameter()
qbo_param.reference_data_path = os.path.join(ref_data_prefix,
'time-series/')
qbo_param.test_data_path = os.path.join(test_data_prefix,
'time-series/E3SM_v1/')
qbo_param.test_name = 'e3sm_v1'
qbo_param.start_yr = '1990'
qbo_param.end_yr = '1999'
ts_param = AreaMeanTimeSeriesParameter()
ts_param.reference_data_path = os.path.join(ref_data_prefix,
'time-series/')
ts_param.test_data_path = os.path.join(test_data_prefix,
'time-series/E3SM_v1/')
ts_param.test_name = 'e3sm_v1'
ts_param.start_yr = '1990'
ts_param.end_yr = '1999'
runner.sets_to_run = [
'lat_lon', 'zonal_mean_xy', 'zonal_mean_2d', 'polar',
'cosp_histogram', 'meridional_mean_2d', 'enso_diags', 'qbo',
'area_mean_time_series'
]
runner.run_diags([param, enso_param, qbo_param, ts_param])
actual_images_dir = param.results_dir
# The expected_images_file lists all 475 images we expect to compare.
# It was generated with the following steps:
# cd /lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/tutorial_2020_all_sets
# find . -type f -name '*.png' > ../expected_images_complete_run.txt
expected_images_file = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/expected_images_complete_run.txt'
expected_images_dir = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/tutorial_2020_all_sets'
mismatched_images = []
with open(expected_images_file) as f:
for line in f:
image = line.strip('./').strip('\n')
path_to_actual_png = os.path.join(actual_images_dir, image)
path_to_expected_png = os.path.join(expected_images_dir, image)
actual_png = Image.open(path_to_actual_png).convert('RGB')
expected_png = Image.open(path_to_expected_png).convert('RGB')
diff = ImageChops.difference(actual_png, expected_png)
bbox = diff.getbbox()
if not bbox:
# If `diff.getbbox()` is None, then the images are in theory equal
self.assertIsNone(diff.getbbox())
else:
# Sometimes, a few pixels will differ, but the two images appear identical.
# https://codereview.stackexchange.com/questions/55902/fastest-way-to-count-non-zero-pixels-using-python-and-pillow
nonzero_pixels = diff.crop(bbox).point(
lambda x: 255
if x else 0).convert("L").point(bool).getdata()
num_nonzero_pixels = sum(nonzero_pixels)
print('\npath_to_actual_png={}'.format(path_to_actual_png))
print(
'path_to_expected_png={}'.format(path_to_expected_png))
print('diff has {} nonzero pixels.'.format(
num_nonzero_pixels))
width, height = expected_png.size
num_pixels = width * height
print('total number of pixels={}'.format(num_pixels))
fraction = num_nonzero_pixels / num_pixels
print('num_nonzero_pixels/num_pixels fraction={}'.format(
fraction))
# Fraction of mismatched pixels should be less than 0.02%
if fraction >= 0.0002:
mismatched_images.append(image)
self.assertEqual(mismatched_images, [])
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter
from acme_diags.run import runner
param = CoreParameter()
#For compy
machine_path_prefix = '/compyfs/e3sm_diags_data/'
#For cori
#machine_path_prefix = '/global/project/projectdirs/acme/acme_diags'
param.test_data_path = os.path.join(
machine_path_prefix,
'test_model_data_for_acme_diags/time-series/E3SM_v1_historical_r1/')
#param.reference_data_path = os.path.join(machine_path_prefix, 'obs_for_e3sm_diags/time-series/')
param.reference_data_path = os.path.join(
machine_path_prefix, 'test_model_data_for_acme_diags/time-series/')
param.test_name = 'e3sm_v1'
param.rune_type = 'model-vs-model'
prefix = '/compyfs/www/zhan429/examples/'
param.results_dir = os.path.join(prefix, 'area_mean_with_models')
param.multiprocessing = True
param.num_workers = 40
# We're passing in this new object as well, in
# addtion to the CoreParameter object.
ts_param = AreaMeanTimeSeriesParameter()
# Running the software with the API:
# python all_sets.py -d all_sets.py
import numpy
from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.enso_diags_parameter import EnsoDiagsParameter
from acme_diags.parameter.meridional_mean_2d_parameter import MeridionalMean2dParameter
from acme_diags.parameter.zonal_mean_2d_parameter import ZonalMean2dParameter
from acme_diags.run import Run
run_object = Run()
param = CoreParameter()
ts_param = AreaMeanTimeSeriesParameter()
m2d_param = MeridionalMean2dParameter()
m2d_param.plevs = [
200.,
500.,
]
z2d_param = ZonalMean2dParameter()
z2d_param.plevs = [
200.,
300.,
]
enso_param = EnsoDiagsParameter()
enso_param.test_name = 'e3sm_v1'
run_object.run_diags([param, ts_param, m2d_param, z2d_param, enso_param])
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.run import runner
param = CoreParameter()
# Location of the data.
param.reference_data_path = (
"/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/climatology/")
param.test_data_path = (
"/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/climatology/"
)
# Name of the test model data, used to find the climo files.
param.test_name = "20161118.beta0.FC5COSP.ne30_ne30.edison"
# An optional, shorter name to be used instead of the test_name.
param.short_test_name = "beta0.FC5COSP.ne30"
# What plotsets to run the diags on.
param.sets = ["lat_lon"]
# Name of the folder where the results are stored.
# Change `prefix` to use your directory.
prefix = "/global/cfs/cdirs/e3sm/www/<your directory>/examples"
param.results_dir = os.path.join(prefix, "ex5_model_to_obs")
# Below are more optional arguments.
# 'mpl' is to create matplotlib plots, 'vcs' is for vcs plots.
param.backend = "mpl"
# Title of the difference plots.
param.diff_title = "Model - Obs."
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter
from acme_diags.run import runner
param = CoreParameter()
#For compy
machine_path_prefix = '/compyfs/e3sm_diags_data/'
#For cori
#machine_path_prefix = '/global/project/projectdirs/acme/acme_diags'
param.reference_data_path = os.path.join(machine_path_prefix,
'obs_for_e3sm_diags/time-series/')
param.test_data_path = os.path.join(
machine_path_prefix, 'test_model_data_for_acme_diags/time-series/E3SM_v1/')
param.test_name = 'e3sm_v1'
param.output_format = ['png', 'pdf', 'svg']
#prefix = '/compyfs/www/zhan429/examples/'
prefix = '/compyfs/www/zhan429/tests/'
param.results_dir = os.path.join(prefix, 'area_mean_with_obs')
#param.multiprocessing = True
#param.num_workers = 40
# We're passing in this new object as well, in
# addtion to the CoreParameter object.
ts_param = AreaMeanTimeSeriesParameter()
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter
from acme_diags.parameter.zonal_mean_2d_parameter import ZonalMean2dParameter
from acme_diags.run import runner
param = CoreParameter()
param.reference_data_path = '/compyfs/e3sm_diags_data/obs_for_e3sm_diags/climatology/'
param.test_data_path = '/compyfs/e3sm_diags_data/test_model_data_for_acme_diags/climatology/'
param.test_name = '20161118.beta0.FC5COSP.ne30_ne30.edison'
param.seasons = ["ANN"]
param.multiprocessing = True
param.num_workers = 40
prefix = '/compyfs/www/zhan429/tests'
param.results_dir = os.path.join(prefix, 'all_sets')
#
##Set specific parameters for new sets
ts_param = AreaMeanTimeSeriesParameter()
ts_param.reference_data_path = '/compyfs/e3sm_diags_data/obs_for_e3sm_diags/time-series/'
ts_param.test_data_path = '/compyfs/e3sm_diags_data/test_model_data_for_acme_diags/time-series/E3SM_v1/'
ts_param.test_name = 'e3sm_v1'
ts_param.start_yr = '2002'
ts_param.end_yr = '2008'
runner.sets_to_run = [
'lat_lon', 'zonal_mean_xy', 'zonal_mean_2d', 'polar', 'cosp_histogram',
'meridional_mean_2d', 'area_mean_time_series'
]
import os from acme_diags.parameter.core_parameter import CoreParameter from acme_diags.run import runner param = CoreParameter() # Location of the data. param.test_data_path = "/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/time-series/CESM1-CAM5_cmip/" param.reference_data_path = "/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/time-series/CESM1-CAM5_cmip" # Set this parameter to True. # By default, e3sm_diags expects the test data to be climo data. param.test_timeseries_input = True # Years to slice the test data, base this off the years in the filenames. param.test_start_yr = "2001" param.test_end_yr = "2003" # Set this parameter to True. # By default, e3sm_diags expects the ref data to be climo data. param.ref_timeseries_input = True # Years to slice the ref data, base this off the years in the filenames. param.ref_start_yr = "1850" param.ref_end_yr = "1852" # When running with time-series data, you don't need to specify the name of the data. # But you should, otherwise nothing is displayed when the test/ref name is needed. param.short_test_name = "CESM1-CAM5-historical" param.short_ref_name = "CESM1-CAM5-historical" # This parameter modifies the software to accommodate model vs model runs.
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.zonal_mean_2d_parameter import ZonalMean2dParameter
from acme_diags.run import runner
param = CoreParameter()
param.reference_data_path = (
"/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/climatology/")
param.test_data_path = (
"/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/climatology/"
)
param.test_name = "20161118.beta0.FC5COSP.ne30_ne30.edison"
param.seasons = ["ANN"]
# Name of the folder where the results are stored.
# Change `prefix` to use your directory.
prefix = "/global/cfs/cdirs/e3sm/www/<your directory>/examples"
param.results_dir = os.path.join(prefix, "ex6_zonal_mean_2d_and_lat_lon_demo")
# Uncomment the two lines below to just
# run the diags with T and PRECT.
# param.selectors += ['variables']
# param.variables = ['T', 'PRECT']
# The new changes are below.
zonal_mean_2d_param = ZonalMean2dParameter()
zonal_mean_2d_param.plevs = [10.0, 20.0, 30.0]
runner.sets_to_run = ["zonal_mean_2d", "lat_lon"]
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.run import runner
param = CoreParameter()
# Location of the ref data.
param.reference_data_path = (
"/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/climatology/"
)
# Name of the ref model data, used to find the climo files.
param.ref_name = "20161118.beta0.F1850COSP.ne30_ne30.edison"
# An optional, shorter name to be used instead of the ref_name.
param.short_ref_name = "Ref: beta0.F1850COSP_ne30"
# Location of the test data.
param.test_data_path = (
"/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/climatology"
)
# Name of the test model data, used to find the climo files.
param.test_name = "20161118.beta0.FC5COSP.ne30_ne30.edison"
# An optional, shorter name to be used instead of the test_name.
param.short_test_name = "Test: beta0_FC5COSP_ne30"
# What plotsets to run the diags on.
param.sets = ["lat_lon"]
# Name of the folder where the results are stored.
# Change `prefix` to use your directory.
prefix = "/global/cfs/cdirs/e3sm/www/<your directory>/examples"
param.results_dir = os.path.join(prefix, "ex4_model_to_model")
import os
from acme_diags.run import runner
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter
from acme_diags.parameter.enso_diags_parameter import EnsoDiagsParameter
from acme_diags.parameter.qbo_parameter import QboParameter
# See https://e3sm-project.github.io/e3sm_diags/docs/html/quickguides/quick-guide-general.html
# Compy
#data_prefix = '/compyfs/e3sm_diags_data'
#html_prefix = '/compyfs/www/<username>' # Change <username>
# Cori
data_prefix = '/global/cfs/cdirs/e3sm/acme_diags'
html_prefix = '/global/cfs/cdirs/e3sm/www/<username>' # Change <username>
param = CoreParameter()
param.reference_data_path = os.path.join(data_prefix,
'obs_for_e3sm_diags/climatology')
param.test_data_path = os.path.join(
data_prefix, 'test_model_data_for_acme_diags/climatology/')
param.test_name = '20161118.beta0.FC5COSP.ne30_ne30.edison'
param.seasons = [
"ANN", "JJA"
] # Default setting: seasons = ["ANN", "DJF", "MAM", "JJA", "SON"]
param.results_dir = os.path.join(html_prefix, 'tutorial_2020_all_sets')
param.multiprocessing = True
param.num_workers = 30
# Additional parameters:
import os from acme_diags.parameter.core_parameter import CoreParameter from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter from acme_diags.run import runner param = CoreParameter() #For compy #machine_path_prefix = '/compyfs/e3sm_diags_data/' #For cori #machine_path_prefix = '/global/project/projectdirs/acme/acme_diags' #for acme1 machine_path_prefix = '/p/user_pub/e3sm/e3sm_diags_data/' #param.reference_data_path = os.path.join(machine_path_prefix, 'obs_for_e3sm_diags/time-series/') #param.test_data_path = os.path.join(machine_path_prefix, 'test_model_data_for_acme_diags/time-series/CESM1-CAM5_cmip/') param.reference_data_path = '/p/user_pub/e3sm/zhang40/e3sm_cmip6_xmls/v1_water_cycle/amip/' param.test_data_path = '/p/user_pub/e3sm/zhang40/e3sm_cmip6_xmls/v1_water_cycle/amip/r1i1p1f1' param.test_name = 'e3sm_v1_amip_r1' #For compy #prefix = '/compyfs/www/zhan429/doc_examples/' #For cori #prefix = '/global/project/projectdirs/acme/www/zhang40/' #For acme1 #prefix = '/var/www/acme/acme-diags/zhang40/tests/' #param.results_dir = os.path.join(prefix, 'area_mean_amip_r1-3') param.multiprocessing = True param.num_workers = 25 # We're passing in this new object as well, in
import os from acme_diags.parameter.core_parameter import CoreParameter from acme_diags.run import runner param = CoreParameter() # Location of the data. param.reference_data_path = '/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/climatology/' param.test_data_path = '/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/climatology/' # Name of the test model data, used to find the climo files. param.test_name = '20161118.beta0.FC5COSP.ne30_ne30.edison' # An optional, shorter name to be used instead of the test_name. param.short_test_name = 'beta0.FC5COSP.ne30' # What plotsets to run the diags on. param.sets = ['lat_lon'] # Name of the folder where the results are stored. # Change `prefix` to use your directory. #prefix = '/global/cfs/cdirs/e3sm/www/<your directory>/examples' param.results_dir = os.path.join(prefix, 'ex5_model_to_obs') # Below are more optional arguments. # 'mpl' is to create matplotlib plots, 'vcs' is for vcs plots. param.backend = 'mpl' # Title of the difference plots. param.diff_title = 'Model - Obs.' # Save the netcdf files for each of the ref, test, and diff plot. param.save_netcdf = True ## For running with multiprocessing. #param.multiprocessing = True
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter
from acme_diags.run import runner
param = CoreParameter()
#For compy
machine_path_prefix = '/compyfs/e3sm_diags_data/'
#For cori
#machine_path_prefix = '/global/project/projectdirs/acme/acme_diags'
param.reference_data_path = os.path.join(machine_path_prefix,
'obs_for_e3sm_diags/time-series/')
param.test_data_path = os.path.join(
machine_path_prefix, 'test_model_data_for_acme_diags/time-series/E3SM_v1/')
param.test_name = 'e3sm_v1'
#param.seasons = ["ANN"] #seasons shouldn't be a parameter for area_mean_time_series, need to investigate while uncomment this line won't work
param.test_timeseries_input = True
param.test_start_yr = '2002'
param.test_end_yr = '2008'
param.ref_timeseries_input = True
param.ref_start_yr = '1002'
param.ref_end_yr = '2008'
prefix = '/compyfs/www/zhan429/run_with_api/'
param.results_dir = os.path.join(prefix, 'area_mean_bad_ref_year')
print(param.results_dir)
attrs_core = vars(param)
print(', '.join("%s: %s" % item for item in attrs_core.items()))
def run_all_sets(html_prefix, d):
param = CoreParameter()
param.reference_data_path = d["obs_climo"]
param.test_data_path = d["test_climo"]
param.test_name = "20161118.beta0.FC5COSP.ne30_ne30.edison"
param.seasons = [
"ANN",
"JJA",
] # Default setting: seasons = ["ANN", "DJF", "MAM", "JJA", "SON"]
param.results_dir = os.path.join(html_prefix, "v2_4_0_all_sets")
param.multiprocessing = True
param.num_workers = 30
# Set specific parameters for new sets
enso_param = EnsoDiagsParameter()
enso_param.reference_data_path = d["obs_ts"]
enso_param.test_data_path = d["test_ts"]
enso_param.test_name = "e3sm_v1"
enso_param.start_yr = "1990"
enso_param.end_yr = "1999"
qbo_param = QboParameter()
qbo_param.reference_data_path = d["obs_ts"]
qbo_param.test_data_path = d["test_ts"]
qbo_param.test_name = "e3sm_v1"
qbo_param.start_yr = "1990"
qbo_param.end_yr = "1999"
ts_param = AreaMeanTimeSeriesParameter()
ts_param.reference_data_path = d["obs_ts"]
ts_param.test_data_path = d["test_ts"]
ts_param.test_name = "e3sm_v1"
ts_param.start_yr = "1990"
ts_param.end_yr = "1999"
dc_param = DiurnalCycleParameter()
dc_param.reference_data_path = d["dc_obs_climo"]
dc_param.test_data_path = d["dc_test_climo"]
dc_param.test_name = "20180215.DECKv1b_H1.ne30_oEC.edison"
dc_param.short_test_name = "DECKv1b_H1.ne30_oEC"
# Plotting diurnal cycle amplitude on different scales. Default is True
dc_param.normalize_test_amp = False
streamflow_param = StreamflowParameter()
streamflow_param.reference_data_path = d["streamflow_obs_ts"]
streamflow_param.test_data_path = d["streamflow_test_ts"]
streamflow_param.test_name = "20180215.DECKv1b_H1.ne30_oEC.edison"
streamflow_param.test_start_yr = "1980"
streamflow_param.test_end_yr = "2014"
# Streamflow gauge station data range from year 1986 to 1995
streamflow_param.ref_start_yr = "1986"
streamflow_param.ref_end_yr = "1995"
arm_param = ARMDiagsParameter()
arm_param.reference_data_path = d["arm_obs"]
arm_param.ref_name = "armdiags"
arm_param.test_data_path = d["arm_test"]
arm_param.test_name = "20210122.F2010.armsites"
arm_param.run_type = "model_vs_obs"
arm_param.test_start_yr = "0001"
arm_param.test_end_yr = "0001"
arm_param.ref_start_yr = "0001"
arm_param.ref_end_yr = "0001"
runner.sets_to_run = [
"lat_lon",
"zonal_mean_xy",
"zonal_mean_2d",
"polar",
"cosp_histogram",
"meridional_mean_2d",
"enso_diags",
"qbo",
"area_mean_time_series",
"diurnal_cycle",
"streamflow",
"arm_diags",
]
runner.run_diags([
param, enso_param, qbo_param, ts_param, dc_param, streamflow_param,
arm_param
])
return param.results_dir
