param.results_dir = os.path.join(prefix, 'area_mean_bad_ref_year')
print(param.results_dir)
attrs_core = vars(param)
print(', '.join("%s: %s" % item for item in attrs_core.items()))
print('-------------------------------')
#ts_param = AreaMeanTimeSeriesParameter()
#ts_param.ref_names = ['']
#attrs = vars(ts_param)
#print (', '.join("%s: %s" % item for item in attrs.items()))
#param.multiprocessing = True
#param.num_workers = 40
# We're passing in this new object as well, in
# addtion to the CoreParameter object.
runner.sets_to_run = ['area_mean_time_series']
print('*******************************')
#ts_param = AreaMeanTimeSeriesParameter()
#ts_param.ref_names = ['none']
#attrs = vars(ts_param)
#print (', '.join("%s: %s" % item for item in attrs.items()))
runner.run_diags([param])
#runner.run_diags([ts_param])
#runner.run_diags([param,ts_param])
def test_complete_run(self):
# Anvil
# Run `source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified.sh`
test_data_prefix = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data'
ref_data_prefix = '/lcrc/group/e3sm/public_html/diagnostics/observations/Atm'
html_prefix = '.'
param = CoreParameter()
param.reference_data_path = os.path.join(ref_data_prefix,
'climatology')
param.test_data_path = os.path.join(test_data_prefix, 'climatology/')
param.test_name = '20161118.beta0.FC5COSP.ne30_ne30.edison'
param.seasons = [
"ANN", "JJA"
] # Default setting: seasons = ["ANN", "DJF", "MAM", "JJA", "SON"]
param.results_dir = os.path.join(html_prefix, 'tutorial_2020_all_sets')
param.multiprocessing = True
param.num_workers = 30
# Additional parameters:
#param.short_test_name = 'beta0.FC5COSP.ne30'
#param.run_type = 'model_vs_model'
#param.diff_title = 'Difference'
#param.output_format = ['png']
#param.output_format_subplot = ['pdf']
#param.save_netcdf = True
# Set specific parameters for new sets
enso_param = EnsoDiagsParameter()
enso_param.reference_data_path = os.path.join(ref_data_prefix,
'time-series/')
enso_param.test_data_path = os.path.join(test_data_prefix,
'time-series/E3SM_v1/')
enso_param.test_name = 'e3sm_v1'
enso_param.start_yr = '1990'
enso_param.end_yr = '1999'
qbo_param = QboParameter()
qbo_param.reference_data_path = os.path.join(ref_data_prefix,
'time-series/')
qbo_param.test_data_path = os.path.join(test_data_prefix,
'time-series/E3SM_v1/')
qbo_param.test_name = 'e3sm_v1'
qbo_param.start_yr = '1990'
qbo_param.end_yr = '1999'
ts_param = AreaMeanTimeSeriesParameter()
ts_param.reference_data_path = os.path.join(ref_data_prefix,
'time-series/')
ts_param.test_data_path = os.path.join(test_data_prefix,
'time-series/E3SM_v1/')
ts_param.test_name = 'e3sm_v1'
ts_param.start_yr = '1990'
ts_param.end_yr = '1999'
runner.sets_to_run = [
'lat_lon', 'zonal_mean_xy', 'zonal_mean_2d', 'polar',
'cosp_histogram', 'meridional_mean_2d', 'enso_diags', 'qbo',
'area_mean_time_series'
]
runner.run_diags([param, enso_param, qbo_param, ts_param])
actual_images_dir = param.results_dir
# The expected_images_file lists all 475 images we expect to compare.
# It was generated with the following steps:
# cd /lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/tutorial_2020_all_sets
# find . -type f -name '*.png' > ../expected_images_complete_run.txt
expected_images_file = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/expected_images_complete_run.txt'
expected_images_dir = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/tutorial_2020_all_sets'
mismatched_images = []
with open(expected_images_file) as f:
for line in f:
image = line.strip('./').strip('\n')
path_to_actual_png = os.path.join(actual_images_dir, image)
path_to_expected_png = os.path.join(expected_images_dir, image)
actual_png = Image.open(path_to_actual_png).convert('RGB')
expected_png = Image.open(path_to_expected_png).convert('RGB')
diff = ImageChops.difference(actual_png, expected_png)
bbox = diff.getbbox()
if not bbox:
# If `diff.getbbox()` is None, then the images are in theory equal
self.assertIsNone(diff.getbbox())
else:
# Sometimes, a few pixels will differ, but the two images appear identical.
# https://codereview.stackexchange.com/questions/55902/fastest-way-to-count-non-zero-pixels-using-python-and-pillow
nonzero_pixels = diff.crop(bbox).point(
lambda x: 255
if x else 0).convert("L").point(bool).getdata()
num_nonzero_pixels = sum(nonzero_pixels)
print('\npath_to_actual_png={}'.format(path_to_actual_png))
print(
'path_to_expected_png={}'.format(path_to_expected_png))
print('diff has {} nonzero pixels.'.format(
num_nonzero_pixels))
width, height = expected_png.size
num_pixels = width * height
print('total number of pixels={}'.format(num_pixels))
fraction = num_nonzero_pixels / num_pixels
print('num_nonzero_pixels/num_pixels fraction={}'.format(
fraction))
# Fraction of mismatched pixels should be less than 0.02%
if fraction >= 0.0002:
mismatched_images.append(image)
self.assertEqual(mismatched_images, [])
data_prefix, 'obs_for_e3sm_diags/time-series/')
enso_param.test_data_path = os.path.join(
data_prefix, 'test_model_data_for_acme_diags/time-series/E3SM_v1/')
enso_param.test_name = 'e3sm_v1'
enso_param.start_yr = '1990'
enso_param.end_yr = '1999'
qbo_param = QboParameter()
qbo_param.reference_data_path = os.path.join(
data_prefix, 'obs_for_e3sm_diags/time-series/')
qbo_param.test_data_path = os.path.join(
data_prefix, 'test_model_data_for_acme_diags/time-series/E3SM_v1/')
qbo_param.test_name = 'e3sm_v1'
qbo_param.start_yr = '1990'
qbo_param.end_yr = '1999'
ts_param = AreaMeanTimeSeriesParameter()
ts_param.reference_data_path = os.path.join(data_prefix,
'obs_for_e3sm_diags/time-series/')
ts_param.test_data_path = os.path.join(
data_prefix, 'test_model_data_for_acme_diags/time-series/E3SM_v1/')
ts_param.test_name = 'e3sm_v1'
ts_param.start_yr = '1990'
ts_param.end_yr = '1999'
runner.sets_to_run = [
'lat_lon', 'zonal_mean_xy', 'zonal_mean_2d', 'polar', 'cosp_histogram',
'meridional_mean_2d', 'enso_diags', 'qbo', 'area_mean_time_series'
]
runner.run_diags([param, enso_param, qbo_param, ts_param])
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.zonal_mean_2d_parameter import ZonalMean2dParameter
from acme_diags.run import runner
param = CoreParameter()
param.reference_data_path = (
"/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/climatology/")
param.test_data_path = (
"/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/climatology/"
)
param.test_name = "20161118.beta0.FC5COSP.ne30_ne30.edison"
param.seasons = ["ANN"]
# Name of the folder where the results are stored.
# Change `prefix` to use your directory.
prefix = "/global/cfs/cdirs/e3sm/www/<your directory>/examples"
param.results_dir = os.path.join(prefix, "ex6_zonal_mean_2d_and_lat_lon_demo")
# Uncomment the two lines below to just
# run the diags with T and PRECT.
# param.selectors += ['variables']
# param.variables = ['T', 'PRECT']
# The new changes are below.
zonal_mean_2d_param = ZonalMean2dParameter()
zonal_mean_2d_param.plevs = [10.0, 20.0, 30.0]
runner.sets_to_run = ["zonal_mean_2d", "lat_lon"]
runner.run_diags([param, zonal_mean_2d_param])
ts_param = AreaMeanTimeSeriesParameter()
ts_param.reference_data_path = obs_ts
ts_param.test_data_path = ts_path
ts_param.test_name = casename
ts_param.start_yr = "1980"
ts_param.end_yr = "2014"
streamflow_param = StreamflowParameter()
streamflow_param.reference_data_path = obs_ts
streamflow_param.test_data_path = ts_path
streamflow_param.test_start_yr = "1980"
streamflow_param.test_end_yr = "2014"
# Streamflow gauge station data range from year 1986 to 1995
streamflow_param.ref_start_yr = "1986"
streamflow_param.ref_end_yr = "1995"
#
runner.sets_to_run = [
"lat_lon",
"zonal_mean_xy",
"zonal_mean_2d",
"polar",
"cosp_histogram",
"meridional_mean_2d",
"enso_diags",
"qbo",
"area_mean_time_series",
"diurnal_cycle",
"streamflow",
]
runner.run_diags([param, enso_param, qbo_param, ts_param, dc_param, streamflow_param])
def run_all_sets(html_prefix, d):
param = CoreParameter()
param.reference_data_path = d["obs_climo"]
param.test_data_path = d["test_climo"]
param.test_name = "20161118.beta0.FC5COSP.ne30_ne30.edison"
param.seasons = [
"ANN",
"JJA",
] # Default setting: seasons = ["ANN", "DJF", "MAM", "JJA", "SON"]
param.results_dir = os.path.join(html_prefix, "v2_4_0_all_sets")
param.multiprocessing = True
param.num_workers = 30
# Set specific parameters for new sets
enso_param = EnsoDiagsParameter()
enso_param.reference_data_path = d["obs_ts"]
enso_param.test_data_path = d["test_ts"]
enso_param.test_name = "e3sm_v1"
enso_param.start_yr = "1990"
enso_param.end_yr = "1999"
qbo_param = QboParameter()
qbo_param.reference_data_path = d["obs_ts"]
qbo_param.test_data_path = d["test_ts"]
qbo_param.test_name = "e3sm_v1"
qbo_param.start_yr = "1990"
qbo_param.end_yr = "1999"
ts_param = AreaMeanTimeSeriesParameter()
ts_param.reference_data_path = d["obs_ts"]
ts_param.test_data_path = d["test_ts"]
ts_param.test_name = "e3sm_v1"
ts_param.start_yr = "1990"
ts_param.end_yr = "1999"
dc_param = DiurnalCycleParameter()
dc_param.reference_data_path = d["dc_obs_climo"]
dc_param.test_data_path = d["dc_test_climo"]
dc_param.test_name = "20180215.DECKv1b_H1.ne30_oEC.edison"
dc_param.short_test_name = "DECKv1b_H1.ne30_oEC"
# Plotting diurnal cycle amplitude on different scales. Default is True
dc_param.normalize_test_amp = False
streamflow_param = StreamflowParameter()
streamflow_param.reference_data_path = d["streamflow_obs_ts"]
streamflow_param.test_data_path = d["streamflow_test_ts"]
streamflow_param.test_name = "20180215.DECKv1b_H1.ne30_oEC.edison"
streamflow_param.test_start_yr = "1980"
streamflow_param.test_end_yr = "2014"
# Streamflow gauge station data range from year 1986 to 1995
streamflow_param.ref_start_yr = "1986"
streamflow_param.ref_end_yr = "1995"
arm_param = ARMDiagsParameter()
arm_param.reference_data_path = d["arm_obs"]
arm_param.ref_name = "armdiags"
arm_param.test_data_path = d["arm_test"]
arm_param.test_name = "20210122.F2010.armsites"
arm_param.run_type = "model_vs_obs"
arm_param.test_start_yr = "0001"
arm_param.test_end_yr = "0001"
arm_param.ref_start_yr = "0001"
arm_param.ref_end_yr = "0001"
runner.sets_to_run = [
"lat_lon",
"zonal_mean_xy",
"zonal_mean_2d",
"polar",
"cosp_histogram",
"meridional_mean_2d",
"enso_diags",
"qbo",
"area_mean_time_series",
"diurnal_cycle",
"streamflow",
"arm_diags",
]
runner.run_diags([
param, enso_param, qbo_param, ts_param, dc_param, streamflow_param,
arm_param
])
return param.results_dir
