Ejemplo n.º 1
0
    def setUp(self):
        self.runner = Run()
        self.core_param = CoreParameter()

        # These parameters don't need to map to real files since we're
        # just testing if the parameters are created correctly.
        self.core_param.reference_data_path = '/this/can/be/whatever/for/'
        self.core_param.reference_data_path += 'testing/obs_for_acme_diags/'
        self.core_param.test_data_path = '/this/can//also/be/whatever/for/testing/'
        self.core_param.test_data_path += 'test_model_data_for_acme_diags/climatology/'
        self.core_param.test_name = 'SomeTimeStamp.beta?.SomeModelThing.ne30_ne30.somemachine'

        self.core_param.results_dir = 'results'
Ejemplo n.º 2
0
# Define data paths for obs
input_prefix = "/global/cfs/cdirs/e3sm/acme_diags"
obs_climo = "/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/climatology"
obs_ts = "/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/time-series"

# Define data paths for test model
test_prefix = "/global/cfs/cdirs/e3sm/zhang40/postprocessing_for_e3sm_diags"
case = "20180215.DECKv1b_H1.ne30_oEC.edison"
casename = case.split(".")[1] + "." + case.split(".")[2]

climo_path = os.path.join(test_prefix, "climo/" + case + "/1980-2014/rgr")
ts_path = os.path.join(test_prefix, "monthly_ts/" + case + "/1980-2014/rgr")
dc_climo_path = os.path.join(test_prefix, "diurnal_climo/" + case + "/1980-2014/rgr")

# Define parameters for core sets: 'lat_lon','zonal_mean_xy', 'zonal_mean_2d', 'polar', 'cosp_histogram', 'meridional_mean_2d'
param = CoreParameter()
param.reference_data_path = obs_climo
param.test_data_path = climo_path
param.test_name = case
param.test_short_name = casename

# Define results path.
prefix = "/global/cfs/cdirs/e3sm/www/zhang40/tutorials/"
param.results_dir = os.path.join(prefix, "run_v230_allsets")

param.multiprocessing = True
param.num_workers = 30
param.seasons = ["ANN", "JJA"]

# Additional parameters:
# Short version of test model name being printed as plot titles
Ejemplo n.º 3
0
def get_parameters(parser=CoreParser()):
    """
    Get the parameters from the parser.
    """
    # A separate parser to just get the args used.
    # The reason it's a separate object than `parser`
    # is so we can parse the known args.
    parser_for_args = CoreParser()
    # The unknown args are _.
    # These are any set-specific args that aren't needed
    # for now, we just want to know what args are used.
    args, _ = parser_for_args.parse_known_args()

    # Below is the legacy way to run this software, pre v2.0.0.
    # There weren't any arguments defined.
    if not any(getattr(args, arg) for arg in vars(args)):
        parser.print_help()
        sys.exit()

    # For when a user runs the software with commands like:
    #    e3sm_diags lat_lon [the other parameters]
    # This use-case is usually ran when the provenance
    # command is copied and pasted from the viewers.
    if args.set_name in SET_TO_PARSER:
        parser = SET_TO_PARSER[args.set_name]()
        parameters = parser.get_parameters(cmd_default_vars=False,
                                           argparse_vals_only=False)

    # The below two clauses are for the legacy way to
    # run this software, pre v2.0.0.
    # Ex: e3sm_diags -p params.py -d diags.cfg
    elif args.parameters and not args.other_parameters:  # -p only
        original_parameter = parser.get_orig_parameters(
            argparse_vals_only=False)

        # Load the default cfg files.
        run_type = getattr(original_parameter, "run_type", "model_vs_obs")
        default_diags_paths = [
            get_default_diags_path(set_name, run_type)
            for set_name in CoreParameter().sets
        ]

        other_parameters = parser.get_other_parameters(
            files_to_open=default_diags_paths, argparse_vals_only=False)

        parameters = parser.get_parameters(
            orig_parameters=original_parameter,
            other_parameters=other_parameters,
            cmd_default_vars=False,
            argparse_vals_only=False,
        )

    else:
        parameters = parser.get_parameters(cmd_default_vars=False,
                                           argparse_vals_only=False)

    parser.check_values_of_params(parameters)

    if not parameters:
        msg = "No parameters were able to be created. Please check your .py "
        msg += "file, and any .cfg files or command line args you're using."
        raise RuntimeError(msg)

    return parameters
Ejemplo n.º 4
0
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter
from acme_diags.run import runner

param = CoreParameter()

#For compy
#machine_path_prefix = '/compyfs/e3sm_diags_data/'
#For cori
#machine_path_prefix = '/global/project/projectdirs/acme/acme_diags'
#for acme1
machine_path_prefix = '/p/user_pub/e3sm/e3sm_diags_data/'

#param.reference_data_path = os.path.join(machine_path_prefix, 'obs_for_e3sm_diags/time-series/')
#param.test_data_path = os.path.join(machine_path_prefix, 'test_model_data_for_acme_diags/time-series/CESM1-CAM5_cmip/')
param.reference_data_path = '/p/user_pub/e3sm/zhang40/e3sm_cmip6_xmls/v1_water_cycle/historical/'
param.test_data_path = '/p/user_pub/e3sm/zhang40/e3sm_cmip6_xmls/v1_water_cycle/historical/r1i1p1f1'
param.test_name = 'e3sm_v1_hist_r1'

#For compy
#prefix = '/compyfs/www/zhan429/doc_examples/'
#For cori
#prefix = '/global/project/projectdirs/acme/www/zhang40/'
#For acme1
prefix = '/var/www/acme/acme-diags/zhang40/tests/'
param.results_dir = os.path.join(prefix, 'area_mean_hist_r1-5_land')
param.multiprocessing = True
param.num_workers = 25

# We're passing in this new object as well, in
Ejemplo n.º 5
0
import os

from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.run import runner

param = CoreParameter()

# Location of the ref data.
param.reference_data_path = (
    "/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/climatology/")
# Name of the ref obs data, used to find the climo files.
param.ref_name = "ceres_ebaf_toa_v2.8"

# Location of the test data.
param.test_data_path = (
    "/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/climatology/")
# Name of the test obs data, used to find the climo files.
param.test_name = "ceres_ebaf_toa_v4.0"

# Name of the folder where the results are stored.
# Change `prefix` to use your directory.
prefix = "/global/cfs/cdirs/e3sm/www/<your directory>/examples"
param.results_dir = os.path.join(prefix, "ex7_obs_vs_obs")
# What plotsets to run the diags on.
param.sets = ["lat_lon"]

# Below are more optional arguments.

# What seasons to run the diags on.
# If not defined, diags is ran on ['ANN', 'DJF', 'MAM', 'JJA', 'SON'].
param.seasons = ["ANN"]
Ejemplo n.º 6
0
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.run import runner

param = CoreParameter()

# Location of the ref data.
param.reference_data_path = '/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/climatology/'
# Name of the ref obs data, used to find the climo files.
param.ref_name = 'ceres_ebaf_toa_v2.8'

# Location of the test data.
param.test_data_path = '/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/climatology/'
# Name of the test obs data, used to find the climo files.
param.test_name = 'ceres_ebaf_toa_v4.0'

# Name of the folder where the results are stored.
# Change `prefix` to use your directory.
#prefix = '/global/cfs/cdirs/e3sm/www/<your directory>/examples'
param.results_dir = os.path.join(prefix, 'ex7_obs_vs_obs')
# What plotsets to run the diags on.
param.sets = ['lat_lon']

# Below are more optional arguments.

# What seasons to run the diags on.
# If not defined, diags is ran on ['ANN', 'DJF', 'MAM', 'JJA', 'SON'].
param.seasons = ['ANN']
# 'mpl' is to create matplotlib plots, 'vcs' is for vcs plots.
param.backend = 'mpl'
Ejemplo n.º 7
0
import os

from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.run import runner

param = CoreParameter()

# Location of the data.
param.test_data_path = "/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/time-series/CESM1-CAM5_cmip/"
param.reference_data_path = (
    "/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/time-series/"
)

# Set this parameter to True.
# By default, e3sm_diags expects the test data to be climo data.
param.test_timeseries_input = True
# Years to slice the test data, base this off the years in the filenames.
param.test_start_yr = "2003"
param.test_end_yr = "2004"

# Set this parameter to True.
# By default, e3sm_diags expects the ref data to be climo data.
param.ref_timeseries_input = True
# Years to slice the ref data, base this off the years in the filenames.
param.ref_start_yr = "2003"
param.ref_end_yr = "2004"

# When running with time-series data, you don't need to specify the name of the data.
# But you should, otherwise nothing is displayed when the test/ref name is needed.
param.short_test_name = "CESM1-CAM5-historical"
# param.short_ref_name = ''
Ejemplo n.º 8
0
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.run import runner

param = CoreParameter()
# Location of the data.
param.test_data_path = '/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/time-series/E3SM_v1'
param.reference_data_path = '/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/time-series/E3SM_v1'

# Set this parameter to True.
# By default, e3sm_diags expects the test data to be climo data.
param.test_timeseries_input = True
# Years to slice the test data, base this off the years in the filenames.
param.test_start_yr = '2011'
param.test_end_yr = '2013'

# Set this parameter to True.
# By default, e3sm_diags expects the ref data to be climo data.
param.ref_timeseries_input = True
# Years to slice the ref data, base this off the years in the filenames.
param.ref_start_yr = '1850'
param.ref_end_yr = '1852'

# When running with time-series data, you don't need to specify the name of the data.
# But you should, otherwise nothing is displayed when the test/ref name is needed.
param.short_test_name = 'historical_H1'
param.short_ref_name = 'historical_H1'

# This parameter modifies the software to accommodate model vs model runs.
# The default setting for run_type is 'model_vs_obs'.
param.run_type = 'model_vs_model'
Ejemplo n.º 9
0
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter
from acme_diags.run import runner

param = CoreParameter()

#For compy
#machine_path_prefix = '/compyfs/e3sm_diags_data/'
#For cori
#machine_path_prefix = '/global/project/projectdirs/acme/acme_diags'
#for acme1
machine_path_prefix = '/p/user_pub/e3sm/e3sm_diags_data/'

#param.reference_data_path = os.path.join(machine_path_prefix, 'obs_for_e3sm_diags/time-series/')
#param.test_data_path = os.path.join(machine_path_prefix, 'test_model_data_for_acme_diags/time-series/CESM1-CAM5_cmip/')
param.reference_data_path = '/p/user_pub/e3sm/zhang40/e3sm_cmip6_xmls/v1_water_cycle/abrupt-4xCO2/'
param.test_data_path = '/p/user_pub/e3sm/zhang40/e3sm_cmip6_xmls/v1_water_cycle/piControl/r1i1p1f1'
param.test_name = 'e3sm_v1_piControl'

#For compy
#prefix = '/compyfs/www/zhan429/doc_examples/'
#For cori
#prefix = '/global/project/projectdirs/acme/www/zhang40/'
#For acme1
#prefix = '/var/www/acme/acme-diags/zhang40/tests/'
#param.results_dir = os.path.join(prefix, 'area_mean_picontrol_r1')
param.multiprocessing = True
param.num_workers = 25

# We're passing in this new object as well, in
Ejemplo n.º 10
0
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter
from acme_diags.run import runner

param = CoreParameter()

#For compy
#machine_path_prefix = '/compyfs/e3sm_diags_data/'
#For cori
#machine_path_prefix = '/global/project/projectdirs/acme/acme_diags'
#for acme1
machine_path_prefix = '/p/user_pub/e3sm/e3sm_diags_data/'

param.reference_data_path = os.path.join(machine_path_prefix,
                                         'obs_for_e3sm_diags/time-series/')
#param.test_data_path = os.path.join(machine_path_prefix, 'test_model_data_for_acme_diags/time-series/CESM1-CAM5_cmip/')
param.test_data_path = '/p/user_pub/e3sm/zhang40/e3sm_cmip6_xmls/v1_water_cycle/historical/r1i1p1f1'
param.test_name = 'e3sm_v1_hist_r1'

#For compy
#prefix = '/compyfs/www/zhan429/doc_examples/'
#For cori
#prefix = '/global/project/projectdirs/acme/www/zhang40/'
#For acme1
prefix = '/var/www/acme/acme-diags/zhang40/tests/'
param.results_dir = os.path.join(prefix, 'area_mean_with_no_obs_hist_r1')
param.multiprocessing = True
param.num_workers = 50

# We're passing in this new object as well, in
Ejemplo n.º 11
0
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.run import runner

param = CoreParameter()

# Location of the ref data.
param.reference_data_path = '/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/climatology/'
# Name of the ref model data, used to find the climo files.
param.ref_name = '20161118.beta0.F1850COSP.ne30_ne30.edison'
# An optional, shorter name to be used instead of the ref_name.
param.short_ref_name = 'Ref: beta0.F1850COSP_ne30'

# Location of the test data.
param.test_data_path = '/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/climatology'
# Name of the test model data, used to find the climo files.
param.test_name = '20161118.beta0.FC5COSP.ne30_ne30.edison'
# An optional, shorter name to be used instead of the test_name.
param.short_test_name = 'Test: beta0_FC5COSP_ne30'

# What plotsets to run the diags on.
param.sets = ['lat_lon']
# Name of the folder where the results are stored.
# Change `prefix` to use your directory.
#prefix = '/global/cfs/cdirs/e3sm/www/<your directory>/examples'
param.results_dir = os.path.join(prefix, 'ex4_model_to_model')
# This parameter modifies the software to accommodate model vs model runs.
# The default setting for run_type is 'model_vs_obs'.
param.run_type = 'model_vs_model'

# Below are more optional arguments.
Ejemplo n.º 12
0
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.run import runner

param = CoreParameter()

# Location of the data.
param.test_data_path = '/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/time-series/CESM1-CAM5_cmip/'
param.reference_data_path = '/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/time-series/'

# Set this parameter to True.
# By default, e3sm_diags expects the test data to be climo data.
param.test_timeseries_input = True
# Years to slice the test data, base this off the years in the filenames.
param.test_start_yr = '2003'
param.test_end_yr = '2004'

# Set this parameter to True.
# By default, e3sm_diags expects the ref data to be climo data.
param.ref_timeseries_input = True
# Years to slice the ref data, base this off the years in the filenames.
param.ref_start_yr = '2003'
param.ref_end_yr = '2004'

# When running with time-series data, you don't need to specify the name of the data.
# But you should, otherwise nothing is displayed when the test/ref name is needed.
param.short_test_name = 'CESM1-CAM5-historical'
#param.short_ref_name = ''

# This parameter modifies the software to accommodate model vs model runs.
# The default setting for run_type is 'model_vs_obs'.
Ejemplo n.º 13
0
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter
from acme_diags.run import runner

param = CoreParameter()

param.reference_data_path = '/compyfs/e3sm_diags_data/obs_for_e3sm_diags/time-series/'
param.test_data_path = '/compyfs/e3sm_diags_data/test_model_data_for_acme_diags/time-series/E3SM_v1/'

param.test_name = 'e3sm_v1'
param.seasons = [
    "ANN"
]  #seasons shouldn't be a parameter for area_mean_time_series, need to investigate while uncomment this line won't work
#param.seasons = []  #seasons shouldn't be a parameter for area_mean_time_series, need to investigate while uncomment this line won't work
param.test_timeseries_input = True
param.test_start_yr = '1998'
param.test_end_yr = '2004'
param.ref_timeseries_input = True
param.ref_start_yr = '1998'
param.ref_end_yr = '2004'

param.multiprocessing = True
param.num_workers = 40
prefix = '/compyfs/www/zhan429/tests'
param.results_dir = os.path.join(prefix, 'multiple_sets_2ts')

# Set specific parameters for new sets
ts_param = AreaMeanTimeSeriesParameter()
ts_param.reference_data_path = '/compyfs/e3sm_diags_data/obs_for_e3sm_diags/time-series/'
ts_param.test_data_path = '/compyfs/e3sm_diags_data/test_model_data_for_acme_diags/time-series/E3SM_v1/'
Ejemplo n.º 14
0
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.run import runner

param = CoreParameter()

#param.reference_data_path = '/p/user_pub/e3sm/zhang40/analysis_data_e3sm_diags/GPCP_OAFLux/climatology'
param.reference_data_path = '/p/user_pub/e3sm/e3sm_diags_data/obs_for_e3sm_diags/climatology'
param.test_data_path = '/p/user_pub/e3sm/e3sm_diags_data/test_model_data_for_acme_diags/climatology'
param.test_name = '20161118.beta0.FC5COSP.ne30_ne30.edison'
#param.seasons = ["ANN"]
param.multiprocessing = True
param.num_workers = 40
prefix = '/var/www/acme/acme-diags/zhang40/tests/'
param.results_dir = os.path.join(prefix, 'all_vars_core_v2')

#
###Set specific parameters for new sets
#ts_param = AreaMeanTimeSeriesParameter()
#ts_param.reference_data_path = '/compyfs/e3sm_diags_data/obs_for_e3sm_diags/time-series/'
#ts_param.test_data_path = '/compyfs/e3sm_diags_data/test_model_data_for_acme_diags/time-series/E3SM_v1/'
#ts_param.test_name = 'e3sm_v1'
#ts_param.start_yr = '2002'
#ts_param.end_yr = '2008'

runner.sets_to_run = ['lat_lon','zonal_mean_xy', 'zonal_mean_2d', 'polar', 'cosp_histogram', 'meridional_mean_2d']#, 'area_mean_time_series']
#runner.run_diags([param, ts_param])

#runner.sets_to_run = ['area_mean_time_series']
#runner.run_diags([param, ts_param])
#runner.sets_to_run = ['lat_lon','zonal_mean_xy']
Ejemplo n.º 15
0
 def __init__(self):
     self.sets_to_run = CoreParameter().sets
     self.parser = CoreParser()
Ejemplo n.º 16
0
 def setUp(self):
     self.parameter = CoreParameter()
Ejemplo n.º 17
0
    def test_complete_run(self):
        # Anvil
        # Run `source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified.sh`
        test_data_prefix = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data'
        ref_data_prefix = '/lcrc/group/e3sm/public_html/diagnostics/observations/Atm'
        html_prefix = '.'

        param = CoreParameter()

        param.reference_data_path = os.path.join(ref_data_prefix,
                                                 'climatology')
        param.test_data_path = os.path.join(test_data_prefix, 'climatology/')
        param.test_name = '20161118.beta0.FC5COSP.ne30_ne30.edison'
        param.seasons = [
            "ANN", "JJA"
        ]  # Default setting: seasons = ["ANN", "DJF", "MAM", "JJA", "SON"]

        param.results_dir = os.path.join(html_prefix, 'tutorial_2020_all_sets')
        param.multiprocessing = True
        param.num_workers = 30

        # Additional parameters:
        #param.short_test_name = 'beta0.FC5COSP.ne30'
        #param.run_type = 'model_vs_model'
        #param.diff_title = 'Difference'
        #param.output_format = ['png']
        #param.output_format_subplot = ['pdf']
        #param.save_netcdf = True

        # Set specific parameters for new sets
        enso_param = EnsoDiagsParameter()
        enso_param.reference_data_path = os.path.join(ref_data_prefix,
                                                      'time-series/')
        enso_param.test_data_path = os.path.join(test_data_prefix,
                                                 'time-series/E3SM_v1/')
        enso_param.test_name = 'e3sm_v1'
        enso_param.start_yr = '1990'
        enso_param.end_yr = '1999'

        qbo_param = QboParameter()
        qbo_param.reference_data_path = os.path.join(ref_data_prefix,
                                                     'time-series/')
        qbo_param.test_data_path = os.path.join(test_data_prefix,
                                                'time-series/E3SM_v1/')
        qbo_param.test_name = 'e3sm_v1'
        qbo_param.start_yr = '1990'
        qbo_param.end_yr = '1999'

        ts_param = AreaMeanTimeSeriesParameter()
        ts_param.reference_data_path = os.path.join(ref_data_prefix,
                                                    'time-series/')
        ts_param.test_data_path = os.path.join(test_data_prefix,
                                               'time-series/E3SM_v1/')
        ts_param.test_name = 'e3sm_v1'
        ts_param.start_yr = '1990'
        ts_param.end_yr = '1999'

        runner.sets_to_run = [
            'lat_lon', 'zonal_mean_xy', 'zonal_mean_2d', 'polar',
            'cosp_histogram', 'meridional_mean_2d', 'enso_diags', 'qbo',
            'area_mean_time_series'
        ]
        runner.run_diags([param, enso_param, qbo_param, ts_param])

        actual_images_dir = param.results_dir
        # The expected_images_file lists all 475 images we expect to compare.
        # It was generated with the following steps:
        # cd /lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/tutorial_2020_all_sets
        # find . -type f -name '*.png' > ../expected_images_complete_run.txt
        expected_images_file = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/expected_images_complete_run.txt'
        expected_images_dir = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/tutorial_2020_all_sets'

        mismatched_images = []

        with open(expected_images_file) as f:
            for line in f:
                image = line.strip('./').strip('\n')
                path_to_actual_png = os.path.join(actual_images_dir, image)
                path_to_expected_png = os.path.join(expected_images_dir, image)

                actual_png = Image.open(path_to_actual_png).convert('RGB')
                expected_png = Image.open(path_to_expected_png).convert('RGB')
                diff = ImageChops.difference(actual_png, expected_png)

                bbox = diff.getbbox()
                if not bbox:
                    # If `diff.getbbox()` is None, then the images are in theory equal
                    self.assertIsNone(diff.getbbox())
                else:
                    # Sometimes, a few pixels will differ, but the two images appear identical.
                    # https://codereview.stackexchange.com/questions/55902/fastest-way-to-count-non-zero-pixels-using-python-and-pillow
                    nonzero_pixels = diff.crop(bbox).point(
                        lambda x: 255
                        if x else 0).convert("L").point(bool).getdata()
                    num_nonzero_pixels = sum(nonzero_pixels)
                    print('\npath_to_actual_png={}'.format(path_to_actual_png))
                    print(
                        'path_to_expected_png={}'.format(path_to_expected_png))
                    print('diff has {} nonzero pixels.'.format(
                        num_nonzero_pixels))
                    width, height = expected_png.size
                    num_pixels = width * height
                    print('total number of pixels={}'.format(num_pixels))
                    fraction = num_nonzero_pixels / num_pixels
                    print('num_nonzero_pixels/num_pixels fraction={}'.format(
                        fraction))
                    # Fraction of mismatched pixels should be less than 0.02%
                    if fraction >= 0.0002:
                        mismatched_images.append(image)

        self.assertEqual(mismatched_images, [])
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter

from acme_diags.run import runner

param = CoreParameter()

#For compy
machine_path_prefix = '/compyfs/e3sm_diags_data/'
#For cori
#machine_path_prefix = '/global/project/projectdirs/acme/acme_diags'

param.test_data_path = os.path.join(
    machine_path_prefix,
    'test_model_data_for_acme_diags/time-series/E3SM_v1_historical_r1/')
#param.reference_data_path = os.path.join(machine_path_prefix, 'obs_for_e3sm_diags/time-series/')
param.reference_data_path = os.path.join(
    machine_path_prefix, 'test_model_data_for_acme_diags/time-series/')
param.test_name = 'e3sm_v1'
param.rune_type = 'model-vs-model'

prefix = '/compyfs/www/zhan429/examples/'
param.results_dir = os.path.join(prefix, 'area_mean_with_models')
param.multiprocessing = True
param.num_workers = 40

# We're passing in this new object as well, in
# addtion to the CoreParameter object.

ts_param = AreaMeanTimeSeriesParameter()
Ejemplo n.º 19
0
# Running the software with the API:
#    python all_sets.py -d all_sets.py
import numpy
from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.enso_diags_parameter import EnsoDiagsParameter
from acme_diags.parameter.meridional_mean_2d_parameter import MeridionalMean2dParameter
from acme_diags.parameter.zonal_mean_2d_parameter import ZonalMean2dParameter
from acme_diags.run import Run

run_object = Run()
param = CoreParameter()
ts_param = AreaMeanTimeSeriesParameter()

m2d_param = MeridionalMean2dParameter()
m2d_param.plevs = [
    200.,
    500.,
]
z2d_param = ZonalMean2dParameter()
z2d_param.plevs = [
    200.,
    300.,
]

enso_param = EnsoDiagsParameter()
enso_param.test_name = 'e3sm_v1'

run_object.run_diags([param, ts_param, m2d_param, z2d_param, enso_param])
Ejemplo n.º 20
0
import os

from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.run import runner

param = CoreParameter()

# Location of the data.
param.reference_data_path = (
    "/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/climatology/")
param.test_data_path = (
    "/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/climatology/"
)
# Name of the test model data, used to find the climo files.
param.test_name = "20161118.beta0.FC5COSP.ne30_ne30.edison"
# An optional, shorter name to be used instead of the test_name.
param.short_test_name = "beta0.FC5COSP.ne30"

# What plotsets to run the diags on.
param.sets = ["lat_lon"]
# Name of the folder where the results are stored.
# Change `prefix` to use your directory.
prefix = "/global/cfs/cdirs/e3sm/www/<your directory>/examples"
param.results_dir = os.path.join(prefix, "ex5_model_to_obs")

# Below are more optional arguments.

# 'mpl' is to create matplotlib plots, 'vcs' is for vcs plots.
param.backend = "mpl"
# Title of the difference plots.
param.diff_title = "Model - Obs."
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter

from acme_diags.run import runner

param = CoreParameter()

#For compy
machine_path_prefix = '/compyfs/e3sm_diags_data/'
#For cori
#machine_path_prefix = '/global/project/projectdirs/acme/acme_diags'

param.reference_data_path = os.path.join(machine_path_prefix,
                                         'obs_for_e3sm_diags/time-series/')
param.test_data_path = os.path.join(
    machine_path_prefix, 'test_model_data_for_acme_diags/time-series/E3SM_v1/')
param.test_name = 'e3sm_v1'
param.output_format = ['png', 'pdf', 'svg']

#prefix = '/compyfs/www/zhan429/examples/'
prefix = '/compyfs/www/zhan429/tests/'

param.results_dir = os.path.join(prefix, 'area_mean_with_obs')
#param.multiprocessing = True
#param.num_workers =  40

# We're passing in this new object as well, in
# addtion to the CoreParameter object.

ts_param = AreaMeanTimeSeriesParameter()
Ejemplo n.º 22
0
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter
from acme_diags.parameter.zonal_mean_2d_parameter import ZonalMean2dParameter

from acme_diags.run import runner

param = CoreParameter()

param.reference_data_path = '/compyfs/e3sm_diags_data/obs_for_e3sm_diags/climatology/'
param.test_data_path = '/compyfs/e3sm_diags_data/test_model_data_for_acme_diags/climatology/'
param.test_name = '20161118.beta0.FC5COSP.ne30_ne30.edison'
param.seasons = ["ANN"]
param.multiprocessing = True
param.num_workers = 40
prefix = '/compyfs/www/zhan429/tests'
param.results_dir = os.path.join(prefix, 'all_sets')

#
##Set specific parameters for new sets
ts_param = AreaMeanTimeSeriesParameter()
ts_param.reference_data_path = '/compyfs/e3sm_diags_data/obs_for_e3sm_diags/time-series/'
ts_param.test_data_path = '/compyfs/e3sm_diags_data/test_model_data_for_acme_diags/time-series/E3SM_v1/'
ts_param.test_name = 'e3sm_v1'
ts_param.start_yr = '2002'
ts_param.end_yr = '2008'

runner.sets_to_run = [
    'lat_lon', 'zonal_mean_xy', 'zonal_mean_2d', 'polar', 'cosp_histogram',
    'meridional_mean_2d', 'area_mean_time_series'
]
Ejemplo n.º 23
0
import os

from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.run import runner

param = CoreParameter()

# Location of the data.
param.test_data_path = "/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/time-series/CESM1-CAM5_cmip/"
param.reference_data_path = "/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/time-series/CESM1-CAM5_cmip"

# Set this parameter to True.
# By default, e3sm_diags expects the test data to be climo data.
param.test_timeseries_input = True
# Years to slice the test data, base this off the years in the filenames.
param.test_start_yr = "2001"
param.test_end_yr = "2003"

# Set this parameter to True.
# By default, e3sm_diags expects the ref data to be climo data.
param.ref_timeseries_input = True
# Years to slice the ref data, base this off the years in the filenames.
param.ref_start_yr = "1850"
param.ref_end_yr = "1852"

# When running with time-series data, you don't need to specify the name of the data.
# But you should, otherwise nothing is displayed when the test/ref name is needed.
param.short_test_name = "CESM1-CAM5-historical"
param.short_ref_name = "CESM1-CAM5-historical"

# This parameter modifies the software to accommodate model vs model runs.
Ejemplo n.º 24
0
import os

from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.zonal_mean_2d_parameter import ZonalMean2dParameter
from acme_diags.run import runner

param = CoreParameter()

param.reference_data_path = (
    "/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/climatology/")
param.test_data_path = (
    "/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/climatology/"
)
param.test_name = "20161118.beta0.FC5COSP.ne30_ne30.edison"
param.seasons = ["ANN"]

# Name of the folder where the results are stored.
# Change `prefix` to use your directory.
prefix = "/global/cfs/cdirs/e3sm/www/<your directory>/examples"
param.results_dir = os.path.join(prefix, "ex6_zonal_mean_2d_and_lat_lon_demo")

# Uncomment the two lines below to just
# run the diags with T and PRECT.
# param.selectors += ['variables']
# param.variables = ['T', 'PRECT']

# The new changes are below.
zonal_mean_2d_param = ZonalMean2dParameter()
zonal_mean_2d_param.plevs = [10.0, 20.0, 30.0]

runner.sets_to_run = ["zonal_mean_2d", "lat_lon"]
Ejemplo n.º 25
0
import os

from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.run import runner

param = CoreParameter()

# Location of the ref data.
param.reference_data_path = (
    "/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/climatology/"
)
# Name of the ref model data, used to find the climo files.
param.ref_name = "20161118.beta0.F1850COSP.ne30_ne30.edison"
# An optional, shorter name to be used instead of the ref_name.
param.short_ref_name = "Ref: beta0.F1850COSP_ne30"

# Location of the test data.
param.test_data_path = (
    "/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/climatology"
)
# Name of the test model data, used to find the climo files.
param.test_name = "20161118.beta0.FC5COSP.ne30_ne30.edison"
# An optional, shorter name to be used instead of the test_name.
param.short_test_name = "Test: beta0_FC5COSP_ne30"

# What plotsets to run the diags on.
param.sets = ["lat_lon"]
# Name of the folder where the results are stored.
# Change `prefix` to use your directory.
prefix = "/global/cfs/cdirs/e3sm/www/<your directory>/examples"
param.results_dir = os.path.join(prefix, "ex4_model_to_model")
Ejemplo n.º 26
0
import os
from acme_diags.run import runner
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter
from acme_diags.parameter.enso_diags_parameter import EnsoDiagsParameter
from acme_diags.parameter.qbo_parameter import QboParameter

# See https://e3sm-project.github.io/e3sm_diags/docs/html/quickguides/quick-guide-general.html
# Compy
#data_prefix = '/compyfs/e3sm_diags_data'
#html_prefix = '/compyfs/www/<username>'  # Change <username>
# Cori
data_prefix = '/global/cfs/cdirs/e3sm/acme_diags'
html_prefix = '/global/cfs/cdirs/e3sm/www/<username>'  # Change <username>

param = CoreParameter()

param.reference_data_path = os.path.join(data_prefix,
                                         'obs_for_e3sm_diags/climatology')
param.test_data_path = os.path.join(
    data_prefix, 'test_model_data_for_acme_diags/climatology/')
param.test_name = '20161118.beta0.FC5COSP.ne30_ne30.edison'
param.seasons = [
    "ANN", "JJA"
]  # Default setting: seasons = ["ANN", "DJF", "MAM", "JJA", "SON"]

param.results_dir = os.path.join(html_prefix, 'tutorial_2020_all_sets')
param.multiprocessing = True
param.num_workers = 30

# Additional parameters:
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter
from acme_diags.run import runner

param = CoreParameter()

#For compy
#machine_path_prefix = '/compyfs/e3sm_diags_data/'
#For cori
#machine_path_prefix = '/global/project/projectdirs/acme/acme_diags'
#for acme1
machine_path_prefix = '/p/user_pub/e3sm/e3sm_diags_data/'

#param.reference_data_path = os.path.join(machine_path_prefix, 'obs_for_e3sm_diags/time-series/')
#param.test_data_path = os.path.join(machine_path_prefix, 'test_model_data_for_acme_diags/time-series/CESM1-CAM5_cmip/')
param.reference_data_path = '/p/user_pub/e3sm/zhang40/e3sm_cmip6_xmls/v1_water_cycle/amip/'
param.test_data_path = '/p/user_pub/e3sm/zhang40/e3sm_cmip6_xmls/v1_water_cycle/amip/r1i1p1f1'
param.test_name = 'e3sm_v1_amip_r1'

#For compy
#prefix = '/compyfs/www/zhan429/doc_examples/'
#For cori
#prefix = '/global/project/projectdirs/acme/www/zhang40/'
#For acme1
#prefix = '/var/www/acme/acme-diags/zhang40/tests/'
#param.results_dir = os.path.join(prefix, 'area_mean_amip_r1-3')
param.multiprocessing = True
param.num_workers =  25

# We're passing in this new object as well, in
Ejemplo n.º 28
0
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.run import runner

param = CoreParameter()

# Location of the data.
param.reference_data_path = '/global/cfs/cdirs/e3sm/acme_diags/obs_for_e3sm_diags/climatology/'
param.test_data_path = '/global/cfs/cdirs/e3sm/acme_diags/test_model_data_for_acme_diags/climatology/'
# Name of the test model data, used to find the climo files.
param.test_name = '20161118.beta0.FC5COSP.ne30_ne30.edison'
# An optional, shorter name to be used instead of the test_name.
param.short_test_name = 'beta0.FC5COSP.ne30'

# What plotsets to run the diags on.
param.sets = ['lat_lon']
# Name of the folder where the results are stored.
# Change `prefix` to use your directory.
#prefix = '/global/cfs/cdirs/e3sm/www/<your directory>/examples'
param.results_dir = os.path.join(prefix, 'ex5_model_to_obs')

# Below are more optional arguments.

# 'mpl' is to create matplotlib plots, 'vcs' is for vcs plots.
param.backend = 'mpl'
# Title of the difference plots.
param.diff_title = 'Model - Obs.'
# Save the netcdf files for each of the ref, test, and diff plot.
param.save_netcdf = True
## For running with multiprocessing.
#param.multiprocessing = True
Ejemplo n.º 29
0
import os
from acme_diags.parameter.core_parameter import CoreParameter
from acme_diags.parameter.area_mean_time_series_parameter import AreaMeanTimeSeriesParameter

from acme_diags.run import runner

param = CoreParameter()

#For compy
machine_path_prefix = '/compyfs/e3sm_diags_data/'
#For cori
#machine_path_prefix = '/global/project/projectdirs/acme/acme_diags'

param.reference_data_path = os.path.join(machine_path_prefix,
                                         'obs_for_e3sm_diags/time-series/')
param.test_data_path = os.path.join(
    machine_path_prefix, 'test_model_data_for_acme_diags/time-series/E3SM_v1/')

param.test_name = 'e3sm_v1'
#param.seasons = ["ANN"]  #seasons shouldn't be a parameter for area_mean_time_series, need to investigate while uncomment this line won't work
param.test_timeseries_input = True
param.test_start_yr = '2002'
param.test_end_yr = '2008'
param.ref_timeseries_input = True
param.ref_start_yr = '1002'
param.ref_end_yr = '2008'
prefix = '/compyfs/www/zhan429/run_with_api/'
param.results_dir = os.path.join(prefix, 'area_mean_bad_ref_year')
print(param.results_dir)
attrs_core = vars(param)
print(', '.join("%s: %s" % item for item in attrs_core.items()))
def run_all_sets(html_prefix, d):
    param = CoreParameter()

    param.reference_data_path = d["obs_climo"]
    param.test_data_path = d["test_climo"]
    param.test_name = "20161118.beta0.FC5COSP.ne30_ne30.edison"
    param.seasons = [
        "ANN",
        "JJA",
    ]  # Default setting: seasons = ["ANN", "DJF", "MAM", "JJA", "SON"]

    param.results_dir = os.path.join(html_prefix, "v2_4_0_all_sets")
    param.multiprocessing = True
    param.num_workers = 30

    # Set specific parameters for new sets
    enso_param = EnsoDiagsParameter()
    enso_param.reference_data_path = d["obs_ts"]
    enso_param.test_data_path = d["test_ts"]
    enso_param.test_name = "e3sm_v1"
    enso_param.start_yr = "1990"
    enso_param.end_yr = "1999"

    qbo_param = QboParameter()
    qbo_param.reference_data_path = d["obs_ts"]
    qbo_param.test_data_path = d["test_ts"]
    qbo_param.test_name = "e3sm_v1"
    qbo_param.start_yr = "1990"
    qbo_param.end_yr = "1999"

    ts_param = AreaMeanTimeSeriesParameter()
    ts_param.reference_data_path = d["obs_ts"]
    ts_param.test_data_path = d["test_ts"]
    ts_param.test_name = "e3sm_v1"
    ts_param.start_yr = "1990"
    ts_param.end_yr = "1999"

    dc_param = DiurnalCycleParameter()
    dc_param.reference_data_path = d["dc_obs_climo"]
    dc_param.test_data_path = d["dc_test_climo"]
    dc_param.test_name = "20180215.DECKv1b_H1.ne30_oEC.edison"
    dc_param.short_test_name = "DECKv1b_H1.ne30_oEC"
    # Plotting diurnal cycle amplitude on different scales. Default is True
    dc_param.normalize_test_amp = False

    streamflow_param = StreamflowParameter()
    streamflow_param.reference_data_path = d["streamflow_obs_ts"]
    streamflow_param.test_data_path = d["streamflow_test_ts"]
    streamflow_param.test_name = "20180215.DECKv1b_H1.ne30_oEC.edison"
    streamflow_param.test_start_yr = "1980"
    streamflow_param.test_end_yr = "2014"
    # Streamflow gauge station data range from year 1986 to 1995
    streamflow_param.ref_start_yr = "1986"
    streamflow_param.ref_end_yr = "1995"

    arm_param = ARMDiagsParameter()
    arm_param.reference_data_path = d["arm_obs"]
    arm_param.ref_name = "armdiags"
    arm_param.test_data_path = d["arm_test"]
    arm_param.test_name = "20210122.F2010.armsites"
    arm_param.run_type = "model_vs_obs"
    arm_param.test_start_yr = "0001"
    arm_param.test_end_yr = "0001"
    arm_param.ref_start_yr = "0001"
    arm_param.ref_end_yr = "0001"

    runner.sets_to_run = [
        "lat_lon",
        "zonal_mean_xy",
        "zonal_mean_2d",
        "polar",
        "cosp_histogram",
        "meridional_mean_2d",
        "enso_diags",
        "qbo",
        "area_mean_time_series",
        "diurnal_cycle",
        "streamflow",
        "arm_diags",
    ]
    runner.run_diags([
        param, enso_param, qbo_param, ts_param, dc_param, streamflow_param,
        arm_param
    ])

    return param.results_dir