Exemple #1
0
def _load_dataset(dataset_config_data):
    """"""
    if dataset_config_data['data_source'] == 'local':
        if dataset_config_data['file_count'] > 1:
            logger.error(
                'Multi-file datasets are currently not supported. Cancelling load '
                'of the following dataset: {}'.format(dataset_config_data)
            )
            return None

        return local.load_file(dataset_config_data['path'],
                               dataset_config_data['variable'],
                               **dataset_config_data.get('optional_args', {}))
    elif dataset_config_data['data_source'] == 'rcmed':
        return rcmed.parameter_dataset(dataset_config_data['dataset_id'],
                                       dataset_config_data['parameter_id'],
                                       dataset_config_data['min_lat'],
                                       dataset_config_data['max_lat'],
                                       dataset_config_data['min_lon'],
                                       dataset_config_data['min_lon'],
                                       dataset_config_data['start_time'],
                                       dataset_config_data['end_time'],
                                       **dataset_config_data.get('optional_args', {}))
    elif dataset_config_data['data_source'] == 'esgf':
        return esgf.load_dataset(dataset_config_data['dataset_id'],
                                 dataset_config_data['variable'],
                                 dataset_config_data['esgf_username'],
                                 dataset_config_data['esgf_password'],
                                 **dataset_config_data.get('optional_args', {}))
    elif dataset_config_data['data_source'] == 'dap':
        return dap.load(dataset_config_data['url'],
                        dataset_config_data['variable'],
                        **dataset_config_data('optional_args', {}))
Exemple #2
0
def load_model_screen(header):
     '''Generates screen to be able to load model file.
     Path to model file (netCDF) and variable name is required.

     :param header: Header of page
     :type header: string

     :returns: Notification
     :rtype: string
     '''

     ready_screen("load_model_screen")
     screen.addstr(1, 1, header + " > Load Model File ")
     screen.addstr(4, 2, "Enter model path: ")
     model_path = screen.getstr()
     try:
          netCDF_file = Dataset(model_path, 'r')
          all_netcdf_variables = [variable.encode() for variable in netCDF_file.variables.keys()]
          netCDF_file.close()
          try:
               screen.addstr(6, 2, "Enter model variable name {0}: ".format(all_netcdf_variables))
               variable_name = screen.getstr()
               model_dataset = load_file(model_path, variable_name)
               model_datasets.append(model_dataset)
               models_info.append({ 'directory': model_path,
                                 'variable_name': variable_name
                              })
               note = "Model file successfully added."
          except:
               note = "WARNING: Model file cannot be added. The variable [{0}] is not accepted. Please try again.".format(variable_name)
     except:
          note = "WARNING: Model file cannot be read. Please check the file directory or format. Only netCDF format is accepted."

     return note
 def test_dataset_origin(self):
     """Test that dataset origin is local."""
     dataset = local.load_file(self.file_path, 'value', elevation_index=1)
     expected_keys = {'source', 'path', 'lat_name', 'lon_name', 'time_name',
                      'elevation_index'}
     self.assertEqual(set(dataset.origin.keys()), expected_keys)
     self.assertEqual(dataset.origin['source'], 'local')
Exemple #4
0
def load_dataset(dataset_id,
                 variable,
                 esgf_username,
                 esgf_password,
                 search_url=DEFAULT_ESGF_SEARCH,
                 elevation_index=0,
                 name='',
                 **additional_constraints):
    ''' Load an ESGF dataset.

    :param dataset_id: The ESGF ID of the dataset to load.
    :type dataset_id: :mod:`string`

    :param variable: The variable to load.
    :type variable: :mod:`string`

    :param esgf_username: ESGF OpenID value to use for authentication.
    :type esgf_username: :mod:`string`

    :param esgf_password: ESGF Password to use for authentication.
    :type esgf_password: :mod:`string`

    :param search_url: (Optional) The ESGF node to use for searching. Defaults
        to the Jet Propulsion Laboratory node.
    :type search_url: :mod:`string`

    :param elevation_index: (Optional) The elevation level to strip out when
        loading the dataset using ocw.data_source.local.
    :type elevation_index: :class:`int`

    :param name: (Optional) A name for the loaded dataset.
    :type name: :mod:`string`

    :param additional_constraints: (Optional) Additional key,value pairs to
        pass as constraints to the search wrapper. These can be anything found
        on the ESGF metadata page for a dataset.

    :returns: A :class:`list` of :class:`dataset.Dataset` contained the
        requested dataset. If the dataset is stored in multiple files each will
        be loaded into a separate :class:`dataset.Dataset`.

    :raises ValueError: If no dataset can be found for the supplied ID and
        variable, or if the requested dataset is a multi-file dataset.
    '''
    download_data = _get_file_download_data(url=search_url,
                                            dataset_id=dataset_id,
                                            variable=variable)

    datasets = []
    for url, var in download_data:
        _download_files([url], esgf_username, esgf_password)
        datasets.append(local.load_file('/tmp/' + url.split('/')[-1],
                                        var,
                                        name=name,
                                        elevation_index=elevation_index))

    return datasets
    def test_that_file_contents_are_valid(self):
        dp.write_netcdf(self.ds, self.file_name)
        new_ds = local.load_file(self.file_name, self.ds.variable)

        self.assertEqual(self.ds.variable, new_ds.variable)
        np.testing.assert_array_equal(self.ds.lats, new_ds.lats)
        np.testing.assert_array_equal(self.ds.lons, new_ds.lons)
        np.testing.assert_array_equal(self.ds.times, new_ds.times)
        np.testing.assert_array_equal(self.ds.values, new_ds.values)
Exemple #6
0
def check_dataset_loading():
    ''' Try loading test dataset '''
    dataset = None
    try:
        file_path = create_netcdf_object()
        dataset = local.load_file(file_path, variable_name='value')
    except Exception as e:
        fail("\nDataset loading")
        print("The following error occured")
        print(e)
        end(dataset)
    success("\nDataset loading")
    return dataset
def loadDataset(urlAndVariable, dir=None):
    '''Load a dataset (variable), returning a Dataset object.'''
    shortName = None
    if len(urlAndVariable) == 3:
        url, variable, shortName = urlAndVariable
    else:
        url, variable = urlAndVariable
    f = retrieveFile(url, dir)
    ds = local.load_file(f, variable)
    if shortName is None or shortName == '':
        shortName = f + '?' + variable
    ds.name = shortName
    shape = ds.values.shape
    if len(shape) == 3:
        coords = '(time, lat, lon)'
    elif len(shape) == 2:
        coords = '(lat, lon)'
    print('loadDataset: File %s has variable %s with shape %s: %s' % (f, variable, coords, shape), file=sys.stderr) 
    return ds
Exemple #8
0
def _load_local_dataset_object(dataset_info):
    ''' Create an ocw.dataset.Dataset object from supplied data.

    .. note: At the moment, data_source.local cannot take advantage of all the
        supplied variable names. This functionality will be added in the future.
        However, in the mean time, it is expected that the dataset_info object
        still contain all the appropriate keys.

    :param dataset_info: The necessary data to load a local dataset with
        ocw.data_source.local. Must be of the form:
        {
            'dataset_id': The path to the local file for loading,
            'var_name': The variable data to pull from the file,
            'lat_name': The latitude variable name,
            'lon_name': The longitude variable name,
            'time_name': The time variable name
            'name': Optional dataset name
        }
    :type dataset_info: Dictionary

    :returns: An ocw.dataset.Dataset object containing the requested information.

    :raises KeyError: If the required keys aren't present in the dataset_info.
    :raises ValueError: If data_source.local could not load the requested file.
    '''
    path = dataset_info['dataset_id']
    var_name = dataset_info['var_name']
    lat_name = dataset_info['lat_name']
    lon_name = dataset_info['lon_name']
    time_name = dataset_info['time_name']
    # If a name is passed for the dataset, use it. Otherwise, use the file name.
    name = (dataset_info['name'] 
            if 'name' in dataset_info.keys() 
            else path.split('/')[-1])

    dataset =  local.load_file(path, var_name)
    dataset.name = name
    return dataset
"""

from __future__ import print_function

import numpy as np
import numpy.ma as ma

import ocw.data_source.local as local
import ocw.dataset as ds
import ocw.dataset_processor as dsp
import ocw.plotter as plotter

# data source: https://dx.doi.org/10.6084/m9.figshare.3753321.v1
#    AOD_monthly_2000-Mar_2016-FEB_from_MISR_L3_JOINT.nc is publicly available.
dataset = local.load_file(
    'AOD_monthly_2000-MAR_2016-FEB_from_MISR_L3_JOINT.nc', 'nonabsorbing_ave')
# Subset the data for East Asia.
Bounds = ds.Bounds(lat_min=20, lat_max=57.7, lon_min=90, lon_max=150)
dataset = dsp.subset(dataset, Bounds)

# The original dataset includes nonabsorbing AOD values between March 2000 and February 2015.
# dsp.temporal_subset will extract data in September-October-November.
dataset_SON = dsp.temporal_subset(dataset,
                                  month_start=9,
                                  month_end=11,
                                  average_each_year=True)

ny, nx = dataset_SON.values.shape[1:]

# multi-year mean aod
clim_aod = ma.zeros([3, ny, nx])
Exemple #10
0
 def test_function_load_file_lons(self):
     """To test load_file function for longitudes"""
     self.assertItemsEqual(local.load_file(
         self.file_path, "value").lons, self.longitudes)
Exemple #11
0
 def test_function_load_file_lons(self):
     '''To test load_file function for longitudes'''
     self.assertItemsEqual(
         local.load_file(self.file_path, "value").lons, self.longitudes)
FILE_1 = "AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax.nc"
FILE_2 = "AFRICA_UC-WRF311_CTL_ERAINT_MM_50km-rg_1989-2008_tasmax.nc"
# Filename for the output image/plot (without file extension)
OUTPUT_PLOT = "wrf_bias_compared_to_knmi"

FILE_1_PATH = path.join('/tmp', FILE_1)
FILE_2_PATH = path.join('/tmp', FILE_2)

if not path.exists(FILE_1_PATH):
    urlretrieve(FILE_LEADER + FILE_1, FILE_1_PATH)
if not path.exists(FILE_2_PATH):
    urlretrieve(FILE_LEADER + FILE_2, FILE_2_PATH)

# Step 1: Load Local NetCDF Files into OCW Dataset Objects.
print("Loading %s into an OCW Dataset Object" % (FILE_1_PATH,))
knmi_dataset = local.load_file(FILE_1_PATH, "tasmax")
print("KNMI_Dataset.values shape: (times, lats, lons) - %s \n" %
      (knmi_dataset.values.shape,))

print("Loading %s into an OCW Dataset Object" % (FILE_2_PATH,))
wrf_dataset = local.load_file(FILE_2_PATH, "tasmax")
print("WRF_Dataset.values shape: (times, lats, lons) - %s \n" %
      (wrf_dataset.values.shape,))

# Step 2: Temporally Rebin the Data into an Annual Timestep.
print("Temporally Rebinning the Datasets to an Annual Timestep")
knmi_dataset = dsp.temporal_rebin(knmi_dataset, temporal_resolution='annual')
wrf_dataset = dsp.temporal_rebin(wrf_dataset, temporal_resolution='annual')
print("KNMI_Dataset.values shape: %s" % (knmi_dataset.values.shape,))
print("WRF_Dataset.values shape: %s \n\n" % (wrf_dataset.values.shape,))
# One Local Model Files
FILE_1 = "AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax.nc"

# Filename for the output image/plot (without file extension)
OUTPUT_PLOT = "knmi_temporal_std"

# Download necessary NetCDF file if needed
if path.exists(FILE_1):
    pass
else:
    urllib.urlretrieve(FILE_LEADER + FILE_1, FILE_1)

""" Step 1: Load Local NetCDF File into OCW Dataset Objects """
print("Loading %s into an OCW Dataset Object" % (FILE_1,))
# 'tasmax' is variable name of values
knmi_dataset = local.load_file(FILE_1, "tasmax")

print("KNMI_Dataset.values shape: (times, lats, lons) - %s \n" % (knmi_dataset.values.shape,))

# Acessing latittudes and longitudes of netCDF file
lats = knmi_dataset.lats
lons = knmi_dataset.lons

""" Step 2:  Build a Metric to use for Evaluation - Temporal STD for this example """
# You can build your own metrics, but OCW also ships with some common metrics
print("Setting up a Temporal STD metric to use for evaluation")
std = metrics.TemporalStdDev()

""" Step 3: Create an Evaluation Object using Datasets and our Metric """
# The Evaluation Class Signature is:
# Evaluation(reference, targets, metrics, subregions=None)
# Download necessary NetCDF file if not present
if not path.exists(FILE_1):
    print('Downloading %s' % (FILE_LEADER + FILE_1))
    urlretrieve(FILE_LEADER + FILE_1, FILE_1)

if not path.exists(FILE_2):
    print('Downloading %s' % (FILE_LEADER + FILE_2))
    urlretrieve(FILE_LEADER + FILE_2, FILE_2)

if not path.exists(FILE_3):
    print('Downloading %s' % (FILE_LEADER + FILE_3))
    urlretrieve(FILE_LEADER + FILE_3, FILE_3)

# Step 1: Load Local NetCDF File into OCW Dataset Objects and store in list
target_datasets.append(local.load_file(FILE_1, varName, name='KNMI'))
target_datasets.append(local.load_file(FILE_2, varName, name='REGCM'))
target_datasets.append(local.load_file(FILE_3, varName, name='UCT'))

# Step 2: Fetch an OCW Dataset Object from the data_source.rcmed module
print('Working with the rcmed interface to get CRU3.1 Daily Precipitation')
# the dataset_id and the parameter id were determined from
# https://rcmes.jpl.nasa.gov/content/data-rcmes-database
CRU31 = rcmed.parameter_dataset(
    10, 37, LAT_MIN, LAT_MAX, LON_MIN, LON_MAX, START, END)

# Step 3: Processing datasets so they are the same shape
print('Processing datasets so they are the same shape')
CRU31 = dsp.water_flux_unit_conversion(CRU31)
CRU31 = dsp.normalize_dataset_datetimes(CRU31, 'monthly')
Exemple #15
0
 def test_function_load_file_alt_lats(self):
     """Test load_file function for lats with different variable names."""
     np.testing.assert_array_equal(
         local.load_file(self.file_path, "value", lat_name="alt_lat").lats,
         self.alt_lats)
    4. plotter

"""

import ocw.dataset as ds
import ocw.data_source.local as local
import ocw.dataset_processor as dsp
import ocw.plotter as plotter

import numpy as np
import numpy.ma as ma


''' data source: https://dx.doi.org/10.6084/m9.figshare.3753321.v1
    AOD_monthly_2000-Mar_2016-FEB_from_MISR_L3_JOINT.nc is publicly available.'''
dataset = local.load_file('AOD_monthly_2000-MAR_2016-FEB_from_MISR_L3_JOINT.nc',
                          'nonabsorbing_ave')
''' Subset the data for East Asia'''
Bounds = ds.Bounds(lat_min=20, lat_max=57.7, lon_min=90, lon_max=150)
dataset = dsp.subset(dataset, Bounds)

'''The original dataset includes nonabsorbing AOD values between March 2000 and February 2015. 
dsp.temporal_subset will extract data in September-October-November.'''
dataset_SON = dsp.temporal_subset(
    dataset, month_start=9, month_end=11, average_each_year=True)

ny, nx = dataset_SON.values.shape[1:]

# multi-year mean aod
clim_aod = ma.zeros([3, ny, nx])

clim_aod[0, :] = ma.mean(dataset_SON.values, axis=0)  # 16-year mean
Exemple #17
0
 def test_function_load_file_lons(self):
     """Test load_file function for longitudes."""
     np.testing.assert_array_equal(
         local.load_file(self.file_path, "value").lons, self.longitudes)
Exemple #18
0
 def test_function_load_file_times(self):
     """Test load_file function for times."""
     new_times = datetime.datetime(2001, 1, 1), datetime.datetime(
         2001, 2, 1), datetime.datetime(2001, 3, 1)
     np.testing.assert_array_equal(
         local.load_file(self.file_path, "value").times, new_times)
Exemple #19
0
 def test_load_invalid_file_path(self):
     """To test load_file an invalid path raises an exception."""
     self.invalid_netcdf_path = '/invalid/path'
     with self.assertRaises(ValueError):
         local.load_file(file_path=self.invalid_netcdf_path,
                         variable_name='test variable')
Exemple #20
0
 def test_custom_dataset_name(self):
     """Test adding a custom name to a dataset."""
     dataset = local.load_file(self.file_path, 'value', name='foo')
     self.assertEqual(dataset.name, 'foo')
Exemple #21
0
 def test_function_load_file_values(self):
     """Test load_file function for values."""
     new_values = self.values[:, 0, :, :]
     self.assertTrue(
         np.allclose(
             local.load_file(self.file_path, "value").values, new_values))
Exemple #22
0
 def test_function_load_file_values(self):
     """To test load_file function for values"""
     new_values = self.values[:, 0, :, :]
     self.assertTrue(numpy.allclose(local.load_file(
         self.file_path, "value").values, new_values))
Exemple #23
0
# Filename for the output image/plot (without file extension)
OUTPUT_PLOT = "tasmax_africa_bias_annual"

# Download necessary NetCDF file if not present
if path.exists(FILE_1):
    pass
else:
    urllib.urlretrieve(FILE_LEADER + FILE_1, FILE_1)

if path.exists(FILE_2):
    pass
else:
    urllib.urlretrieve(FILE_LEADER + FILE_2, FILE_2)
""" Step 1: Load Local NetCDF File into OCW Dataset Objects """
# Load local knmi model data
knmi_dataset = local.load_file(FILE_1, "tasmax")
knmi_dataset.name = "AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax"

wrf311_dataset = local.load_file(FILE_2, "tasmax")
wrf311_dataset.name = "AFRICA_UC-WRF311_CTL_ERAINT_MM_50km-rg_1989-2008_tasmax"
""" Step 2: Fetch an OCW Dataset Object from the data_source.rcmed module """
print("Working with the rcmed interface to get CRU3.1 Daily-Max Temp")
metadata = rcmed.get_parameters_metadata()

cru_31 = [m for m in metadata if m['parameter_id'] == "39"][0]
""" The RCMED API uses the following function to query, subset and return the 
raw data from the database:

rcmed.parameter_dataset(dataset_id, parameter_id, min_lat, max_lat, min_lon, 
                        max_lon, start_time, end_time)
# This way we can easily adjust the time span of the retrievals
YEARS = 3
# Two Local Model Files 
MODEL = "AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax.nc"
# Filename for the output image/plot (without file extension)
OUTPUT_PLOT = "cru_31_tmax_knmi_africa_bias_full"

# Download necessary NetCDF file if not present
if path.exists(MODEL):
    pass
else:
    urllib.urlretrieve(FILE_LEADER + MODEL, MODEL)

""" Step 1: Load Local NetCDF File into OCW Dataset Objects """
print("Loading %s into an OCW Dataset Object" % (MODEL,))
knmi_dataset = local.load_file(MODEL, "tasmax")
print("KNMI_Dataset.values shape: (times, lats, lons) - %s \n" % (knmi_dataset.values.shape,))

""" Step 2: Fetch an OCW Dataset Object from the data_source.rcmed module """
print("Working with the rcmed interface to get CRU3.1 Daily-Max Temp")
metadata = rcmed.get_parameters_metadata()

cru_31 = [m for m in metadata if m['parameter_id'] == "39"][0]

""" The RCMED API uses the following function to query, subset and return the 
raw data from the database:

rcmed.parameter_dataset(dataset_id, parameter_id, min_lat, max_lat, min_lon, 
                        max_lon, start_time, end_time)

The first two required params are in the cru_31 variable we defined earlier
 def test_function_load_file_alt_lats(self):
     """Test load_file function for lats with different variable names."""
     np.testing.assert_array_equal(local.load_file(
         self.file_path, "value", lat_name="alt_lat").lats, self.alt_lats)
Exemple #26
0
 def test_function_load_file_values(self):
     '''To test load_file function for values'''
     new_values = self.values[0, :, :, :]
     self.assertTrue(
         numpy.allclose(
             local.load_file(self.file_path, "value").values, new_values))
Exemple #27
0
 def test_function_load_file_lats(self):
     '''To test load_file function for latitudes'''
     self.assertItemsEqual(local.load_file(self.file_path, "value").lats, self.latitudes)
Exemple #28
0
 def test_function_load_file_times(self):
     '''To test load_file function for times'''
     newTimes = datetime.datetime(2001, 01, 01), datetime.datetime(
         2001, 02, 01), datetime.datetime(2001, 03, 01)
     self.assertItemsEqual(
         local.load_file(self.file_path, "value").times, newTimes)
Exemple #29
0
CORDEX_AF_TAS = "AFRICA_UQAM-CRCM5_CTL_ERAINT_MM_50km_1989-2008_tas.nc"
CORDEX_AF_PR = "AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_pr.nc"
EU_CORDEX_PR = "pr_EUR-44i_ECMWF-ERAINT_evaluation_r1i1p1_KNMI-RACMO22E_v1_mon.nc"
EU_CORDEX_TAS = "tas_EUR-44i_ECMWF-ERAINT_evaluation_r1i1p1_KNMI-RACMO22E_v1_mon.nc"
CRU_31_PR = "GLOBAL_CRU_CTL_CRU-TS31_MM_0.5deg_1980-2008_pr.nc"
CRU_31_TAS = "GLOBAL_CRU_CTL_CRU-TS31_MM_0.5deg_1980-2008_tas.nc"
TRMM_PR = "TRMM_199801-201212_pr.nc"

# Configure your evaluation here. The evaluation bounds are determined by
# the lat/lon/time values that are set here. If you set the lat/lon values
# outside of the range of the datasets' values you will get an error. Your
# start/end time values should be in 12 month intervals due to the metrics
# being used.
#
# Africa Evaluation Settings
ref_dataset = local.load_file(CORDEX_AF_TAS, "tas")
ref_dataset.name = "cordex_af_tas"
target_dataset = local.load_file(CRU_31_TAS, "tas")
target_dataset.name = "cru_31_tas"
LAT_MIN = -40
LAT_MAX = 40
LON_MIN = -20
LON_MAX = 55
START = datetime.datetime(1999, 1, 1)
END = datetime.datetime(2000, 12, 1)
SEASON_MONTH_START = 1
SEASON_MONTH_END = 12

EVAL_BOUNDS = Bounds(LAT_MIN, LAT_MAX, LON_MIN, LON_MAX, START, END)

# Normalize the time values of our datasets so they fall on expected days
month_index = config['month_index']
month_start = month_index[0]
month_end = month_index[-1]    

ref_info = config['reference']
model_info = config['model']

# Filename for the output data/plot (without file extension)
OUTPUT = "%s_%s_%s_%s_%s" %(location['name'], ref_info['variable'], model_info['data_name'], ref_info['data_name'],model_info['future']['scenario_name'])

print("Processing "+ ref_info['data_name'] + "  data")
""" Step 1: Load Local NetCDF Files into OCW Dataset Objects """

print("Loading %s into an OCW Dataset Object" % (ref_info['path'],))
ref_dataset = local.load_file(ref_info['path'], ref_info['variable'])
print(ref_info['data_name'] +" values shape: (times, lats, lons) - %s \n" % (ref_dataset.values.shape,))

print("Loading %s into an OCW Dataset Object" % (model_info['present']['path'],))
model_dataset_present = local.load_file(model_info['present']['path'], model_info['variable'])
print(model_info['data_name'] +" values shape: (times, lats, lons) - %s \n" % (model_dataset_present.values.shape,))
dy = model_dataset_present.spatial_resolution()[0]
dx = model_dataset_present.spatial_resolution()[1]

model_dataset_future = local.load_file(model_info['future']['path'], model_info['variable'])
print(model_info['future']['scenario_name']+':'+model_info['data_name'] +" values shape: (times, lats, lons) - %s \n" % (model_dataset_future.values.shape,))

""" Step 2: Temporal subsetting """
print("Temporal subsetting for the selected month(s)")
ref_temporal_subset = dsp.temporal_subset(ref_dataset, month_start, month_end)
model_temporal_subset_present = dsp.temporal_subset(model_dataset_present, month_start, month_end)
Exemple #31
0
 def test_dataset_origin(self):
     ds = local.load_file(self.file_path, 'value', elevation_index=1)
     expected_keys = set(['source', 'path', 'lat_name', 'lon_name',
                          'time_name', 'elevation_index'])
     self.assertEqual(set(ds.origin.keys()), expected_keys)
     self.assertEqual(ds.origin['source'], 'local')
results = []
labels = []  # could just as easily b the names for each subregion
region_counter = 0

# Download necessary NetCDF file if not present
if not path.exists(FILE_1):
    urllib.urlretrieve(FILE_LEADER + FILE_1, FILE_1)

if not path.exists(FILE_2):
    urllib.urlretrieve(FILE_LEADER + FILE_2, FILE_2)

if not path.exists(FILE_3):
    urllib.urlretrieve(FILE_LEADER + FILE_3, FILE_3)

""" Step 1: Load Local NetCDF File into OCW Dataset Objects and store in list"""
target_datasets.append(local.load_file(FILE_1, varName, name="KNMI"))
target_datasets.append(local.load_file(FILE_2, varName, name="REGCM"))
target_datasets.append(local.load_file(FILE_3, varName, name="UCT"))


""" Step 2: Fetch an OCW Dataset Object from the data_source.rcmed module """
print("Working with the rcmed interface to get CRU3.1 Daily Precipitation")
# the dataset_id and the parameter id were determined from
# https://rcmes.jpl.nasa.gov/content/data-rcmes-database
CRU31 = rcmed.parameter_dataset(
    10, 37, LAT_MIN, LAT_MAX, LON_MIN, LON_MAX, START, END)


""" Step 3: Processing datasets so they are the same shape ... """
print("Processing datasets so they are the same shape")
CRU31 = dsp.water_flux_unit_conversion(CRU31)
# Download necessary NetCDF file if not present
if path.exists(FILE_1):
    pass
else:
    urllib.urlretrieve(FILE_LEADER + FILE_1, FILE_1)

if path.exists(FILE_2):
    pass
else:
    urllib.urlretrieve(FILE_LEADER + FILE_2, FILE_2)


""" Step 1: Load Local NetCDF File into OCW Dataset Objects """
# Load local knmi model data
knmi_dataset = local.load_file(FILE_1, "tasmax")
knmi_dataset.name = "AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax"

wrf311_dataset = local.load_file(FILE_2, "tasmax")
wrf311_dataset.name = "AFRICA_UC-WRF311_CTL_ERAINT_MM_50km-rg_1989-2008_tasmax"



""" Step 2: Fetch an OCW Dataset Object from the data_source.rcmed module """
print("Working with the rcmed interface to get CRU3.1 Daily-Max Temp")
metadata = rcmed.get_parameters_metadata()

cru_31 = [m for m in metadata if m['parameter_id'] == "39"][0]

""" The RCMED API uses the following function to query, subset and return the 
raw data from the database:
Exemple #34
0
min_lon = space_info['min_lon']
max_lon = space_info['max_lon']

""" Step 1: Load the reference data """
ref_data_info = config['datasets']['reference']
ref_lat_name = None
ref_lon_name = None
if 'latitude_name' in ref_data_info.keys():
    ref_lat_name = ref_data_info['latitude_name']
if 'longitude_name' in ref_data_info.keys():
    ref_lon_name = ref_data_info['longitude_name']
print 'Loading observation dataset:\n',ref_data_info
ref_name = ref_data_info['data_name']
if ref_data_info['data_source'] == 'local':
    ref_dataset = local.load_file(ref_data_info['path'],
                                  ref_data_info['variable'], name=ref_name,
                                  lat_name=ref_lat_name, lon_name=ref_lon_name)
elif ref_data_info['data_source'] == 'rcmed':
      ref_dataset = rcmed.parameter_dataset(ref_data_info['dataset_id'],
                                            ref_data_info['parameter_id'],
                                            min_lat, max_lat, min_lon, max_lon,
                                            start_time, end_time)
elif ref_data_info['data_source'] == 'ESGF':
      username=raw_input('Enter your ESGF OpenID:\n')
      password=raw_input('Enter your ESGF password:\n')
      ds = esgf.load_dataset(dataset_id = ref_data_info['dataset_id'],
                             variable = ref_data_info['variable'],
                             esgf_username=username,
                             esgf_password=password)
      ref_dataset = ds[0]
else:
Exemple #35
0
 def test_custom_dataset_name(self):
     '''Test adding a custom name to a dataset'''
     ds = local.load_file(self.file_path, 'value', name='foo')
     self.assertEqual(ds.name, 'foo')
Exemple #36
0
 def test_load_invalid_file_path(self):
     self.invalid_netcdf_path = '/invalid/path'
     with self.assertRaises(ValueError):
         local.load_file(file_path=self.invalid_netcdf_path,
                         variable_name='test variable')
Exemple #37
0
 def test_function_load_file_alt_lons(self):
     """To test load_file function for lons with different variable names"""
     np.testing.assert_array_equal(local.load_file(
         self.file_path, "value", lon_name="alt_lon").lons, self.alt_lons)
Exemple #38
0
 def test_function_load_file_times(self):
     """To test load_file function for times"""
     newTimes = datetime.datetime(2001, 01, 01), datetime.datetime(
         2001, 02, 01), datetime.datetime(2001, 03, 01)
     self.assertItemsEqual(local.load_file(
         self.file_path, "value").times, newTimes)
Exemple #39
0
 def test_function_load_file_alt_times(self):
     """To test load_file function for times with different variable names"""
     newTimes = datetime.datetime(2001, 4, 1), datetime.datetime(
         2001, 5, 1), datetime.datetime(2001, 6, 1)
     np.testing.assert_array_equal(local.load_file(
         self.file_path, "value", time_name="alt_time").times, newTimes)
Exemple #40
0
 def test_custom_dataset_name(self):
     """Test adding a custom name to a dataset"""
     ds = local.load_file(self.file_path, 'value', name='foo')
     self.assertEqual(ds.name, 'foo')
target_datasets_ensemble = []
target_datasets = []
allNames = []

# Download necessary NetCDF file if not present
if not path.exists(FILE_1):
    urllib.urlretrieve(FILE_LEADER + FILE_1, FILE_1)

if not path.exists(FILE_2):
    urllib.urlretrieve(FILE_LEADER + FILE_2, FILE_2)

if not path.exists(FILE_3):
    urllib.urlretrieve(FILE_LEADER + FILE_3, FILE_3)

# Step 1: Load Local NetCDF File into OCW Dataset Objects and store in list
target_datasets.append(local.load_file(FILE_1, varName, name='KNMI'))
target_datasets.append(local.load_file(FILE_2, varName, name='REGCM'))
target_datasets.append(local.load_file(FILE_3, varName, name='UCT'))

# Step 2: Fetch an OCW Dataset Object from the data_source.rcmed module
print('Working with the rcmed interface to get CRU3.1 Monthly Mean Precipitation')
# the dataset_id and the parameter id were determined from
# https://rcmes.jpl.nasa.gov/content/data-rcmes-database
CRU31 = rcmed.parameter_dataset(
    10, 37, LAT_MIN, LAT_MAX, LON_MIN, LON_MAX, START, END)

# Step 3: Processing Datasets so they are the same shape
print('Processing datasets ...')
CRU31 = dsp.normalize_dataset_datetimes(CRU31, 'monthly')
print('... on units')
CRU31 = dsp.water_flux_unit_conversion(CRU31)
# This way we can easily adjust the time span of the retrievals
YEARS = 3
# Two Local Model Files 
MODEL = "AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax.nc"
# Filename for the output image/plot (without file extension)
OUTPUT_PLOT = "cru_31_tmax_knmi_africa_bias_full"

# Download necessary NetCDF file if not present
if path.exists(MODEL):
    pass
else:
    urllib.urlretrieve(FILE_LEADER + MODEL, MODEL)

""" Step 1: Load Local NetCDF File into OCW Dataset Objects """
print("Loading %s into an OCW Dataset Object" % (MODEL,))
knmi_dataset = local.load_file(MODEL, "tasmax")
print("KNMI_Dataset.values shape: (times, lats, lons) - %s \n" % (knmi_dataset.values.shape,))

""" Step 2: Fetch an OCW Dataset Object from the data_source.rcmed module """
print("Working with the rcmed interface to get CRU3.1 Daily-Max Temp")
metadata = rcmed.get_parameters_metadata()

cru_31 = [m for m in metadata if m['parameter_id'] == "39"][0]

""" The RCMED API uses the following function to query, subset and return the 
raw data from the database:

rcmed.parameter_dataset(dataset_id, parameter_id, min_lat, max_lat, min_lon, 
                        max_lon, start_time, end_time)

The first two required params are in the cru_31 variable we defined earlier
Exemple #43
0
 def test_function_load_file_lons(self):
     """To test load_file function for longitudes"""
     np.testing.assert_array_equal(local.load_file(
         self.file_path, "value").lons, self.longitudes)
Exemple #44
0
 def test_function_load_file_times(self):
     """To test load_file function for times"""
     newTimes = datetime.datetime(2001, 1, 1), datetime.datetime(
         2001, 2, 1), datetime.datetime(2001, 3, 1)
     np.testing.assert_array_equal(local.load_file(
         self.file_path, "value").times, newTimes)
Exemple #45
0
target_datasets = []
#list for names for all the datasets
allNames = []

# Download necessary NetCDF file if not present
if path.exists(FILE_1):
    pass
else:
    urllib.urlretrieve(FILE_LEADER + FILE_1, FILE_1)

if path.exists(FILE_2):
    pass
else:
    urllib.urlretrieve(FILE_LEADER + FILE_2, FILE_2)
""" Step 1: Load Local NetCDF File into OCW Dataset Objects and store in list"""
target_datasets.append(local.load_file(FILE_1, varName, name="KNMI"))
target_datasets.append(local.load_file(FILE_2, varName, name="UCT"))
""" Step 2: Fetch an OCW Dataset Object from the data_source.rcmed module """
print(
    "Working with the rcmed interface to get CRU3.1 Monthly Mean Precipitation"
)
# the dataset_id and the parameter id were determined from
# https://rcmes.jpl.nasa.gov/content/data-rcmes-database
CRU31 = rcmed.parameter_dataset(10, 37, LAT_MIN, LAT_MAX, LON_MIN, LON_MAX,
                                START, END)
""" Step 3: Resample Datasets so they are the same shape """
print("Resampling datasets")
CRU31 = dsp.water_flux_unit_conversion(CRU31)
CRU31 = dsp.temporal_rebin(CRU31, datetime.timedelta(days=30))

for member, each_target_dataset in enumerate(target_datasets):