def _load_dataset(dataset_config_data): """""" if dataset_config_data['data_source'] == 'local': if dataset_config_data['file_count'] > 1: logger.error( 'Multi-file datasets are currently not supported. Cancelling load ' 'of the following dataset: {}'.format(dataset_config_data) ) return None return local.load_file(dataset_config_data['path'], dataset_config_data['variable'], **dataset_config_data.get('optional_args', {})) elif dataset_config_data['data_source'] == 'rcmed': return rcmed.parameter_dataset(dataset_config_data['dataset_id'], dataset_config_data['parameter_id'], dataset_config_data['min_lat'], dataset_config_data['max_lat'], dataset_config_data['min_lon'], dataset_config_data['min_lon'], dataset_config_data['start_time'], dataset_config_data['end_time'], **dataset_config_data.get('optional_args', {})) elif dataset_config_data['data_source'] == 'esgf': return esgf.load_dataset(dataset_config_data['dataset_id'], dataset_config_data['variable'], dataset_config_data['esgf_username'], dataset_config_data['esgf_password'], **dataset_config_data.get('optional_args', {})) elif dataset_config_data['data_source'] == 'dap': return dap.load(dataset_config_data['url'], dataset_config_data['variable'], **dataset_config_data('optional_args', {}))
def load_model_screen(header): '''Generates screen to be able to load model file. Path to model file (netCDF) and variable name is required. :param header: Header of page :type header: string :returns: Notification :rtype: string ''' ready_screen("load_model_screen") screen.addstr(1, 1, header + " > Load Model File ") screen.addstr(4, 2, "Enter model path: ") model_path = screen.getstr() try: netCDF_file = Dataset(model_path, 'r') all_netcdf_variables = [variable.encode() for variable in netCDF_file.variables.keys()] netCDF_file.close() try: screen.addstr(6, 2, "Enter model variable name {0}: ".format(all_netcdf_variables)) variable_name = screen.getstr() model_dataset = load_file(model_path, variable_name) model_datasets.append(model_dataset) models_info.append({ 'directory': model_path, 'variable_name': variable_name }) note = "Model file successfully added." except: note = "WARNING: Model file cannot be added. The variable [{0}] is not accepted. Please try again.".format(variable_name) except: note = "WARNING: Model file cannot be read. Please check the file directory or format. Only netCDF format is accepted." return note
def test_dataset_origin(self): """Test that dataset origin is local.""" dataset = local.load_file(self.file_path, 'value', elevation_index=1) expected_keys = {'source', 'path', 'lat_name', 'lon_name', 'time_name', 'elevation_index'} self.assertEqual(set(dataset.origin.keys()), expected_keys) self.assertEqual(dataset.origin['source'], 'local')
def load_dataset(dataset_id, variable, esgf_username, esgf_password, search_url=DEFAULT_ESGF_SEARCH, elevation_index=0, name='', **additional_constraints): ''' Load an ESGF dataset. :param dataset_id: The ESGF ID of the dataset to load. :type dataset_id: :mod:`string` :param variable: The variable to load. :type variable: :mod:`string` :param esgf_username: ESGF OpenID value to use for authentication. :type esgf_username: :mod:`string` :param esgf_password: ESGF Password to use for authentication. :type esgf_password: :mod:`string` :param search_url: (Optional) The ESGF node to use for searching. Defaults to the Jet Propulsion Laboratory node. :type search_url: :mod:`string` :param elevation_index: (Optional) The elevation level to strip out when loading the dataset using ocw.data_source.local. :type elevation_index: :class:`int` :param name: (Optional) A name for the loaded dataset. :type name: :mod:`string` :param additional_constraints: (Optional) Additional key,value pairs to pass as constraints to the search wrapper. These can be anything found on the ESGF metadata page for a dataset. :returns: A :class:`list` of :class:`dataset.Dataset` contained the requested dataset. If the dataset is stored in multiple files each will be loaded into a separate :class:`dataset.Dataset`. :raises ValueError: If no dataset can be found for the supplied ID and variable, or if the requested dataset is a multi-file dataset. ''' download_data = _get_file_download_data(url=search_url, dataset_id=dataset_id, variable=variable) datasets = [] for url, var in download_data: _download_files([url], esgf_username, esgf_password) datasets.append(local.load_file('/tmp/' + url.split('/')[-1], var, name=name, elevation_index=elevation_index)) return datasets
def test_that_file_contents_are_valid(self): dp.write_netcdf(self.ds, self.file_name) new_ds = local.load_file(self.file_name, self.ds.variable) self.assertEqual(self.ds.variable, new_ds.variable) np.testing.assert_array_equal(self.ds.lats, new_ds.lats) np.testing.assert_array_equal(self.ds.lons, new_ds.lons) np.testing.assert_array_equal(self.ds.times, new_ds.times) np.testing.assert_array_equal(self.ds.values, new_ds.values)
def check_dataset_loading(): ''' Try loading test dataset ''' dataset = None try: file_path = create_netcdf_object() dataset = local.load_file(file_path, variable_name='value') except Exception as e: fail("\nDataset loading") print("The following error occured") print(e) end(dataset) success("\nDataset loading") return dataset
def loadDataset(urlAndVariable, dir=None): '''Load a dataset (variable), returning a Dataset object.''' shortName = None if len(urlAndVariable) == 3: url, variable, shortName = urlAndVariable else: url, variable = urlAndVariable f = retrieveFile(url, dir) ds = local.load_file(f, variable) if shortName is None or shortName == '': shortName = f + '?' + variable ds.name = shortName shape = ds.values.shape if len(shape) == 3: coords = '(time, lat, lon)' elif len(shape) == 2: coords = '(lat, lon)' print('loadDataset: File %s has variable %s with shape %s: %s' % (f, variable, coords, shape), file=sys.stderr) return ds
def _load_local_dataset_object(dataset_info): ''' Create an ocw.dataset.Dataset object from supplied data. .. note: At the moment, data_source.local cannot take advantage of all the supplied variable names. This functionality will be added in the future. However, in the mean time, it is expected that the dataset_info object still contain all the appropriate keys. :param dataset_info: The necessary data to load a local dataset with ocw.data_source.local. Must be of the form: { 'dataset_id': The path to the local file for loading, 'var_name': The variable data to pull from the file, 'lat_name': The latitude variable name, 'lon_name': The longitude variable name, 'time_name': The time variable name 'name': Optional dataset name } :type dataset_info: Dictionary :returns: An ocw.dataset.Dataset object containing the requested information. :raises KeyError: If the required keys aren't present in the dataset_info. :raises ValueError: If data_source.local could not load the requested file. ''' path = dataset_info['dataset_id'] var_name = dataset_info['var_name'] lat_name = dataset_info['lat_name'] lon_name = dataset_info['lon_name'] time_name = dataset_info['time_name'] # If a name is passed for the dataset, use it. Otherwise, use the file name. name = (dataset_info['name'] if 'name' in dataset_info.keys() else path.split('/')[-1]) dataset = local.load_file(path, var_name) dataset.name = name return dataset
""" from __future__ import print_function import numpy as np import numpy.ma as ma import ocw.data_source.local as local import ocw.dataset as ds import ocw.dataset_processor as dsp import ocw.plotter as plotter # data source: https://dx.doi.org/10.6084/m9.figshare.3753321.v1 # AOD_monthly_2000-Mar_2016-FEB_from_MISR_L3_JOINT.nc is publicly available. dataset = local.load_file( 'AOD_monthly_2000-MAR_2016-FEB_from_MISR_L3_JOINT.nc', 'nonabsorbing_ave') # Subset the data for East Asia. Bounds = ds.Bounds(lat_min=20, lat_max=57.7, lon_min=90, lon_max=150) dataset = dsp.subset(dataset, Bounds) # The original dataset includes nonabsorbing AOD values between March 2000 and February 2015. # dsp.temporal_subset will extract data in September-October-November. dataset_SON = dsp.temporal_subset(dataset, month_start=9, month_end=11, average_each_year=True) ny, nx = dataset_SON.values.shape[1:] # multi-year mean aod clim_aod = ma.zeros([3, ny, nx])
def test_function_load_file_lons(self): """To test load_file function for longitudes""" self.assertItemsEqual(local.load_file( self.file_path, "value").lons, self.longitudes)
def test_function_load_file_lons(self): '''To test load_file function for longitudes''' self.assertItemsEqual( local.load_file(self.file_path, "value").lons, self.longitudes)
FILE_1 = "AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax.nc" FILE_2 = "AFRICA_UC-WRF311_CTL_ERAINT_MM_50km-rg_1989-2008_tasmax.nc" # Filename for the output image/plot (without file extension) OUTPUT_PLOT = "wrf_bias_compared_to_knmi" FILE_1_PATH = path.join('/tmp', FILE_1) FILE_2_PATH = path.join('/tmp', FILE_2) if not path.exists(FILE_1_PATH): urlretrieve(FILE_LEADER + FILE_1, FILE_1_PATH) if not path.exists(FILE_2_PATH): urlretrieve(FILE_LEADER + FILE_2, FILE_2_PATH) # Step 1: Load Local NetCDF Files into OCW Dataset Objects. print("Loading %s into an OCW Dataset Object" % (FILE_1_PATH,)) knmi_dataset = local.load_file(FILE_1_PATH, "tasmax") print("KNMI_Dataset.values shape: (times, lats, lons) - %s \n" % (knmi_dataset.values.shape,)) print("Loading %s into an OCW Dataset Object" % (FILE_2_PATH,)) wrf_dataset = local.load_file(FILE_2_PATH, "tasmax") print("WRF_Dataset.values shape: (times, lats, lons) - %s \n" % (wrf_dataset.values.shape,)) # Step 2: Temporally Rebin the Data into an Annual Timestep. print("Temporally Rebinning the Datasets to an Annual Timestep") knmi_dataset = dsp.temporal_rebin(knmi_dataset, temporal_resolution='annual') wrf_dataset = dsp.temporal_rebin(wrf_dataset, temporal_resolution='annual') print("KNMI_Dataset.values shape: %s" % (knmi_dataset.values.shape,)) print("WRF_Dataset.values shape: %s \n\n" % (wrf_dataset.values.shape,))
# One Local Model Files FILE_1 = "AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax.nc" # Filename for the output image/plot (without file extension) OUTPUT_PLOT = "knmi_temporal_std" # Download necessary NetCDF file if needed if path.exists(FILE_1): pass else: urllib.urlretrieve(FILE_LEADER + FILE_1, FILE_1) """ Step 1: Load Local NetCDF File into OCW Dataset Objects """ print("Loading %s into an OCW Dataset Object" % (FILE_1,)) # 'tasmax' is variable name of values knmi_dataset = local.load_file(FILE_1, "tasmax") print("KNMI_Dataset.values shape: (times, lats, lons) - %s \n" % (knmi_dataset.values.shape,)) # Acessing latittudes and longitudes of netCDF file lats = knmi_dataset.lats lons = knmi_dataset.lons """ Step 2: Build a Metric to use for Evaluation - Temporal STD for this example """ # You can build your own metrics, but OCW also ships with some common metrics print("Setting up a Temporal STD metric to use for evaluation") std = metrics.TemporalStdDev() """ Step 3: Create an Evaluation Object using Datasets and our Metric """ # The Evaluation Class Signature is: # Evaluation(reference, targets, metrics, subregions=None)
# Download necessary NetCDF file if not present if not path.exists(FILE_1): print('Downloading %s' % (FILE_LEADER + FILE_1)) urlretrieve(FILE_LEADER + FILE_1, FILE_1) if not path.exists(FILE_2): print('Downloading %s' % (FILE_LEADER + FILE_2)) urlretrieve(FILE_LEADER + FILE_2, FILE_2) if not path.exists(FILE_3): print('Downloading %s' % (FILE_LEADER + FILE_3)) urlretrieve(FILE_LEADER + FILE_3, FILE_3) # Step 1: Load Local NetCDF File into OCW Dataset Objects and store in list target_datasets.append(local.load_file(FILE_1, varName, name='KNMI')) target_datasets.append(local.load_file(FILE_2, varName, name='REGCM')) target_datasets.append(local.load_file(FILE_3, varName, name='UCT')) # Step 2: Fetch an OCW Dataset Object from the data_source.rcmed module print('Working with the rcmed interface to get CRU3.1 Daily Precipitation') # the dataset_id and the parameter id were determined from # https://rcmes.jpl.nasa.gov/content/data-rcmes-database CRU31 = rcmed.parameter_dataset( 10, 37, LAT_MIN, LAT_MAX, LON_MIN, LON_MAX, START, END) # Step 3: Processing datasets so they are the same shape print('Processing datasets so they are the same shape') CRU31 = dsp.water_flux_unit_conversion(CRU31) CRU31 = dsp.normalize_dataset_datetimes(CRU31, 'monthly')
def test_function_load_file_alt_lats(self): """Test load_file function for lats with different variable names.""" np.testing.assert_array_equal( local.load_file(self.file_path, "value", lat_name="alt_lat").lats, self.alt_lats)
4. plotter """ import ocw.dataset as ds import ocw.data_source.local as local import ocw.dataset_processor as dsp import ocw.plotter as plotter import numpy as np import numpy.ma as ma ''' data source: https://dx.doi.org/10.6084/m9.figshare.3753321.v1 AOD_monthly_2000-Mar_2016-FEB_from_MISR_L3_JOINT.nc is publicly available.''' dataset = local.load_file('AOD_monthly_2000-MAR_2016-FEB_from_MISR_L3_JOINT.nc', 'nonabsorbing_ave') ''' Subset the data for East Asia''' Bounds = ds.Bounds(lat_min=20, lat_max=57.7, lon_min=90, lon_max=150) dataset = dsp.subset(dataset, Bounds) '''The original dataset includes nonabsorbing AOD values between March 2000 and February 2015. dsp.temporal_subset will extract data in September-October-November.''' dataset_SON = dsp.temporal_subset( dataset, month_start=9, month_end=11, average_each_year=True) ny, nx = dataset_SON.values.shape[1:] # multi-year mean aod clim_aod = ma.zeros([3, ny, nx]) clim_aod[0, :] = ma.mean(dataset_SON.values, axis=0) # 16-year mean
def test_function_load_file_lons(self): """Test load_file function for longitudes.""" np.testing.assert_array_equal( local.load_file(self.file_path, "value").lons, self.longitudes)
def test_function_load_file_times(self): """Test load_file function for times.""" new_times = datetime.datetime(2001, 1, 1), datetime.datetime( 2001, 2, 1), datetime.datetime(2001, 3, 1) np.testing.assert_array_equal( local.load_file(self.file_path, "value").times, new_times)
def test_load_invalid_file_path(self): """To test load_file an invalid path raises an exception.""" self.invalid_netcdf_path = '/invalid/path' with self.assertRaises(ValueError): local.load_file(file_path=self.invalid_netcdf_path, variable_name='test variable')
def test_custom_dataset_name(self): """Test adding a custom name to a dataset.""" dataset = local.load_file(self.file_path, 'value', name='foo') self.assertEqual(dataset.name, 'foo')
def test_function_load_file_values(self): """Test load_file function for values.""" new_values = self.values[:, 0, :, :] self.assertTrue( np.allclose( local.load_file(self.file_path, "value").values, new_values))
def test_function_load_file_values(self): """To test load_file function for values""" new_values = self.values[:, 0, :, :] self.assertTrue(numpy.allclose(local.load_file( self.file_path, "value").values, new_values))
# Filename for the output image/plot (without file extension) OUTPUT_PLOT = "tasmax_africa_bias_annual" # Download necessary NetCDF file if not present if path.exists(FILE_1): pass else: urllib.urlretrieve(FILE_LEADER + FILE_1, FILE_1) if path.exists(FILE_2): pass else: urllib.urlretrieve(FILE_LEADER + FILE_2, FILE_2) """ Step 1: Load Local NetCDF File into OCW Dataset Objects """ # Load local knmi model data knmi_dataset = local.load_file(FILE_1, "tasmax") knmi_dataset.name = "AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax" wrf311_dataset = local.load_file(FILE_2, "tasmax") wrf311_dataset.name = "AFRICA_UC-WRF311_CTL_ERAINT_MM_50km-rg_1989-2008_tasmax" """ Step 2: Fetch an OCW Dataset Object from the data_source.rcmed module """ print("Working with the rcmed interface to get CRU3.1 Daily-Max Temp") metadata = rcmed.get_parameters_metadata() cru_31 = [m for m in metadata if m['parameter_id'] == "39"][0] """ The RCMED API uses the following function to query, subset and return the raw data from the database: rcmed.parameter_dataset(dataset_id, parameter_id, min_lat, max_lat, min_lon, max_lon, start_time, end_time)
# This way we can easily adjust the time span of the retrievals YEARS = 3 # Two Local Model Files MODEL = "AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax.nc" # Filename for the output image/plot (without file extension) OUTPUT_PLOT = "cru_31_tmax_knmi_africa_bias_full" # Download necessary NetCDF file if not present if path.exists(MODEL): pass else: urllib.urlretrieve(FILE_LEADER + MODEL, MODEL) """ Step 1: Load Local NetCDF File into OCW Dataset Objects """ print("Loading %s into an OCW Dataset Object" % (MODEL,)) knmi_dataset = local.load_file(MODEL, "tasmax") print("KNMI_Dataset.values shape: (times, lats, lons) - %s \n" % (knmi_dataset.values.shape,)) """ Step 2: Fetch an OCW Dataset Object from the data_source.rcmed module """ print("Working with the rcmed interface to get CRU3.1 Daily-Max Temp") metadata = rcmed.get_parameters_metadata() cru_31 = [m for m in metadata if m['parameter_id'] == "39"][0] """ The RCMED API uses the following function to query, subset and return the raw data from the database: rcmed.parameter_dataset(dataset_id, parameter_id, min_lat, max_lat, min_lon, max_lon, start_time, end_time) The first two required params are in the cru_31 variable we defined earlier
def test_function_load_file_alt_lats(self): """Test load_file function for lats with different variable names.""" np.testing.assert_array_equal(local.load_file( self.file_path, "value", lat_name="alt_lat").lats, self.alt_lats)
def test_function_load_file_values(self): '''To test load_file function for values''' new_values = self.values[0, :, :, :] self.assertTrue( numpy.allclose( local.load_file(self.file_path, "value").values, new_values))
def test_function_load_file_lats(self): '''To test load_file function for latitudes''' self.assertItemsEqual(local.load_file(self.file_path, "value").lats, self.latitudes)
def test_function_load_file_times(self): '''To test load_file function for times''' newTimes = datetime.datetime(2001, 01, 01), datetime.datetime( 2001, 02, 01), datetime.datetime(2001, 03, 01) self.assertItemsEqual( local.load_file(self.file_path, "value").times, newTimes)
CORDEX_AF_TAS = "AFRICA_UQAM-CRCM5_CTL_ERAINT_MM_50km_1989-2008_tas.nc" CORDEX_AF_PR = "AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_pr.nc" EU_CORDEX_PR = "pr_EUR-44i_ECMWF-ERAINT_evaluation_r1i1p1_KNMI-RACMO22E_v1_mon.nc" EU_CORDEX_TAS = "tas_EUR-44i_ECMWF-ERAINT_evaluation_r1i1p1_KNMI-RACMO22E_v1_mon.nc" CRU_31_PR = "GLOBAL_CRU_CTL_CRU-TS31_MM_0.5deg_1980-2008_pr.nc" CRU_31_TAS = "GLOBAL_CRU_CTL_CRU-TS31_MM_0.5deg_1980-2008_tas.nc" TRMM_PR = "TRMM_199801-201212_pr.nc" # Configure your evaluation here. The evaluation bounds are determined by # the lat/lon/time values that are set here. If you set the lat/lon values # outside of the range of the datasets' values you will get an error. Your # start/end time values should be in 12 month intervals due to the metrics # being used. # # Africa Evaluation Settings ref_dataset = local.load_file(CORDEX_AF_TAS, "tas") ref_dataset.name = "cordex_af_tas" target_dataset = local.load_file(CRU_31_TAS, "tas") target_dataset.name = "cru_31_tas" LAT_MIN = -40 LAT_MAX = 40 LON_MIN = -20 LON_MAX = 55 START = datetime.datetime(1999, 1, 1) END = datetime.datetime(2000, 12, 1) SEASON_MONTH_START = 1 SEASON_MONTH_END = 12 EVAL_BOUNDS = Bounds(LAT_MIN, LAT_MAX, LON_MIN, LON_MAX, START, END) # Normalize the time values of our datasets so they fall on expected days
month_index = config['month_index'] month_start = month_index[0] month_end = month_index[-1] ref_info = config['reference'] model_info = config['model'] # Filename for the output data/plot (without file extension) OUTPUT = "%s_%s_%s_%s_%s" %(location['name'], ref_info['variable'], model_info['data_name'], ref_info['data_name'],model_info['future']['scenario_name']) print("Processing "+ ref_info['data_name'] + " data") """ Step 1: Load Local NetCDF Files into OCW Dataset Objects """ print("Loading %s into an OCW Dataset Object" % (ref_info['path'],)) ref_dataset = local.load_file(ref_info['path'], ref_info['variable']) print(ref_info['data_name'] +" values shape: (times, lats, lons) - %s \n" % (ref_dataset.values.shape,)) print("Loading %s into an OCW Dataset Object" % (model_info['present']['path'],)) model_dataset_present = local.load_file(model_info['present']['path'], model_info['variable']) print(model_info['data_name'] +" values shape: (times, lats, lons) - %s \n" % (model_dataset_present.values.shape,)) dy = model_dataset_present.spatial_resolution()[0] dx = model_dataset_present.spatial_resolution()[1] model_dataset_future = local.load_file(model_info['future']['path'], model_info['variable']) print(model_info['future']['scenario_name']+':'+model_info['data_name'] +" values shape: (times, lats, lons) - %s \n" % (model_dataset_future.values.shape,)) """ Step 2: Temporal subsetting """ print("Temporal subsetting for the selected month(s)") ref_temporal_subset = dsp.temporal_subset(ref_dataset, month_start, month_end) model_temporal_subset_present = dsp.temporal_subset(model_dataset_present, month_start, month_end)
def test_dataset_origin(self): ds = local.load_file(self.file_path, 'value', elevation_index=1) expected_keys = set(['source', 'path', 'lat_name', 'lon_name', 'time_name', 'elevation_index']) self.assertEqual(set(ds.origin.keys()), expected_keys) self.assertEqual(ds.origin['source'], 'local')
results = [] labels = [] # could just as easily b the names for each subregion region_counter = 0 # Download necessary NetCDF file if not present if not path.exists(FILE_1): urllib.urlretrieve(FILE_LEADER + FILE_1, FILE_1) if not path.exists(FILE_2): urllib.urlretrieve(FILE_LEADER + FILE_2, FILE_2) if not path.exists(FILE_3): urllib.urlretrieve(FILE_LEADER + FILE_3, FILE_3) """ Step 1: Load Local NetCDF File into OCW Dataset Objects and store in list""" target_datasets.append(local.load_file(FILE_1, varName, name="KNMI")) target_datasets.append(local.load_file(FILE_2, varName, name="REGCM")) target_datasets.append(local.load_file(FILE_3, varName, name="UCT")) """ Step 2: Fetch an OCW Dataset Object from the data_source.rcmed module """ print("Working with the rcmed interface to get CRU3.1 Daily Precipitation") # the dataset_id and the parameter id were determined from # https://rcmes.jpl.nasa.gov/content/data-rcmes-database CRU31 = rcmed.parameter_dataset( 10, 37, LAT_MIN, LAT_MAX, LON_MIN, LON_MAX, START, END) """ Step 3: Processing datasets so they are the same shape ... """ print("Processing datasets so they are the same shape") CRU31 = dsp.water_flux_unit_conversion(CRU31)
# Download necessary NetCDF file if not present if path.exists(FILE_1): pass else: urllib.urlretrieve(FILE_LEADER + FILE_1, FILE_1) if path.exists(FILE_2): pass else: urllib.urlretrieve(FILE_LEADER + FILE_2, FILE_2) """ Step 1: Load Local NetCDF File into OCW Dataset Objects """ # Load local knmi model data knmi_dataset = local.load_file(FILE_1, "tasmax") knmi_dataset.name = "AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax" wrf311_dataset = local.load_file(FILE_2, "tasmax") wrf311_dataset.name = "AFRICA_UC-WRF311_CTL_ERAINT_MM_50km-rg_1989-2008_tasmax" """ Step 2: Fetch an OCW Dataset Object from the data_source.rcmed module """ print("Working with the rcmed interface to get CRU3.1 Daily-Max Temp") metadata = rcmed.get_parameters_metadata() cru_31 = [m for m in metadata if m['parameter_id'] == "39"][0] """ The RCMED API uses the following function to query, subset and return the raw data from the database:
min_lon = space_info['min_lon'] max_lon = space_info['max_lon'] """ Step 1: Load the reference data """ ref_data_info = config['datasets']['reference'] ref_lat_name = None ref_lon_name = None if 'latitude_name' in ref_data_info.keys(): ref_lat_name = ref_data_info['latitude_name'] if 'longitude_name' in ref_data_info.keys(): ref_lon_name = ref_data_info['longitude_name'] print 'Loading observation dataset:\n',ref_data_info ref_name = ref_data_info['data_name'] if ref_data_info['data_source'] == 'local': ref_dataset = local.load_file(ref_data_info['path'], ref_data_info['variable'], name=ref_name, lat_name=ref_lat_name, lon_name=ref_lon_name) elif ref_data_info['data_source'] == 'rcmed': ref_dataset = rcmed.parameter_dataset(ref_data_info['dataset_id'], ref_data_info['parameter_id'], min_lat, max_lat, min_lon, max_lon, start_time, end_time) elif ref_data_info['data_source'] == 'ESGF': username=raw_input('Enter your ESGF OpenID:\n') password=raw_input('Enter your ESGF password:\n') ds = esgf.load_dataset(dataset_id = ref_data_info['dataset_id'], variable = ref_data_info['variable'], esgf_username=username, esgf_password=password) ref_dataset = ds[0] else:
def test_custom_dataset_name(self): '''Test adding a custom name to a dataset''' ds = local.load_file(self.file_path, 'value', name='foo') self.assertEqual(ds.name, 'foo')
def test_load_invalid_file_path(self): self.invalid_netcdf_path = '/invalid/path' with self.assertRaises(ValueError): local.load_file(file_path=self.invalid_netcdf_path, variable_name='test variable')
def test_function_load_file_alt_lons(self): """To test load_file function for lons with different variable names""" np.testing.assert_array_equal(local.load_file( self.file_path, "value", lon_name="alt_lon").lons, self.alt_lons)
def test_function_load_file_times(self): """To test load_file function for times""" newTimes = datetime.datetime(2001, 01, 01), datetime.datetime( 2001, 02, 01), datetime.datetime(2001, 03, 01) self.assertItemsEqual(local.load_file( self.file_path, "value").times, newTimes)
def test_function_load_file_alt_times(self): """To test load_file function for times with different variable names""" newTimes = datetime.datetime(2001, 4, 1), datetime.datetime( 2001, 5, 1), datetime.datetime(2001, 6, 1) np.testing.assert_array_equal(local.load_file( self.file_path, "value", time_name="alt_time").times, newTimes)
def test_custom_dataset_name(self): """Test adding a custom name to a dataset""" ds = local.load_file(self.file_path, 'value', name='foo') self.assertEqual(ds.name, 'foo')
target_datasets_ensemble = [] target_datasets = [] allNames = [] # Download necessary NetCDF file if not present if not path.exists(FILE_1): urllib.urlretrieve(FILE_LEADER + FILE_1, FILE_1) if not path.exists(FILE_2): urllib.urlretrieve(FILE_LEADER + FILE_2, FILE_2) if not path.exists(FILE_3): urllib.urlretrieve(FILE_LEADER + FILE_3, FILE_3) # Step 1: Load Local NetCDF File into OCW Dataset Objects and store in list target_datasets.append(local.load_file(FILE_1, varName, name='KNMI')) target_datasets.append(local.load_file(FILE_2, varName, name='REGCM')) target_datasets.append(local.load_file(FILE_3, varName, name='UCT')) # Step 2: Fetch an OCW Dataset Object from the data_source.rcmed module print('Working with the rcmed interface to get CRU3.1 Monthly Mean Precipitation') # the dataset_id and the parameter id were determined from # https://rcmes.jpl.nasa.gov/content/data-rcmes-database CRU31 = rcmed.parameter_dataset( 10, 37, LAT_MIN, LAT_MAX, LON_MIN, LON_MAX, START, END) # Step 3: Processing Datasets so they are the same shape print('Processing datasets ...') CRU31 = dsp.normalize_dataset_datetimes(CRU31, 'monthly') print('... on units') CRU31 = dsp.water_flux_unit_conversion(CRU31)
def test_function_load_file_lons(self): """To test load_file function for longitudes""" np.testing.assert_array_equal(local.load_file( self.file_path, "value").lons, self.longitudes)
def test_function_load_file_times(self): """To test load_file function for times""" newTimes = datetime.datetime(2001, 1, 1), datetime.datetime( 2001, 2, 1), datetime.datetime(2001, 3, 1) np.testing.assert_array_equal(local.load_file( self.file_path, "value").times, newTimes)
target_datasets = [] #list for names for all the datasets allNames = [] # Download necessary NetCDF file if not present if path.exists(FILE_1): pass else: urllib.urlretrieve(FILE_LEADER + FILE_1, FILE_1) if path.exists(FILE_2): pass else: urllib.urlretrieve(FILE_LEADER + FILE_2, FILE_2) """ Step 1: Load Local NetCDF File into OCW Dataset Objects and store in list""" target_datasets.append(local.load_file(FILE_1, varName, name="KNMI")) target_datasets.append(local.load_file(FILE_2, varName, name="UCT")) """ Step 2: Fetch an OCW Dataset Object from the data_source.rcmed module """ print( "Working with the rcmed interface to get CRU3.1 Monthly Mean Precipitation" ) # the dataset_id and the parameter id were determined from # https://rcmes.jpl.nasa.gov/content/data-rcmes-database CRU31 = rcmed.parameter_dataset(10, 37, LAT_MIN, LAT_MAX, LON_MIN, LON_MAX, START, END) """ Step 3: Resample Datasets so they are the same shape """ print("Resampling datasets") CRU31 = dsp.water_flux_unit_conversion(CRU31) CRU31 = dsp.temporal_rebin(CRU31, datetime.timedelta(days=30)) for member, each_target_dataset in enumerate(target_datasets):