Example #1
0
 def test_load_multiple_files_custom_dataset_name(self):
     """Test adding a custom name to a dataset."""
     dataset = local.load_multiple_files(self.file_path,
                                         "value",
                                         generic_dataset_name=True,
                                         dataset_name=['foo'])
     self.assertEqual(dataset[0].name, 'foo')
Example #2
0
    def test_function_load_multiple_files_times(self):
        """To test load_multiple_files function for times"""
        dataset, data_name = local.load_multiple_files(self.file_path, "value")

        newTimes = datetime.datetime(2001, 01, 01), datetime.datetime(
            2001, 02, 01), datetime.datetime(2001, 03, 01)
        self.assertItemsEqual(dataset[0].times, newTimes)
Example #3
0
    def test_function_load_multiple_files_times(self):
        """To test load_multiple_files function for times"""
        dataset, data_name = local.load_multiple_files(self.file_path, "value")

        newTimes = datetime.datetime(2001, 01, 01), datetime.datetime(
            2001, 02, 01), datetime.datetime(2001, 03, 01)
        self.assertItemsEqual(dataset[0].times, newTimes)
Example #4
0
    def test_function_load_multiple_files_times(self):
        """To test load_multiple_files function for times"""
        dataset = local.load_multiple_files(self.file_path, "value")

        newTimes = datetime.datetime(2001, 1, 1), datetime.datetime(
            2001, 2, 1), datetime.datetime(2001, 3, 1)
        np.testing.assert_array_equal(dataset[0].times, newTimes)
 def test_load_multiple_files_custom_dataset_name(self):
     """Test adding a custom name to a dataset."""
     dataset = local.load_multiple_files(self.file_path,
                                         "value",
                                         generic_dataset_name=True,
                                         dataset_name=['foo'])
     self.assertEqual(dataset[0].name, 'foo')
Example #6
0
    def test_function_load_multiple_files_times(self):
        """Test load_multiple_files function for times."""
        dataset = local.load_multiple_files(self.file_path, "value")

        new_times = datetime.datetime(2001, 1, 1), datetime.datetime(
            2001, 2, 1), datetime.datetime(2001, 3, 1)
        np.testing.assert_array_equal(dataset[0].times, new_times)
Example #7
0
 def test_load_multiple_files_custom_dataset_name(self):
     """Test adding a custom name to a dataset"""
     dataset, data_name = local.load_multiple_files(self.file_path,
                                                    "value",
                                                    dataset_name='foo')
     self.assertEqual(dataset[0].name, 'foo')
Example #8
0
 def test_function_load_multiple_files_values(self):
     """To test load_multiple_files function for values"""
     new_values = self.values[:, 0, :, :]
     dataset, data_name = local.load_multiple_files(self.file_path, "value")
     self.assertTrue(numpy.allclose(dataset[0].values, new_values))
                   us_states=regions[3])
MW_bounds = Bounds(boundary_type='us_states',
                   us_states=regions[4])
NE_bounds = Bounds(boundary_type='us_states',
                   us_states=regions[5])
SE_bounds = Bounds(boundary_type='us_states',
                   us_states=regions[6])

regional_bounds = [NW_bounds, SW_bounds, NGP_bounds,
                   SGP_bounds, MW_bounds, NE_bounds, SE_bounds]

""" Load nClimGrid file into OCW Dataset """
obs_dataset = local.load_file(file_obs, variable_name='tave')

""" Load CMIP5 simulations into a list of OCW Datasets"""
model_dataset = local.load_multiple_files(file_path=model_file_path, variable_name='tas',
                                          dataset_name=dataset_name, variable_unit='K')

""" Temporal subset of obs_dataset """
obs_dataset_subset = dsp.temporal_slice(obs_dataset,
                  start_time=start_date, end_time=end_date)
obs_dataset_season = dsp.temporal_subset(obs_dataset_subset, month_start, month_end,
                      average_each_year=True)

""" Temporal subset of model_dataset """
model_dataset_subset = [dsp.temporal_slice(dataset,start_time=start_date, end_time=end_date)
                        for dataset in model_dataset]
model_dataset_season = [dsp.temporal_subset(dataset, month_start, month_end,
                      average_each_year=True) for dataset in model_dataset_subset]


""" Spatial subset of obs_dataset and generate time series """
Example #10
0
 def test_dataset_origin(self):
     dataset, data_name = local.load_multiple_files(self.file_path, 'value')
     expected_keys = set(['source', 'path', 'lat_name', 'lon_name',
                          'time_name'])
     self.assertEqual(set(dataset[0].origin.keys()), expected_keys)
     self.assertEqual(dataset[0].origin['source'], 'local')
Example #11
0
""" Step 2: Load model NetCDF Files into OCW Dataset Objects """
model_data_info = config['datasets']['targets']
model_lat_name = None
model_lon_name = None
if 'latitude_name' in model_data_info.keys():
    model_lat_name = model_data_info['latitude_name']
if 'longitude_name' in model_data_info.keys():
    model_lon_name = model_data_info['longitude_name']
boundary_check_model = True
if 'GCM_data' in model_data_info.keys():
    if model_data_info['GCM_data']:
        boundary_check_model = False                                           
print 'Loading model datasets:\n',model_data_info
if model_data_info['data_source'] == 'local':
    model_datasets, model_names = local.load_multiple_files(file_path = model_data_info['path'],
                                                            variable_name =model_data_info['variable'], 
                                                            lat_name=model_lat_name, lon_name=model_lon_name)
elif model_data_info['data_source'] == 'ESGF':
      md = esgf.load_dataset(dataset_id=model_data_info['dataset_id'],
                             variable=model_data_info['variable'],
                             esgf_username=username,
                             esgf_password=password)
      model_datasets, model_names = md[0]
else:
    print ' '
    # TO DO: support RCMED
if temporal_resolution == 'daily' or temporal_resolution == 'monthly':
    for idata,dataset in enumerate(model_datasets):
        model_datasets[idata] = dsp.normalize_dataset_datetimes(dataset, temporal_resolution)

""" Step 3: Subset the data for temporal and spatial domain """
Example #12
0
 def test_function_load_multiple_files_lons(self):
     """Test load_multiple_file function for longitudes."""
     dataset = local.load_multiple_files(self.file_path, "value")
     np.testing.assert_array_equal(dataset[0].lons, self.longitudes)
Example #13
0
 def test_function_load_multiple_files_data_name(self):
     """Test load_multiple_file function for dataset name."""
     dataset = local.load_multiple_files(self.file_path, "value")
     self.assertEqual([dataset[0].name], [''])
Example #14
0
 def test_function_load_multiple_files_lons(self):
     """To test load_multiple_file function for longitudes"""
     dataset = local.load_multiple_files(self.file_path, "value")
     np.testing.assert_array_equal(dataset[0].lons, self.longitudes)
Example #15
0
 def test_function_load_multiple_files_data_name(self):
     dataset = local.load_multiple_files(self.file_path, "value")
     self.assertEqual([dataset[0].name], ['data'])
 def test_dataset_origin(self):
     """Test that dataset origin is local."""
     dataset = local.load_multiple_files(self.file_path, 'value')
     expected_keys = {'source', 'path', 'lat_name', 'lon_name', 'time_name'}
     self.assertEqual(set(dataset[0].origin.keys()), expected_keys)
     self.assertEqual(dataset[0].origin['source'], 'local')
Example #17
0
 def test_dataset_origin(self):
     dataset, data_name = local.load_multiple_files(self.file_path, 'value')
     expected_keys = set(
         ['source', 'path', 'lat_name', 'lon_name', 'time_name'])
     self.assertEqual(set(dataset[0].origin.keys()), expected_keys)
     self.assertEqual(dataset[0].origin['source'], 'local')
Example #18
0
 def test_function_load_multiple_files_lons(self):
     """To test load_multiple_file function for longitudes"""
     dataset, data_name = local.load_multiple_files(self.file_path, "value")
     self.assertItemsEqual(dataset[0].lons, self.longitudes)
Example #19
0
                                          min_lat, max_lat, min_lon, max_lon,
                                          start_time, end_time)
else:
    print ' '
    # TO DO: support ESGF

ref_dataset = dsp.normalize_dataset_datetimes(ref_dataset, temporal_resolution)
if 'multiplying_factor' in ref_data_info.keys():
    ref_dataset.values = ref_dataset.values * ref_data_info[
        'multiplying_factor']
""" Step 2: Load model NetCDF Files into OCW Dataset Objects """
model_data_info = config['datasets']['targets']
print 'Loading model datasets:\n', model_data_info
if model_data_info['data_source'] == 'local':
    model_datasets, model_names = local.load_multiple_files(
        file_path=model_data_info['path'],
        variable_name=model_data_info['variable'])
else:
    print ' '
    # TO DO: support RCMED and ESGF
for idata, dataset in enumerate(model_datasets):
    model_datasets[idata] = dsp.normalize_dataset_datetimes(
        dataset, temporal_resolution)
""" Step 3: Subset the data for temporal and spatial domain """
# Create a Bounds object to use for subsetting
if time_info['maximum_overlap_period']:
    start_time, end_time = utils.get_temporal_overlap([ref_dataset] +
                                                      model_datasets)
    print 'Maximum overlap period'
    print 'start_time:', start_time
    print 'end_time:', end_time
Example #20
0
 def test_function_load_multiple_files_values(self):
     """To test load_multiple_files function for values"""
     new_values = self.values[:, 0, :, :]
     dataset, data_name = local.load_multiple_files(
         self.file_path, "value")
     self.assertTrue(numpy.allclose(dataset[0].values, new_values))
SW_bounds = Bounds(boundary_type='us_states', us_states=regions[1])
NGP_bounds = Bounds(boundary_type='us_states', us_states=regions[2])
SGP_bounds = Bounds(boundary_type='us_states', us_states=regions[3])
MW_bounds = Bounds(boundary_type='us_states', us_states=regions[4])
NE_bounds = Bounds(boundary_type='us_states', us_states=regions[5])
SE_bounds = Bounds(boundary_type='us_states', us_states=regions[6])

regional_bounds = [
    NW_bounds, SW_bounds, NGP_bounds, SGP_bounds, MW_bounds, NE_bounds,
    SE_bounds
]
""" Load nClimDiv file into OCW Dataset """
obs_dataset = local.load_file(file_obs, variable_name='tave')
""" Load CMIP5 simulations into a list of OCW Datasets"""
model_dataset = local.load_multiple_files(file_path=model_file_path,
                                          variable_name='tas',
                                          dataset_name=dataset_name,
                                          variable_unit='K')
""" Temporal subset of obs_dataset """
obs_dataset_subset = dsp.temporal_slice(obs_dataset,
                                        start_time=start_date,
                                        end_time=end_date)
obs_dataset_season = dsp.temporal_subset(obs_dataset_subset,
                                         month_start,
                                         month_end,
                                         average_each_year=True)
""" Temporal subset of model_dataset """
model_dataset_subset = [
    dsp.temporal_slice(dataset, start_time=start_date, end_time=end_date)
    for dataset in model_dataset
]
model_dataset_season = [
 def test_function_load_multiple_files_data_name(self):
     """Test load_multiple_file function for dataset name."""
     dataset = local.load_multiple_files(self.file_path, "value")
     self.assertEqual([dataset[0].name], [''])
Example #23
0
 def test_dataset_origin(self):
     """Test that dataset origin is local."""
     dataset = local.load_multiple_files(self.file_path, 'value')
     expected_keys = {'source', 'path', 'lat_name', 'lon_name', 'time_name'}
     self.assertEqual(set(dataset[0].origin.keys()), expected_keys)
     self.assertEqual(dataset[0].origin['source'], 'local')
Example #24
0
 def test_function_load_multiple_files_data_name(self):
     dataset, data_name = local.load_multiple_files(self.file_path, "value")
     self.assertEqual(data_name, ['model'])
Example #25
0
 def test_load_multiple_files_custom_dataset_name(self):
     """Test adding a custom name to a dataset"""
     dataset, data_name = local.load_multiple_files(self.file_path,
                                                    "value",
                                                    dataset_name='foo')
     self.assertEqual(dataset[0].name, 'foo')
Example #26
0
 def test_function_load_multiple_files_lons(self):
     """To test load_multiple_file function for longitudes"""
     dataset, data_name = local.load_multiple_files(self.file_path, "value")
     self.assertItemsEqual(dataset[0].lons, self.longitudes)
Example #27
0
 def test_function_load_multiple_files_data_name(self):
     dataset, data_name = local.load_multiple_files(self.file_path, "value")
     self.assertEqual(data_name, ['model'])
Example #28
0
with time_block(bcdp_results, 'Seasonal Subsetting'):
    ens = ens.select_season(season='DJF')

with time_block(bcdp_results, 'Resampling'):
    ens = ens.resample(freq='Y')

with time_block(bcdp_results, 'Regridding'):
    ens.regrid(backend='scipy', method='linear', output_grid=output_grid)

print(f'BCDP Results: {bcdp_results}')

### OCW SECTION
print('OCW Benchmarks')
ocw_results = {}
with time_block(ocw_results, 'Dataset Loading'):
    datasets = local.load_multiple_files(paths, 'clt')

with time_block(ocw_results, 'Domain Subsetting'):
    for i, ds in enumerate(datasets):
        datasets[i] = dsp.subset(ds, bnds)

with time_block(ocw_results, 'Seasonal Subsetting'):
    for i, ds in enumerate(datasets):
        datasets[i] = dsp.temporal_subset(ds, 9, 11)

with time_block(ocw_results, 'Resampling'):
    for i, ds in enumerate(datasets):
        datasets[i] = dsp.temporal_rebin(ds, 'annual')

with time_block(ocw_results, 'Regridding'):
    for i, ds in enumerate(datasets):