def test_load_multiple_files_custom_dataset_name(self): """Test adding a custom name to a dataset.""" dataset = local.load_multiple_files(self.file_path, "value", generic_dataset_name=True, dataset_name=['foo']) self.assertEqual(dataset[0].name, 'foo')
def test_function_load_multiple_files_times(self): """To test load_multiple_files function for times""" dataset, data_name = local.load_multiple_files(self.file_path, "value") newTimes = datetime.datetime(2001, 01, 01), datetime.datetime( 2001, 02, 01), datetime.datetime(2001, 03, 01) self.assertItemsEqual(dataset[0].times, newTimes)
def test_function_load_multiple_files_times(self): """To test load_multiple_files function for times""" dataset = local.load_multiple_files(self.file_path, "value") newTimes = datetime.datetime(2001, 1, 1), datetime.datetime( 2001, 2, 1), datetime.datetime(2001, 3, 1) np.testing.assert_array_equal(dataset[0].times, newTimes)
def test_function_load_multiple_files_times(self): """Test load_multiple_files function for times.""" dataset = local.load_multiple_files(self.file_path, "value") new_times = datetime.datetime(2001, 1, 1), datetime.datetime( 2001, 2, 1), datetime.datetime(2001, 3, 1) np.testing.assert_array_equal(dataset[0].times, new_times)
def test_load_multiple_files_custom_dataset_name(self): """Test adding a custom name to a dataset""" dataset, data_name = local.load_multiple_files(self.file_path, "value", dataset_name='foo') self.assertEqual(dataset[0].name, 'foo')
def test_function_load_multiple_files_values(self): """To test load_multiple_files function for values""" new_values = self.values[:, 0, :, :] dataset, data_name = local.load_multiple_files(self.file_path, "value") self.assertTrue(numpy.allclose(dataset[0].values, new_values))
us_states=regions[3]) MW_bounds = Bounds(boundary_type='us_states', us_states=regions[4]) NE_bounds = Bounds(boundary_type='us_states', us_states=regions[5]) SE_bounds = Bounds(boundary_type='us_states', us_states=regions[6]) regional_bounds = [NW_bounds, SW_bounds, NGP_bounds, SGP_bounds, MW_bounds, NE_bounds, SE_bounds] """ Load nClimGrid file into OCW Dataset """ obs_dataset = local.load_file(file_obs, variable_name='tave') """ Load CMIP5 simulations into a list of OCW Datasets""" model_dataset = local.load_multiple_files(file_path=model_file_path, variable_name='tas', dataset_name=dataset_name, variable_unit='K') """ Temporal subset of obs_dataset """ obs_dataset_subset = dsp.temporal_slice(obs_dataset, start_time=start_date, end_time=end_date) obs_dataset_season = dsp.temporal_subset(obs_dataset_subset, month_start, month_end, average_each_year=True) """ Temporal subset of model_dataset """ model_dataset_subset = [dsp.temporal_slice(dataset,start_time=start_date, end_time=end_date) for dataset in model_dataset] model_dataset_season = [dsp.temporal_subset(dataset, month_start, month_end, average_each_year=True) for dataset in model_dataset_subset] """ Spatial subset of obs_dataset and generate time series """
def test_dataset_origin(self): dataset, data_name = local.load_multiple_files(self.file_path, 'value') expected_keys = set(['source', 'path', 'lat_name', 'lon_name', 'time_name']) self.assertEqual(set(dataset[0].origin.keys()), expected_keys) self.assertEqual(dataset[0].origin['source'], 'local')
""" Step 2: Load model NetCDF Files into OCW Dataset Objects """ model_data_info = config['datasets']['targets'] model_lat_name = None model_lon_name = None if 'latitude_name' in model_data_info.keys(): model_lat_name = model_data_info['latitude_name'] if 'longitude_name' in model_data_info.keys(): model_lon_name = model_data_info['longitude_name'] boundary_check_model = True if 'GCM_data' in model_data_info.keys(): if model_data_info['GCM_data']: boundary_check_model = False print 'Loading model datasets:\n',model_data_info if model_data_info['data_source'] == 'local': model_datasets, model_names = local.load_multiple_files(file_path = model_data_info['path'], variable_name =model_data_info['variable'], lat_name=model_lat_name, lon_name=model_lon_name) elif model_data_info['data_source'] == 'ESGF': md = esgf.load_dataset(dataset_id=model_data_info['dataset_id'], variable=model_data_info['variable'], esgf_username=username, esgf_password=password) model_datasets, model_names = md[0] else: print ' ' # TO DO: support RCMED if temporal_resolution == 'daily' or temporal_resolution == 'monthly': for idata,dataset in enumerate(model_datasets): model_datasets[idata] = dsp.normalize_dataset_datetimes(dataset, temporal_resolution) """ Step 3: Subset the data for temporal and spatial domain """
def test_function_load_multiple_files_lons(self): """Test load_multiple_file function for longitudes.""" dataset = local.load_multiple_files(self.file_path, "value") np.testing.assert_array_equal(dataset[0].lons, self.longitudes)
def test_function_load_multiple_files_data_name(self): """Test load_multiple_file function for dataset name.""" dataset = local.load_multiple_files(self.file_path, "value") self.assertEqual([dataset[0].name], [''])
def test_function_load_multiple_files_lons(self): """To test load_multiple_file function for longitudes""" dataset = local.load_multiple_files(self.file_path, "value") np.testing.assert_array_equal(dataset[0].lons, self.longitudes)
def test_function_load_multiple_files_data_name(self): dataset = local.load_multiple_files(self.file_path, "value") self.assertEqual([dataset[0].name], ['data'])
def test_dataset_origin(self): """Test that dataset origin is local.""" dataset = local.load_multiple_files(self.file_path, 'value') expected_keys = {'source', 'path', 'lat_name', 'lon_name', 'time_name'} self.assertEqual(set(dataset[0].origin.keys()), expected_keys) self.assertEqual(dataset[0].origin['source'], 'local')
def test_dataset_origin(self): dataset, data_name = local.load_multiple_files(self.file_path, 'value') expected_keys = set( ['source', 'path', 'lat_name', 'lon_name', 'time_name']) self.assertEqual(set(dataset[0].origin.keys()), expected_keys) self.assertEqual(dataset[0].origin['source'], 'local')
def test_function_load_multiple_files_lons(self): """To test load_multiple_file function for longitudes""" dataset, data_name = local.load_multiple_files(self.file_path, "value") self.assertItemsEqual(dataset[0].lons, self.longitudes)
min_lat, max_lat, min_lon, max_lon, start_time, end_time) else: print ' ' # TO DO: support ESGF ref_dataset = dsp.normalize_dataset_datetimes(ref_dataset, temporal_resolution) if 'multiplying_factor' in ref_data_info.keys(): ref_dataset.values = ref_dataset.values * ref_data_info[ 'multiplying_factor'] """ Step 2: Load model NetCDF Files into OCW Dataset Objects """ model_data_info = config['datasets']['targets'] print 'Loading model datasets:\n', model_data_info if model_data_info['data_source'] == 'local': model_datasets, model_names = local.load_multiple_files( file_path=model_data_info['path'], variable_name=model_data_info['variable']) else: print ' ' # TO DO: support RCMED and ESGF for idata, dataset in enumerate(model_datasets): model_datasets[idata] = dsp.normalize_dataset_datetimes( dataset, temporal_resolution) """ Step 3: Subset the data for temporal and spatial domain """ # Create a Bounds object to use for subsetting if time_info['maximum_overlap_period']: start_time, end_time = utils.get_temporal_overlap([ref_dataset] + model_datasets) print 'Maximum overlap period' print 'start_time:', start_time print 'end_time:', end_time
def test_function_load_multiple_files_values(self): """To test load_multiple_files function for values""" new_values = self.values[:, 0, :, :] dataset, data_name = local.load_multiple_files( self.file_path, "value") self.assertTrue(numpy.allclose(dataset[0].values, new_values))
SW_bounds = Bounds(boundary_type='us_states', us_states=regions[1]) NGP_bounds = Bounds(boundary_type='us_states', us_states=regions[2]) SGP_bounds = Bounds(boundary_type='us_states', us_states=regions[3]) MW_bounds = Bounds(boundary_type='us_states', us_states=regions[4]) NE_bounds = Bounds(boundary_type='us_states', us_states=regions[5]) SE_bounds = Bounds(boundary_type='us_states', us_states=regions[6]) regional_bounds = [ NW_bounds, SW_bounds, NGP_bounds, SGP_bounds, MW_bounds, NE_bounds, SE_bounds ] """ Load nClimDiv file into OCW Dataset """ obs_dataset = local.load_file(file_obs, variable_name='tave') """ Load CMIP5 simulations into a list of OCW Datasets""" model_dataset = local.load_multiple_files(file_path=model_file_path, variable_name='tas', dataset_name=dataset_name, variable_unit='K') """ Temporal subset of obs_dataset """ obs_dataset_subset = dsp.temporal_slice(obs_dataset, start_time=start_date, end_time=end_date) obs_dataset_season = dsp.temporal_subset(obs_dataset_subset, month_start, month_end, average_each_year=True) """ Temporal subset of model_dataset """ model_dataset_subset = [ dsp.temporal_slice(dataset, start_time=start_date, end_time=end_date) for dataset in model_dataset ] model_dataset_season = [
def test_function_load_multiple_files_data_name(self): dataset, data_name = local.load_multiple_files(self.file_path, "value") self.assertEqual(data_name, ['model'])
with time_block(bcdp_results, 'Seasonal Subsetting'): ens = ens.select_season(season='DJF') with time_block(bcdp_results, 'Resampling'): ens = ens.resample(freq='Y') with time_block(bcdp_results, 'Regridding'): ens.regrid(backend='scipy', method='linear', output_grid=output_grid) print(f'BCDP Results: {bcdp_results}') ### OCW SECTION print('OCW Benchmarks') ocw_results = {} with time_block(ocw_results, 'Dataset Loading'): datasets = local.load_multiple_files(paths, 'clt') with time_block(ocw_results, 'Domain Subsetting'): for i, ds in enumerate(datasets): datasets[i] = dsp.subset(ds, bnds) with time_block(ocw_results, 'Seasonal Subsetting'): for i, ds in enumerate(datasets): datasets[i] = dsp.temporal_subset(ds, 9, 11) with time_block(ocw_results, 'Resampling'): for i, ds in enumerate(datasets): datasets[i] = dsp.temporal_rebin(ds, 'annual') with time_block(ocw_results, 'Regridding'): for i, ds in enumerate(datasets):