def test_startMonth_greater_than_endMonth(self):
self.dataset_times = np.array([datetime.datetime(year, month, 1)
for year in range(2000, 2010)
for month in [1, 8, 9, 10, 11, 12]])
self.tempSubset = dp.temporal_subset(self.ten_year_dataset, 8, 1)
np.testing.assert_array_equal(
self.dataset_times, self.tempSubset.times)
def test_startMonth_greater_than_endMonth(self):
self.dataset_times = np.array([datetime.datetime(year, month, 1)
for year in range(2000, 2010)
for month in [1, 8, 9, 10, 11, 12]])
self.tempSubset = dp.temporal_subset(self.ten_year_dataset, 8, 1)
np.testing.assert_array_equal(
self.dataset_times, self.tempSubset.times)
def test_returned_dataset(self):
self.dataset_times = np.array([datetime.datetime(year, month, 1)
for year in range(2000, 2010)
for month in range(1, 6)])
self.tempSubset = dp.temporal_subset(self.ten_year_dataset, 1, 5)
np.testing.assert_array_equal(
self.dataset_times, self.tempSubset.times)
def test_returned_dataset(self):
self.dataset_times = np.array([datetime.datetime(year, month, 1)
for year in range(2000, 2010)
for month in range(1, 6)])
self.tempSubset = dp.temporal_subset(self.ten_year_dataset, 1, 5)
np.testing.assert_array_equal(
self.dataset_times, self.tempSubset.times)
def test_temporal_subset_equal_start_end_month(self):
self.dataset_times = np.array([datetime.datetime(year, 1, 1)
for year in range(2000, 2010)])
self.tempSubset = dp.temporal_subset(self.ten_year_dataset,
1, 1,
average_each_year=True)
np.testing.assert_array_equal(self.dataset_times,
self.tempSubset.times)
def test_temporal_subset_equal_start_end_month(self):
self.dataset_times = np.array([datetime.datetime(year, 1, 1)
for year in range(2000, 2010)])
self.tempSubset = dp.temporal_subset(self.ten_year_dataset,
1, 1,
average_each_year=True)
np.testing.assert_array_equal(self.dataset_times,
self.tempSubset.times)
def test_temporal_subset_with_average_values(self):
self.tempSubset = dp.temporal_subset(self.ten_year_dataset,
1, 3,
average_each_year=True)
self.dataset_values = np.ones([len(self.tempSubset.times),
len(self.ten_year_dataset.lats),
len(self.ten_year_dataset.lons)])
np.testing.assert_array_equal(self.dataset_values,
self.tempSubset.values)
def test_temporal_subset_with_average_values(self):
self.tempSubset = dp.temporal_subset(self.ten_year_dataset,
1, 3,
average_each_year=True)
self.dataset_values = np.ones([len(self.tempSubset.times),
len(self.ten_year_dataset.lats),
len(self.ten_year_dataset.lons)])
np.testing.assert_array_equal(self.dataset_values,
self.tempSubset.values)
def test_temporal_subset_attributes(self):
self.tempSubset = dp.temporal_subset(self.ten_year_dataset,
1, 3,
average_each_year=True)
self.assertEqual(self.tempSubset.name, self.ten_year_dataset.name)
self.assertEqual(self.tempSubset.variable,
self.ten_year_dataset.variable)
self.assertEqual(self.tempSubset.units, self.ten_year_dataset.units)
np.testing.assert_array_equal(self.tempSubset.lats,
self.ten_year_dataset.lats)
np.testing.assert_array_equal(self.tempSubset.lons,
self.ten_year_dataset.lons)
def test_temporal_subset_attributes(self):
self.tempSubset = dp.temporal_subset(self.ten_year_dataset,
1, 3,
average_each_year=True)
self.assertEqual(self.tempSubset.name, self.ten_year_dataset.name)
self.assertEqual(self.tempSubset.variable,
self.ten_year_dataset.variable)
self.assertEqual(self.tempSubset.units, self.ten_year_dataset.units)
np.testing.assert_array_equal(self.tempSubset.lats,
self.ten_year_dataset.lats)
np.testing.assert_array_equal(self.tempSubset.lons,
self.ten_year_dataset.lons)
us_states=regions[6])
regional_bounds = [NW_bounds, SW_bounds, NGP_bounds,
SGP_bounds, MW_bounds, NE_bounds, SE_bounds]
""" Load nClimGrid file into OCW Dataset """
obs_dataset = local.load_file(file_obs, variable_name='tave')
""" Load CMIP5 simulations into a list of OCW Datasets"""
model_dataset = local.load_multiple_files(file_path=model_file_path, variable_name='tas',
dataset_name=dataset_name, variable_unit='K')
""" Temporal subset of obs_dataset """
obs_dataset_subset = dsp.temporal_slice(obs_dataset,
start_time=start_date, end_time=end_date)
obs_dataset_season = dsp.temporal_subset(obs_dataset_subset, month_start, month_end,
average_each_year=True)
""" Temporal subset of model_dataset """
model_dataset_subset = [dsp.temporal_slice(dataset,start_time=start_date, end_time=end_date)
for dataset in model_dataset]
model_dataset_season = [dsp.temporal_subset(dataset, month_start, month_end,
average_each_year=True) for dataset in model_dataset_subset]
""" Spatial subset of obs_dataset and generate time series """
obs_timeseries = np.zeros([nyear, n_region]) # region index 0-6: NW, SW, NGP, SGP, MW, NE, SE
model_timeseries = np.zeros([nmodel, nyear, n_region])
for iregion in np.arange(n_region):
obs_timeseries[:, iregion] = utils.calc_time_series(
dsp.subset(obs_dataset_season, regional_bounds[iregion]))
model_info['variable'])
print(
model_info['data_name'] + " values shape: (times, lats, lons) - %s \n" %
(model_dataset_present.values.shape, ))
dy = model_dataset_present.spatial_resolution()[0]
dx = model_dataset_present.spatial_resolution()[1]
model_dataset_future = local.load_file(model_info['future']['path'],
model_info['variable'])
print(
model_info['future']['scenario_name'] + ':' + model_info['data_name'] +
" values shape: (times, lats, lons) - %s \n" %
(model_dataset_future.values.shape, ))
""" Step 2: Temporal subsetting """
print("Temporal subsetting for the selected month(s)")
ref_temporal_subset = dsp.temporal_subset(ref_dataset, month_start, month_end)
model_temporal_subset_present = dsp.temporal_subset(model_dataset_present,
month_start, month_end)
model_temporal_subset_future = dsp.temporal_subset(model_dataset_future,
month_start, month_end)
""" Step 3: Spatial aggregation of observational data into the model grid """
print(
"Spatial aggregation of observational data near latitude %0.2f and longitude %0.2f "
% (grid_lat, grid_lon))
# There are two options to aggregate observational data near a model grid point
#ref_subset = spatial_aggregation(ref_temporal_subset, grid_lon-0.5*dx, grid_lon+0.5*dx, grid_lat-0.5*dy, grid_lat+0.5*dy)
#model_subset_present = spatial_aggregation(model_temporal_subset_present, grid_lon-0.5*dx, grid_lon+0.5*dx, grid_lat-0.5*dy, grid_lat+0.5*dy)
#model_subset_future = spatial_aggregation(model_temporal_subset_future, grid_lon-0.5*dx, grid_lon+0.5*dx, grid_lat-0.5*dy, grid_lat+0.5*dy)
ref_subset = extract_data_at_nearest_grid_point(ref_temporal_subset, grid_lon,
grid_lat)
model_subset_present = extract_data_at_nearest_grid_point(
NW_bounds, SW_bounds, NGP_bounds, SGP_bounds, MW_bounds, NE_bounds,
SE_bounds
]
""" Load nClimDiv file into OCW Dataset """
obs_dataset = local.load_file(file_obs, variable_name='tave')
""" Load CMIP5 simulations into a list of OCW Datasets"""
model_dataset = local.load_multiple_files(file_path=model_file_path,
variable_name='tas',
dataset_name=dataset_name,
variable_unit='K')
""" Temporal subset of obs_dataset """
obs_dataset_subset = dsp.temporal_slice(obs_dataset,
start_time=start_date,
end_time=end_date)
obs_dataset_season = dsp.temporal_subset(obs_dataset_subset,
month_start,
month_end,
average_each_year=True)
""" Temporal subset of model_dataset """
model_dataset_subset = [
dsp.temporal_slice(dataset, start_time=start_date, end_time=end_date)
for dataset in model_dataset
]
model_dataset_season = [
dsp.temporal_subset(dataset,
month_start,
month_end,
average_each_year=True)
for dataset in model_dataset_subset
]
""" Spatial subset of obs_dataset and generate time series """
obs_timeseries = np.zeros([nyear, n_region
bounds = Bounds(min_lat, max_lat, min_lon, max_lon, start_time, end_time)
ref_dataset = dsp.subset(bounds,ref_dataset)
if ref_dataset.temporal_resolution() != temporal_resolution:
ref_dataset = dsp.temporal_rebin(ref_dataset, temporal_resolution)
for idata,dataset in enumerate(model_datasets):
model_datasets[idata] = dsp.subset(bounds,dataset)
if dataset.temporal_resolution() != temporal_resolution:
model_datasets[idata] = dsp.temporal_rebin(dataset, temporal_resolution)
# Temporaly subset both observation and model datasets for the user specified season
month_start = time_info['month_start']
month_end = time_info['month_end']
average_each_year = time_info['average_each_year']
ref_dataset = dsp.temporal_subset(month_start, month_end,ref_dataset,average_each_year)
for idata,dataset in enumerate(model_datasets):
model_datasets[idata] = dsp.temporal_subset(month_start, month_end,dataset,average_each_year)
# generate grid points for regridding
if config['regrid']['regrid_on_reference']:
new_lat = ref_dataset.lats
new_lon = ref_dataset.lons
else:
delta_lat = config['regrid']['regrid_dlat']
delta_lon = config['regrid']['regrid_dlon']
nlat = (max_lat - min_lat)/delta_lat+1
nlon = (max_lon - min_lon)/delta_lon+1
new_lat = np.linspace(min_lat, max_lat, nlat)
new_lon = np.linspace(min_lon, max_lon, nlon)
import numpy as np
import numpy.ma as ma
''' data source: https://dx.doi.org/10.6084/m9.figshare.3753321.v1
AOD_monthly_2000-Mar_2016-FEB_from_MISR_L3_JOINT.nc is publicly available.'''
dataset = local.load_file('AOD_monthly_2000-MAR_2016-FEB_from_MISR_L3_JOINT.nc',
'nonabsorbing_ave')
''' Subset the data for East Asia'''
Bounds = ds.Bounds(lat_min=20, lat_max=57.7, lon_min=90, lon_max=150)
dataset = dsp.subset(dataset, Bounds)
'''The original dataset includes nonabsorbing AOD values between March 2000 and February 2015.
dsp.temporal_subset will extract data in September-October-November.'''
dataset_SON = dsp.temporal_subset(
dataset, month_start=9, month_end=11, average_each_year=True)
ny, nx = dataset_SON.values.shape[1:]
# multi-year mean aod
clim_aod = ma.zeros([3, ny, nx])
clim_aod[0, :] = ma.mean(dataset_SON.values, axis=0) # 16-year mean
clim_aod[1, :] = ma.mean(dataset_SON.values[-5:, :],
axis=0) # the last 5-year mean
clim_aod[2, :] = dataset_SON.values[-1, :] # the last year's value
# plot clim_aod (3 subplots)
plotter.draw_contour_map(clim_aod, dataset_SON.lats, dataset_SON.lons,
fname='nonabsorbing_AOD_clim_East_Asia_Sep-Nov',
gridshape=[1, 3], subtitles=['2000-2015: 16 years', '2011-2015: 5 years', '2015: 1 year'],
print(f'BCDP Results: {bcdp_results}')
### OCW SECTION
print('OCW Benchmarks')
ocw_results = {}
with time_block(ocw_results, 'Dataset Loading'):
datasets = local.load_multiple_files(paths, 'clt')
with time_block(ocw_results, 'Domain Subsetting'):
for i, ds in enumerate(datasets):
datasets[i] = dsp.subset(ds, bnds)
with time_block(ocw_results, 'Seasonal Subsetting'):
for i, ds in enumerate(datasets):
datasets[i] = dsp.temporal_subset(ds, 9, 11)
with time_block(ocw_results, 'Resampling'):
for i, ds in enumerate(datasets):
datasets[i] = dsp.temporal_rebin(ds, 'annual')
with time_block(ocw_results, 'Regridding'):
for i, ds in enumerate(datasets):
datasets[i] = dsp.spatial_regrid(ds, new_lats, new_lons)
print(f'OCW Results: {ocw_results}')
# Plot results
matplotlib.style.use('ggplot')
df = pd.DataFrame({'OCW': ocw_results, 'BCDP': bcdp_results})
df.plot.bar(logy=True, rot=12)
print("Loading %s into an OCW Dataset Object" % (ref_info['path'],))
ref_dataset = local.load_file(ref_info['path'], ref_info['variable'])
print(ref_info['data_name'] +" values shape: (times, lats, lons) - %s \n" % (ref_dataset.values.shape,))
print("Loading %s into an OCW Dataset Object" % (model_info['present']['path'],))
model_dataset_present = local.load_file(model_info['present']['path'], model_info['variable'])
print(model_info['data_name'] +" values shape: (times, lats, lons) - %s \n" % (model_dataset_present.values.shape,))
dy = model_dataset_present.spatial_resolution()[0]
dx = model_dataset_present.spatial_resolution()[1]
model_dataset_future = local.load_file(model_info['future']['path'], model_info['variable'])
print(model_info['future']['scenario_name']+':'+model_info['data_name'] +" values shape: (times, lats, lons) - %s \n" % (model_dataset_future.values.shape,))
""" Step 2: Temporal subsetting """
print("Temporal subsetting for the selected month(s)")
ref_temporal_subset = dsp.temporal_subset(ref_dataset, month_start, month_end)
model_temporal_subset_present = dsp.temporal_subset(model_dataset_present, month_start, month_end)
model_temporal_subset_future = dsp.temporal_subset(model_dataset_future, month_start, month_end)
""" Step 3: Spatial aggregation of observational data into the model grid """
print("Spatial aggregation of observational data near latitude %0.2f and longitude %0.2f " % (grid_lat, grid_lon))
# There are two options to aggregate observational data near a model grid point
#ref_subset = spatial_aggregation(ref_temporal_subset, grid_lon-0.5*dx, grid_lon+0.5*dx, grid_lat-0.5*dy, grid_lat+0.5*dy)
#model_subset_present = spatial_aggregation(model_temporal_subset_present, grid_lon-0.5*dx, grid_lon+0.5*dx, grid_lat-0.5*dy, grid_lat+0.5*dy)
#model_subset_future = spatial_aggregation(model_temporal_subset_future, grid_lon-0.5*dx, grid_lon+0.5*dx, grid_lat-0.5*dy, grid_lat+0.5*dy)
ref_subset = extract_data_at_nearest_grid_point(ref_temporal_subset, grid_lon, grid_lat)
model_subset_present = extract_data_at_nearest_grid_point(model_temporal_subset_present, grid_lon, grid_lat)
model_subset_future = extract_data_at_nearest_grid_point(model_temporal_subset_future, grid_lon, grid_lat)
""" Step 4: Create a statistical downscaling object and downscaling model output """
min_lon = np.max([min_lon, ref_dataset.lons.min()])
max_lon = np.min([max_lon, ref_dataset.lons.max()])
bounds = Bounds(min_lat, max_lat, min_lon, max_lon, start_time, end_time)
if ref_dataset.lats.ndim != 2 and ref_dataset.lons.ndim != 2:
ref_dataset = dsp.subset(bounds, ref_dataset)
for idata, dataset in enumerate(model_datasets):
if dataset.lats.ndim != 2 and dataset.lons.ndim != 2:
model_datasets[idata] = dsp.subset(bounds, dataset)
# Temporaly subset both observation and model datasets for the user specified season
month_start = time_info['month_start']
month_end = time_info['month_end']
average_each_year = time_info['average_each_year']
ref_dataset = dsp.temporal_subset(month_start, month_end, ref_dataset,
average_each_year)
for idata, dataset in enumerate(model_datasets):
model_datasets[idata] = dsp.temporal_subset(month_start, month_end,
dataset, average_each_year)
# generate grid points for regridding
if config['regrid']['regrid_on_reference']:
new_lat = ref_dataset.lats
new_lon = ref_dataset.lons
else:
delta_lat = config['regrid']['regrid_dlat']
delta_lon = config['regrid']['regrid_dlon']
nlat = (max_lat - min_lat) / delta_lat + 1
nlon = (max_lon - min_lon) / delta_lon + 1
new_lat = np.linspace(min_lat, max_lat, nlat)
new_lon = np.linspace(min_lon, max_lon, nlon)
for i, dataset in enumerate(datasets):
datasets[i] = dsp.subset(dataset, bounds)
if dataset.temporal_resolution() != temporal_resolution:
datasets[i] = dsp.temporal_rebin(datasets[i], temporal_resolution)
# Temporally subset both observation and model datasets
# for the user specified season
month_start = time_info['month_start']
month_end = time_info['month_end']
average_each_year = time_info['average_each_year']
# For now we will treat the first listed dataset as the reference dataset for
# evaluation purposes.
for i, dataset in enumerate(datasets):
datasets[i] = dsp.temporal_subset(dataset, month_start, month_end,
average_each_year)
reference_dataset = datasets[0]
target_datasets = datasets[1:]
reference_name = names[0]
target_names = names[1:]
# generate grid points for regridding
if config['regrid']['regrid_on_reference']:
new_lat = reference_dataset.lats
new_lon = reference_dataset.lons
else:
delta_lat = config['regrid']['regrid_dlat']
delta_lon = config['regrid']['regrid_dlon']
nlat = (max_lat - min_lat)/delta_lat+1
nlon = (max_lon - min_lon)/delta_lon+1
import ocw.dataset_processor as dsp
import ocw.plotter as plotter
import numpy as np
import numpy.ma as ma
''' data source: https://dx.doi.org/10.6084/m9.figshare.3753321.v1
AOD_monthly_2000-Mar_2016-FEB_from_MISR_L3_JOINT.nc is publicly available.'''
dataset = local.load_file(
'AOD_monthly_2000-MAR_2016-FEB_from_MISR_L3_JOINT.nc', 'nonabsorbing_ave')
''' Subset the data for East Asia'''
Bounds = ds.Bounds(lat_min=20, lat_max=57.7, lon_min=90, lon_max=150)
dataset = dsp.subset(dataset, Bounds)
'''The original dataset includes nonabsorbing AOD values between March 2000 and February 2015.
dsp.temporal_subset will extract data in September-October-November.'''
dataset_SON = dsp.temporal_subset(dataset,
month_start=9,
month_end=11,
average_each_year=True)
ny, nx = dataset_SON.values.shape[1:]
# multi-year mean aod
clim_aod = ma.zeros([3, ny, nx])
clim_aod[0, :] = ma.mean(dataset_SON.values, axis=0) # 16-year mean
clim_aod[1, :] = ma.mean(dataset_SON.values[-5:, :],
axis=0) # the last 5-year mean
clim_aod[2, :] = dataset_SON.values[-1, :] # the last year's value
# plot clim_aod (3 subplots)
plotter.draw_contour_map(
clim_aod,
for i, dataset in enumerate(model_datasets):
model_datasets[i] = dsp.subset(dataset, bounds)
if dataset.temporal_resolution() != temporal_resolution:
model_datasets[i] = dsp.temporal_rebin(dataset, temporal_resolution)
# Temporally subset both observation and model datasets
# for the user specified season
month_start = time_info['month_start']
month_end = time_info['month_end']
average_each_year = time_info['average_each_year']
# TODO: Fully support multiple observation / reference datasets.
# For now we will only use the first reference dataset listed in the config file
obs_dataset = obs_datasets[0]
obs_name = obs_names[0]
obs_dataset = dsp.temporal_subset(obs_dataset, month_start, month_end,
average_each_year)
for i, dataset in enumerate(model_datasets):
model_datasets[i] = dsp.temporal_subset(dataset, month_start, month_end,
average_each_year)
# generate grid points for regridding
if config['regrid']['regrid_on_reference']:
new_lat = obs_dataset.lats
new_lon = obs_dataset.lons
else:
delta_lat = config['regrid']['regrid_dlat']
delta_lon = config['regrid']['regrid_dlon']
nlat = (max_lat - min_lat) / delta_lat + 1
nlon = (max_lon - min_lon) / delta_lon + 1
new_lat = np.linspace(min_lat, max_lat, nlat)
new_lon = np.linspace(min_lon, max_lon, nlon)
FILE_2 = "AFRICA_UC-WRF311_CTL_ERAINT_MM_50km-rg_1989-2008_tasmax.nc"
# Filename for the output image/plot (without file extension)
OUTPUT_PLOT = "wrf_bias_compared_to_knmi"
FILE_1_PATH = path.join('/tmp', FILE_1)
FILE_2_PATH = path.join('/tmp', FILE_2)
if not path.exists(FILE_1_PATH):
urllib.urlretrieve(FILE_LEADER + FILE_1, FILE_1_PATH)
if not path.exists(FILE_2_PATH):
urllib.urlretrieve(FILE_LEADER + FILE_2, FILE_2_PATH)
""" Step 1: Load Local NetCDF Files into OCW Dataset Objects """
print("Loading %s into an OCW Dataset Object" % (FILE_1_PATH,))
knmi_dataset = local.load_file(FILE_1_PATH, "tasmax")
print("KNMI_Dataset.values shape: (times, lats, lons) - %s \n" % (knmi_dataset.values.shape,))
print("Loading %s into an OCW Dataset Object" % (FILE_2_PATH,))
wrf_dataset = local.load_file(FILE_2_PATH, "tasmax")
print("WRF_Dataset.values shape: (times, lats, lons) - %s \n" % (wrf_dataset.values.shape,))
""" Step 2: Calculate seasonal average """
print("Calculate seasonal average")
knmi_DJF_mean = utils.calc_temporal_mean(dsp.temporal_subset(month_start=12, month_end=2, target_dataset=knmi_dataset))
wrf_DJF_mean = utils.calc_temporal_mean(dsp.temporal_subset(month_start=12, month_end=2, target_dataset=wrf_dataset))
print("Seasonally averaged KNMI_Dataset.values shape: (times, lats, lons) - %s \n" % (knmi_DJF_mean.shape,))
print("Seasonally averaged wrf_Dataset.values shape: (times, lats, lons) - %s \n" % (wrf_DJF_mean.shape,))
knmi_JJA_mean = utils.calc_temporal_mean(dsp.temporal_subset(month_start=6, month_end=8, target_dataset=knmi_dataset))
wrf_JJA_mean = utils.calc_temporal_mean(dsp.temporal_subset(month_start=6, month_end=8, target_dataset=wrf_dataset))
