def testLoadDataset(self):
''' test universal dataset loading function '''
from datasets.common import loadDataset, loadClim, loadStnTS
# test climtology
ds = loadClim(name='PCIC', grid='arb2_d02', varlist=['precip'])
assert isinstance(ds, Dataset)
assert ds.name == 'PCIC'
assert 'precip' in ds
assert ds.gdal and ds.isProjected
# test CVDP
ds = loadDataset(name='HadISST',
period=None,
varlist=None,
mode='CVDP')
assert isinstance(ds, Dataset)
assert ds.name == 'HadISST'
assert 'PDO' in ds
assert ds.gdal and not ds.isProjected
# test CVDP with WRF
ds = loadDataset(name='phys-ens-2100',
period=None,
varlist=None,
mode='CVDP')
assert isinstance(ds, Dataset)
assert ds.name == 'phys-ens-2100'
assert 'PDO' in ds
assert ds.gdal and not ds.isProjected
# test WRF station time-series
ds = loadStnTS(name='ctrl-1_d02',
varlist=['MaxPrecip_1d'],
station='ecprecip',
filetypes='hydro')
assert isinstance(ds, Dataset)
assert ds.name == 'ctrl-1_d02'
assert 'MaxPrecip_1d' in ds
# test example with list expansion
# test EC station time-series
dss = loadStnTS(name=['EC', 'ctrl-1'],
varlist=['MaxPrecip_1d', 'precip'],
station='ecprecip',
filetypes='hydro',
load_list=['name', 'varlist'],
lproduct='outer')
assert len(dss) == 4
assert isinstance(ds, Dataset)
assert dss[1].name == 'EC' and dss[2].name == 'ctrl-1'
assert 'MaxPrecip_1d' in dss[0] and 'precip' in dss[1]
assert 'MaxPrecip_1d' not in dss[3] and 'precip' not in dss[2]
def loadShapeObservations(obs=None, seasons=None, basins=None, provs=None, shapes=None, varlist=None, slices=None,
aggregation='mean', shapetype=None, period=None, variable_list=None, **kwargs):
''' convenience function to load shape observations; the main function is to select sensible defaults
based on 'varlist', if no 'obs' are specified '''
# prepare arguments
if shapetype is None: shapetype = 'shpavg' # really only one in use
# resolve variable list (no need to maintain order)
if isinstance(varlist,basestring): varlist = [varlist]
variables = set(shp_params)
for name in varlist:
if name in variable_list: variables.update(variable_list[name].vars)
else: variables.add(name)
variables = list(variables)
# figure out default datasets
if obs is None: obs = 'Observations'
lUnity = lCRU = lWSC = False
if obs[:3].lower() in ('obs','wsc'):
if any(var in CRU_vars for var in variables):
if aggregation == 'mean' and seasons is None:
lUnity = True; obs = []
if basins and any([var in WSC_vars for var in variables]):
if aggregation.lower() in ('mean','std','sem','min','max') and seasons is None:
lWSC = True; obs = []
if not isinstance(obs,(list,tuple)): obs = (obs,)
# configure slicing (extract basin/province/shape and period)
slices = _configSlices(slices=slices, basins=basins, provs=provs, shapes=shapes, period=period)
if slices is not None:
noyears = slices.copy(); noyears.pop('years',None) # slices for climatologies
# prepare and load ensemble of observations
obsens = Ensemble(name='obs',title='Observations', basetype=Dataset)
if len(obs) > 0: # regular operations with user-defined dataset
try:
ensemble = loadEnsembleTS(names=obs, season=seasons, aggregation=aggregation, slices=slices,
varlist=variables, shape=shapetype, ldataset=False, **kwargs)
for ens in ensemble: obsens += ens
except EmptyDatasetError: pass
if lUnity: # load Unity data instead of averaging CRU data
if period is None: period = (1979,1994)
dataset = loadDataset(name='Unity', varlist=variables, mode='climatology', period=period, shape=shapetype)
if slices is not None: dataset = dataset(**noyears) # slice immediately
obsens += dataset.load()
if lCRU: # this is basically regular operations with CRU as default
obsens += loadEnsembleTS(names='CRU', season=seasons, aggregation=aggregation, slices=slices,
varlist=variables, shape=shapetype, ldataset=True, **kwargs)
if lWSC: # another special case: river hydrographs
# from datasets.WSC import loadGageStation, GageStationError
try:
dataset = loadGageStation(basin=basins, varlist=['runoff'], aggregation=aggregation, mode='climatology', filetype='monthly')
if slices is not None: dataset = dataset(**noyears) # slice immediately
obsens += dataset.load()
except GageStationError:
pass # just ignore, if gage station if data is missing
# return ensembles (will be wrapped in a list, if BatchLoad is used)
return obsens
def testLoadDataset(self):
''' test universal dataset loading function '''
from datasets.common import loadDataset, loadClim, loadStnTS
# test climtology
ds = loadClim(name='PCIC', grid='arb2_d02', varlist=['precip'])
assert isinstance(ds, Dataset)
assert ds.name == 'PCIC'
assert 'precip' in ds
assert ds.gdal and ds.isProjected
# test CVDP
ds = loadDataset(name='HadISST', period=None, varlist=None, mode='CVDP')
assert isinstance(ds, Dataset)
assert ds.name == 'HadISST'
assert 'PDO' in ds
assert ds.gdal and not ds.isProjected
# test CVDP with WRF
ds = loadDataset(name='phys-ens-2100', period=None, varlist=None, mode='CVDP')
assert isinstance(ds, Dataset)
assert ds.name == 'phys-ens-2100'
assert 'PDO' in ds
assert ds.gdal and not ds.isProjected
# test WRF station time-series
ds = loadStnTS(name='ctrl-1_d02', varlist=['MaxPrecip_1d'], station='ecprecip', filetypes='hydro')
assert isinstance(ds, Dataset)
assert ds.name == 'ctrl-1_d02'
assert 'MaxPrecip_1d' in ds
# test example with list expansion
# test EC station time-series
dss = loadStnTS(name=['EC','ctrl-1'], varlist=['MaxPrecip_1d','precip'],
station='ecprecip', filetypes='hydro',
load_list=['name','varlist'], lproduct='outer')
assert len(dss) == 4
assert isinstance(ds, Dataset)
assert dss[1].name == 'EC' and dss[2].name == 'ctrl-1'
assert 'MaxPrecip_1d' in dss[0] and 'precip' in dss[1]
assert 'MaxPrecip_1d' not in dss[3] and 'precip' not in dss[2]
if isinstance(periods, (np.integer, int)): periods = [periods]
# check and expand WRF experiment list
WRF_experiments = getExperimentList(WRF_experiments, WRF_project, 'WRF')
if isinstance(WRF_domains, (np.integer, int)): WRF_domains = [WRF_domains]
# check and expand CESM experiment list
CESM_experiments = getExperimentList(CESM_experiments, CESM_project,
'CESM')
# expand datasets and resolutions
if datasets is None: datasets = gridded_datasets
# update some dependencies
unity_grid = grid # trivial in this case
obs_args['grid'] = grid
obs_args['varlist'] = load_list
## load observations/reference dataset
obs_dataset = loadDataset(name=obs_name, mode=obs_mode, **obs_args).load()
# print an announcement
if len(WRF_experiments) > 0:
print('\n Bias-correcting WRF Datasets:')
print([exp.name for exp in WRF_experiments])
if len(CESM_experiments) > 0:
print('\n Bias-correcting CESM Datasets:')
print([exp.name for exp in CESM_experiments])
if len(datasets) > 0:
print('\n Bias-correcting Other Datasets:')
print(datasets)
print('\n On Grid/Resolution:\n {:s}'.format(grid))
print('\n Variable List: {:s}'.format(
'All' if varlist is None else printList(varlist)))
print('\n Bias-Correction Method: {:s}'.format(bc_method))
def loadShapeObservations(obs=None,
seasons=None,
basins=None,
provs=None,
shapes=None,
varlist=None,
slices=None,
aggregation='mean',
shapetype=None,
period=None,
variable_list=None,
**kwargs):
''' convenience function to load shape observations; the main function is to select sensible defaults
based on 'varlist', if no 'obs' are specified '''
# prepare arguments
if shapetype is None: shapetype = 'shpavg' # really only one in use
# resolve variable list (no need to maintain order)
if isinstance(varlist, basestring): varlist = [varlist]
variables = set(shp_params)
for name in varlist:
if name in variable_list: variables.update(variable_list[name].vars)
else: variables.add(name)
variables = list(variables)
# figure out default datasets
if obs is None: obs = 'Observations'
lUnity = lCRU = lWSC = False
if obs[:3].lower() in ('obs', 'wsc'):
if any(var in CRU_vars for var in variables):
if aggregation == 'mean' and seasons is None:
lUnity = True
obs = []
if basins and any([var in WSC_vars for var in variables]):
if aggregation.lower() in ('mean', 'std', 'sem', 'min',
'max') and seasons is None:
lWSC = True
obs = []
if not isinstance(obs, (list, tuple)): obs = (obs, )
# configure slicing (extract basin/province/shape and period)
slices = _configSlices(slices=slices,
basins=basins,
provs=provs,
shapes=shapes,
period=period)
if slices is not None:
noyears = slices.copy()
noyears.pop('years', None) # slices for climatologies
# prepare and load ensemble of observations
obsens = Ensemble(name='obs', title='Observations', basetype=Dataset)
if len(obs) > 0: # regular operations with user-defined dataset
try:
ensemble = loadEnsembleTS(names=obs,
season=seasons,
aggregation=aggregation,
slices=slices,
varlist=variables,
shape=shapetype,
ldataset=False,
**kwargs)
for ens in ensemble:
obsens += ens
except EmptyDatasetError:
pass
if lUnity: # load Unity data instead of averaging CRU data
if period is None: period = (1979, 1994)
dataset = loadDataset(name='Unity',
varlist=variables,
mode='climatology',
period=period,
shape=shapetype)
if slices is not None:
dataset = dataset(**noyears) # slice immediately
obsens += dataset.load()
if lCRU: # this is basically regular operations with CRU as default
obsens += loadEnsembleTS(names='CRU',
season=seasons,
aggregation=aggregation,
slices=slices,
varlist=variables,
shape=shapetype,
ldataset=True,
**kwargs)
if lWSC: # another special case: river hydrographs
# from datasets.WSC import loadGageStation, GageStationError
try:
dataset = loadGageStation(basin=basins,
varlist=['runoff'],
aggregation=aggregation,
mode='climatology',
filetype='monthly')
if slices is not None:
dataset = dataset(**noyears) # slice immediately
obsens += dataset.load()
except GageStationError:
pass # just ignore, if gage station data is missing
# return ensembles (will be wrapped in a list, if BatchLoad is used)
return obsens
## process arguments
if isinstance(periods, (np.integer,int)): periods = [periods]
# check and expand WRF experiment list
WRF_experiments = getExperimentList(WRF_experiments, WRF_project, 'WRF')
if isinstance(WRF_domains, (np.integer,int)): WRF_domains = [WRF_domains]
# check and expand CESM experiment list
CESM_experiments = getExperimentList(CESM_experiments, CESM_project, 'CESM')
# expand datasets and resolutions
if datasets is None: datasets = gridded_datasets
# update some dependencies
unity_grid = grid # trivial in this case
obs_args['grid'] = grid
obs_args['varlist'] = load_list
## load observations/reference dataset
obs_dataset = loadDataset(name=obs_name, mode=obs_mode, **obs_args).load()
# print an announcement
if len(WRF_experiments) > 0:
print('\n Bias-correcting WRF Datasets:')
print([exp.name for exp in WRF_experiments])
if len(CESM_experiments) > 0:
print('\n Bias-correcting CESM Datasets:')
print([exp.name for exp in CESM_experiments])
if len(datasets) > 0:
print('\n Bias-correcting Other Datasets:')
print(datasets)
print('\n On Grid/Resolution:\n {:s}'.format(grid))
print('\n Variable List: {:s}'.format('All' if varlist is None else printList(varlist)))
print('\n Bias-Correction Method: {:s}'.format(bc_method))
print('\n Observationa/Reference Dataset:\n {:s}'.format(obs_dataset if ldebug else obs_dataset.name))