def test_RequestDataset(self):
rd = RequestDataset(self.uri, self.variable)
self.assertEqual(rd['uri'], self.uri)
self.assertEqual(rd['alias'], self.variable)
rd = RequestDataset(self.uri, self.variable, alias='an_alias')
self.assertEqual(rd.alias, 'an_alias')
def test_RequestDatasetCollection(self):
env.DIR_DATA = ocgis.env.DIR_TEST_DATA
daymet = self.test_data.get_rd('daymet_tmax')
tas = self.test_data.get_rd('cancm4_tas')
uris = [daymet.uri, tas.uri]
variables = ['foo1', 'foo2']
rdc = RequestDatasetCollection()
for uri, variable in zip(uris, variables):
rd = RequestDataset(uri, variable)
rdc.update(rd)
self.assertEqual([1, 2], [rd.did for rd in rdc])
variables = ['foo1', 'foo1']
rdc = RequestDatasetCollection()
for ii, (uri, variable) in enumerate(zip(uris, variables)):
rd = RequestDataset(uri, variable)
if ii == 1:
with self.assertRaises(KeyError):
rdc.update(rd)
else:
rdc.update(rd)
aliases = ['a1', 'a2']
for uri, variable, alias in zip(uris, variables, aliases):
rd = RequestDataset(uri, variable, alias=alias)
rdc.update(rd)
for row in rdc:
self.assertIsInstance(row, RequestDataset)
self.assertIsInstance(rdc[0], RequestDataset)
self.assertIsInstance(rdc['a2'], RequestDataset)
def test_inspect_as_dct(self):
variables = [self.variable, None, 'foo', 'time']
for variable in variables:
rd = RequestDataset(self.uri, variable)
try:
ret = rd.inspect_as_dct()
except KeyError:
if variable == 'foo':
continue
else:
raise
except ValueError:
if variable == 'time':
continue
else:
raise
ref = ret['derived']
if variable is None:
self.assertEqual(
ref, {
'End Date': '2020-12-31 12:00:00',
'Start Date': '2011-01-01 12:00:00'
})
else:
self.assertEqual(ref['End Date'], '2020-12-31 12:00:00')
def test_RequestDataset_level_range(self):
lr = '1|1'
rd = RequestDataset(self.uri, self.variable, level_range=lr)
self.assertEqual(rd.level_range, [1, 1])
with self.assertRaises(DefinitionValidationError):
rd = RequestDataset(self.uri, self.variable, level_range=[2, 1])
def test_RequestDatasetCollection(self):
env.DIR_DATA = ocgis.env.DIR_TEST_DATA
uris = [
'tas_day_CanCM4_decadal2011_r2i1p1_20120101-20211231.nc',
'albisccp_cfDay_CCSM4_1pctCO2_r2i1p1_00200101-00391231.nc'
]
variables = ['foo1', 'foo2']
rdc = RequestDatasetCollection()
for uri, variable in zip(uris, variables):
rd = RequestDataset(uri, variable)
rdc.update(rd)
self.assertEqual([1, 2], [rd.did for rd in rdc])
variables = ['foo1', 'foo1']
rdc = RequestDatasetCollection()
for ii, (uri, variable) in enumerate(zip(uris, variables)):
rd = RequestDataset(uri, variable)
if ii == 1:
with self.assertRaises(KeyError):
rdc.update(rd)
else:
rdc.update(rd)
aliases = ['a1', 'a2']
for uri, variable, alias in zip(uris, variables, aliases):
rd = RequestDataset(uri, variable, alias=alias)
rdc.update(rd)
for row in rdc:
self.assertIsInstance(row, RequestDataset)
self.assertIsInstance(rdc[0], RequestDataset)
self.assertIsInstance(rdc['a2'], RequestDataset)
def test_RequestDataset_time_range(self):
tr = [dt(2000, 1, 1), dt(2000, 12, 31)]
rd = RequestDataset(self.uri, self.variable, time_range=tr)
self.assertEqual(rd.time_range, tr)
out = [dt(2000, 1, 1, 0, 0), dt(2000, 12, 31, 23, 59, 59)]
tr = '2000-1-1|2000-12-31'
rd = RequestDataset(self.uri, self.variable, time_range=tr)
self.assertEqual(rd.time_range, out)
tr = '2000-12-31|2000-1-1'
with self.assertRaises(DefinitionValidationError):
rd = RequestDataset(self.uri, self.variable, time_range=tr)
def test_output_data(self):
uri = [
'/home/local/WX/ben.koziol/climate_data/QED-2013/sfwe/maurer02v2/sfwe_p/sfwe_p.nc',
'/home/local/WX/ben.koziol/climate_data/QED-2013/sfwe/maurer02v2/sfwe/sfwe.nc',
'/home/local/WX/ben.koziol/climate_data/QED-2013/sfwe/maurer02v2/pr/pr.nc'
]
variable = [
'sfwe_p',
'sfwe',
'p',
]
rds = [
RequestDataset(u,
v,
time_region={
'month': [7],
'year': range(1990, 2000)
}) for u, v in zip(uri, variable)
]
ops = ocgis.OcgOperations(dataset=rds,
geom='state_boundaries',
select_ugid=[16],
spatial_operation='clip',
aggregate=True)
ret = ops.execute()
ref = ret[16]
sfwe_p = ref.variables['sfwe_p'].value
p = ref.variables['p'].value
sfwe = ref.variables['sfwe'].value
idx_bad = sfwe_p > 1
bad_sfwe_p = sfwe_p[idx_bad]
self.assertEqual(bad_sfwe_p.compressed().shape[0], 0)
idx_bad = sfwe_p > 1
bad_sfwe = sfwe[idx_bad]
bad_p = p[idx_bad]
rds = [RequestDataset(u, v) for u, v in zip(uri, variable)]
ops = ocgis.OcgOperations(dataset=rds,
geom='WBDHU8_June2013',
select_ugid=[378],
spatial_operation='clip',
aggregate=True)
ret = ops.execute()
ref = ret[378]
import ipdb
ipdb.set_trace()
def test_dataset(self):
env.DIR_DATA = ocgis.env.DIR_TEST_DATA
rd = RequestDataset(
'tas_day_CanCM4_decadal2011_r2i1p1_20120101-20211231.nc', 'tas')
ds = definition.Dataset(rd)
self.assertEqual(ds.value, RequestDatasetCollection([rd]))
dsa = {
'uri': 'tas_day_CanCM4_decadal2011_r2i1p1_20120101-20211231.nc',
'variable': 'tas'
}
ds = definition.Dataset(dsa)
self.assertEqual(
ds.get_url_string(),
'uri=/usr/local/climate_data/CanCM4/tas_day_CanCM4_decadal2011_r2i1p1_20120101-20211231.nc&variable=tas&alias=tas&t_units=none&t_calendar=none&s_proj=none'
)
dsb = [
dsa, {
'uri':
'albisccp_cfDay_CCSM4_1pctCO2_r2i1p1_00200101-00391231.nc',
'variable': 'albisccp',
'alias': 'knight'
}
]
ds = definition.Dataset(dsb)
str_cmp = 'uri1=/usr/local/climate_data/CanCM4/tas_day_CanCM4_decadal2011_r2i1p1_20120101-20211231.nc&variable1=tas&alias1=tas&t_units1=none&t_calendar1=none&s_proj1=none&uri2=/usr/local/climate_data/CCSM4/albisccp_cfDay_CCSM4_1pctCO2_r2i1p1_00200101-00391231.nc&variable2=albisccp&alias2=knight&t_units2=none&t_calendar2=none&s_proj2=none'
self.assertEqual(ds.get_url_string(), str_cmp)
def __init__(self,
uri=None,
variable=None,
interface_overload={},
request_dataset=None):
self.request_dataset = request_dataset
if self.request_dataset is None:
self.uri = uri
self.variable = variable
self.alias = None
self.did = None
if self.variable is None:
try:
self.ds = None
rootgrp = nc.Dataset(uri)
self.meta = NcMetadata(rootgrp)
finally:
rootgrp.close()
else:
from ocgis.api.request import RequestDataset
kwds = {'uri': uri, 'variable': variable}
kwds.update(interface_overload)
rd = RequestDataset(**kwds)
self.ds = NcDataset(request_dataset=rd)
self.meta = self.ds.metadata
else:
self.uri = self.request_dataset.uri
self.variable = self.request_dataset.variable
self.ds = self.request_dataset.ds
self.meta = self.request_dataset.ds.metadata
self.alias = self.request_dataset.alias
self.did = self.request_dataset.did
def test_pickle(self):
rd = RequestDataset(uri=self.uri, variable=self.variable)
rd_path = os.path.join(ocgis.env.DIR_OUTPUT, 'rd.pkl')
with open(rd_path, 'w') as f:
pickle.dump(rd, f)
with open(rd_path, 'r') as f:
rd2 = pickle.load(f)
self.assertTrue(rd == rd2)
def test_inspect_as_dct(self):
variables = [self.variable,None,'foo','time']
for variable in variables:
rd = RequestDataset(self.uri,variable)
try:
ret = rd.inspect_as_dct()
except KeyError:
if variable == 'foo':
continue
else:
raise
except ValueError:
if variable == 'time':
continue
else:
raise
ref = ret['derived']
if variable is None:
self.assertEqual(ref,{'End Date': '2020-12-31 12:00:00', 'Start Date': '2011-01-01 12:00:00'})
else:
self.assertEqual(ref['End Date'],'2020-12-31 12:00:00')
def test_env_dir_data(self):
## test setting the var to a single directory
env.DIR_DATA = ocgis.env.DIR_TEST_DATA
rd = self.test_data.get_rd('cancm4_rhs')
target = os.path.join(
env.DIR_DATA, 'CanCM4',
'rhs_day_CanCM4_decadal2010_r2i1p1_20110101-20201231.nc')
try:
self.assertEqual(rd.uri, target)
## attempt to normalize the path
except AssertionError:
self.assertEqual(rd.uid, os.path.normpath(target))
## test none and not finding the data
env.DIR_DATA = None
with self.assertRaises(ValueError):
RequestDataset('does_not_exists.nc', variable='foo')
## set data directory and not find it.
env.DIR_DATA = os.path.join(ocgis.env.DIR_TEST_DATA, 'CCSM4')
with self.assertRaises(ValueError):
RequestDataset(
'rhs_day_CanCM4_decadal2010_r2i1p1_20110101-20201231.nc',
variable='rhs')
def test_dataset(self):
env.DIR_DATA = ocgis.env.DIR_TEST_DATA
reference_rd = self.test_data.get_rd('cancm4_tas')
rd = RequestDataset(reference_rd.uri, reference_rd.variable)
ds = definition.Dataset(rd)
self.assertEqual(ds.value, RequestDatasetCollection([rd]))
dsa = {'uri': reference_rd.uri, 'variable': reference_rd.variable}
ds = definition.Dataset(dsa)
reference_rd2 = self.test_data.get_rd('narccap_crcm')
dsb = [
dsa, {
'uri': reference_rd2.uri,
'variable': reference_rd2.variable,
'alias': 'knight'
}
]
ds = definition.Dataset(dsb)
def test_time_region(self):
tr1 = {'month': [6], 'year': [2001]}
rd = RequestDataset(uri=self.uri,
variable=self.variable,
time_region=tr1)
self.assertEqual(rd.time_region, tr1)
tr2 = {'bad': 15}
with self.assertRaises(DefinitionValidationError):
RequestDataset(uri=self.uri,
variable=self.variable,
time_region=tr2)
tr_str = 'month~6|year~2001'
rd = RequestDataset(uri=self.uri,
variable=self.variable,
time_region=tr_str)
self.assertEqual(rd.time_region, tr1)
tr_str = 'month~6-8|year~2001-2003'
rd = RequestDataset(uri=self.uri,
variable=self.variable,
time_region=tr_str)
self.assertEqual(rd.time_region, {
'month': [6, 7, 8],
'year': [2001, 2002, 2003]
})
tr_str = 'month~6-8'
rd = RequestDataset(uri=self.uri,
variable=self.variable,
time_region=tr_str)
self.assertEqual(rd.time_region, {'month': [6, 7, 8], 'year': None})
tr_str = 'month~6-8|year~none'
rd = RequestDataset(uri=self.uri,
variable=self.variable,
time_region=tr_str)
self.assertEqual(rd.time_region, {'month': [6, 7, 8], 'year': None})
def test_inspect_method(self):
rd = RequestDataset(self.uri, self.variable)
rd.inspect()
def test_inspect_method(self):
rd = RequestDataset(self.uri,self.variable)
rd.inspect()
def test_multiple_uris(self):
env.DIR_DATA = '/usr/local/climate_data/narccap'
uris = ['pr_CRCM_ccsm_1981010103.nc','pr_CRCM_ccsm_1986010103.nc']
variable = 'pr'
rd = RequestDataset(uri=uris,variable=variable)
rd.inspect()
def test_multiple_uris(self):
env.DIR_DATA = '/usr/local/climate_data/narccap'
uris = ['pr_CRCM_ccsm_1981010103.nc', 'pr_CRCM_ccsm_1986010103.nc']
variable = 'pr'
rd = RequestDataset(uri=uris, variable=variable)
rd.inspect()
def compute_sfwe_other():
ocgis.env.DIR_DATA = '/data/ben.koziol/sfwe/data'
ocgis.env.DIR_OUTPUT = '/home/local/WX/ben.koziol'
ocgis.env.OVERWRITE = False
tas = [[
'bcca_gfdl_cm2_1.gregorian.20c3m.run1.tas.1971-2000.nc', 'tas', None,
'bcca_gfdl'
],
[
'bcca_cccma_cgcm3_1.gregorian.20c3m.run1.tas.1971-2000.nc',
'tas', None, 'bcca_cccma_cgcm3'
],
[
'arrm_cgcm3_t63.20c3m.tas.NAm.1971-2000.nc', 'tas', '365_day',
'arrm_cgcm3'
],
[
'arrm_gfdl_2.1.20c3m.tas.NAm.1971-2000.nc', 'tas', '365_day',
'arrm_gfdl'
]]
pr = [[
'bcca_gfdl_cm2_1.gregorian.20c3m.run1.pr.1971-2000.nc', 'pr', None,
'bcca_gfdl'
],
[
'bcca_cccma_cgcm3_1.gregorian.20c3m.run1.pr.1971-2000.nc', 'pr',
None, 'bcca_cccma_cgcm3'
],
[
'arrm_cgcm3_t63.20c3m.pr.NAm.1971-2000.nc', 'pr', '365_day',
'arrm_cgcm3'
],
[
'arrm_gfdl_2.1.20c3m.pr.NAm.1971-2000.nc', 'pr', '365_day',
'arrm_gfdl'
]]
for t, p in zip(tas, pr):
tas_rd = ocgis.RequestDataset(t[0], t[1], t_calendar=t[2])
pr_rd = ocgis.RequestDataset(p[0], p[1], t_calendar=p[2])
calc = [{
'func': 'sfwe',
'name': 'sfwe',
'kwds': {
'tas': 'tas',
'pr': 'pr'
}
}]
rdc = RequestDatasetCollection([tas_rd, pr_rd])
sfwe = compute(rdc,
calc, ['month', 'year'],
175,
verbose=True,
prefix='sfwe_' + t[3])
print(sfwe)
#sfwe = '/home/local/WX/ben.koziol/climate_data/QED-2013/sfwe/maurer02v2/sfwe/sfwe.nc'
calc = [{'func': 'sum', 'name': 'p'}]
rds = [pr_rd]
rdc = RequestDatasetCollection(rds)
pr = compute(rdc,
calc, ['month', 'year'],
175,
verbose=True,
prefix='pr_' + t[3])
print(pr)
#pr = '/home/local/WX/ben.koziol/climate_data/QED-2013/sfwe/maurer02v2/pr/pr.nc'
calc = [{
'func': 'ratio_sfwe_p',
'name': 'sfwe_p',
'kwds': {
'sfwe': 'sfwe',
'p': 'p'
}
}]
rds = [
RequestDataset(sfwe, 'sfwe', t_calendar=t[2]),
RequestDataset(pr, 'p', t_calendar=t[2])
]
rdc = RequestDatasetCollection(rds)
sfwe_p = compute(rdc,
calc,
None,
175,
verbose=True,
prefix='sfwe_p_' + t[3])
print(sfwe_p)
def test_same_projection(self):
daymet_uri = self.test_data.get_rd('daymet_tmax').uri
rd1 = RequestDataset(uri=daymet_uri, variable='tmax', alias='tmax1')
rd2 = RequestDataset(uri=daymet_uri, variable='tmax', alias='tmax2')
ops = OcgOperations(dataset=[rd1, rd2], snippet=True)
ops.execute()
def compute_sfwe_maurer():
def maurer_pr():
ret = {
'uri': 'Maurer02new_OBS_pr_daily.1971-2000.nc',
'variable': 'pr'
}
return (ret)
def maurer_tas():
ret = {
'uri': 'Maurer02new_OBS_tas_daily.1971-2000.nc',
'variable': 'tas'
}
return (ret)
ocgis.env.DIR_DATA = '/usr/local/climate_data/'
ocgis.env.DIR_OUTPUT = '/home/local/WX/ben.koziol/climate_data/QED-2013/sfwe/maurer02v2'
ocgis.env.OVERWRITE = True
calc = [{
'func': 'sfwe',
'name': 'sfwe',
'kwds': {
'tas': 'tas',
'pr': 'pr'
}
}]
time_range = None
rds = []
for var in [maurer_pr(), maurer_tas()]:
var.update({'time_range': time_range})
rds.append(var)
rdc = RequestDatasetCollection(rds)
sfwe = compute(rdc,
calc, ['month', 'year'],
175,
verbose=True,
prefix='sfwe')
print(sfwe)
#sfwe = '/home/local/WX/ben.koziol/climate_data/QED-2013/sfwe/maurer02v2/sfwe/sfwe.nc'
calc = [{'func': 'sum', 'name': 'p'}]
time_range = None
rds = [maurer_pr()]
rdc = RequestDatasetCollection(rds)
pr = compute(rdc, calc, ['month', 'year'], 175, verbose=True, prefix='pr')
print(pr)
#pr = '/home/local/WX/ben.koziol/climate_data/QED-2013/sfwe/maurer02v2/pr/pr.nc'
calc = [{
'func': 'ratio_sfwe_p',
'name': 'sfwe_p',
'kwds': {
'sfwe': 'sfwe',
'p': 'p'
}
}]
rds = [RequestDataset(sfwe, 'sfwe'), RequestDataset(pr, 'p')]
rdc = RequestDatasetCollection(rds)
sfwe_p = compute(rdc, calc, None, 175, verbose=True, prefix='sfwe_p')
print(sfwe_p)