def test_HeatIndex_keyed_output(self):
raise (SkipTest)
ds = [
self.test_data.get_rd('cancm4_tasmax_2011'),
self.test_data.get_rd('cancm4_rhsmax')
]
calc = [{
'func': 'heat_index',
'name': 'heat_index',
'kwds': {
'tas': 'tasmax',
'rhs': 'rhsmax',
'units': 'k'
}
}]
ops = OcgOperations(dataset=ds,
calc=calc,
snippet=False,
output_format='numpy')
self.assertEqual(ops.calc_grouping, None)
ret = ops.execute()
it = KeyedIterator(ret[1], mode='calc')
for ii, row in enumerate(it.iter_rows(ret[1])):
if ii < 1000:
self.assertEqual(row['cid'], 1)
self.assertEqual(row['tgid'], None)
self.assertNotEqual(row['tid'], None)
else:
break
ops = OcgOperations(dataset=ds,
calc=calc,
snippet=True,
output_format='keyed')
ops.execute()
def test_csv_conversion(self):
ocgis.env.OVERWRITE = True
ops = OcgOperations(dataset=self.get_dataset(), output_format='csv')
ret = self.get_ret(ops)
## test with a geometry to check writing of user-geometry overview shapefile
geom = make_poly((38, 39), (-104, -103))
ops = OcgOperations(dataset=self.get_dataset(),
output_format='csv',
geom=geom)
ret = ops.execute()
def test_HeatIndex(self):
kwds = {'time_range': [dt(2011, 1, 1), dt(2011, 12, 31, 23, 59, 59)]}
ds = [
self.test_data.get_rd('cancm4_tasmax_2011', kwds=kwds),
self.test_data.get_rd('cancm4_rhsmax', kwds=kwds)
]
calc = [{
'func': 'heat_index',
'name': 'heat_index',
'kwds': {
'tas': 'tasmax',
'rhs': 'rhsmax',
'units': 'k'
}
}]
## operations on entire data arrays
ops = OcgOperations(dataset=ds, calc=calc)
self.assertEqual(ops.calc_grouping, None)
ret = ops.execute()
ref = ret[1]
self.assertEqual(ref.variables.keys(), ['tasmax', 'rhsmax'])
self.assertEqual(ref.calc.keys(), ['heat_index'])
hi = ref.calc['heat_index']
self.assertEqual(hi.shape, (365, 1, 64, 128))
## confirm no masked geometries
self.assertFalse(ref._archetype.spatial.vector.geom.mask.any())
## confirm some masked data in calculation output
self.assertTrue(hi.mask.any())
## snippet-based testing
ops = OcgOperations(dataset=ds, calc=calc, snippet=True)
ret = ops.execute()
self.assertEqual(ret[1].calc['heat_index'].shape, (1, 1, 64, 128))
ops = OcgOperations(dataset=ds,
calc=calc,
snippet=True,
output_format='csv')
ret = ops.execute()
# subprocess.check_call(['loffice',ret])
# try temporal grouping
ops = OcgOperations(dataset=ds, calc=calc, calc_grouping=['month'])
ret = ops.execute()
self.assertEqual(ret[1].calc['heat_index'].shape, (12, 1, 64, 128))
ret = OcgOperations(dataset=ds,
calc=calc,
calc_grouping=['month'],
output_format='csv',
snippet=True).execute()
def test_get_meta(self):
ops = OcgOperations(dataset=self.datasets)
meta = ops.get_meta()
self.assertTrue(len(meta) > 100)
self.assertTrue('\n' in meta)
ops = OcgOperations(dataset=self.datasets,
calc=[{
'func': 'mean',
'name': 'my_mean'
}])
meta = ops.get_meta()
self.assertTrue(len(meta) > 100)
self.assertTrue('\n' in meta)
def test_differing_projections(self):
rd1 = self.test_data.get_rd('daymet_tmax')
# rd2 = RequestDataset(uri=self.hostetler,variable='TG',t_calendar='noleap')
rd2 = self.test_data.get_rd('cancm4_tas')
## for numpy formats, different projections are allowed.
ops = OcgOperations(dataset=[rd1, rd2], snippet=True)
ret = ops.execute()
## it is not okay for other formats
with self.assertRaises(ValueError):
ops = OcgOperations(dataset=[rd1, rd2],
snippet=True,
output_format='csv+')
ops.execute()
def test_calculation(self):
calc = [{
'func': 'mean',
'name': 'mean'
}, {
'func': 'std',
'name': 'std'
}]
calc_grouping = ['year']
kwds = {
'aggregate': True,
'spatial_operation': 'clip',
'calc': calc,
'calc_grouping': calc_grouping,
'output_format': 'numpy',
'geom': self.california,
'dataset': self.dataset,
'snippet': False
}
ops = OcgOperations(**kwds)
ret = ops.execute()
ref = ret[25].calc['Prcp']
self.assertEquals(ref.keys(), ['mean', 'std', 'n'])
for value in ref.itervalues():
self.assertEqual(value.shape, (1, 1, 1, 1))
ref = ret[25].calc['tasmax']
self.assertEquals(ref.keys(), ['mean', 'std', 'n'])
for value in ref.itervalues():
self.assertEqual(value.shape, (10, 1, 1, 1))
def test_keyed(self):
raise(SkipTest)
ds = self.dataset
# ds.append(self.albisccp.copy())
ds.append(self.tasmin.copy())
ops = OcgOperations(dataset=ds,geom=self.california,output_format='numpy')
ret = ops.execute()
ref = ret[25].variables
self.assertEqual(ref['tasmax']._use_for_id,['gid','tid'])
self.assertEqual(ref['tasmin']._use_for_id,[])
# for key in ['albisccp','Prcp']:
# self.assertEqual(ret[25].variables[key]._use_for_id,['gid','tid'])
ops = OcgOperations(dataset=ds,geom=self.california,output_format='keyed',snippet=True)
ret = ops.execute()
def test_differing_projections(self):
rd1 = self.test_data.get_rd('daymet_tmax')
# rd2 = RequestDataset(uri=self.hostetler,variable='TG',t_calendar='noleap')
rd2 = self.test_data.get_rd('cancm4_tas')
ops = OcgOperations(dataset=[rd1, rd2], snippet=True)
with self.assertRaises(ValueError):
ops.execute()
def test_calculate(self):
# ocgis.env.VERBOSE = True
# ocgis.env.DEBUG = True
calc = [{
'func': 'sfwe',
'name': 'sfwe',
'kwds': {
'tas': 'tas',
'pr': 'pr'
}
}]
time_range = [dt(1990, 1, 1), dt(1990, 3, 31)]
rds = []
for var in [self.maurer_pr, self.maurer_tas]:
var.update({'time_range': time_range})
rds.append(var)
geom = 'state_boundaries'
select_ugid = [16]
ops = OcgOperations(dataset=rds,
geom=geom,
select_ugid=select_ugid,
calc=calc,
calc_grouping=['month'],
output_format='nc')
ret = ops.execute()
def get_ops(self, kwds={}, time_range=None, level_range=None):
dataset = self.get_dataset(time_range, level_range)
if 'output_format' not in kwds:
kwds.update({'output_format': 'numpy'})
kwds.update({'dataset': dataset})
ops = OcgOperations(**kwds)
return (ops)
def test_calc_grouping(self):
_cg = [None, ['day', 'month'], 'day']
for cg in _cg:
if cg is not None:
eq = tuple(cg)
else:
eq = cg
obj = definition.CalcGrouping(cg)
try:
self.assertEqual(obj.value, eq)
except AssertionError:
self.assertEqual(obj.value, ('day', ))
## only month, year, and day combinations are currently supported
rd = self.test_data.get_rd('cancm4_tas')
calcs = [None, [{'func': 'mean', 'name': 'mean'}]]
acceptable = ['day', 'month', 'year']
for calc in calcs:
for length in [1, 2, 3, 4, 5]:
for combo in itertools.combinations(
['day', 'month', 'year', 'hour', 'minute'], length):
try:
ops = OcgOperations(dataset=rd,
calc=calc,
calc_grouping=combo)
except DefinitionValidationError:
reraise = True
for c in combo:
if c not in acceptable:
reraise = False
if reraise:
raise
def test_same_variable_name(self):
ds = [self.cancm4.copy(), self.cancm4.copy()]
with self.assertRaises(KeyError):
OcgOperations(dataset=ds)
ds[0].alias = 'foo'
ds[1].alias = 'foo'
with self.assertRaises(KeyError):
OcgOperations(dataset=ds)
ds = [self.cancm4.copy(), self.cancm4.copy()]
ds[0].alias = 'foo_var'
ops = OcgOperations(dataset=ds, snippet=True)
ret = ops.execute()
self.assertEqual(ret[1].variables.keys(), ['foo_var', 'tasmax'])
values = ret[1].variables.values()
self.assertTrue(np.all(values[0].value == values[1].value))
def test_null_parms(self):
ops = OcgOperations(dataset=self.datasets_no_range)
self.assertEqual(ops.geom, None)
self.assertEqual(len(ops.dataset), 3)
for ds in ops.dataset:
self.assertEqual(ds.time_range, None)
self.assertEqual(ds.level_range, None)
ops.__repr__()
def get_ops(self, kwds={}):
geom = self.california
ops = OcgOperations(dataset=self.dataset,
snippet=True,
geom=geom,
output_format='numpy')
for k, v in kwds.iteritems():
setattr(ops, k, v)
return (ops)
def test_keyed_conversion(self):
raise (SkipTest)
calc = [None, [{'func': 'mean', 'name': 'my_mean'}]]
group = ['month', 'year']
for c in calc:
ops = OcgOperations(dataset=self.get_dataset(),
output_format='keyed',
calc=c,
calc_grouping=group)
ret = self.get_ret(ops)
def test_point_shapefile_subset(self):
_output_format = ['numpy', 'nc', 'csv', 'csv+']
for output_format in _output_format:
rd = self.test_data.get_rd('cancm4_tas')
ops = OcgOperations(dataset=rd,
geom='qed_city_centroids',
output_format=output_format,
prefix=output_format)
ret = ops.execute()
if output_format == 'numpy':
self.assertEqual(len(ret), 4)
def test_daymet(self):
# uri = 'http://daymet.ornl.gov/thredds//dodsC/allcf/2011/9947_2011/tmax.nc'
rd = self.test_data.get_rd('daymet_tmax')
geom = 'state_boundaries'
select_ugid = [32]
snippet = True
ops = OcgOperations(dataset=rd,
geom=geom,
snippet=snippet,
select_ugid=select_ugid,
output_format='numpy')
ops.execute()
def test_geom_string(self):
ops = OcgOperations(dataset=self.datasets, geom='state_boundaries')
self.assertEqual(len(ops.geom), 51)
ops.geom = None
self.assertEqual(ops.geom, None)
ops.geom = 'mi_watersheds'
self.assertEqual(len(ops.geom), 60)
ops.geom = '-120|40|-110|50'
self.assertEqual(ops.geom.spatial.geom[0].bounds,
(-120.0, 40.0, -110.0, 50.0))
ops.geom = [-120, 40, -110, 50]
self.assertEqual(ops.geom.spatial.geom[0].bounds,
(-120.0, 40.0, -110.0, 50.0))
def get_collection(self, aggregate=False):
if aggregate:
spatial_operation = 'clip'
else:
spatial_operation = 'intersects'
rd = self.test_data.get_rd('cancm4_tas')
ops = OcgOperations(dataset=rd,
geom='state_boundaries',
select_ugid=[25],
spatial_operation=spatial_operation,
aggregate=aggregate)
ret = ops.execute()
return (ret[25])
def test_calculate(self):
ocgis.env.DIR_BIN = '/home/local/WX/ben.koziol/links/ocgis/bin/QED_2013_dynamic_percentiles'
percentiles = [90, 92.5, 95, 97.5]
operations = ['gt', 'gte', 'lt', 'lte']
calc_groupings = [
['month'],
# ['month','year'],
# ['year']
]
uris_variables = [
[
'/home/local/WX/ben.koziol/climate_data/maurer/2010-concatenated/Maurer02new_OBS_tasmax_daily.1971-2000.nc',
'tasmax'
],
[
'/home/local/WX/ben.koziol/climate_data/maurer/2010-concatenated/Maurer02new_OBS_tasmin_daily.1971-2000.nc',
'tasmin'
]
]
geoms_select_ugids = [
['qed_city_centroids', None],
['state_boundaries', [39]],
# ['us_counties',[2416,1335]]
]
for tup in itertools.product(percentiles, operations, calc_groupings,
uris_variables, geoms_select_ugids):
print(tup)
percentile, operation, calc_grouping, uri_variable, geom_select_ugid = tup
ops = OcgOperations(dataset={
'uri': uri_variable[0],
'variable': uri_variable[1],
'time_region': {
'year': [1990],
'month': [6, 7, 8]
}
},
geom=geom_select_ugid[0],
select_ugid=geom_select_ugid[1],
calc=[{
'func': 'qed_dynamic_percentile_threshold',
'kwds': {
'operation': operation,
'percentile': percentile
},
'name': 'dp'
}],
calc_grouping=calc_grouping,
output_format='numpy')
ret = ops.execute()
def test_high_res(self):
ocgis.env.OVERWRITE = True
nc_spatial = NcSpatial(0.5,(-90.0,90.0),(0.0,360.0))
path = self.make_data(nc_spatial)
dataset = {'uri':path,'variable':'foo'}
output_format = 'nc'
snippet = True
geom = self.nebraska
for s_abstraction in ['point','polygon']:
interface = {'s_abstraction':s_abstraction}
ops = OcgOperations(dataset=dataset,output_format=output_format,
geom=geom,snippet=snippet,abstraction=s_abstraction)
ret = OcgInterpreter(ops).execute()
def test_clip_aggregate(self):
## this geometry was hanging
# ocgis.env.VERBOSE = True
# ocgis.env.DEBUG = True
rd = self.test_data.get_rd('cancm4_tas',
kwds={'time_region': {
'year': [2003]
}})
ops = OcgOperations(dataset=rd,
geom='state_boundaries',
select_ugid=[14, 16],
aggregate=False,
spatial_operation='clip',
output_format='csv+')
ret = ops.execute()
def test_shp_conversion(self):
ocgis.env.OVERWRITE = True
calc = [
None,
[{
'func': 'mean',
'name': 'my_mean'
}],
]
group = ['month', 'year']
for c in calc:
ops = OcgOperations(dataset=self.get_dataset(),
output_format='shp',
calc_grouping=group,
calc=c)
ret = self.get_ret(ops)
def run_standard_operations(self, calc, capture=False, output_format=None):
_aggregate = [False, True]
_calc_grouping = [['month'], ['month', 'year']]
_output_format = output_format or ['numpy', 'csv+', 'nc']
captured = []
for ii, tup in enumerate(
itertools.product(_aggregate, _calc_grouping, _output_format)):
aggregate, calc_grouping, output_format = tup
if aggregate is True and output_format == 'nc':
continue
rd = self.test_data.get_rd(
'cancm4_tas', kwds={'time_region': {
'year': [2001, 2002]
}})
try:
ops = OcgOperations(dataset=rd,
geom='state_boundaries',
select_ugid=[25],
calc=calc,
calc_grouping=calc_grouping,
output_format=output_format,
aggregate=aggregate,
prefix=('standard_ops_' + str(ii)))
ret = ops.execute()
if output_format == 'numpy':
ref = ret[25].calc['tas'][calc[0]['name']]
if aggregate:
space_shape = [1, 1]
else:
space_shape = [5, 4]
if calc_grouping == ['month']:
shp1 = [12]
else:
shp1 = [24]
test_shape = shp1 + [1] + space_shape
self.assertEqual(ref.shape, tuple(test_shape))
if not aggregate:
self.assertTrue(np.ma.is_masked(ref[0, 0, 0, 0]))
except Exception as e:
if capture:
parms = dict(aggregate=aggregate,
calc_grouping=calc_grouping,
output_format=output_format)
captured.append({'exception': e, 'parms': parms})
else:
raise
return (captured)
def test_low_res(self):
ocgis.env.OVERWRITE = True
nc_spatial = NcSpatial(5.0,(-90.0,90.0),(0.0,360.0))
path = self.make_data(nc_spatial)
dataset = {'uri':path,'variable':'foo'}
output_format = 'shp'
geom = self.nebraska
ip = Inspect(dataset['uri'],dataset['variable'])
for s_abstraction in ['point','polygon']:
interface = {'s_abstraction':s_abstraction}
ops = OcgOperations(dataset=dataset,
output_format=output_format,
geom=geom,
abstraction=s_abstraction)
ret = OcgInterpreter(ops).execute()
def get_ret(self,
ops=None,
kwds={},
shp=False,
time_range=None,
level_range=None):
if ops is None:
ops = self.get_ops(kwds,
time_range=time_range,
level_range=level_range)
self.ops = ops
ret = OcgInterpreter(ops).execute()
if shp or self.return_shp:
kwds2 = kwds.copy()
kwds2.update({'output_format': 'shp'})
ops2 = OcgOperations(**kwds2)
OcgInterpreter(ops2).execute()
return (ret)
def test_bad_time_dimension(self):
ocgis.env.DIR_DATA = '/usr/local/climate_data'
uri = 'seasonalbias.nc'
variable = 'bias'
for output_format in ['csv', 'csv+', 'shp', 'numpy']:
ops = OcgOperations(dataset={
'uri': uri,
'variable': variable
},
output_format=output_format,
format_time=False,
prefix=output_format)
ret = ops.execute()
if output_format == 'numpy':
self.assertNumpyAll(
ret[1].variables['bias'].temporal.value,
np.array([-712208.5, -712117., -712025., -711933.5]))
self.assertNumpyAll(
ret[1].variables['bias'].temporal.bounds,
np.array([[-712254., -712163.], [-712163., -712071.],
[-712071., -711979.], [-711979., -711888.]]))
def test_frequency_duration_real_data(self):
uri = 'Maurer02new_OBS_tasmax_daily.1971-2000.nc'
variable = 'tasmax'
ocgis.env.DIR_DATA = '/usr/local/climate_data'
for output_format in ['csv+', 'shp', 'csv']:
ops = OcgOperations(
dataset={
'uri': uri,
'variable': variable,
'time_region': {
'year': [1991],
'month': [7]
}
},
output_format=output_format,
prefix=output_format,
calc=[{
'name': 'Frequency Duration',
'func': 'freq_duration',
'kwds': {
'threshold': 25.0,
'operation': 'gte'
}
}],
calc_grouping=['month', 'year'],
geom='us_counties',
select_ugid=[2778],
aggregate=True,
calc_raw=False,
spatial_operation='clip',
headers=[
'did', 'ugid', 'gid', 'year', 'month', 'day', 'variable',
'calc_name', 'value'
],
)
ret = ops.execute()
def test_date_groups(self):
calc = [{'func': 'mean', 'name': 'mean'}]
rd = self.test_data.get_rd('cancm4_tasmax_2011')
calc_grouping = ['month']
ops = OcgOperations(dataset=rd,
calc=calc,
calc_grouping=calc_grouping,
geom='state_boundaries',
select_ugid=[25])
ret = ops.execute()
ref = ret[25].variables['tasmax'].temporal
rdt = ref.group.representative_datetime
self.assertTrue(
np.all(rdt == np.array(
[dt(2011, month, 16) for month in range(1, 13)])))
calc_grouping = ['year']
ops = OcgOperations(dataset=rd,
calc=calc,
calc_grouping=calc_grouping,
geom='state_boundaries',
select_ugid=[25])
ret = ops.execute()
ref = ret[25].variables['tasmax'].temporal
rdt = ref.group.representative_datetime
self.assertTrue(
np.all(rdt == [dt(year, 7, 1) for year in range(2011, 2021)]))
calc_grouping = ['month', 'year']
ops = OcgOperations(dataset=rd,
calc=calc,
calc_grouping=calc_grouping,
geom='state_boundaries',
select_ugid=[25])
ret = ops.execute()
ref = ret[25].variables['tasmax'].temporal
rdt = ref.group.representative_datetime
self.assertTrue(
np.all(rdt == [
dt(year, month, 16) for year, month in itertools.product(
range(2011, 2021), range(1, 13))
]))
calc_grouping = ['day']
ops = OcgOperations(dataset=rd,
calc=calc,
calc_grouping=calc_grouping,
geom='state_boundaries',
select_ugid=[25])
ret = ops.execute()
ref = ret[25].variables['tasmax'].temporal
rdt = ref.group.representative_datetime
self.assertTrue(
np.all(rdt == [dt(2011, 1, day, 12) for day in range(1, 32)]))
calc_grouping = ['month', 'day']
ops = OcgOperations(dataset=rd,
calc=calc,
calc_grouping=calc_grouping,
geom='state_boundaries',
select_ugid=[25])
ret = ops.execute()
ref = ret[25].variables['tasmax'].temporal
rdt = ref.group.representative_datetime
self.assertEqual(rdt[0], dt(2011, 1, 1, 12))
self.assertEqual(rdt[12], dt(2011, 1, 13, 12))
calc_grouping = ['year', 'day']
ops = OcgOperations(dataset=rd,
calc=calc,
calc_grouping=calc_grouping,
geom='state_boundaries',
select_ugid=[25])
ret = ops.execute()
ref = ret[25].variables['tasmax'].temporal
rdt = ref.group.representative_datetime
self.assertEqual(rdt[0], dt(2011, 1, 1, 12))
rd = self.test_data.get_rd(
'cancm4_tasmax_2011',
kwds={'time_region': {
'month': [1],
'year': [2011]
}})
calc_grouping = ['month', 'day', 'year']
ops = OcgOperations(dataset=rd,
calc=calc,
calc_grouping=calc_grouping,
geom='state_boundaries',
select_ugid=[25])
ret = ops.execute()
ref = ret[25].variables['tasmax'].temporal
rdt = ref.group.representative_datetime
self.assertTrue(np.all(rdt == ref.value_datetime))
self.assertTrue(np.all(ref.bounds_datetime == ref.group.bounds))
def run():
dataset = {'uri':'/usr/local/climate_data/CanCM4/tasmax_day_CanCM4_decadal2000_r2i1p1_20010101-20101231.nc','variable':'tasmax'}
ops = OcgOperations(dataset=dataset,output_format='keyed')
ret = ops.execute()
