def test_calculation_iteration(self):
field = self.get_field(with_value=True,month_count=2)
field.variables.add_variable(Variable(value=field.variables['tmax'].value+5,
name='tmin',alias='tmin'))
field.temporal.name_uid = 'tid'
field.level.name_uid = 'lid'
field.spatial.geom.name_uid = 'gid'
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
mu = Mean(field=field,tgd=tgd,alias='my_mean')
ret = mu.execute()
kwds = copy(field.__dict__)
kwds.pop('_raw')
kwds.pop('_variables')
kwds['temporal'] = tgd
kwds['variables'] = ret
cfield = DerivedField(**kwds)
cfield.temporal.name_uid = 'tid'
cfield.temporal.name_value = 'time'
cfield.spatial.name_uid = 'gid'
sc = ShpCabinet()
meta = sc.get_meta('state_boundaries')
sp = SpatialCollection(meta=meta,key='state_boundaries',headers=constants.calc_headers)
for row in sc.iter_geoms('state_boundaries'):
sp.add_field(row['properties']['UGID'],row['geom'],cfield.variables.keys()[0],
cfield,properties=row['properties'])
for ii,row in enumerate(sp.get_iter_dict()):
if ii == 0:
self.assertEqual(row[0].bounds,(-100.5, 39.5, -99.5, 40.5))
self.assertDictEqual(row[1],{'lid': 1, 'ugid': 1, 'vid': 1, 'cid': 1, 'did': 1, 'year': 2000, 'time': datetime.datetime(2000, 1, 16, 0, 0), 'calc_alias': 'my_mean_tmax', 'value': 0.44808476666433006, 'month': 1, 'alias': 'tmax', 'variable': 'tmax', 'gid': 1, 'calc_key': 'mean', 'tid': 1, 'level': 50, 'day': 16})
self.assertEqual(len(row),2)
self.assertEqual(len(row[1]),len(constants.calc_headers))
def test_use_raw_values(self):
field = self.get_field(with_value=True, month_count=2)
field.variables.add_variable(Variable(value=field.variables['tmax'].value + 5,
name='tmin', alias='tmin'))
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
ur = [True, False]
agg = [
True,
False
]
for u, a in itertools.product(ur, agg):
if a:
cfield = field.get_spatially_aggregated()
self.assertNotEqual(cfield.shape, cfield._raw.shape)
self.assertEqual(set([r.value.shape for r in cfield.variables.values()]), set([(2, 60, 2, 1, 1)]))
self.assertEqual(cfield.shape, (2, 60, 2, 1, 1))
else:
cfield = field
self.assertEqual(set([r.value.shape for r in cfield.variables.values()]), set([(2, 60, 2, 3, 4)]))
self.assertEqual(cfield.shape, (2, 60, 2, 3, 4))
mu = Mean(field=cfield, tgd=tgd, alias='my_mean', use_raw_values=u)
ret = mu.execute()
if a:
self.assertEqual(set([r.value.shape for r in ret.values()]), set([(2, 2, 2, 1, 1)]))
else:
self.assertEqual(set([r.value.shape for r in ret.values()]), set([(2, 2, 2, 3, 4)]))
def test_sample_size(self):
field = self.get_field(with_value=True, month_count=2)
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
mu = Mean(field=field, tgd=tgd, alias='my_mean', calc_sample_size=True,
dtype=np.float64)
dvc = mu.execute()
dv = dvc['my_mean']
self.assertEqual(dv.name, 'mean')
self.assertEqual(dv.alias, 'my_mean')
self.assertIsInstance(dv, DerivedVariable)
self.assertEqual(dv.value.shape, (2, 2, 2, 3, 4))
self.assertNumpyAll(np.ma.mean(field.variables['tmax'].value[1, tgd.dgroups[1], 0, :, :], axis=0),
dv.value[1, 1, 0, :, :])
ret = dvc['n_my_mean']
self.assertNumpyAll(ret.value[0, 0, 0],
np.ma.array(data=[[31, 31, 31, 31], [31, 31, 31, 31], [31, 31, 31, 31]],
mask=[[False, False, False, False], [False, False, False, False],
[False, False, False, False]],
fill_value=999999,
dtype=ret.dtype))
mu = Mean(field=field, tgd=tgd, alias='my_mean', calc_sample_size=False)
dvc = mu.execute()
self.assertNotIn('n_my_mean', dvc.keys())
def test_execute_units_are_maintained(self):
field = self.get_field(with_value=True, month_count=2)
units_kelvin = get_units_object('kelvin')
self.assertEqual(field['tmax'].cfunits, units_kelvin)
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
mu = Mean(field=field, tgd=tgd, alias='my_mean', calc_sample_size=False, dtype=np.float64)
dvc = mu.execute()
self.assertEqual(dvc['my_mean'].cfunits, units_kelvin)
def test_two_variables(self):
field = self.get_field(with_value=True, month_count=2)
field.variables.add_variable(Variable(value=field.variables['tmax'].value + 5,
name='tmin', alias='tmin'))
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
mu = Mean(field=field, tgd=tgd, alias='my_mean', dtype=np.float64)
ret = mu.execute()
self.assertEqual(len(ret), 2)
self.assertAlmostEqual(5.0, abs(ret['my_mean_tmax'].value.mean() - ret['my_mean_tmin'].value.mean()))
def test_execute(self):
field = self.get_field(with_value=True, month_count=2)
self.assertIsInstance(field, Field)
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
mu = Mean(field=field, tgd=tgd, alias='my_mean', dtype=np.float64)
dvc = mu.execute()
dv = dvc['my_mean']
self.assertEqual(dv.name, 'my_mean')
self.assertEqual(dv.get_value().shape, (2, 2, 2, 3, 4))
self.assertNumpyAll(np.mean(field['tmax'].get_value()[1, tgd.dgroups[1], 0, :, :], axis=0),
dv.get_value()[1, 1, 0, :, :])
def test_calculate(self):
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
field = field.get_field_slice({'y': slice(0, 10), 'x': slice(0, 10)})
klasses = [IcclimTG, IcclimTN, IcclimTX]
for klass in klasses:
for calc_grouping in [['month'], ['month', 'year']]:
tgd = field.temporal.get_grouping(calc_grouping)
itg = klass(field=field, tgd=tgd)
ret_icclim = itg.execute()
mean = Mean(field=field, tgd=tgd)
ret_ocgis = mean.execute()
self.assertNumpyAll(ret_icclim[klass.key].get_value(), ret_ocgis['mean'].get_value())
def test_execute_units_are_maintained(self):
field = self.get_field(with_value=True, month_count=2)
units_kelvin = get_units_object('kelvin')
self.assertEqual(field['tmax'].cfunits, units_kelvin)
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
mu = Mean(field=field,
tgd=tgd,
alias='my_mean',
calc_sample_size=False,
dtype=np.float64)
dvc = mu.execute()
self.assertEqual(dvc['my_mean'].cfunits, units_kelvin)
def test_Mean(self):
field = self.get_field(with_value=True,month_count=2)
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
mu = Mean(field=field,tgd=tgd,alias='my_mean')
dvc = mu.execute()
dv = dvc['my_mean_tmax']
self.assertEqual(dv.name,'mean')
self.assertEqual(dv.alias,'my_mean_tmax')
self.assertIsInstance(dv,DerivedVariable)
self.assertEqual(dv.value.shape,(2,2,2,3,4))
self.assertNumpyAll(np.ma.mean(field.variables['tmax'].value[1,tgd.dgroups[1],0,:,:],axis=0),
dv.value[1,1,0,:,:])
def test_execute_two_variables(self):
"""Test running a field with two variables through the mean calculation."""
field = self.get_field(with_value=True, month_count=2)
field.add_variable(
Variable(value=field['tmax'].get_value() + 5, name='tmin', dimensions=field['tmax'].dimensions),
is_data=True)
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
mu = Mean(field=field, tgd=tgd, alias='my_mean', dtype=np.float64)
ret = mu.execute()
self.assertEqual(len(ret), 2)
self.assertAlmostEqual(5.0,
abs(ret['my_mean_tmax'].get_value().mean() - ret['my_mean_tmin'].get_value().mean()))
def test_execute_file_only(self):
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
field = field.get_field_slice({'time': slice(10, 20), 'y': slice(20, 30), 'x': slice(40, 50)})
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
# Value should not be loaded at this point.
self.assertEqual(field['tas']._value, None)
mu = Mean(field=field, tgd=tgd, alias='my_mean', file_only=True)
ret = mu.execute()
# Value should still not be loaded.
self.assertIsNone(field['tas']._value)
# No value should be calculated for the calculation.
self.assertIsNone(ret['my_mean']._value)
def test_execute(self):
field = self.get_field(with_value=True, month_count=2)
self.assertIsInstance(field, Field)
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
mu = Mean(field=field, tgd=tgd, alias='my_mean', dtype=np.float64)
dvc = mu.execute()
dv = dvc['my_mean']
self.assertEqual(dv.name, 'my_mean')
self.assertEqual(dv.get_value().shape, (2, 2, 2, 3, 4))
self.assertNumpyAll(
np.mean(field['tmax'].get_value()[1, tgd.dgroups[1], 0, :, :],
axis=0),
dv.get_value()[1, 1, 0, :, :])
def test_execute_two_variables_sample_size(self):
field = self.get_field(with_value=True, month_count=2)
field.add_variable(
Variable(value=field['tmax'].get_value() + 5, name='tmin', dimensions=field['tmax'].dimensions),
is_data=True)
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
mu = Mean(field=field, tgd=tgd, alias='my_mean', dtype=np.float64, calc_sample_size=True)
ret = mu.execute()
self.assertEqual(len(ret), 4)
self.assertAlmostEqual(5.0,
abs(ret['my_mean_tmax'].get_value().mean() - ret['my_mean_tmin'].get_value().mean()))
self.assertEqual({'my_mean_tmax', 'n_my_mean_tmax', 'my_mean_tmin', 'n_my_mean_tmin'},
set(ret.keys()))
def test_calculation_iteration_two_calculations(self):
field = self.get_field(with_value=True, month_count=2)
field.variables.add_variable(Variable(value=field.variables['tmax'].value + 5, name='tmin', alias='tmin'))
field.temporal.name_uid = 'tid'
field.level.name_uid = 'lid'
field.spatial.geom.name_uid = 'gid'
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
mu = Mean(field=field, tgd=tgd, alias='my_mean', dtype=np.float64, add_parents=True)
ret = mu.execute()
thresh = Threshold(field=field, vc=ret, tgd=tgd, alias='a_treshold', add_parents=True,
parms={'operation': 'gte', 'threshold': 0.5})
ret = thresh.execute()
kwds = copy(field.__dict__)
kwds.pop('_raw')
kwds.pop('_variables')
kwds.pop('_should_regrid')
kwds.pop('_has_assigned_coordinate_system')
kwds.pop('_attrs')
kwds['name'] = kwds.pop('_name')
kwds['temporal'] = tgd
kwds['variables'] = ret
cfield = DerivedField(**kwds)
cfield.temporal.name_uid = 'tid'
cfield.temporal.name_value = 'time'
cfield.spatial.name_uid = 'gid'
sc = ShpCabinet()
meta = sc.get_meta('state_boundaries')
sp = SpatialCollection(meta=meta, key='state_boundaries', headers=constants.HEADERS_CALC)
for row in sc.iter_geoms('state_boundaries', as_spatial_dimension=True):
sp.add_field(cfield, ugeom=row)
cids = set()
for ii, row in enumerate(sp.get_iter_dict(melted=True)):
cids.update([row[1]['cid']])
if ii == 0:
self.assertEqual(row[0].bounds, (-100.5, 39.5, -99.5, 40.5))
self.assertDictEqual(row[1], {'lid': 1, 'ugid': 1, 'vid': 1, 'cid': 1, 'did': 1, 'year': 2000,
'time': datetime.datetime(2000, 1, 16, 0, 0),
'calc_alias': 'my_mean_tmax', 'value': 0.44808476666433006, 'month': 1,
'alias': 'tmax', 'variable': 'tmax', 'gid': 1, 'calc_key': 'mean',
'tid': 1, 'level': 50, 'day': 16})
self.assertEqual(len(row), 2)
self.assertEqual(len(row[1]), len(constants.HEADERS_CALC))
self.assertEqual(ii + 1, 2 * 2 * 2 * 3 * 4 * 51 * 4)
self.assertEqual(len(cids), 4)
def test_file_only(self):
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
field = field[:, 10:20, :, 20:30, 40:50]
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
## value should not be loaded at this point
self.assertEqual(field.variables['tas']._value, None)
mu = Mean(field=field, tgd=tgd, alias='my_mean', file_only=True)
ret = mu.execute()
## value should still not be loaded
self.assertEqual(field.variables['tas']._value, None)
## there should be no value in the variable present and attempts to load
## it should fail.
with self.assertRaises(Exception):
ret['my_mean_tas'].value
def test_execute_two_variables(self):
"""Test running a field with two variables through the mean calculation."""
field = self.get_field(with_value=True, month_count=2)
field.add_variable(Variable(value=field['tmax'].get_value() + 5,
name='tmin',
dimensions=field['tmax'].dimensions),
is_data=True)
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
mu = Mean(field=field, tgd=tgd, alias='my_mean', dtype=np.float64)
ret = mu.execute()
self.assertEqual(len(ret), 2)
self.assertAlmostEqual(
5.0,
abs(ret['my_mean_tmax'].get_value().mean() -
ret['my_mean_tmin'].get_value().mean()))
def test_execute_file_only(self):
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
field = field.get_field_slice({
'time': slice(10, 20),
'y': slice(20, 30),
'x': slice(40, 50)
})
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
# Value should not be loaded at this point.
self.assertEqual(field['tas']._value, None)
mu = Mean(field=field, tgd=tgd, alias='my_mean', file_only=True)
ret = mu.execute()
# Value should still not be loaded.
self.assertIsNone(field['tas']._value)
# No value should be calculated for the calculation.
self.assertIsNone(ret['my_mean']._value)
def test_execute_sample_size(self):
field = self.get_field(with_value=True, month_count=2)
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
mu = Mean(field=field,
tgd=tgd,
alias='my_mean',
calc_sample_size=True,
dtype=np.float64)
dvc = mu.execute()
dv = dvc['my_mean']
self.assertEqual(dv.name, 'my_mean')
self.assertEqual(dv.get_value().shape, (2, 2, 2, 3, 4))
self.assertNumpyAll(
np.mean(field['tmax'].get_value()[1, tgd.dgroups[1], 0, :, :],
axis=0),
dv.get_value()[1, 1, 0, :, :])
ret = dvc['n_my_mean']
self.assertNumpyAll(
ret.get_masked_value()[0, 0, 0],
np.ma.array(data=[[31, 31, 31, 31], [31, 31, 31, 31],
[31, 31, 31, 31]],
mask=[[False, False, False, False],
[False, False, False, False],
[False, False, False, False]],
fill_value=999999,
dtype=ret.dtype))
mu = Mean(field=field,
tgd=tgd,
alias='my_mean',
calc_sample_size=False)
dvc = mu.execute()
self.assertNotIn('n_my_mean', list(dvc.keys()))
def test_execute_two_variables_sample_size(self):
field = self.get_field(with_value=True, month_count=2)
field.add_variable(Variable(value=field['tmax'].get_value() + 5,
name='tmin',
dimensions=field['tmax'].dimensions),
is_data=True)
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
mu = Mean(field=field,
tgd=tgd,
alias='my_mean',
dtype=np.float64,
calc_sample_size=True)
ret = mu.execute()
self.assertEqual(len(ret), 4)
self.assertAlmostEqual(
5.0,
abs(ret['my_mean_tmax'].get_value().mean() -
ret['my_mean_tmin'].get_value().mean()))
self.assertEqual(
{
'my_mean_tmax', 'n_my_mean_tmax', 'my_mean_tmin',
'n_my_mean_tmin'
}, set(ret.keys()))
