def test_calculation_file_only_one_variable(self):
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
field = field.get_field_slice({'time': slice(0, 10)})
expr = 'es=6.1078*exp(17.08085*(tas-273.16)/(234.175+(tas-273.16)))'
ef = EvalFunction(expr=expr, field=field, file_only=True)
ret = ef.execute()
self.assertEqual(ret['es']._value, None)
def test_calculation_file_only_one_variable(self):
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
field = field.get_field_slice({'time': slice(0, 10)})
expr = 'es=6.1078*exp(17.08085*(tas-273.16)/(234.175+(tas-273.16)))'
ef = EvalFunction(expr=expr, field=field, file_only=True)
ret = ef.execute()
self.assertEqual(ret['es']._value, None)
def test_is_multivariate(self):
expr = 'tas2=tas+2'
self.assertFalse(EvalFunction.is_multivariate(expr))
expr = 'tas2=log(tas+2)'
self.assertFalse(EvalFunction.is_multivariate(expr))
expr = 'tas4=tas+exp(tasmax)'
self.assertTrue(EvalFunction.is_multivariate(expr))
def test_is_multivariate(self):
expr = 'tas2=tas+2'
self.assertFalse(EvalFunction.is_multivariate(expr))
expr = 'tas2=log(tas+2)'
self.assertFalse(EvalFunction.is_multivariate(expr))
expr = 'tas4=tas+exp(tasmax)'
self.assertTrue(EvalFunction.is_multivariate(expr))
def test_calculation_file_only_one_variable(self):
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
field = field[:,0:10,:,:,:]
expr = 'es=6.1078*exp(17.08085*(tas-273.16)/(234.175+(tas-273.16)))'
ef = EvalFunction(expr=expr,field=field,file_only=True)
ret = ef.execute()
self.assertEqual(ret['es']._value,None)
self.assertEqual(ret['es'].dtype,field.variables['tas'].dtype)
self.assertEqual(ret['es'].fill_value,field.variables['tas'].fill_value)
def test_get_eval_string_bad_string(self):
# this string has no equals sign
expr = 'es6.1078*exp(log(17.08085)*(tas-273.16)/(234.175+(tas-273.16)))'
with self.assertRaises(ValueError):
EvalFunction._get_eval_string_(expr, {'tas': 'var.value'})
# this string has a numpy function "foo" that is not enabled (and does not exist)
expr = 'es=6.1078*exp(foo(17.08085)*(tas-273.16)/(234.175+(tas-273.16)))'
with self.assertRaises(ValueError):
EvalFunction._get_eval_string_(expr, {'tas': 'var.value'})
def test_get_eval_string_bad_string(self):
## this string has no equals sign
expr = 'es6.1078*exp(log(17.08085)*(tas-273.16)/(234.175+(tas-273.16)))'
with self.assertRaises(ValueError):
EvalFunction._get_eval_string_(expr,{'tas':'var.value'})
## this string has a numpy function "foo" that is not enabled (and does not exist)
expr = 'es=6.1078*exp(foo(17.08085)*(tas-273.16)/(234.175+(tas-273.16)))'
with self.assertRaises(ValueError):
EvalFunction._get_eval_string_(expr,{'tas':'var.value'})
def test_calculation_one_variable_exp_and_log(self):
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
field = field[:,0:10,:,:,:]
expr = 'es=6.1078*exp(log(17.08085)*(tas-273.16)/(234.175+(tas-273.16)))'
ef = EvalFunction(expr=expr,field=field)
ret = ef.execute()
var = field.variables['tas']
actual_value = 6.1078*np.exp(np.log(17.08085)*(var.value-273.16)/(234.175+(var.value-273.16)))
self.assertNumpyAll(ret['es'].value,actual_value)
def test_calculation_file_only_two_variables(self):
rd = self.test_data.get_rd('cancm4_tas')
rd2 = self.test_data.get_rd('cancm4_tasmax_2001')
field = rd.get()
field2 = rd2.get()
field.variables.add_variable(field2.variables['tasmax'],assign_new_uid=True)
field = field[:,0:10,:,:,:]
expr = 'foo=log(1000*(tasmax-tas))/3'
ef = EvalFunction(expr=expr,field=field,file_only=True)
ret = ef.execute()
self.assertEqual(ret['foo']._value,None)
def test_calculation_file_only_two_variables(self):
rd = self.test_data.get_rd('cancm4_tas')
rd2 = self.test_data.get_rd('cancm4_tasmax_2001')
field = rd.get()
field2 = rd2.get()
with orphaned(field2['tasmax']):
field.add_variable(field2['tasmax'], is_data=True)
field = field.get_field_slice({'time': slice(0, 10)})
expr = 'foo=log(1000*(tasmax-tas))/3'
ef = EvalFunction(expr=expr, field=field, file_only=True)
ret = ef.execute()
self.assertEqual(ret['foo']._value, None)
def test_calculation_file_only_two_variables(self):
rd = self.test_data.get_rd('cancm4_tas')
rd2 = self.test_data.get_rd('cancm4_tasmax_2001')
field = rd.get()
field2 = rd2.get()
with orphaned(field2['tasmax']):
field.add_variable(field2['tasmax'], is_data=True)
field = field.get_field_slice({'time': slice(0, 10)})
expr = 'foo=log(1000*(tasmax-tas))/3'
ef = EvalFunction(expr=expr, field=field, file_only=True)
ret = ef.execute()
self.assertEqual(ret['foo']._value, None)
def test_calculation_one_variable_exp_and_log(self):
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
# field = field[:, 0:10, :, :, :]
field = field.get_field_slice({'time': slice(0, 10)})
expr = 'es=6.1078*exp(log(17.08085)*(tas-273.16)/(234.175+(tas-273.16)))'
ef = EvalFunction(expr=expr, field=field)
ret = ef.execute()
var = field['tas']
actual_value = 6.1078 * np.exp(
np.log(17.08085) * (var.get_value() - 273.16) /
(234.175 + (var.get_value() - 273.16)))
self.assertNumpyAll(ret['es'].get_value(), actual_value)
def test_calculation_one_variable_exp_only(self):
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
# field = field[:, 0:10, :, :, :]
field = field.get_field_slice({'time': slice(0, 10)})
expr = 'es=6.1078*exp(17.08085*(tas-273.16)/(234.175+(tas-273.16)))'
ef = EvalFunction(expr=expr, field=field)
ret = ef.execute()
self.assertEqual(list(ret.keys()), ['es'])
self.assertEqual(ret['es'].units, None)
self.assertEqual(ret['es'].name, 'es')
var = field['tas']
actual_value = 6.1078 * np.exp(17.08085 * (var.get_value() - 273.16) / (234.175 + (var.get_value() - 273.16)))
self.assertNumpyAll(ret['es'].get_value(), actual_value)
def test_get_eval_string_two_variables(self):
map_vars = {'tasmax': 'tasmax.value', 'tas': 'tas.value'}
expr = 'foo=log(1000*(tasmax-tas))/3'
to_eval, out_variable_name = EvalFunction._get_eval_string_(
expr, map_vars)
self.assertEqual(to_eval, 'np.log(1000*(tasmax.value-tas.value))/3')
self.assertEqual(out_variable_name, 'foo')
def test_calculation_one_variable_exp_only(self):
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
field = field[:,0:10,:,:,:]
expr = 'es=6.1078*exp(17.08085*(tas-273.16)/(234.175+(tas-273.16)))'
ef = EvalFunction(expr=expr,field=field)
ret = ef.execute()
self.assertEqual(ret.keys(),['es'])
self.assertEqual(ret['es'].units,None)
self.assertEqual(ret['es'].alias,'es')
self.assertEqual(ret['es'].name,'es')
self.assertEqual(ret['es'].parents.keys(),['tas'])
var = field.variables['tas']
actual_value = 6.1078*np.exp(17.08085*(var.value-273.16)/(234.175+(var.value-273.16)))
self.assertNumpyAll(ret['es'].value,actual_value)
def test_calculation_one_variable_exp_only(self):
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
# field = field[:, 0:10, :, :, :]
field = field.get_field_slice({'time': slice(0, 10)})
expr = 'es=6.1078*exp(17.08085*(tas-273.16)/(234.175+(tas-273.16)))'
ef = EvalFunction(expr=expr, field=field)
ret = ef.execute()
self.assertEqual(list(ret.keys()), ['es'])
self.assertEqual(ret['es'].units, None)
self.assertEqual(ret['es'].name, 'es')
var = field['tas']
actual_value = 6.1078 * np.exp(17.08085 * (var.get_value() - 273.16) /
(234.175 + (var.get_value() - 273.16)))
self.assertNumpyAll(ret['es'].get_value(), actual_value)
def test_calculation_two_variables_exp_only(self):
rd = self.test_data.get_rd('cancm4_tas')
rd2 = self.test_data.get_rd('cancm4_tasmax_2001')
field = rd.get()
field2 = rd2.get()
field.variables.add_variable(field2.variables['tasmax'],assign_new_uid=True)
field = field[:,0:10,:,:,:]
expr = 'foo=log(1000*(tasmax-tas))/3'
ef = EvalFunction(expr=expr,field=field)
ret = ef.execute()
self.assertEqual(ret.keys(),['foo'])
self.assertEqual(set(ret['foo'].parents.keys()),set(['tas','tasmax']))
tas = field.variables['tas']
tasmax = field.variables['tasmax']
actual_value = np.log(1000*(tasmax.value-tas.value))/3
self.assertNumpyAll(ret['foo'].value,actual_value)
def test_get_eval_string_power(self):
"""Test the power ufunc is appropriately parsed from the eval string."""
expr = 'es=power(foo, 4)'
map_vars = {'foo': '_exec_foo.value'}
parsed, output_variable = EvalFunction._get_eval_string_(expr, map_vars)
self.assertEqual(output_variable, 'es')
self.assertEqual(parsed, 'np.power(_exec_foo.value, 4)')
def test_get_eval_string(self):
expr = 'es=6.1078*exp(log(17.08085)*(tas-273.16)/(234.175+(tas-273.16)))'
to_eval, out_variable_name = EvalFunction._get_eval_string_(
expr, {'tas': 'var.value'})
self.assertEqual(
to_eval,
'6.1078*np.exp(np.log(17.08085)*(var.value-273.16)/(234.175+(var.value-273.16)))'
)
self.assertEqual(out_variable_name, 'es')
expr = 'tas=tas-4'
to_eval, out_variable_name = EvalFunction._get_eval_string_(
expr, {'tas': 'var.value'})
self.assertEqual(to_eval, 'var.value-4')
self.assertEqual(out_variable_name, 'tas')
expr = 'tas=4-tas'
to_eval, out_variable_name = EvalFunction._get_eval_string_(
expr, {'tas': 'var.value'})
self.assertEqual(to_eval, '4-var.value')
self.assertEqual(out_variable_name, 'tas')
expr = 'tas=tas'
to_eval, out_variable_name = EvalFunction._get_eval_string_(
expr, {'tas': 'var.value'})
self.assertEqual(to_eval, 'var.value')
self.assertEqual(out_variable_name, 'tas')
expr = 'tas=tas-tas-tas'
to_eval, out_variable_name = EvalFunction._get_eval_string_(
expr, {'tas': 'var.value'})
self.assertEqual(to_eval, 'var.value-var.value-var.value')
self.assertEqual(out_variable_name, 'tas')
expr = 'tas=tas-tas-tas-tas'
to_eval, out_variable_name = EvalFunction._get_eval_string_(
expr, {'tas': 'var.value'})
self.assertEqual(to_eval, 'var.value-var.value-var.value-var.value')
self.assertEqual(out_variable_name, 'tas')
expr = 'tas_2=tas_1-tas_1-tas_1-tas_1'
to_eval, out_variable_name = EvalFunction._get_eval_string_(
expr, {'tas_1': 'var.value'})
self.assertEqual(to_eval, 'var.value-var.value-var.value-var.value')
self.assertEqual(out_variable_name, 'tas_2')
expr = 'tas=tas-tas-tas-tas-tasmax-tas'
to_eval, out_variable_name = EvalFunction._get_eval_string_(
expr, {
'tas': 'var.value',
'tasmax': 'var2.value'
})
self.assertEqual(
to_eval,
'var.value-var.value-var.value-var.value-var2.value-var.value')
self.assertEqual(out_variable_name, 'tas')
def test_get_eval_string_power(self):
"""Test the power ufunc is appropriately parsed from the eval string."""
expr = 'es=power(foo, 4)'
map_vars = {'foo': '_exec_foo.value'}
parsed, output_variable = EvalFunction._get_eval_string_(
expr, map_vars)
self.assertEqual(output_variable, 'es')
self.assertEqual(parsed, 'np.power(_exec_foo.value, 4)')
def test_calculation_two_variables_exp_only(self):
rd = self.test_data.get_rd('cancm4_tas')
rd2 = self.test_data.get_rd('cancm4_tasmax_2001')
field = rd.get()
field2 = rd2.get()
with orphaned(field2['tasmax']):
field.add_variable(field2['tasmax'], is_data=True)
field = field.get_field_slice({'time': slice(0, 10)})
expr = 'foo=log(1000*(tasmax-tas))/3'
ef = EvalFunction(expr=expr, field=field)
ret = ef.execute()
self.assertEqual(list(ret.keys()), ['foo'])
tas = field['tas']
tasmax = field['tasmax']
actual_value = np.log(1000 * (tasmax.get_value() - tas.get_value())) / 3
self.assertNumpyAll(ret['foo'].get_value(), actual_value)
def test_calculation_two_variables_exp_only(self):
rd = self.test_data.get_rd('cancm4_tas')
rd2 = self.test_data.get_rd('cancm4_tasmax_2001')
field = rd.get()
field2 = rd2.get()
with orphaned(field2['tasmax']):
field.add_variable(field2['tasmax'], is_data=True)
field = field.get_field_slice({'time': slice(0, 10)})
expr = 'foo=log(1000*(tasmax-tas))/3'
ef = EvalFunction(expr=expr, field=field)
ret = ef.execute()
self.assertEqual(list(ret.keys()), ['foo'])
tas = field['tas']
tasmax = field['tasmax']
actual_value = np.log(1000 *
(tasmax.get_value() - tas.get_value())) / 3
self.assertNumpyAll(ret['foo'].get_value(), actual_value)
def _parse_(self,value):
## if this is not an eval function (a string to interpret as a function)
## then construct the function dictionaries. otherwise, pass through
if '=' in value:
self._is_eval_function = True
if EvalFunction.is_multivariate(value):
eval_klass = MultivariateEvalFunction
else:
eval_klass = EvalFunction
value = {'func':value,'ref':eval_klass}
## if it is not an eval function, then do the standard argument parsing
if not self._is_eval_function:
fr = register.FunctionRegistry()
## get the function key string form the calculation definition dictionary
function_key = value['func']
## this is the message for the DefinitionValidationError if this key
## may not be found.
dve_msg = 'The function key "{0}" is not available in the function registry.'.format(function_key)
## retrieve the calculation class reference from the function registry
try:
value['ref'] = fr[function_key]
## if the function cannot be found, it may be part of a contributed
## library of calculations not registered by default as the external
## library is an optional dependency.
except KeyError:
## this will register the icclim indices.
if function_key.startswith('{0}_'.format(constants.prefix_icclim_function_key)):
register.register_icclim(fr)
else:
raise(DefinitionValidationError(self,dve_msg))
## make another attempt to register the function
try:
value['ref'] = fr[function_key]
except KeyError:
raise(DefinitionValidationError(self,dve_msg))
## parameters will be set to empty if none are present in the calculation
## dictionary.
if 'kwds' not in value:
value['kwds'] = OrderedDict()
## make the keyword parameter definitions lowercase.
else:
value['kwds'] = OrderedDict(value['kwds'])
for k,v in value['kwds'].iteritems():
try:
value['kwds'][k] = v.lower()
except AttributeError:
pass
return(value)
def test_get_eval_string(self):
expr = 'es=6.1078*exp(log(17.08085)*(tas-273.16)/(234.175+(tas-273.16)))'
to_eval,out_variable_name = EvalFunction._get_eval_string_(expr,{'tas':'var.value'})
self.assertEqual(to_eval,'6.1078*np.exp(np.log(17.08085)*(var.value-273.16)/(234.175+(var.value-273.16)))')
self.assertEqual(out_variable_name,'es')
expr = 'tas=tas-4'
to_eval,out_variable_name = EvalFunction._get_eval_string_(expr,{'tas':'var.value'})
self.assertEqual(to_eval,'var.value-4')
self.assertEqual(out_variable_name,'tas')
expr = 'tas=4-tas'
to_eval,out_variable_name = EvalFunction._get_eval_string_(expr,{'tas':'var.value'})
self.assertEqual(to_eval,'4-var.value')
self.assertEqual(out_variable_name,'tas')
expr = 'tas=tas'
to_eval,out_variable_name = EvalFunction._get_eval_string_(expr,{'tas':'var.value'})
self.assertEqual(to_eval,'var.value')
self.assertEqual(out_variable_name,'tas')
expr = 'tas=tas-tas-tas'
to_eval,out_variable_name = EvalFunction._get_eval_string_(expr,{'tas':'var.value'})
self.assertEqual(to_eval,'var.value-var.value-var.value')
self.assertEqual(out_variable_name,'tas')
expr = 'tas=tas-tas-tas-tas'
to_eval,out_variable_name = EvalFunction._get_eval_string_(expr,{'tas':'var.value'})
self.assertEqual(to_eval,'var.value-var.value-var.value-var.value')
self.assertEqual(out_variable_name,'tas')
expr = 'tas_2=tas_1-tas_1-tas_1-tas_1'
to_eval,out_variable_name = EvalFunction._get_eval_string_(expr,{'tas_1':'var.value'})
self.assertEqual(to_eval,'var.value-var.value-var.value-var.value')
self.assertEqual(out_variable_name,'tas_2')
expr = 'tas=tas-tas-tas-tas-tasmax-tas'
to_eval,out_variable_name = EvalFunction._get_eval_string_(expr,{'tas':'var.value','tasmax':'var2.value'})
self.assertEqual(to_eval,'var.value-var.value-var.value-var.value-var2.value-var.value')
self.assertEqual(out_variable_name,'tas')
def execute(self, coll, file_only=False, tgds=None):
"""
:param :class:~`ocgis.SpatialCollection` coll:
:param bool file_only:
:param dict tgds: {'field_alias': :class:`ocgis.interface.base.dimension.temporal.TemporalGroupDimension`,...}
"""
from ocgis import VariableCollection
# Select which dictionary will hold the temporal group dimensions.
if tgds is None:
tgds_to_use = self._tgds
tgds_overloaded = False
else:
tgds_to_use = tgds
tgds_overloaded = True
# Group the variables. If grouping is None, calculations are performed on each element.
if self.grouping is not None:
ocgis_lh('Setting temporal groups: {0}'.format(self.grouping), 'calc.engine')
for field in coll.iter_fields():
if tgds_overloaded:
assert field.name in tgds_to_use
else:
if field.name not in tgds_to_use:
tgds_to_use[field.name] = field.time.get_grouping(self.grouping)
# Iterate over functions.
for ugid, container in list(coll.children.items()):
for field_name, field in list(container.children.items()):
new_temporal = tgds_to_use.get(field_name)
if new_temporal is not None:
new_temporal = new_temporal.copy()
# If the engine has a grouping, ensure it is equivalent to the new temporal dimension.
if self.grouping is not None:
try:
compare = set(new_temporal.grouping) == set(self.grouping)
# Types may be unhashable, compare directly.
except TypeError:
compare = new_temporal.grouping == self.grouping
if not compare:
msg = 'Engine temporal grouping and field temporal grouping are not equivalent. Perhaps ' \
'optimizations are incorrect?'
ocgis_lh(logger='calc.engine', exc=ValueError(msg))
out_vc = VariableCollection()
for f in self.funcs:
try:
ocgis_lh('Calculating: {0}'.format(f['func']), logger='calc.engine')
# Initialize the function.
function = f['ref'](alias=f['name'], dtype=None, field=field, file_only=file_only, vc=out_vc,
parms=f['kwds'], tgd=new_temporal, calc_sample_size=self.calc_sample_size,
meta_attrs=f.get('meta_attrs'),
spatial_aggregation=self.spatial_aggregation)
# Allow a calculation to create a temporal aggregation after initialization.
if new_temporal is None and function.tgd is not None:
new_temporal = function.tgd.extract()
except KeyError:
# Likely an eval function which does not have the name key.
function = EvalFunction(field=field, file_only=file_only, vc=out_vc, expr=self.funcs[0]['func'],
meta_attrs=self.funcs[0].get('meta_attrs'))
ocgis_lh('calculation initialized', logger='calc.engine', level=logging.DEBUG)
# Return the variable collection from the calculations.
out_vc = function.execute()
for dv in out_vc.values():
# Any outgoing variables from a calculation must have an associated data type.
try:
assert dv.dtype is not None
except AssertionError:
assert isinstance(dv.dtype, np.dtype)
# If this is a file only operation, there should be no computed values.
if file_only:
assert dv._value is None
ocgis_lh('calculation finished', logger='calc.engine', level=logging.DEBUG)
# Try to mark progress. Okay if it is not there.
try:
self._progress.mark()
except AttributeError:
pass
out_field = function.field.copy()
function_tag = function.tag
# Format the returned field. Doing things like removing original data variables and modifying the
# time dimension if necessary. Field functions handle all field modifications on their own, so bypass
# in that case.
if new_temporal is not None:
new_temporal = new_temporal.extract()
format_return_field(function_tag, out_field, new_temporal=new_temporal)
# Add the calculation variables.
for variable in list(out_vc.values()):
variable = variable.extract()
out_field.add_variable(variable)
# Tag the calculation data as data variables.
out_field.append_to_tags(function_tag, list(out_vc.keys()))
# Update the field if there is a CRS. This will ensure accurate tagging of data variables.
if out_field.crs is not None:
# print 'here'
out_field.crs.format_spatial_object(out_field)
coll.children[ugid].children[field_name] = out_field
return coll
def test_init(self):
expr = 'es=6.1078*exp(17.08085*(tas-273.16)/(234.175+(tas-273.16)))'
ef = EvalFunction(expr=expr)
self.assertEqual(ef.expr, expr)
def test_get_eval_string_two_variables(self):
map_vars = {'tasmax':'tasmax.value','tas':'tas.value'}
expr = 'foo=log(1000*(tasmax-tas))/3'
to_eval,out_variable_name = EvalFunction._get_eval_string_(expr,map_vars)
self.assertEqual(to_eval,'np.log(1000*(tasmax.value-tas.value))/3')
self.assertEqual(out_variable_name,'foo')
def execute(self,coll,file_only=False,tgds=None):
'''
:param :class:~`ocgis.SpatialCollection` coll:
:param bool file_only:
:param dict tgds: {'field_alias': :class:`ocgis.interface.base.dimension.temporal.TemporalGroupDimension`,...}
'''
## switch field type based on the types of calculations present
if self._check_calculation_members_(self.funcs,AbstractMultivariateFunction):
klass = DerivedMultivariateField
elif self._check_calculation_members_(self.funcs,EvalFunction):
## if the input field has more than one variable, assumed this is a
## multivariate calculation
klass = DerivedField
for field_container in coll.itervalues():
for field in field_container.itervalues():
if len(field.variables.keys()) > 1:
klass = DerivedMultivariateField
break
else:
klass = DerivedField
## select which dictionary will hold the temporal group dimensions
if tgds == None:
tgds_to_use = self._tgds
tgds_overloaded = False
else:
tgds_to_use = tgds
tgds_overloaded = True
## group the variables. if grouping is None, calculations are performed
## on each element. array computations are taken advantage of.
if self.grouping is not None:
ocgis_lh('Setting temporal groups: {0}'.format(self.grouping),'calc.engine')
for v in coll.itervalues():
for k2,v2 in v.iteritems():
if tgds_overloaded:
assert(k2 in tgds_to_use)
else:
if k2 not in tgds_to_use:
tgds_to_use[k2] = v2.temporal.get_grouping(self.grouping)
## iterate over functions
for ugid,dct in coll.iteritems():
for alias_field,field in dct.iteritems():
## choose a representative data type based on the first variable
dtype = field.variables.values()[0].dtype
new_temporal = tgds_to_use.get(alias_field)
## if the engine has a grouping, ensure it is equivalent to the
## new temporal dimension.
if self.grouping is not None:
try:
compare = set(new_temporal.grouping) == set(self.grouping)
## types may be unhashable, compare directly
except TypeError:
compare = new_temporal.grouping == self.grouping
if compare == False:
msg = ('Engine temporal grouping and field temporal grouping '
'are not equivalent. Perhaps optimizations are incorrect?')
ocgis_lh(logger='calc.engine',exc=ValueError(msg))
out_vc = VariableCollection()
for f in self.funcs:
try:
ocgis_lh('Calculating: {0}'.format(f['func']),logger='calc.engine')
## initialize the function
function = f['ref'](alias=f['name'],dtype=dtype,field=field,file_only=file_only,vc=out_vc,
parms=f['kwds'],tgd=new_temporal,use_raw_values=self.use_raw_values,
calc_sample_size=self.calc_sample_size)
except KeyError:
## likely an eval function which does not have the name
## key
function = EvalFunction(field=field,file_only=file_only,vc=out_vc,
expr=self.funcs[0]['func'])
ocgis_lh('calculation initialized',logger='calc.engine',level=logging.DEBUG)
## return the variable collection from the calculations
out_vc = function.execute()
for dv in out_vc.itervalues():
## any outgoing variables from a calculation must have a
## data type associated with it
try:
assert(dv.dtype != None)
except AssertionError:
assert(isinstance(dv.dtype,np.dtype))
## if this is a file only operation, then there should
## be no values.
if file_only:
assert(dv._value == None)
ocgis_lh('calculation finished',logger='calc.engine',level=logging.DEBUG)
## try to mark progress
try:
self._progress.mark()
except AttributeError:
pass
new_temporal = new_temporal or field.temporal
new_field = klass(variables=out_vc,temporal=new_temporal,spatial=field.spatial,
level=field.level,realization=field.realization,meta=field.meta,
uid=field.uid,name=field.name)
coll[ugid][alias_field] = new_field
return(coll)
def _parse_(self, value):
# test if the value is an eval function and set internal flag
if '=' in value:
self._is_eval_function = True
elif isinstance(value, dict) and value.get(
'func') is not None and '=' in value['func']:
self._is_eval_function = True
else:
self._is_eval_function = False
# format the output dictionary
if self._is_eval_function:
# select the function reference...
try:
eval_string = value['func']
except TypeError:
eval_string = value
# determine if the eval string is multivariate
if EvalFunction.is_multivariate(eval_string):
eval_klass = MultivariateEvalFunction
else:
eval_klass = EvalFunction
# reset the output dictionary
new_value = {
'func': eval_string,
'ref': eval_klass,
'name': None,
'kwds': OrderedDict()
}
# attempt to update the meta_attrs if they are present
try:
new_value.update({'meta_attrs': value['meta_attrs']})
# attempting to index a string incorrectly
except TypeError:
new_value.update({'meta_attrs': None})
# adjust the reference
value = new_value
# if it is not an eval function, then do the standard argument parsing
else:
# check for required keys
if isinstance(value, dict):
for key in self._required_keys_initial:
if key not in value:
msg = 'The key "{0}" is required for calculation dictionaries.'.format(
key)
raise DefinitionValidationError(self, msg)
fr = register.FunctionRegistry()
# get the function key string form the calculation definition dictionary
function_key = value['func']
# this is the message for the DefinitionValidationError if this key may not be found.
dve_msg = 'The function key "{0}" is not available in the function registry.'.format(
function_key)
# retrieve the calculation class reference from the function registry
try:
value['ref'] = fr[function_key]
# if the function cannot be found, it may be part of a contributed library of calculations not registered by
# default as the external library is an optional dependency.
except KeyError:
# this will register the icclim indices.
if function_key.startswith('{0}_'.format(
constants.ICCLIM_PREFIX_FUNCTION_KEY)):
register.register_icclim(fr)
else:
raise DefinitionValidationError(self, dve_msg)
# make another attempt to register the function
try:
value['ref'] = fr[function_key]
except KeyError:
raise DefinitionValidationError(self, dve_msg)
# parameters will be set to empty if none are present in the calculation dictionary.
if 'kwds' not in value:
value['kwds'] = OrderedDict()
# make the keyword parameter definitions lowercase.
else:
value['kwds'] = OrderedDict(value['kwds'])
for k, v in value['kwds'].items():
try:
value['kwds'][k] = v.lower()
except AttributeError:
pass
# add placeholder for meta_attrs if it is not present
if 'meta_attrs' not in value:
value['meta_attrs'] = None
else:
# replace with the metadata attributes class if the attributes are not none
ma = value['meta_attrs']
if ma is not None:
value['meta_attrs'] = MetadataAttributes(ma)
return value
def _parse_(self, value):
# test if the value is an eval function and set internal flag
if '=' in value:
self._is_eval_function = True
elif isinstance(value, dict) and value.get('func') is not None and '=' in value['func']:
self._is_eval_function = True
else:
self._is_eval_function = False
# format the output dictionary
if self._is_eval_function:
# select the function reference...
try:
eval_string = value['func']
except TypeError:
eval_string = value
# determine if the eval string is multivariate
if EvalFunction.is_multivariate(eval_string):
eval_klass = MultivariateEvalFunction
else:
eval_klass = EvalFunction
# reset the output dictionary
new_value = {'func': eval_string, 'ref': eval_klass, 'name': None, 'kwds': OrderedDict()}
# attempt to update the meta_attrs if they are present
try:
new_value.update({'meta_attrs': value['meta_attrs']})
# attempting to index a string incorrectly
except TypeError:
new_value.update({'meta_attrs': None})
# adjust the reference
value = new_value
# if it is not an eval function, then do the standard argument parsing
else:
# check for required keys
if isinstance(value, dict):
for key in self._required_keys_initial:
if key not in value:
msg = 'The key "{0}" is required for calculation dictionaries.'.format(key)
raise DefinitionValidationError(self, msg)
fr = register.FunctionRegistry()
# get the function key string form the calculation definition dictionary
function_key = value['func']
# this is the message for the DefinitionValidationError if this key may not be found.
dve_msg = 'The function key "{0}" is not available in the function registry.'.format(function_key)
# retrieve the calculation class reference from the function registry
try:
value['ref'] = fr[function_key]
# if the function cannot be found, it may be part of a contributed library of calculations not registered by
# default as the external library is an optional dependency.
except KeyError:
# this will register the icclim indices.
if function_key.startswith('{0}_'.format(constants.ICCLIM_PREFIX_FUNCTION_KEY)):
register.register_icclim(fr)
else:
raise DefinitionValidationError(self, dve_msg)
# make another attempt to register the function
try:
value['ref'] = fr[function_key]
except KeyError:
raise DefinitionValidationError(self, dve_msg)
# parameters will be set to empty if none are present in the calculation dictionary.
if 'kwds' not in value:
value['kwds'] = OrderedDict()
# make the keyword parameter definitions lowercase.
else:
value['kwds'] = OrderedDict(value['kwds'])
for k, v in value['kwds'].iteritems():
try:
value['kwds'][k] = v.lower()
except AttributeError:
pass
# add placeholder for meta_attrs if it is not present
if 'meta_attrs' not in value:
value['meta_attrs'] = None
else:
# replace with the metadata attributes class if the attributes are not none
ma = value['meta_attrs']
if ma is not None:
value['meta_attrs'] = MetadataAttributes(ma)
return value
def execute(self, coll, file_only=False, tgds=None):
"""
:param :class:~`ocgis.SpatialCollection` coll:
:param bool file_only:
:param dict tgds: {'field_alias': :class:`ocgis.interface.base.dimension.temporal.TemporalGroupDimension`,...}
"""
from ocgis import VariableCollection
# Select which dictionary will hold the temporal group dimensions.
if tgds is None:
tgds_to_use = self._tgds
tgds_overloaded = False
else:
tgds_to_use = tgds
tgds_overloaded = True
# Group the variables. If grouping is None, calculations are performed on each element.
if self.grouping is not None:
ocgis_lh('Setting temporal groups: {0}'.format(self.grouping),
'calc.engine')
for field in coll.iter_fields():
if tgds_overloaded:
assert field.name in tgds_to_use
else:
if field.name not in tgds_to_use:
tgds_to_use[field.name] = field.time.get_grouping(
self.grouping)
# Iterate over functions.
for ugid, container in list(coll.children.items()):
for field_name, field in list(container.children.items()):
new_temporal = tgds_to_use.get(field_name)
if new_temporal is not None:
new_temporal = new_temporal.copy()
# If the engine has a grouping, ensure it is equivalent to the new temporal dimension.
if self.grouping is not None:
try:
compare = set(new_temporal.grouping) == set(
self.grouping)
# Types may be unhashable, compare directly.
except TypeError:
compare = new_temporal.grouping == self.grouping
if not compare:
msg = 'Engine temporal grouping and field temporal grouping are not equivalent. Perhaps ' \
'optimizations are incorrect?'
ocgis_lh(logger='calc.engine', exc=ValueError(msg))
out_vc = VariableCollection()
for f in self.funcs:
try:
ocgis_lh('Calculating: {0}'.format(f['func']),
logger='calc.engine')
# Initialize the function.
function = f['ref'](
alias=f['name'],
dtype=None,
field=field,
file_only=file_only,
vc=out_vc,
parms=f['kwds'],
tgd=new_temporal,
calc_sample_size=self.calc_sample_size,
meta_attrs=f.get('meta_attrs'),
spatial_aggregation=self.spatial_aggregation)
# Allow a calculation to create a temporal aggregation after initialization.
if new_temporal is None and function.tgd is not None:
new_temporal = function.tgd.extract()
except KeyError:
# Likely an eval function which does not have the name key.
function = EvalFunction(
field=field,
file_only=file_only,
vc=out_vc,
expr=self.funcs[0]['func'],
meta_attrs=self.funcs[0].get('meta_attrs'))
ocgis_lh('calculation initialized',
logger='calc.engine',
level=logging.DEBUG)
# Return the variable collection from the calculations.
out_vc = function.execute()
for dv in out_vc.values():
# Any outgoing variables from a calculation must have an associated data type.
try:
assert dv.dtype is not None
except AssertionError:
assert isinstance(dv.dtype, np.dtype)
# If this is a file only operation, there should be no computed values.
if file_only:
assert dv._value is None
ocgis_lh('calculation finished',
logger='calc.engine',
level=logging.DEBUG)
# Try to mark progress. Okay if it is not there.
try:
self._progress.mark()
except AttributeError:
pass
out_field = function.field.copy()
function_tag = function.tag
# Format the returned field. Doing things like removing original data variables and modifying the
# time dimension if necessary. Field functions handle all field modifications on their own, so bypass
# in that case.
if new_temporal is not None:
new_temporal = new_temporal.extract()
format_return_field(function_tag,
out_field,
new_temporal=new_temporal)
# Add the calculation variables.
for variable in list(out_vc.values()):
variable = variable.extract()
out_field.add_variable(variable)
# Tag the calculation data as data variables.
out_field.append_to_tags(function_tag, list(out_vc.keys()))
# Update the field if there is a CRS. This will ensure accurate tagging of data variables.
if out_field.crs is not None:
# print 'here'
out_field.crs.format_spatial_object(out_field)
coll.children[ugid].children[field_name] = out_field
return coll