def test_replace_values_jagged_integer_array(self):
name = 'cg_eng_alarm_at'
param = Parameter.query.filter(Parameter.name == name).first()
data_slice = np.array([
'\x91\x04', '\xc0', '\x91\x04', '\x92\x04\x04', '\x91\x04', '\xc0',
'\xc0', '\xc0', '\xc0', '\xc0'
])
value_encoding = 'int16'
fill_value = _get_fill_value(param)
is_array = True
rval = _replace_values(data_slice, value_encoding, fill_value,
is_array, name)
self.assertEqual(rval.dtype.kind, 'i')
np.testing.assert_equal(
rval,
np.array([
[4, fill_value],
[fill_value, fill_value],
[4, fill_value],
[4, 4],
[4, fill_value],
[fill_value, fill_value],
[fill_value, fill_value],
[fill_value, fill_value],
[fill_value, fill_value],
[fill_value, fill_value],
]))
def _insert_data(dataset, param, data, provenance_metadata=None, request_id=None):
"""
Insert the specified parameter into this dataset. If data is None, use the fill value
:param dataset:
:param param:
:param data:
:return:
"""
dims = ['obs']
# IF dimensions are defined in preload, use those
# otherwise, create dimensions dynamically based on the
# shape of the data
if param.dimensions:
dims += [d.value for d in param.dimensions]
else:
if data is not None:
for index, _ in enumerate(data.shape[1:]):
name = '%s_dim_%d' % (param.name, index)
dims.append(name)
# IF data is missing and specified dimensions aren't already defined
# we cannot determine the correct shape, limit dimensions to obs
missing = [d for d in dims if d not in dataset.dims]
if missing and data is None:
log.error('Unable to resolve all dimensions for derived parameter: %r. Filling as scalar', missing)
dims = ['obs']
fill_value = _get_fill_value(param)
# Data is None, replace with fill values
if data is None:
shape = tuple([len(dataset[d]) for d in dims])
data = np.zeros(shape)
data[:] = fill_value
try:
attrs = param.attrs
# Override the fill value supplied by preload if necessary
attrs['_FillValue'] = fill_value
coord_columns = 'time lat lon'
if param.name not in coord_columns:
attrs['coordinates'] = coord_columns
dataset[param.name] = (dims, data, attrs)
except ValueError as e:
message = 'Unable to insert parameter: %r. Data shape (%r) does not match expected shape (%r)' % \
(param, data.shape, e)
to_attach = {'type': 'FunctionError', "parameter": str(param),
'function': str(param.parameter_function), 'message': message}
if provenance_metadata:
provenance_metadata.calculated_metadata.errors.append(to_attach)
log.error('<%s> %s', request_id, message)
raise
def test_replace_values_jagged_integer_array(self):
name = 'cg_eng_alarm_at'
param = Parameter.query.filter(Parameter.name == name).first()
data_slice = np.array(['\x91\x04', '\xc0', '\x91\x04', '\x92\x04\x04', '\x91\x04',
'\xc0', '\xc0', '\xc0', '\xc0', '\xc0'])
value_encoding = 'int16'
fill_value = _get_fill_value(param)
is_array = True
rval = _replace_values(data_slice, value_encoding, fill_value, is_array, name)
self.assertEqual(rval.dtype.kind, 'i')
np.testing.assert_equal(rval, np.array([[4, fill_value],
[fill_value, fill_value],
[4, fill_value],
[4, 4],
[4, fill_value],
[fill_value, fill_value],
[fill_value, fill_value],
[fill_value, fill_value],
[fill_value, fill_value],
[fill_value, fill_value],
]))
def _insert_data(dataset, param, data, provenance_metadata=None, request_id=None):
"""
Insert the specified parameter into this dataset. If data is None, use the fill value
:param dataset:
:param param:
:param data:
:return:
"""
dims = ['obs']
# the preload defined parameter dimensions
param_dimensions = []
if param.dimensions:
param_dimensions = [d.value for d in param.dimensions]
if '-obs' in param_dimensions:
# remove obs dimension from parameter's dimensions and data (13025 AC2)
param_dimensions.remove('-obs')
dims = param_dimensions
if data is not None:
# remove the obs dimension if it is present - in such a case, the data dimensions will be greater than
# the number of items in param_dimensions
data = data[0] if data.ndim > len(param_dimensions) else data
elif param_dimensions:
# append parameter dimensions onto obs
dims += param_dimensions
else:
# create dimensions dynamically based on the
# shape of the data
if data is not None:
for index, _ in enumerate(data.shape[1:]):
name = '%s_dim_%d' % (param.name, index)
dims.append(name)
# IF data is missing and specified dimensions aren't already defined
# we cannot determine the correct shape, limit dimensions to obs
missing = [d for d in dims if d not in dataset.dims]
if missing and data is None:
log.error('Unable to resolve all dimensions for derived parameter: %r. Filling as scalar', missing)
dims = ['obs']
fill_value = _get_fill_value(param)
# Data is None, replace with fill values
if data is None:
shape = tuple([len(dataset[d]) for d in dims])
data = np.zeros(shape)
data[:] = fill_value
try:
attrs = param.attrs
# Override the fill value supplied by preload if necessary
attrs['_FillValue'] = fill_value
coord_columns = 'time lat lon'
if param.name not in coord_columns:
attrs['coordinates'] = coord_columns
dataset[param.name] = (dims, data, attrs)
except ValueError as e:
message = 'Unable to insert parameter: %r. Data shape (%r) does not match expected shape (%r)' % \
(param, data.shape, e)
to_attach = {'type': 'FunctionError', "parameter": str(param),
'function': str(param.parameter_function), 'message': message}
if provenance_metadata:
provenance_metadata.calculated_metadata.errors.append(to_attach)
log.error('<%s> %s', request_id, message)
raise
def test_get_fill_value(self):
for param in Parameter.query:
fill = _get_fill_value(param)
self.assertIsNotNone(fill)
if isinstance(fill, basestring):
self.assertEqual(fill, '')
def test_get_fill_value(self):
for param in Parameter.query:
fill = _get_fill_value(param)
self.assertIsNotNone(fill)
if isinstance(fill, basestring):
self.assertEqual(fill, '')