def test_update_param_attributes(self):
# save initial Parameter to file
self.hdf_file.set_param(Parameter('Blah', np.ma.arange(1)))
# Update the invalidity flag only
self.hdf_file.set_param(Parameter('Blah', np.ma.arange(1), invalid=1),
save_data=False, save_mask=False)
self.assertEqual(self.hdf_file['Blah'].invalid, 1)
def test_get_params_valid_only(self):
hdf = self.hdf_file
hdf['Valid Param'] = Parameter('Valid Param', array=np.ma.arange(10))
hdf['Invalid Param'] = Parameter('Invalid Param',
array=np.ma.arange(10),
invalid=True)
self.assertIn('Invalid Param', hdf)
# check the params that are valid are listed correctly
self.assertEqual(hdf.valid_param_names(),
['TEST_PARAM10', 'TEST_PARAM11', 'Valid Param'])
# request all params inc. invalid
all_params = hdf.get_params(valid_only=False)
self.assertEqual(
sorted(all_params.keys()),
['Invalid Param', 'TEST_PARAM10', 'TEST_PARAM11', 'Valid Param'])
# request only valid params
valid = hdf.get_params(valid_only=True)
self.assertEqual(sorted(valid.keys()),
['TEST_PARAM10', 'TEST_PARAM11', 'Valid Param'])
# request params by name
valid_few = hdf.get_params(param_names=['Valid Param'],
valid_only=True)
self.assertEqual(list(valid_few.keys()), ['Valid Param'])
# try to request the invalid param, but only accepting valid raises keyerror
self.assertRaises(KeyError,
hdf.get_params,
param_names=['Invalid Param', 'Valid Param'],
valid_only=True,
raise_keyerror=True)
def test_combine_submasks(self):
p = Parameter('Submasks',
submasks={
'mask1': np.array([1, 0, 0], dtype=np.bool_),
'mask2': np.array([1, 1, 0], dtype=np.bool_),
})
self.assertEqual(p.combine_submasks().tolist(), [1, 1, 0])
def test___set_item__(self):
'''
set_item uses set_param
'''
set_param_data = self.hdf_file.__setitem__
hdf_file = self.hdf_file
series = hdf_file.hdf['series']
# Create new parameter with np.array.
name1 = 'TEST_PARAM1'
array = np.arange(100)
set_param_data(name1, Parameter(name1, array))
self.assertTrue(np.all(series[name1]['data'].value == array))
self.assertFalse('arinc_429' in series[name1].attrs)
# Create new parameter with np.ma.masked_array.
name2 = 'TEST_PARAM2'
mask = [False] * len(array)
masked_array = np.ma.masked_array(data=array, mask=mask)
param2_frequency = 0.5
param2_offset = 2
param2_arinc_429 = True
set_param_data(
name2,
Parameter(name2,
masked_array,
frequency=param2_frequency,
offset=param2_offset,
arinc_429=param2_arinc_429))
self.assertTrue(np.all(series[name2]['data'].value == array))
self.assertTrue(np.all(series[name2]['mask'].value == mask))
self.assertEqual(series[name2].attrs['frequency'], param2_frequency)
self.assertEqual(series[name2].attrs['supf_offset'], param2_offset)
self.assertEqual(series[name2].attrs['arinc_429'], param2_arinc_429)
# Set existing parameter's data with np.array.
array = np.arange(200)
set_param_data(name1, Parameter(name1, array))
self.assertTrue(np.all(series[name1]['data'].value == array))
# Set existing parameter's data with np.ma.masked_array.
mask = [bool(random.randint(0, 1)) for x in range(len(array))]
masked_array = np.ma.masked_array(data=array, mask=mask)
set_param_data(name1, Parameter(name1, masked_array))
self.assertTrue(np.all(series[name1]['data'].value == array))
self.assertTrue(np.all(series[name1]['mask'].value == mask))
# Save submasks.
submasks = {
'mask1': np.array([False, True, False]),
'mask2': np.array([True, False, False])
}
set_param_data(name1, Parameter(name1, masked_array,
submasks=submasks))
submask_map = simplejson.loads(series[name1].attrs['submasks'])
submask_arrays = series[name1]['submasks'][:]
self.assertEqual(submasks['mask1'].tolist(),
submask_arrays[:, submask_map['mask1']].tolist())
self.assertEqual(submasks['mask2'].tolist(),
submask_arrays[:, submask_map['mask2']].tolist())
def test_get_param_valid_only(self):
hdf = self.hdf_file
hdf['Valid Param'] = Parameter('Valid Param', array=np.ma.arange(10))
hdf['Invalid Param'] = Parameter('Invalid Param', array=np.ma.arange(10), invalid=True)
self.assertIn('Invalid Param', hdf)
# request Invalid param without filtering
self.assertTrue(hdf.get_param('Invalid Param', valid_only=False))
# filtering only valid raises keyerror
self.assertRaises(KeyError, hdf.get_param, 'Invalid Param', valid_only=True)
def test_get_array(self):
array = np.ma.array([10, 20, 30], mask=[0,1,1])
p = Parameter('Submasks', array=array, submasks={
'mask1': np.array([1,0,0], dtype=np.bool_),
'mask2': np.array([1,1,0], dtype=np.bool_),
})
self.assertEqual(p.get_array().tolist(), [10,None,None])
self.assertEqual(p.get_array('mask1').tolist(), [None,20,30])
self.assertEqual(p.get_array('mask2').tolist(), [None,None,30])
def test_get_array(self):
array = np.ma.array([10, 20, 30], mask=[0, 1, 1])
p = Parameter('Submasks',
array=array,
submasks={
'mask1': np.array([1, 0, 0], dtype=np.bool_),
'mask2': np.array([1, 1, 0], dtype=np.bool_),
})
self.assertEqual(p.get_array().tolist(), [10, None, None])
self.assertEqual(p.get_array('mask1').tolist(), [None, 20, 30])
self.assertEqual(p.get_array('mask2').tolist(), [None, None, 30])
def test_get_array__mapped(self):
array = np.ma.array([1, 2, 3], mask=[0,1,1])
values_mapping = {1: 'One', 2: 'Two', 3:'Three'}
p = Parameter('Submasks', array=array, submasks={
'mask1': np.array([1,0,0], dtype=np.bool_),
'mask2': np.array([1,1,0], dtype=np.bool_),
}, values_mapping=values_mapping)
self.assertEqual(p.get_array().tolist(), [1,None,None])
self.assertEqual(p.get_array('mask1').tolist(), [None,2,3])
self.assertEqual(p.get_array('mask2').tolist(), [None,None,3])
self.assertTrue(isinstance(p.get_array('mask1'), MappedArray))
def test_get_array__mapped(self):
array = np.ma.array([1, 2, 3], mask=[0, 1, 1])
values_mapping = {1: 'One', 2: 'Two', 3: 'Three'}
p = Parameter('Submasks',
array=array,
submasks={
'mask1': np.array([1, 0, 0], dtype=np.bool_),
'mask2': np.array([1, 1, 0], dtype=np.bool_),
},
values_mapping=values_mapping)
self.assertEqual(p.get_array().raw.tolist(), [1, None, None])
self.assertEqual(p.get_array('mask1').raw.tolist(), [None, 2, 3])
self.assertEqual(p.get_array('mask2').raw.tolist(), [None, None, 3])
self.assertTrue(isinstance(p.get_array('mask1'), MappedArray))
def test_open_and_close_and_full_masks(self):
self.hdf_file.close()
with hdf_file(self.hdf_path) as hdf:
# check it's open
self.assertFalse(hdf.hdf.id is None)
hdf['sample'] = Parameter('sample', np.array(range(10)))
self.assertEqual(list(hdf['sample'].array.data), range(10))
self.assertTrue(hasattr(hdf['sample'].array, 'mask'))
hdf['masked sample'] = Parameter('masked sample', np.ma.array(range(10)))
self.assertEqual(list(hdf['masked sample'].array.data), range(10))
# check masks are returned in full (not just a single False)
self.assertEqual(list(hdf['masked sample'].array.mask), [False] * 10)
# check it's closed
self.assertEqual(hdf.hdf.__repr__(), '<Closed HDF5 file>')
def test_mapped_array(self):
# created mapped array
mapping = {0: 'zero', 2: 'two', 3: 'three'}
array = np.ma.array(range(5) + range(5), mask=[1, 1, 1, 0, 0, 0, 0, 0, 1, 1])
multi_p = Parameter('multi', array, values_mapping=mapping)
multi_p.array[0] = 'three'
# save array to hdf
self.hdf_file['multi'] = multi_p
self.hdf_file.close()
# check hdf has mapping and integer values stored
with hdf_file(self.hdf_path) as hdf:
saved = hdf['multi']
self.assertEqual(str(saved.array[:]),
"['three' -- -- 'three' '?' 'zero' '?' 'two' -- --]")
self.assertEqual(saved.array.data.dtype, np.int)
def test_mapped_array(self):
# created mapped array
mapping = {0: 'zero', 2: 'two', 3: 'three'}
array = np.ma.array(range(5) + range(5), mask=[1, 1, 1, 0, 0, 0, 0, 0, 1, 1])
multi_p = Parameter('multi', array, values_mapping=mapping)
multi_p.array[0] = 'three'
# save array to hdf
self.hdf_file['multi'] = multi_p
self.hdf_file.close()
# check hdf has mapping and integer values stored
with hdf_file(self.hdf_path) as hdf:
saved = hdf['multi']
self.assertEqual(str(saved.array[:]),
"['three' -- -- 'three' '?' 'zero' '?' 'two' -- --]")
self.assertEqual(saved.array.data.dtype, np.int)
def test_update_param_mask(self):
# setup original array
name1 = 'Airspeed Minus Vref'
array = np.arange(200)
self.hdf_file.set_param(Parameter(name1, array))
series = self.hdf_file.hdf['series']
# Set mask changes, but not Data array
new_array = np.arange(200)[::-1]
new_mask = [bool(random.randint(0, 1)) for x in range(len(new_array))]
masked_array = np.ma.masked_array(data=new_array, mask=new_mask)
self.hdf_file.set_param(Parameter(name1, masked_array),
save_data=False, save_mask=True)
# assert new mask is saved
self.assertTrue(np.all(series[name1]['mask'].value == new_mask))
# asssert data remains same as original array
self.assertTrue(np.all(series[name1]['data'].value == array))
def test_parameter(self):
p_name = 'param'
p = Parameter(p_name)
self.assertEqual(p.name, p_name)
self.assertEqual(p.array, [])
self.assertEqual(p.frequency, 1)
self.assertEqual(p.offset, 0)
self.assertEqual(p.arinc_429, None)
array = np.ma.arange(10)
frequency = 8
offset = 1
arinc_429 = True
p = Parameter('param',
array=array,
frequency=frequency,
offset=offset,
arinc_429=arinc_429)
np.testing.assert_array_equal(p.array, array)
self.assertEqual(p.frequency, frequency)
self.assertEqual(p.offset, offset)
self.assertEqual(p.arinc_429, arinc_429)
def test_multivalue_parameter(self):
values = [1, 2, 3]
mask = [False, True, False]
array = np.ma.MaskedArray(values, mask)
mapping = {1: 'One', 2: 'Two'}
p = Parameter('param', array=array, values_mapping=mapping)
self.assertEqual(p.array[0], 'One')
self.assertEqual(p.raw_array[0], 1)
self.assertTrue(p.array[1] is np.ma.masked)
self.assertTrue(p.raw_array[1] is np.ma.masked)
# Get a value not in the mapping
self.assertEqual(p.array[2], '?')
self.assertEqual(p.raw_array[2], 3)
def test_multivalue_parameter_float_values(self):
values = [17.5, 10.5, 9]
mask = [False, True, False]
array = np.ma.MaskedArray(values, mask)
mapping = {'17.5': 'One', 10.5: 'Two', 5: 'Three'}
p = Parameter('param', array=array, values_mapping=mapping)
self.assertEqual(p.array[0], 'One')
self.assertEqual(p.raw_array[0], 17.5)
self.assertTrue(p.array[1] is np.ma.masked)
self.assertTrue(p.raw_array[1] is np.ma.masked)
# Get a value not in the mapping
self.assertEqual(p.array[2], '?')
self.assertEqual(p.raw_array[2], 9)
def test_combine_submasks(self):
p = Parameter('Submasks', submasks={
'mask1': np.array([1,0,0], dtype=np.bool_),
'mask2': np.array([1,1,0], dtype=np.bool_),
})
self.assertEqual(p.combine_submasks().tolist(), [1,1,0])
def get_param(self, name, valid_only=False, _slice=None, load_submasks=False, copy_param=True):
'''
name e.g. "Heading"
Returns a masked_array. If 'mask' is stored it will be the mask of the
returned masked_array, otherwise it will be False.
:param name: Name of parameter with 'series'.
:type name: str
:param valid_only: Only return valid parameters, default is to include invalid params
:type valid_only: bool
:param _slice: Only read a slice of the parameter's data. The slice indices are 1Hz.
:type _slice: slice
:param load_submasks: Load parameter submasks into a submasks dictionary.
:type load_submasks: bool
:param copy_param: Return a copy of the parameter if found in cache.
:type copy_param: bool
:returns: Parameter object containing HDF data and attrs.
:rtype: Parameter
Warning: caching in this method does not work with slicing. If you
want to slice the data and use parameter cache at the same time, don't
pass _slice to the method (will cause the cache to be skipped), use
copy_param=False and slice the array in your procedure instead. The
overhead of using deepcopy on huge arrays is substantial and the tests
of generalised slicing of the cached arrays in this method proved less
effective than specialised slicing of the returned cached data.
The other problem is loading of submasks: this feature is off by
default but the parameter will be cached with the submasks if that's
what the user requested on first fetch. If the requested submasks are
missing in the cache, the parameter will be refetched from the disk and
stored in the cache with the submasks. This bit can be optimised but we
need to consider what are the use cases of submasks caching.
'''
if name not in self.keys(valid_only):
# Exception should be caught otherwise HDF will crash and close!
raise KeyError("%s" % name)
elif name in self._params_cache:
if not _slice:
logging.debug("Retrieving param '%s' from HDF cache", name)
# XXX: Caching breaks later loading of submasks!
parameter = self._params_cache[name]
if parameter.submasks or not load_submasks:
return deepcopy(parameter) if copy_param else parameter
logging.debug(
'Skipping returning parameter `%s` from cache as slice of '
'data was requested.', name)
group = self.hdf['series'][name]
attrs = group.attrs
data = group['data']
mask = group.get('mask', False)
kwargs = {}
frequency = attrs.get('frequency', 1)
kwargs['frequency'] = frequency
if _slice:
slice_start = int((_slice.start or 0) * frequency)
slice_stop = int((_slice.stop or len(data)) * frequency)
_slice = slice(slice_start, slice_stop)
data = data[_slice]
mask = mask[_slice] if mask else mask
if load_submasks and 'submasks' in attrs and 'submasks' in group.keys():
kwargs['submasks'] = {}
submask_map = attrs['submasks']
if submask_map.strip():
submask_map = simplejson.loads(submask_map)
for sub_name, index in submask_map.items():
kwargs['submasks'][sub_name] = group['submasks'][_slice or slice(None), index]
array = np.ma.masked_array(data, mask=mask)
if 'values_mapping' in attrs:
values_mapping = attrs['values_mapping']
if values_mapping.strip():
mapping = simplejson.loads(values_mapping)
kwargs['values_mapping'] = mapping
if 'values_mapping' not in kwargs and data.dtype == np.int_:
# Force float for non-values_mapped types.
array = array.astype(np.float_)
# Backwards compatibility. Q: When can this be removed?
if 'supf_offset' in attrs:
kwargs['offset'] = attrs['supf_offset']
if 'arinc_429' in attrs:
kwargs['arinc_429'] = attrs['arinc_429']
if 'invalid' in attrs:
kwargs['invalid'] = attrs['invalid']
if kwargs['invalid'] and 'invalidity_reason' in attrs:
kwargs['invalidity_reason'] = attrs['invalidity_reason']
# Units
if 'units' in attrs:
kwargs['units'] = attrs['units']
if 'lfl' in attrs:
kwargs['lfl'] = attrs['lfl']
elif 'description' in attrs:
# Backwards compatibility for HDF files converted from AGS where
# the units are stored in the description. Units will be invalid if
# parameters from a FlightDataAnalyser HDF do not have 'units'
# attributes.
description = attrs['description']
if description:
kwargs['units'] = description
if 'data_type' in attrs:
kwargs['data_type'] = attrs['data_type']
if 'source_name' in attrs:
kwargs['source_name'] = attrs['source_name']
if 'description' in attrs:
kwargs['description'] = attrs['description']
parameter = Parameter(name, array, **kwargs)
# add to cache if required
if name in self.cache_param_list:
if not _slice:
self._params_cache[name] = parameter
else:
logging.debug('Skipping saving parameter `%s` to cache as '
'slice of data was requested.', name)
return parameter