def test_update_param_attributes(self): # save initial Parameter to file self.hdf_file.set_param(Parameter('Blah', np.ma.arange(1))) # Update the invalidity flag only self.hdf_file.set_param(Parameter('Blah', np.ma.arange(1), invalid=1), save_data=False, save_mask=False) self.assertEqual(self.hdf_file['Blah'].invalid, 1)
def test_get_params_valid_only(self): hdf = self.hdf_file hdf['Valid Param'] = Parameter('Valid Param', array=np.ma.arange(10)) hdf['Invalid Param'] = Parameter('Invalid Param', array=np.ma.arange(10), invalid=True) self.assertIn('Invalid Param', hdf) # check the params that are valid are listed correctly self.assertEqual(hdf.valid_param_names(), ['TEST_PARAM10', 'TEST_PARAM11', 'Valid Param']) # request all params inc. invalid all_params = hdf.get_params(valid_only=False) self.assertEqual( sorted(all_params.keys()), ['Invalid Param', 'TEST_PARAM10', 'TEST_PARAM11', 'Valid Param']) # request only valid params valid = hdf.get_params(valid_only=True) self.assertEqual(sorted(valid.keys()), ['TEST_PARAM10', 'TEST_PARAM11', 'Valid Param']) # request params by name valid_few = hdf.get_params(param_names=['Valid Param'], valid_only=True) self.assertEqual(list(valid_few.keys()), ['Valid Param']) # try to request the invalid param, but only accepting valid raises keyerror self.assertRaises(KeyError, hdf.get_params, param_names=['Invalid Param', 'Valid Param'], valid_only=True, raise_keyerror=True)
def test_combine_submasks(self): p = Parameter('Submasks', submasks={ 'mask1': np.array([1, 0, 0], dtype=np.bool_), 'mask2': np.array([1, 1, 0], dtype=np.bool_), }) self.assertEqual(p.combine_submasks().tolist(), [1, 1, 0])
def test___set_item__(self): ''' set_item uses set_param ''' set_param_data = self.hdf_file.__setitem__ hdf_file = self.hdf_file series = hdf_file.hdf['series'] # Create new parameter with np.array. name1 = 'TEST_PARAM1' array = np.arange(100) set_param_data(name1, Parameter(name1, array)) self.assertTrue(np.all(series[name1]['data'].value == array)) self.assertFalse('arinc_429' in series[name1].attrs) # Create new parameter with np.ma.masked_array. name2 = 'TEST_PARAM2' mask = [False] * len(array) masked_array = np.ma.masked_array(data=array, mask=mask) param2_frequency = 0.5 param2_offset = 2 param2_arinc_429 = True set_param_data( name2, Parameter(name2, masked_array, frequency=param2_frequency, offset=param2_offset, arinc_429=param2_arinc_429)) self.assertTrue(np.all(series[name2]['data'].value == array)) self.assertTrue(np.all(series[name2]['mask'].value == mask)) self.assertEqual(series[name2].attrs['frequency'], param2_frequency) self.assertEqual(series[name2].attrs['supf_offset'], param2_offset) self.assertEqual(series[name2].attrs['arinc_429'], param2_arinc_429) # Set existing parameter's data with np.array. array = np.arange(200) set_param_data(name1, Parameter(name1, array)) self.assertTrue(np.all(series[name1]['data'].value == array)) # Set existing parameter's data with np.ma.masked_array. mask = [bool(random.randint(0, 1)) for x in range(len(array))] masked_array = np.ma.masked_array(data=array, mask=mask) set_param_data(name1, Parameter(name1, masked_array)) self.assertTrue(np.all(series[name1]['data'].value == array)) self.assertTrue(np.all(series[name1]['mask'].value == mask)) # Save submasks. submasks = { 'mask1': np.array([False, True, False]), 'mask2': np.array([True, False, False]) } set_param_data(name1, Parameter(name1, masked_array, submasks=submasks)) submask_map = simplejson.loads(series[name1].attrs['submasks']) submask_arrays = series[name1]['submasks'][:] self.assertEqual(submasks['mask1'].tolist(), submask_arrays[:, submask_map['mask1']].tolist()) self.assertEqual(submasks['mask2'].tolist(), submask_arrays[:, submask_map['mask2']].tolist())
def test_get_param_valid_only(self): hdf = self.hdf_file hdf['Valid Param'] = Parameter('Valid Param', array=np.ma.arange(10)) hdf['Invalid Param'] = Parameter('Invalid Param', array=np.ma.arange(10), invalid=True) self.assertIn('Invalid Param', hdf) # request Invalid param without filtering self.assertTrue(hdf.get_param('Invalid Param', valid_only=False)) # filtering only valid raises keyerror self.assertRaises(KeyError, hdf.get_param, 'Invalid Param', valid_only=True)
def test_get_array(self): array = np.ma.array([10, 20, 30], mask=[0,1,1]) p = Parameter('Submasks', array=array, submasks={ 'mask1': np.array([1,0,0], dtype=np.bool_), 'mask2': np.array([1,1,0], dtype=np.bool_), }) self.assertEqual(p.get_array().tolist(), [10,None,None]) self.assertEqual(p.get_array('mask1').tolist(), [None,20,30]) self.assertEqual(p.get_array('mask2').tolist(), [None,None,30])
def test_get_array(self): array = np.ma.array([10, 20, 30], mask=[0, 1, 1]) p = Parameter('Submasks', array=array, submasks={ 'mask1': np.array([1, 0, 0], dtype=np.bool_), 'mask2': np.array([1, 1, 0], dtype=np.bool_), }) self.assertEqual(p.get_array().tolist(), [10, None, None]) self.assertEqual(p.get_array('mask1').tolist(), [None, 20, 30]) self.assertEqual(p.get_array('mask2').tolist(), [None, None, 30])
def test_get_array__mapped(self): array = np.ma.array([1, 2, 3], mask=[0,1,1]) values_mapping = {1: 'One', 2: 'Two', 3:'Three'} p = Parameter('Submasks', array=array, submasks={ 'mask1': np.array([1,0,0], dtype=np.bool_), 'mask2': np.array([1,1,0], dtype=np.bool_), }, values_mapping=values_mapping) self.assertEqual(p.get_array().tolist(), [1,None,None]) self.assertEqual(p.get_array('mask1').tolist(), [None,2,3]) self.assertEqual(p.get_array('mask2').tolist(), [None,None,3]) self.assertTrue(isinstance(p.get_array('mask1'), MappedArray))
def test_get_array__mapped(self): array = np.ma.array([1, 2, 3], mask=[0, 1, 1]) values_mapping = {1: 'One', 2: 'Two', 3: 'Three'} p = Parameter('Submasks', array=array, submasks={ 'mask1': np.array([1, 0, 0], dtype=np.bool_), 'mask2': np.array([1, 1, 0], dtype=np.bool_), }, values_mapping=values_mapping) self.assertEqual(p.get_array().raw.tolist(), [1, None, None]) self.assertEqual(p.get_array('mask1').raw.tolist(), [None, 2, 3]) self.assertEqual(p.get_array('mask2').raw.tolist(), [None, None, 3]) self.assertTrue(isinstance(p.get_array('mask1'), MappedArray))
def test_open_and_close_and_full_masks(self): self.hdf_file.close() with hdf_file(self.hdf_path) as hdf: # check it's open self.assertFalse(hdf.hdf.id is None) hdf['sample'] = Parameter('sample', np.array(range(10))) self.assertEqual(list(hdf['sample'].array.data), range(10)) self.assertTrue(hasattr(hdf['sample'].array, 'mask')) hdf['masked sample'] = Parameter('masked sample', np.ma.array(range(10))) self.assertEqual(list(hdf['masked sample'].array.data), range(10)) # check masks are returned in full (not just a single False) self.assertEqual(list(hdf['masked sample'].array.mask), [False] * 10) # check it's closed self.assertEqual(hdf.hdf.__repr__(), '<Closed HDF5 file>')
def test_mapped_array(self): # created mapped array mapping = {0: 'zero', 2: 'two', 3: 'three'} array = np.ma.array(range(5) + range(5), mask=[1, 1, 1, 0, 0, 0, 0, 0, 1, 1]) multi_p = Parameter('multi', array, values_mapping=mapping) multi_p.array[0] = 'three' # save array to hdf self.hdf_file['multi'] = multi_p self.hdf_file.close() # check hdf has mapping and integer values stored with hdf_file(self.hdf_path) as hdf: saved = hdf['multi'] self.assertEqual(str(saved.array[:]), "['three' -- -- 'three' '?' 'zero' '?' 'two' -- --]") self.assertEqual(saved.array.data.dtype, np.int)
def test_update_param_mask(self): # setup original array name1 = 'Airspeed Minus Vref' array = np.arange(200) self.hdf_file.set_param(Parameter(name1, array)) series = self.hdf_file.hdf['series'] # Set mask changes, but not Data array new_array = np.arange(200)[::-1] new_mask = [bool(random.randint(0, 1)) for x in range(len(new_array))] masked_array = np.ma.masked_array(data=new_array, mask=new_mask) self.hdf_file.set_param(Parameter(name1, masked_array), save_data=False, save_mask=True) # assert new mask is saved self.assertTrue(np.all(series[name1]['mask'].value == new_mask)) # asssert data remains same as original array self.assertTrue(np.all(series[name1]['data'].value == array))
def test_parameter(self): p_name = 'param' p = Parameter(p_name) self.assertEqual(p.name, p_name) self.assertEqual(p.array, []) self.assertEqual(p.frequency, 1) self.assertEqual(p.offset, 0) self.assertEqual(p.arinc_429, None) array = np.ma.arange(10) frequency = 8 offset = 1 arinc_429 = True p = Parameter('param', array=array, frequency=frequency, offset=offset, arinc_429=arinc_429) np.testing.assert_array_equal(p.array, array) self.assertEqual(p.frequency, frequency) self.assertEqual(p.offset, offset) self.assertEqual(p.arinc_429, arinc_429)
def test_multivalue_parameter(self): values = [1, 2, 3] mask = [False, True, False] array = np.ma.MaskedArray(values, mask) mapping = {1: 'One', 2: 'Two'} p = Parameter('param', array=array, values_mapping=mapping) self.assertEqual(p.array[0], 'One') self.assertEqual(p.raw_array[0], 1) self.assertTrue(p.array[1] is np.ma.masked) self.assertTrue(p.raw_array[1] is np.ma.masked) # Get a value not in the mapping self.assertEqual(p.array[2], '?') self.assertEqual(p.raw_array[2], 3)
def test_multivalue_parameter_float_values(self): values = [17.5, 10.5, 9] mask = [False, True, False] array = np.ma.MaskedArray(values, mask) mapping = {'17.5': 'One', 10.5: 'Two', 5: 'Three'} p = Parameter('param', array=array, values_mapping=mapping) self.assertEqual(p.array[0], 'One') self.assertEqual(p.raw_array[0], 17.5) self.assertTrue(p.array[1] is np.ma.masked) self.assertTrue(p.raw_array[1] is np.ma.masked) # Get a value not in the mapping self.assertEqual(p.array[2], '?') self.assertEqual(p.raw_array[2], 9)
def test_combine_submasks(self): p = Parameter('Submasks', submasks={ 'mask1': np.array([1,0,0], dtype=np.bool_), 'mask2': np.array([1,1,0], dtype=np.bool_), }) self.assertEqual(p.combine_submasks().tolist(), [1,1,0])
def get_param(self, name, valid_only=False, _slice=None, load_submasks=False, copy_param=True): ''' name e.g. "Heading" Returns a masked_array. If 'mask' is stored it will be the mask of the returned masked_array, otherwise it will be False. :param name: Name of parameter with 'series'. :type name: str :param valid_only: Only return valid parameters, default is to include invalid params :type valid_only: bool :param _slice: Only read a slice of the parameter's data. The slice indices are 1Hz. :type _slice: slice :param load_submasks: Load parameter submasks into a submasks dictionary. :type load_submasks: bool :param copy_param: Return a copy of the parameter if found in cache. :type copy_param: bool :returns: Parameter object containing HDF data and attrs. :rtype: Parameter Warning: caching in this method does not work with slicing. If you want to slice the data and use parameter cache at the same time, don't pass _slice to the method (will cause the cache to be skipped), use copy_param=False and slice the array in your procedure instead. The overhead of using deepcopy on huge arrays is substantial and the tests of generalised slicing of the cached arrays in this method proved less effective than specialised slicing of the returned cached data. The other problem is loading of submasks: this feature is off by default but the parameter will be cached with the submasks if that's what the user requested on first fetch. If the requested submasks are missing in the cache, the parameter will be refetched from the disk and stored in the cache with the submasks. This bit can be optimised but we need to consider what are the use cases of submasks caching. ''' if name not in self.keys(valid_only): # Exception should be caught otherwise HDF will crash and close! raise KeyError("%s" % name) elif name in self._params_cache: if not _slice: logging.debug("Retrieving param '%s' from HDF cache", name) # XXX: Caching breaks later loading of submasks! parameter = self._params_cache[name] if parameter.submasks or not load_submasks: return deepcopy(parameter) if copy_param else parameter logging.debug( 'Skipping returning parameter `%s` from cache as slice of ' 'data was requested.', name) group = self.hdf['series'][name] attrs = group.attrs data = group['data'] mask = group.get('mask', False) kwargs = {} frequency = attrs.get('frequency', 1) kwargs['frequency'] = frequency if _slice: slice_start = int((_slice.start or 0) * frequency) slice_stop = int((_slice.stop or len(data)) * frequency) _slice = slice(slice_start, slice_stop) data = data[_slice] mask = mask[_slice] if mask else mask if load_submasks and 'submasks' in attrs and 'submasks' in group.keys(): kwargs['submasks'] = {} submask_map = attrs['submasks'] if submask_map.strip(): submask_map = simplejson.loads(submask_map) for sub_name, index in submask_map.items(): kwargs['submasks'][sub_name] = group['submasks'][_slice or slice(None), index] array = np.ma.masked_array(data, mask=mask) if 'values_mapping' in attrs: values_mapping = attrs['values_mapping'] if values_mapping.strip(): mapping = simplejson.loads(values_mapping) kwargs['values_mapping'] = mapping if 'values_mapping' not in kwargs and data.dtype == np.int_: # Force float for non-values_mapped types. array = array.astype(np.float_) # Backwards compatibility. Q: When can this be removed? if 'supf_offset' in attrs: kwargs['offset'] = attrs['supf_offset'] if 'arinc_429' in attrs: kwargs['arinc_429'] = attrs['arinc_429'] if 'invalid' in attrs: kwargs['invalid'] = attrs['invalid'] if kwargs['invalid'] and 'invalidity_reason' in attrs: kwargs['invalidity_reason'] = attrs['invalidity_reason'] # Units if 'units' in attrs: kwargs['units'] = attrs['units'] if 'lfl' in attrs: kwargs['lfl'] = attrs['lfl'] elif 'description' in attrs: # Backwards compatibility for HDF files converted from AGS where # the units are stored in the description. Units will be invalid if # parameters from a FlightDataAnalyser HDF do not have 'units' # attributes. description = attrs['description'] if description: kwargs['units'] = description if 'data_type' in attrs: kwargs['data_type'] = attrs['data_type'] if 'source_name' in attrs: kwargs['source_name'] = attrs['source_name'] if 'description' in attrs: kwargs['description'] = attrs['description'] parameter = Parameter(name, array, **kwargs) # add to cache if required if name in self.cache_param_list: if not _slice: self._params_cache[name] = parameter else: logging.debug('Skipping saving parameter `%s` to cache as ' 'slice of data was requested.', name) return parameter