def test___init__imutable_cache_param_list(self): temp = 'temp_new_file.hdf5' if os.path.exists(temp): os.remove(temp) hdf_a = hdf_file(temp, create=True) hdf_a.cache_param_list.append('test') hdf_b = hdf_file(temp, create=True) self.assertEqual(hdf_b.cache_param_list, []) self.assertNotEqual(id(hdf_a.cache_param_list), id(hdf_b.cache_param_list))
def concat_hdf(hdf_paths, dest=None): ''' Takes in a list of HDF file paths and concatenates the parameter datasets which match the path 'series/<Param Name>/data'. The first file in the list of paths is the template for the output file, with only the 'series/<Param Name>/data' datasets being replaced with the concatenated versions. :param hdf_paths: File paths. :type hdf_paths: list of strings :param dest: optional destination path, which will be the first path in 'paths' :type dest: dict :return: path to concatenated hdf file. :rtype: str ''' # copy hdf to temp area to build upon hdf_master_path = copy_file(hdf_paths[0]) with hdf_file(hdf_master_path) as hdf_master: master_keys = hdf_master.keys() for hdf_path in hdf_paths[1:]: with hdf_file(hdf_path) as hdf: # check that all parameters match (avoids mismatching array lengths) param_keys = hdf.keys() assert set(param_keys) == set(master_keys) logging.debug("Copying parameters from file %s", hdf_path) for param_name in param_keys: param = hdf[param_name] master_param = hdf_master[param_name] assert param.frequency == master_param.frequency assert param.offset == master_param.offset assert param.units == master_param.units # join arrays together master_param.array = np.ma.concatenate( (master_param.raw_array, param.raw_array)) # re-save parameter hdf_master[param_name] = master_param # extend the master's duration hdf_master.duration += hdf.duration #endwith logging.debug("Completed extending parameters from %s", hdf_path) #endfor #endwith if dest: shutil.move(hdf_master_path, dest) return dest else: return hdf_master_path
def setUp(self): self.hdf_path = os.path.join(TEST_DATA_DIR, 'test_hdf_access.hdf') hdf = h5py.File(self.hdf_path, 'w') series = hdf.create_group('series') self.param_name = 'TEST_PARAM10' param_group = series.create_group(self.param_name) self.param_frequency = 2 self.param_supf_offset = 1.5 self.param_arinc_429 = True param_group.attrs['frequency'] = self.param_frequency param_group.attrs['supf_offset'] = self.param_supf_offset param_group.attrs['arinc_429'] = self.param_arinc_429 param_group.attrs['lfl'] = 1 self.param_data = np.arange(100) param_group.create_dataset('data', data=self.param_data) self.masked_param_name = 'TEST_PARAM11' masked_param_group = series.create_group(self.masked_param_name) self.masked_param_frequency = 4 self.masked_param_supf_offset = 2.5 masked_param_group.attrs['frequency'] = self.masked_param_frequency masked_param_group.attrs['supf_offset'] = self.masked_param_supf_offset self.param_mask = [bool(random.randint(0, 1)) for x in range(len(self.param_data))] masked_param_group.create_dataset('data', data=self.param_data) masked_param_group.create_dataset('mask', data=self.param_mask) self.masked_param_submask_arrays = np.array([[False, True], [True, False], [False, False]]) self.masked_param_submask_map = {'mask1': 0, 'mask2': 1} masked_param_group.attrs['submasks'] = \ simplejson.dumps(self.masked_param_submask_map) masked_param_group.create_dataset( 'submasks', data=self.masked_param_submask_arrays) hdf.close() self.hdf_file = hdf_file(self.hdf_path)
def test_split_segments_multiple_types(self, settings): ''' Test data has multiple segments of differing segment types. Test data has already been validated ''' # Overriding MINIMUM_FAST_DURATION. settings.AIRSPEED_THRESHOLD = 80 settings.AIRSPEED_THRESHOLD_TIME = 3 * 60 settings.HEADING_CHANGE_TAXI_THRESHOLD = 60 settings.MINIMUM_SPLIT_DURATION = 100 settings.MINIMUM_FAST_DURATION = 0 settings.MINIMUM_SPLIT_PARAM_VALUE = 0.175 settings.HEADING_RATE_SPLITTING_THRESHOLD = 0.1 settings.MAX_TIMEBASE_AGE = 365 * 10 settings.MIN_FAN_RUNNING = 10 hdf_path = os.path.join(test_data_path, "split_segments_multiple_types.hdf5") temp_path = copy_file(hdf_path) hdf = hdf_file(temp_path) self.maxDiff = None segment_tuples = split_segments(hdf, {}) self.assertEqual(len(segment_tuples), 16, msg="Unexpected number of segments detected") segment_types = tuple(x[0] for x in segment_tuples) self.assertEqual( segment_types, ('STOP_ONLY', 'START_ONLY', 'START_AND_STOP', 'START_AND_STOP', 'START_AND_STOP', 'START_AND_STOP', 'STOP_ONLY', 'START_AND_STOP', 'STOP_ONLY', 'START_ONLY', 'START_ONLY', 'START_AND_STOP', 'START_ONLY', 'START_AND_STOP', 'START_AND_STOP', 'START_ONLY'))
def plot_essential(hdf_path): """ Plot the essential parameters for flight analysis. Assumes hdf_path file contains the parameter series: Frame Counter, Airspeed, Altitude STD, Head True show() is to be called elsewhere (from matplotlib.pyplot import show) :param hdf_path: Path to HDF file. :type hdf_path: string """ fig = plt.figure() ##figsize=(10,8)) plt.title(os.path.basename(hdf_path)) with hdf_file(hdf_path) as hdf: ax1 = fig.add_subplot(4, 1, 1) #ax1.set_title('Frame Counter') ax1.plot(hdf['Frame Counter'].array, 'k--') ax2 = fig.add_subplot(4, 1, 2) ax2.plot(hdf['Airspeed'].array, 'r-') ax3 = fig.add_subplot(4, 1, 3, sharex=ax2) ax3.plot(hdf['Altitude STD'].array, 'g-') ax4 = fig.add_subplot(4, 1, 4, sharex=ax2) ax4.plot(hdf['Heading'].array, 'b-')
def test_split_segments_data_3(self): '''Splits on both Engine and Heading parameters.''' hdf_path = os.path.join(test_data_path, "split_segments_3.hdf5") temp_path = copy_file(hdf_path) hdf = hdf_file(temp_path) segment_tuples = split_segments(hdf, {}) #for a, e in zip(segment_tuples, expected): #print(a,e,a==e) self.assertEqual(segment_tuples, [('START_AND_STOP', slice(0, 3989.0, None), 0), ('START_AND_STOP', slice(3989.0, 7049.0, None), 1), ('START_AND_STOP', slice(7049.0, 9569.0, None), 1), ('START_AND_STOP', slice(9569.0, 12889.0, None), 1), ('START_AND_STOP', slice(12889.0, 15867.0, None), 1), ('START_AND_STOP', slice(15867.0, 18526.0, None), 3), ('START_AND_STOP', slice(18526.0, 21726.0, None), 2), ('START_AND_STOP', slice(21726.0, 24209.0, None), 2), ('START_AND_STOP', slice(24209.0, 26607.0, None), 1), ('START_AND_STOP', slice(26607.0, 28534.0, None), 3), ('START_AND_STOP', slice(28534.0, 30875.0, None), 2), ('START_AND_STOP', slice(30875.0, 33488.0, None), 3), ('NO_MOVEMENT', slice(33488.0, 33680.0, None), 0),])
def _design(self): ''' ''' form = FieldStorage( self.rfile, headers=self.headers, environ={'REQUEST_METHOD': 'POST'}, ) # Handle uploading of an HDF file: file_upload = form['hdf_file'] if not file_upload.filename: self._index(error='Please select a file to upload.') return # Create a temporary file for the upload: file_desc, file_path = mkstemp() file_obj = os.fdopen(file_desc, 'w') file_obj.write(file_upload.file.read()) file_obj.close() try: with hdf_file(file_path) as hdf_file_obj: parameter_names = hdf_file_obj.keys() except IOError: self._index(error='Please select a valid HDF file.') return self._respond_with_template('design.html', { 'parameter_names': parameter_names, 'file_path': file_path, })
def setUp(self): self.hdf_path = os.path.join(TEST_DATA_DIR, 'test_hdf_access.hdf') hdf = h5py.File(self.hdf_path, 'w') series = hdf.create_group('series') self.param_name = 'TEST_PARAM10' param_group = series.create_group(self.param_name) self.param_frequency = 2 self.param_supf_offset = 1.5 self.param_arinc_429 = True param_group.attrs['frequency'] = self.param_frequency param_group.attrs['supf_offset'] = self.param_supf_offset param_group.attrs['arinc_429'] = self.param_arinc_429 self.param_data = np.arange(100) param_group.create_dataset('data', data=self.param_data) self.masked_param_name = 'TEST_PARAM11' masked_param_group = series.create_group(self.masked_param_name) self.masked_param_frequency = 4 self.masked_param_supf_offset = 2.5 masked_param_group.attrs['frequency'] = self.masked_param_frequency masked_param_group.attrs['supf_offset'] = self.masked_param_supf_offset self.param_mask = [bool(random.randint(0, 1)) for x in range(len(self.param_data))] masked_param_group.create_dataset('data', data=self.param_data) masked_param_group.create_dataset('mask', data=self.param_mask) hdf.close() self.hdf_file = hdf_file(self.hdf_path)
def plot_loop(self): ''' The plotting loop. ''' while True: # For some strange reason it appears that printing the following # line affects the plotting window being shown on windows. if self.exit_loop.is_set(): return error_message = self._get_error_message() if error_message: show_error_dialog(*error_message) continue if self._ready_to_plot.is_set(): self._ready_to_plot.clear() try: with hdf_file(self._hdf_path) as hdf: # iterate over whole file as only those params # required were converted earlier into the HDF file params = hdf.get_params() title = os.path.basename(self._hdf_path) plot_parameters(params, self._axes, title=title) except ValueError as err: print 'Waiting for you to fix this error: %s' % err except Exception as err: # traceback required? print 'Exception raised! %s: %s' % (err.__class__.__name__, err) else: time.sleep(1)
def derived_trimmer(hdf_path, node_names, dest): ''' Trims an HDF file of parameters which are not dependencies of nodes in node_names. :param hdf_path: file path of hdf file. :type hdf_path: str :param node_names: A list of Node names which are required. :type node_names: list of str :param dest: destination path for trimmed output file :type dest: str :return: parameters in stripped hdf file :rtype: [str] ''' params = [] with hdf_file(hdf_path) as hdf: derived_nodes = get_derived_nodes(settings.NODE_MODULES) node_mgr = NodeManager( datetime.now(), hdf.duration, hdf.valid_param_names(), [], [], derived_nodes, {}, {}) _graph = graph_nodes(node_mgr) for node_name in node_names: deps = dependencies3(_graph, node_name, node_mgr) params.extend(filter(lambda d: d in node_mgr.hdf_keys, deps)) return strip_hdf(hdf_path, params, dest)
def _code_run(self): postvars = self._parse_post() import numpy as np from analysis_engine.node import A, KTI, KPV, S, P from analysis_engine.library import * data = [] try: with hdf_file(postvars['file_path'][0]) as hdf: if postvars['var_name_1'][0]: exec "%s = hdf['%s']" % (postvars['var_name_1'][0], postvars['hdf_name_1'][0]) data.append(self._prepare_array(locals()[postvars['var_name_1'][0]].array)) if postvars['var_name_2'][0]: exec "%s = hdf['%s']" % (postvars['var_name_2'][0], postvars['hdf_name_2'][0]) data.append(self._prepare_array(locals()[postvars['var_name_2'][0]].array)) if postvars['var_name_3'][0]: exec "%s = hdf['%s']" % (postvars['var_name_3'][0], postvars['hdf_name_3'][0]) data.append(self._prepare_array(locals()[postvars['var_name_3'][0]].array)) exec postvars['code'][0] data.insert(0, self._prepare_array(result)) except Exception as err: return self._respond_with_json({'error': str(err)}) # TODO: Align. # TODO: Remove invalid/masked dependencies # TODO: Use ast module to parse code. return self._respond_with_json({'data': data})
def revert_masks(hdf_path, params=None, delete_derived=False): ''' :type hdf_path: str :type params: params to revert or delete. :type params: [str] or None :type delete_derived: bool ''' with hdf_file(hdf_path) as hdf: if not params: params = hdf.keys() if delete_derived else hdf.lfl_keys() for param_name in params: param = hdf.get_param(param_name, load_submasks=True) if not param.lfl: if delete_derived: del hdf[param_name] continue if 'padding' not in param.submasks: continue param.array = param.get_array(submask='padding') param.submasks = {'padding': param.submasks['padding']} param.invalid = False hdf[param_name] = param
def test_split_segments_data_3(self): '''Splits on both Engine and Heading parameters.''' hdf_path = os.path.join(test_data_path, "split_segments_3.hdf5") temp_path = copy_file(hdf_path) hdf = hdf_file(temp_path) segment_tuples = split_segments(hdf, {}) #for a, e in zip(segment_tuples, expected): #print(a,e,a==e) self.assertEqual(segment_tuples, [ ('START_AND_STOP', slice(0, 3989.0, None), 0), ('START_AND_STOP', slice(3989.0, 7049.0, None), 1), ('START_AND_STOP', slice(7049.0, 9569.0, None), 1), ('START_AND_STOP', slice(9569.0, 12889.0, None), 1), ('START_AND_STOP', slice(12889.0, 15867.0, None), 1), ('START_AND_STOP', slice(15867.0, 18526.0, None), 3), ('START_AND_STOP', slice(18526.0, 21726.0, None), 2), ('START_AND_STOP', slice(21726.0, 24209.0, None), 2), ('START_AND_STOP', slice(24209.0, 26607.0, None), 1), ('START_AND_STOP', slice(26607.0, 28534.0, None), 3), ('START_AND_STOP', slice(28534.0, 30875.0, None), 2), ('START_AND_STOP', slice(30875.0, 33488.0, None), 3), ('NO_MOVEMENT', slice(33488.0, 33680.0, None), 0), ])
def test_split_segments_multiple_types(self): ''' Test data has multiple segments of differing segment types. Test data has already been validated ''' hdf_path = os.path.join(test_data_path, "split_segments_multiple_types.hdf5") temp_path = copy_file(hdf_path) hdf = hdf_file(temp_path) self.maxDiff = None segment_tuples = split_segments(hdf) self.assertEqual(len(segment_tuples), 16, msg="Unexpected number of segments detected") segment_types = tuple(x[0] for x in segment_tuples) self.assertEqual(segment_types, ('STOP_ONLY', 'START_ONLY', 'START_AND_STOP', 'START_AND_STOP', 'START_AND_STOP', 'START_AND_STOP', 'STOP_ONLY', 'START_AND_STOP', 'STOP_ONLY', 'START_ONLY', 'START_ONLY', 'START_AND_STOP', 'START_ONLY', 'START_AND_STOP', 'START_AND_STOP', 'START_ONLY'))
def plot_essential(hdf_path): """ Plot the essential parameters for flight analysis. Assumes hdf_path file contains the parameter series: Frame Counter, Airspeed, Altitude STD, Head True show() is to be called elsewhere (from matplotlib.pyplot import show) :param hdf_path: Path to HDF file. :type hdf_path: string """ fig = plt.figure() ##figsize=(10,8)) plt.title(os.path.basename(hdf_path)) with hdf_file(hdf_path) as hdf: ax1 = fig.add_subplot(4,1,1) #ax1.set_title('Frame Counter') ax1.plot(hdf['Frame Counter'].array, 'k--') ax2 = fig.add_subplot(4,1,2) ax2.plot(hdf['Airspeed'].array, 'r-') ax3 = fig.add_subplot(4,1,3,sharex=ax2) ax3.plot(hdf['Altitude STD'].array, 'g-') ax4 = fig.add_subplot(4,1,4,sharex=ax2) ax4.plot(hdf['Heading'].array, 'b-')
def get_params(self, hdf_path, _slice, phase_name): import shutil import tempfile from hdfaccess.file import hdf_file with tempfile.NamedTemporaryFile() as temp_file: shutil.copy(hdf_path, temp_file.name) with hdf_file(hdf_path) as hdf: pitch_capt = hdf.get('Pitch (Capt)') pitch_fo = hdf.get('Pitch (FO)') roll_capt = hdf.get('Roll (Capt)') roll_fo = hdf.get('Roll (FO)') cc_capt = hdf.get('Control Column Force (Capt)') cc_fo = hdf.get('Control Column Force (FO)') for par in pitch_capt, pitch_fo, roll_capt, roll_fo, cc_capt, \ cc_fo: if par is not None: ref_par = par break phase = S(name=phase_name, frequency=1) phase.create_section(_slice) phase = phase.get_aligned(ref_par)[0] # Align the arrays, usually done in the Nodes for par in pitch_fo, roll_capt, roll_fo, cc_capt, cc_fo: if par is None: continue par.array = align(par, ref_par) par.hz = ref_par.hz return pitch_capt, pitch_fo, roll_capt, roll_fo, cc_capt, cc_fo, phase
def append_segment_info(hdf_segment_path, segment_type, segment_slice, part, fallback_dt=None): """ Get information about a segment such as type, hash, etc. and return a named tuple. If a valid timestamp can't be found, it creates start_dt as epoch(0) i.e. datetime(1970,1,1,1,0). Go-fast dt and Stop dt are relative to this point in time. :param hdf_segment_path: path to HDF segment to analyse :type hdf_segment_path: string :param segment_slice: Slice of this segment relative to original file. :type segment_slice: slice :param part: Numeric part this segment was in the original data file (1 indexed) :type part: Integer :param fallback_dt: Used to replace elements of datetimes which are not available in the hdf file (e.g. YEAR not being recorded) :type fallback_dt: datetime :returns: Segment named tuple :rtype: Segment """ # build information about a slice with hdf_file(hdf_segment_path) as hdf: airspeed = hdf['Airspeed'].array duration = hdf.duration # For now, raise TimebaseError up rather than using EPOCH # TODO: Review whether to revert to epoch again. ##try: start_datetime = _calculate_start_datetime(hdf, fallback_dt) ##except TimebaseError: ##logger.warning("Unable to calculate timebase, using epoch " ##"1.1.1970!") ##start_datetime = datetime.fromtimestamp(0) stop_datetime = start_datetime + timedelta(seconds=duration) hdf.start_datetime = start_datetime if segment_type in ('START_AND_STOP', 'START_ONLY', 'STOP_ONLY'): # we went fast, so get the index spd_above_threshold = \ np.ma.where(airspeed > settings.AIRSPEED_THRESHOLD) go_fast_index = spd_above_threshold[0][0] go_fast_datetime = \ start_datetime + timedelta(seconds=int(go_fast_index)) # Identification of raw data airspeed hash airspeed_hash_sections = runs_of_ones(airspeed.data > settings.AIRSPEED_THRESHOLD) airspeed_hash = hash_array(airspeed.data,airspeed_hash_sections, settings.AIRSPEED_HASH_MIN_SAMPLES) #elif segment_type == 'GROUND_ONLY': ##Q: Create a groundspeed hash? #pass else: go_fast_index = None go_fast_datetime = None # if not go_fast, create hash from entire file airspeed_hash = sha_hash_file(hdf_segment_path) # ('slice type part path hash start_dt go_fast_dt stop_dt') segment = Segment(segment_slice, segment_type, part, hdf_segment_path, airspeed_hash, start_datetime, go_fast_datetime, stop_datetime) return segment
def test_split_segments_146_300(self): hdf = hdf_file(os.path.join(test_data_path, "4_3377853_146-301.hdf5")) segment_tuples = split_segments(hdf, {}) self.assertEqual(segment_tuples, [('START_AND_STOP', slice(0, 24801.0, None)), ('START_AND_STOP', slice(24801.0, 30000.0, None)), ('START_AND_STOP', slice(30000.0, 49999.0, None)), ('START_AND_STOP', slice(49999.0, 69999.0, None)), ('START_AND_STOP', slice(69999.0, 73552.0, None))])
def test_split_segments_146_300(self): hdf = hdf_file(os.path.join(test_data_path, "4_3377853_146-301.hdf5")) segment_tuples = split_segments(hdf) self.assertEqual(segment_tuples, [('START_AND_STOP', slice(0, 24801.0, None)), ('START_AND_STOP', slice(24801.0, 30000.0, None)), ('START_AND_STOP', slice(30000.0, 49999.0, None)), ('START_AND_STOP', slice(49999.0, 69999.0, None)), ('START_AND_STOP', slice(69999.0, 73552.0, None))])
def get_esns(flight): try: with hdf_file(flight.segment.file, read_only=True) as hdf: e1esn = most_common_value(hdf['Eng (1) ESN'].array) e2esn = most_common_value(hdf['Eng (2) ESN'].array) return e1esn, e2esn except: import traceback traceback.print_exc() return '', ''
def _spacetree(self): ''' ''' form = FieldStorage( self.rfile, headers=self.headers, environ={'REQUEST_METHOD': 'POST'}, ) # Handle uploading of an HDF file: file_upload = form['hdf_file'] if not file_upload.filename: self._index(error='Please select a file to upload.') return # Create a temporary file for the upload: file_desc, file_path = mkstemp() file_obj = os.fdopen(file_desc, 'w') file_obj.write(file_upload.file.read()) file_obj.close() try: with hdf_file(file_path) as hdf_file_obj: #lfl_params = hdf_file_obj.keys() json_dep_tree = json.loads(hdf_file_obj.dependency_tree) from networkx.readwrite import json_graph gr_st = json_graph.node_link_graph(json_dep_tree) except IOError: self._index(error='Please select a valid HDF file.') return # Fetch parameters to display in a grid: #self._generate_json(lfl_params) # Save the dependency tree to tree.json: tree = os.path.join(AJAX_DIR, 'tree.json') with open(tree, 'w') as fh: json.dump(graph_adjacencies(gr_st), fh, indent=4) # Save the list of nodes to node_list.json: node_list = os.path.join(AJAX_DIR, 'node_list.json') spanning_tree_params = sorted(gr_st.nodes()) with open(node_list, 'w') as fh: json.dump(spanning_tree_params, fh, indent=4) #polaris_query, params, missing_lfl_params = self._fetch_params(lfl_params) polaris_query, params, missing_lfl_params = '', {}, [] # Render the spacetree: self._respond_with_template( 'spacetree.html', { 'missing_lfl_params': missing_lfl_params, 'params': sorted(params.items()), 'polaris_query': polaris_query, 'server': BASE_URL, 'year': date.today().year, })
def strip_hdf(hdf_path, params_to_keep, dest, deidentify=True): """ Strip an HDF file of all parameters apart from those in params_to_keep. Does not raise an exception if any of the params_to_keep are not in the HDF file. :param hdf_path: file path of hdf file. :type hdf_path: str :param params_to_keep: parameter names to keep. :type param_to_keep: list of str :param dest: destination path for stripped output file :type dest: str :return: all parameters names within the output hdf file :rtype: [str] """ with hdf_file(hdf_path) as hdf, hdf_file(dest, create=True) as hdf_dest: _copy_attrs(hdf.hdf, hdf_dest.hdf, deidentify=deidentify) # Copy top-level attrs. params = hdf.get_params(params_to_keep) for param_name, param in params.iteritems(): hdf_dest[param_name] = param return params.keys()
def test_split_segments_737_3C(self): '''Splits on both DFC Jump and Engine parameters.''' hdf = hdf_file(os.path.join(test_data_path, "1_7295949_737-3C.hdf5")) segment_tuples = split_segments(hdf) self.assertEqual(segment_tuples, [('START_AND_STOP', slice(0, 3168.0, None)), ('START_AND_STOP', slice(3168.0, 6260.0, None)), ('START_AND_STOP', slice(6260.0, 9504.0, None)), ('START_AND_STOP', slice(9504.0, 12680.0, None)), ('START_AND_STOP', slice(12680.0, 15571.0, None)), ('START_AND_STOP', slice(15571.0, 18752.0, None))])
def test_split_segments_737_3C(self): '''Splits on both DFC Jump and Engine parameters.''' hdf = hdf_file(os.path.join(test_data_path, "1_7295949_737-3C.hdf5")) segment_tuples = split_segments(hdf, {}) self.assertEqual(segment_tuples, [('START_AND_STOP', slice(0, 3168.0, None)), ('START_AND_STOP', slice(3168.0, 6260.0, None)), ('START_AND_STOP', slice(6260.0, 9504.0, None)), ('START_AND_STOP', slice(9504.0, 12680.0, None)), ('START_AND_STOP', slice(12680.0, 15571.0, None)), ('START_AND_STOP', slice(15571.0, 18752.0, None))])
def test_create_file(self): temp = 'temp_new_file.hdf5' if os.path.exists(temp): os.remove(temp) # cannot create file without specifying 'create=True' self.assertRaises(IOError, hdf_file, temp) self.assertFalse(os.path.exists(temp)) # this one will create the file hdf = hdf_file(temp, create=True) self.assertTrue(os.path.exists(temp)) self.assertEqual(hdf.hdfaccess_version, 1) os.remove(temp)
def test_split_segments_data_1(self): '''Splits on both DFC Jump and Engine parameters.''' hdf_path = os.path.join(test_data_path, "split_segments_1.hdf5") temp_path = copy_file(hdf_path) hdf = hdf_file(temp_path) segment_tuples = split_segments(hdf, {}) self.assertEqual(segment_tuples, [('START_AND_STOP', slice(0, 9952.0, None), 0), ('START_AND_STOP', slice(9952.0, 21799.0, None), 0), ('START_AND_STOP', slice(21799.0, 24665.0, None), 3), ('START_AND_STOP', slice(24665.0, 27898.0, None), 1), ('START_AND_STOP', slice(27898.0, 31424.0, None), 2)])
def make_kml_file(start_datetime, flight_attrs, kti, kpv, flight_file, REPORTS_DIR, output_path_and_file): '''adapted from FDS process_flight. As of 2013/6/6 we do not geolocate unless KML was requested, to save time.''' from analysis_engine.plot_flight import track_to_kml with hdf_file(output_path_and_file) as hdf: # geo locate KTIs kti = geo_locate(hdf, kti) kti = _timestamp(start_datetime, kti) # geo locate KPVs kpv = geo_locate(hdf, kpv) kpv = _timestamp(start_datetime, kpv) report_path_and_file = REPORTS_DIR + flight_file.replace('.','_')+'.kml' track_to_kml(output_path_and_file, kti, kpv, flight_attrs, dest_path=report_path_and_file)
def strip_hdf(hdf_path, params_to_keep, dest, deidentify=True): ''' Strip an HDF file of all parameters apart from those in params_to_keep. Does not raise an exception if any of the params_to_keep are not in the HDF file. :param hdf_path: file path of hdf file. :type hdf_path: str :param params_to_keep: parameter names to keep. :type param_to_keep: list of str :param dest: destination path for stripped output file :type dest: str :return: all parameters names within the output hdf file :rtype: [str] ''' with hdf_file(hdf_path) as hdf, hdf_file(dest, create=True) as hdf_dest: _copy_attrs(hdf.hdf, hdf_dest.hdf, deidentify=deidentify) # Copy top-level attrs. params = hdf.get_params(params_to_keep) for param_name, param in params.items(): hdf_dest[param_name] = param return params.keys()
def test_split_segments_data_2(self): '''Splits on both DFC Jump and Engine parameters.''' hdf_path = os.path.join(test_data_path, "split_segments_2.hdf5") temp_path = copy_file(hdf_path) hdf = hdf_file(temp_path) segment_tuples = split_segments(hdf, {}) self.assertEqual(segment_tuples, [('START_AND_STOP', slice(0, 3407.0, None)), ('START_AND_STOP', slice(3407.0, 6362.0, None)), ('START_AND_STOP', slice(6362.0, 9912.0, None)), ('START_AND_STOP', slice(9912.0, 13064.0, None)), ('START_AND_STOP', slice(13064.0, 16467.0, None)), ('START_AND_STOP', slice(16467.0, 19200.0, None))])
def save_to_hdf5(self, hdf5_path): '''this will overwrite any existing file with the same name''' with hdf_file(hdf5_path, cache_param_list=[], create=True) as hfile: # set attributes hfile.hdf.attrs['duration'] = self.duration hfile.hdf.attrs['start_datetime '] = self.start_epoch hfile.hdf.attrs['superframe_present']=self.superframe_present hfile.hdf.attrs['hdfaccess_version']=hdfaccess.file.HDFACCESS_VERSION hfile.hdf.attrs['reliable_frame_counter']=self.reliable_frame_counter hfile.hdf.attrs['aircraft_info'] = json.dumps(self.aircraft_info) # save time series for k in self.parameters.keys(): hfile.set_param( self.parameters[k]) return
def test_split_segments_data_2(self): '''Splits on both DFC Jump and Engine parameters.''' hdf_path = os.path.join(test_data_path, "split_segments_2.hdf5") temp_path = copy_file(hdf_path) hdf = hdf_file(temp_path) segment_tuples = split_segments(hdf) self.assertEqual(segment_tuples, [('START_AND_STOP', slice(0, 3583.0, None)), ('START_AND_STOP', slice(3583.0, 6446.0, None)), ('START_AND_STOP', slice(6446.0, 9912.0, None)), ('START_AND_STOP', slice(9912.0, 13064.0, None)), ('START_AND_STOP', slice(13064.0, 16467.0, None)), ('START_AND_STOP', slice(16467.0, 19200.0, None))])
def test_open_and_close_and_full_masks(self): self.hdf_file.close() with hdf_file(self.hdf_path) as hdf: # check it's open self.assertFalse(hdf.hdf.id is None) hdf['sample'] = Parameter('sample', np.array(range(10))) self.assertEqual(list(hdf['sample'].array.data), range(10)) self.assertTrue(hasattr(hdf['sample'].array, 'mask')) hdf['masked sample'] = Parameter('masked sample', np.ma.array(range(10))) self.assertEqual(list(hdf['masked sample'].array.data), range(10)) # check masks are returned in full (not just a single False) self.assertEqual(list(hdf['masked sample'].array.mask), [False] * 10) # check it's closed self.assertEqual(hdf.hdf.__repr__(), '<Closed HDF5 file>')
def test_split_segments_data_2(self): '''Splits on both DFC Jump and Engine parameters.''' hdf_path = os.path.join(test_data_path, "split_segments_2.hdf5") temp_path = copy_file(hdf_path) hdf = hdf_file(temp_path) segment_tuples = split_segments(hdf, {}) self.assertEqual(segment_tuples, [('START_AND_STOP', slice(0, 3407.0, None), 0), ('START_AND_STOP', slice(3407.0, 6362.0, None), 15), ('START_AND_STOP', slice(6362.0, 9912.0, None), 26), ('START_AND_STOP', slice(9912.0, 13064.0, None), 56), ('START_AND_STOP', slice(13064.0, 16467.0, None), 8), ('START_AND_STOP', slice(16467.0, 19065.0, None), 19), ('GROUND_ONLY', slice(19065.0, 19200.0, None), 57)])
def process_raw_hdf(hdf, axes): with hdf_file(hdf) as h: params = h.get_params() params_to_plot = {} for axis in axes: if axis is not None: for param in axis: try: params_to_plot[param] = params[param] except KeyError: print('Parameter %s was not found in the HDF file.' % param) filtered_axes = dict(enumerate(filter(None, axes), start=1)) return params_to_plot, filtered_axes
def _generate_graph(self): postvars = self._parse_post() data = [] with hdf_file(postvars['file_path'][0]) as hdf: params = hdf.get_params(postvars['parameters[]']).values() align_param = params[0] arrays = [align_param[0]] for param in align_param: arrays.append(align(param, align_param)) # TODO: Get parameter data and return it as AJAX. for param in params.values(): data.append(zip(range(len(param.array)), param.array.data.tolist())) return self._respond_with_json({'data': data})
def save_to_hdf5(self, hdf5_path): '''this will overwrite any existing file with the same name''' with hdf_file(hdf5_path, cache_param_list=[], create=True) as hfile: # set attributes hfile.hdf.attrs['duration'] = self.duration hfile.hdf.attrs['start_datetime '] = self.start_epoch hfile.hdf.attrs['superframe_present'] = self.superframe_present hfile.hdf.attrs[ 'hdfaccess_version'] = hdfaccess.file.HDFACCESS_VERSION hfile.hdf.attrs[ 'reliable_frame_counter'] = self.reliable_frame_counter hfile.hdf.attrs['aircraft_info'] = json.dumps(self.aircraft_info) # save time series for k in self.parameters.keys(): hfile.set_param(self.parameters[k]) return
def test_mapped_array(self): # created mapped array mapping = {0: 'zero', 2: 'two', 3: 'three'} array = np.ma.array(range(5) + range(5), mask=[1, 1, 1, 0, 0, 0, 0, 0, 1, 1]) multi_p = Parameter('multi', array, values_mapping=mapping) multi_p.array[0] = 'three' # save array to hdf self.hdf_file['multi'] = multi_p self.hdf_file.close() # check hdf has mapping and integer values stored with hdf_file(self.hdf_path) as hdf: saved = hdf['multi'] self.assertEqual(str(saved.array[:]), "['three' -- -- 'three' '?' 'zero' '?' 'two' -- --]") self.assertEqual(saved.array.data.dtype, np.int)
def make_kml_file(start_datetime, flight_attrs, kti, kpv, flight_file, REPORTS_DIR, output_path_and_file): '''adapted from FDS process_flight. As of 2013/6/6 we do not geolocate unless KML was requested, to save time.''' from analysis_engine.plot_flight import track_to_kml with hdf_file(output_path_and_file) as hdf: # geo locate KTIs kti = geo_locate(hdf, kti) kti = _timestamp(start_datetime, kti) # geo locate KPVs kpv = geo_locate(hdf, kpv) kpv = _timestamp(start_datetime, kpv) report_path_and_file = REPORTS_DIR + flight_file.replace('.', '_') + '.kml' track_to_kml(output_path_and_file, kti, kpv, flight_attrs, dest_path=report_path_and_file)
def test_rto_correct_side_of_split(self): ''' Test to ensure that RTO's are on the correct side of splitting, i.e. at the beginning of a flight. This example HDF5 file appears to have two stationary engine activities and an RTO between the two flights. This creates 6 sizeable slices (potential splitting points) where the engine parameters normalised to 0. We're interested in making the segment split within the first of these eng_min_slices slices (Between indices 11959.5 to 12336.5). Ideally the segment split should be halfway between this, at 12148.0. ''' hdf = hdf_file(os.path.join(test_data_path, "rto_split_segment.hdf5")) segment_tuples = split_segments(hdf, {}) split_idx = 12148.0 self.assertEqual(segment_tuples, [ ('START_AND_STOP', slice(0, split_idx, None), 0), ('START_AND_STOP', slice(split_idx, 21997.0, None), 52), ('GROUND_ONLY', slice(21997.0, 22784.0, None), 45), ])
def validate_file(hdffile, helicopter=False): """ Attempts to open the HDF5 file in using FlightDataAccessor and run all the validation tests. If the file cannot be opened, it will attempt to open the file using the h5py package and validate the namespace to test the HDF5 group structure. """ filename = hdffile.split(os.sep)[-1] open_with_h5py = False hdf = None LOGGER.info("Verifying file '%s' with FlightDataAccessor.", filename) try: hdf = hdf_file(hdffile, read_only=True) except Exception as err: LOGGER.error("FlightDataAccessor cannot open '%s'. " "Exception(%s: %s)", filename, type(err).__name__, err) open_with_h5py = True # If FlightDataAccessor errors upon opening it maybe because '/series' # is not included in the file. hdf_file attempts to create and fails # because we opening it as readonly. Verify the group structure by # using H5PY if open_with_h5py: LOGGER.info("Checking that H5PY package can read the file.") try: hdf_alt = h5py.File(hdffile, 'r') except Exception as err: LOGGER.error("cannot open '%s' using H5PY. Exception(%s: %s)", filename, type(err).__name__, err) return LOGGER.info("File %s can be opened by H5PY, suggesting the format " "is not compatible for POLARIS to use.", filename) LOGGER.info("Will just verify the HDF5 structure and exit.") validate_namespace(hdf_alt) hdf_alt.close() else: validate_namespace(hdf.hdf) # continue testing using FlightDataAccessor validate_root_attribute(hdf) validate_parameters(hdf, helicopter) if hdf: hdf.close()
def test_split_segments_data_3(self): '''Splits on both Engine and Heading parameters.''' hdf_path = os.path.join(test_data_path, "split_segments_3.hdf5") temp_path = copy_file(hdf_path) hdf = hdf_file(temp_path) segment_tuples = split_segments(hdf, {}) self.assertEqual(segment_tuples, [('START_AND_STOP', slice(0, 3989.0, None)), ('START_AND_STOP', slice(3989.0, 7049.0, None)), ('START_AND_STOP', slice(7049.0, 9569.0, None)), ('START_AND_STOP', slice(9569.0, 12889.0, None)), ('START_AND_STOP', slice(12889.0, 15867.0, None)), ('START_AND_STOP', slice(15867.0, 18526.0, None)), ('START_AND_STOP', slice(18526.0, 21726.0, None)), ('START_AND_STOP', slice(21726.0, 24209.0, None)), ('START_AND_STOP', slice(24209.0, 26607.0, None)), ('START_AND_STOP', slice(26607.0, 28534.0, None)), ('START_AND_STOP', slice(28534.0, 30875.0, None)), ('START_AND_STOP', slice(30875.0, 33680.0, None))])
def test_rto_correct_side_of_split(self): ''' Test to ensure that RTO's are on the correct side of splitting, i.e. at the beginning of a flight. This example HDF5 file appears to have two stationary engine activities and an RTO between the two flights. This creates 6 sizeable slices (potential splitting points) where the engine parameters normalised to 0. We're interested in making the segment split within the first of these eng_min_slices slices (Between indices 11959.5 to 12336.5). Ideally the segment split should be halfway between this, at 12148.0. ''' hdf = hdf_file(os.path.join(test_data_path, "rto_split_segment.hdf5")) segment_tuples = split_segments(hdf, {}) split_idx = 12148.0 self.assertEqual( segment_tuples, [('START_AND_STOP', slice(0, split_idx, None), 0), ('START_AND_STOP', slice(split_idx, 21997.0, None), 52), ('GROUND_ONLY', slice(21997.0, 22784.0, None), 45),] )
def test_split_segments_data_3(self): '''Splits on both Engine and Heading parameters.''' hdf_path = os.path.join(test_data_path, "split_segments_3.hdf5") temp_path = copy_file(hdf_path) hdf = hdf_file(temp_path) segment_tuples = split_segments(hdf) self.assertEqual(segment_tuples, [('START_AND_STOP', slice(0, 3987.0, None)), ('START_AND_STOP', slice(3987.0, 7049.0, None)), ('START_AND_STOP', slice(7049.0, 9563.0, None)), ('START_AND_STOP', slice(9563.0, 12921.0, None)), ('START_AND_STOP', slice(12921.0, 15858.0, None)), ('START_AND_STOP', slice(15858.0, 18526.0, None)), ('START_AND_STOP', slice(18526.0, 21728.0, None)), ('START_AND_STOP', slice(21728.0, 24208.0, None)), ('START_AND_STOP', slice(24208.0, 26607.0, None)), ('START_AND_STOP', slice(26607.0, 28534.0, None)), ('START_AND_STOP', slice(28534.0, 30875.0, None)), ('START_AND_STOP', slice(30875.0, 33680.0, None))])
def _spacetree(self): ''' ''' form = FieldStorage( self.rfile, headers=self.headers, environ={'REQUEST_METHOD': 'POST'}, ) # Handle uploading of an HDF file: file_upload = form['hdf_file'] if not file_upload.filename: self._index(error='Please select a file to upload.') return # Create a temporary file for the upload: file_desc, file_path = mkstemp() file_obj = os.fdopen(file_desc, 'w') file_obj.write(file_upload.file.read()) file_obj.close() try: with hdf_file(file_path) as hdf_file_obj: lfl_params = hdf_file_obj.keys() except IOError: self._index(error='Please select a valid HDF file.') return # Fetch parameters to display in a grid: self._generate_json(lfl_params) polaris_query, params, missing_lfl_params = self._fetch_params( lfl_params) # Render the spacetree: self._respond_with_template( 'spacetree.html', { 'missing_lfl_params': missing_lfl_params, 'params': sorted(params.items()), 'polaris_query': polaris_query, 'server': BASE_URL, 'year': date.today().year, })
def test_split_segments_multiple_types(self, settings): ''' Test data has multiple segments of differing segment types. Test data has already been validated ''' # Overriding MINIMUM_FAST_DURATION. settings.AIRSPEED_THRESHOLD = 80 settings.AIRSPEED_THRESHOLD_TIME = 3 * 60 settings.HEADING_CHANGE_TAXI_THRESHOLD = 60 settings.MINIMUM_SPLIT_DURATION = 100 settings.MINIMUM_FAST_DURATION = 0 settings.MINIMUM_SPLIT_PARAM_VALUE = 0.175 settings.HEADING_RATE_SPLITTING_THRESHOLD = 0.1 settings.MAX_TIMEBASE_AGE = 365 * 10 settings.MIN_FAN_RUNNING = 10 hdf_path = os.path.join(test_data_path, "split_segments_multiple_types.hdf5") temp_path = copy_file(hdf_path) hdf = hdf_file(temp_path) self.maxDiff = None segment_tuples = split_segments(hdf, {}) self.assertEqual(len(segment_tuples), 16, msg="Unexpected number of segments detected") segment_types = tuple(x[0] for x in segment_tuples) self.assertEqual(segment_types, ('STOP_ONLY', 'START_ONLY', 'START_AND_STOP', 'START_AND_STOP', 'START_AND_STOP', 'START_AND_STOP', 'STOP_ONLY', 'START_AND_STOP', 'STOP_ONLY', 'START_ONLY', 'START_ONLY', 'START_AND_STOP', 'START_ONLY', 'START_AND_STOP', 'START_AND_STOP', 'START_ONLY'))