def test_graph_nodes_using_sample_tree(self): requested = ['P7', 'P8'] mgr2 = NodeManager(datetime.now(), 10, self.lfl_params, requested, [], self.derived_nodes, {}, {}) gr = graph_nodes(mgr2) self.assertEqual(len(gr), 11) self.assertEqual(gr.neighbors('root'), ['P8', 'P7'])
def derived_trimmer(hdf_path, node_names, dest): ''' Trims an HDF file of parameters which are not dependencies of nodes in node_names. :param hdf_path: file path of hdf file. :type hdf_path: str :param node_names: A list of Node names which are required. :type node_names: list of str :param dest: destination path for trimmed output file :type dest: str :return: parameters in stripped hdf file :rtype: [str] ''' params = [] with hdf_file(hdf_path) as hdf: derived_nodes = get_derived_nodes(settings.NODE_MODULES) node_mgr = NodeManager( datetime.now(), hdf.duration, hdf.valid_param_names(), [], [], derived_nodes, {}, {}) _graph = graph_nodes(node_mgr) for node_name in node_names: deps = dependencies3(_graph, node_name, node_mgr) params.extend(filter(lambda d: d in node_mgr.hdf_keys, deps)) return strip_hdf(hdf_path, params, dest)
def test_invalid_requirement_raises(self): lfl_params = [] requested = ['Smoothed Track', 'Moment of Takeoff'] #it's called Moment Of Takeoff derived = get_derived_nodes([import_module('sample_derived_parameters')]) mgr = NodeManager({'Start Datetime': datetime.now()}, 10, lfl_params, requested, [], derived, {}, {}) self.assertRaises(nx.NetworkXError, dependency_order, mgr, draw=False)
def test_graph_middle_level_depenency_builds_partial_tree(self): requested = ['P5'] mgr = NodeManager(datetime.now(), 1, self.lfl_params, requested, [], self.derived_nodes, {}, {}) gr = graph_nodes(mgr) # should only be linked to P5 self.assertEqual(gr.neighbors('root'), ['P5'])
def test_graph_nodes_with_duplicate_key_in_lfl_and_derived(self): # Test that LFL nodes are used in place of Derived where available. # Tests a few of the colours class One(DerivedParameterNode): # Hack to allow objects rather than classes to be added to the tree. __base__ = DerivedParameterNode __bases__ = [__base__] def derive(self, dep=P('DepOne')): pass class Four(DerivedParameterNode): # Hack to allow objects rather than classes to be added to the tree. __base__ = DerivedParameterNode __bases__ = [__base__] def derive(self, dep=P('DepFour')): pass one = One('overridden') four = Four('used') mgr1 = NodeManager({'Start Datetime': datetime.now()}, 10, [1, 2], [2, 4], [], {1:one, 4:four},{}, {}) gr = graph_nodes(mgr1) self.assertEqual(len(gr), 5) # LFL self.assertEqual(gr.edges(1), []) # as it's in LFL, it shouldn't have any edges self.assertEqual(gr.node[1], {'color': '#72f4eb', 'node_type': 'HDFNode'}) # Derived self.assertEqual(gr.edges(4), [(4,'DepFour')]) self.assertEqual(gr.node[4], {'color': '#72cdf4', 'node_type': 'DerivedParameterNode'}) # Root from analysis_engine.dependency_graph import draw_graph draw_graph(gr, 'test_graph_nodes_with_duplicate_key_in_lfl_and_derived') self.assertEqual(gr.successors('root'), [2,4]) # only the two requested are linked self.assertEqual(gr.node['root'], {'color': '#ffffff'})
def test_sample_parameter_module(self): """Tests many options: can_operate on SmoothedTrack works with """ requested = ['Smoothed Track', 'Moment Of Takeoff', 'Vertical Speed', 'Slip On Runway'] lfl_params = ['Indicated Airspeed', 'Groundspeed', 'Pressure Altitude', 'Heading', 'TAT', 'Latitude', 'Longitude', 'Longitudinal g', 'Lateral g', 'Normal g', 'Pitch', 'Roll', ] derived = get_derived_nodes([import_module('sample_derived_parameters')]) nodes = NodeManager({'Start Datetime': datetime.now()}, 10, lfl_params, requested, [], derived, {}, {}) order, _ = dependency_order(nodes, draw=False) pos = order.index self.assertTrue(len(order)) self.assertNotIn('Moment Of Takeoff', order) # not available self.assertTrue(pos('Vertical Speed') > pos('Pressure Altitude')) self.assertTrue(pos('Slip On Runway') > pos('Groundspeed')) self.assertTrue(pos('Slip On Runway') > pos('Horizontal g Across Track')) self.assertTrue(pos('Horizontal g Across Track') > pos('Roll')) self.assertFalse('Mach' in order) # Mach wasn't requested! self.assertFalse('Radio Altimeter' in order) self.assertEqual(len(nodes.hdf_keys), 12) self.assertEqual(len(nodes.requested), 4) self.assertEqual(len(nodes.derived_nodes), 13)
def test_dependency_with_lowlevel_dependencies_requested(self): """ Simulate requesting a Raw Parameter as a dependency. This requires the requested node to be removed when it is not at the top of the dependency tree. """ requested = [ 'P7', 'P8', # top level nodes 'P4', 'P5', 'P6', # middle level node 'Raw3', # bottom level node ] mgr = NodeManager(datetime.now(), 10, self.lfl_params + ['Floating'], requested, [], self.derived_nodes, {}, {}) gr = graph_nodes(mgr) gr_all, gr_st, order = process_order(gr, mgr) self.assertEqual(len(gr_st), 11) pos = order.index self.assertTrue(pos('P8') > pos('Raw5')) self.assertTrue(pos('P7') > pos('P4')) self.assertTrue(pos('P7') > pos('P5')) self.assertTrue(pos('P7') > pos('P6')) self.assertTrue(pos('P6') > pos('Raw3')) self.assertTrue(pos('P5') > pos('Raw3')) self.assertTrue(pos('P5') > pos('Raw4')) self.assertTrue(pos('P4') > pos('Raw1')) self.assertTrue(pos('P4') > pos('Raw2')) self.assertFalse('Floating' in order) self.assertFalse('root' in order) #don't include the root!
def _get_dependency_order(self, requested, aircraft_info, lfl_params, draw=False, raise_cir_dep=True, segment_info={}): #derived_nodes = get_derived_nodes(settings.NODE_MODULES) if aircraft_info['Aircraft Type'] == 'helicopter': node_modules = settings.NODE_MODULES + settings.NODE_HELICOPTER_MODULE_PATHS else: node_modules = settings.NODE_MODULES # go through modules to get derived nodes derived_nodes = get_derived_nodes(node_modules) if requested == []: # Use all derived nodes if requested is empty requested = [ p for p in derived_nodes.keys() if p not in lfl_params ] if not segment_info: segment_info = { 'Start Datetime': datetime.now(), 'Segment Type': 'START_AND_STOP', } node_mgr = NodeManager(segment_info, 10, lfl_params, requested, [], derived_nodes, aircraft_info, {}) order, gr_st = dependency_order(node_mgr, draw=draw, raise_cir_dep=raise_cir_dep) return order, gr_st
def pre_process_parameters(hdf, segment_info, param_names, required, aircraft_info, achieved_flight_record, force=False): ''' Perform actions prior to main processing run. Actions such as merging parameters up front simplify processing paths by removing circular dependacies. ''' pre_processing_nodes = get_derived_nodes( settings.PRE_PROCESSING_MODULE_PATHS) requested = list(pre_processing_nodes.keys()) node_mgr = NodeManager(segment_info, hdf.duration, param_names, requested, required, pre_processing_nodes, aircraft_info, achieved_flight_record) process_order, gr_st = dependency_order(node_mgr, draw=False) ktis, kpvs, sections, approaches, flight_attrs = \ derive_parameters(hdf, node_mgr, process_order, force=force)
def test_graph_nodes_with_duplicate_key_in_lfl_and_derived(self): # Test that LFL nodes are used in place of Derived where available. # Tests a few of the colours class One(DerivedParameterNode): # Hack to allow objects rather than classes to be added to the tree. __base__ = DerivedParameterNode __bases__ = [__base__] def derive(self, dep=P('DepOne')): pass class Four(DerivedParameterNode): # Hack to allow objects rather than classes to be added to the tree. __base__ = DerivedParameterNode __bases__ = [__base__] def derive(self, dep=P('DepFour')): pass one = One('overridden') four = Four('used') mgr1 = NodeManager({'Start Datetime': datetime.now()}, 10, ['1', '2'], ['2', '4'], [], {'1':one, '4':four},{}, {}) gr = graph_nodes(mgr1) self.assertEqual(len(gr), 5) # LFL self.assertEqual(list(gr.edges('1')), []) # as it's in LFL, it shouldn't have any edges self.assertEqual(gr.node['1'], {'node_type': 'HDFNode'}) # Derived self.assertEqual(list(gr.edges('4')), [('4','DepFour')]) self.assertEqual(gr.node['4'], {'node_type': 'DerivedParameterNode'}) # Root self.assertEqual(list(gr.successors('root')), ['2','4']) # only the two requested are linked self.assertEqual(gr.node['root'], {})
def _get_dependency_order(self, requested, aircraft_info, lfl_params, segment_info={}): if not segment_info: segment_info = { 'Start Datetime': datetime.now(), 'Segment Type': 'START_AND_STOP', } rel_path = Path('dummy_nodes') pre_processing_modules = ['merge_multistate_parameters', 'merge_parameters'] node_modules = [ import_module(rel_path / 'pre_processing' / f'{mod}') for mod in pre_processing_modules ] pre_processing_nodes = get_derived_nodes(node_modules) pre_processing_requested = list(pre_processing_nodes.keys()) node_mgr = NodeManager( segment_info, 10, lfl_params, pre_processing_requested, [], pre_processing_nodes, aircraft_info, {}) order, _ = dependency_order(node_mgr) modules = { rel_path: [ 'derived_parameters', 'flight_phase', 'key_point_values', 'key_time_instances', 'approaches', 'multistate_parameters', 'flight_attribute' ] } if aircraft_info['Aircraft Type'] == 'helicopter': modules[rel_path / 'helicopter'] = [ 'derived_parameters', 'flight_phase', 'key_point_values', 'key_time_instances', 'multistate_parameters', ] node_modules = [ import_module(path / f'{mod}') for path, mods in modules.items() for mod in mods ] # go through modules to get derived nodes derived_nodes = get_derived_nodes(node_modules) if requested == []: # Use all derived nodes if requested is empty requested = [p for p in derived_nodes.keys() if p not in lfl_params] node_mgr= NodeManager(segment_info, 10, lfl_params + order, requested, [], derived_nodes, aircraft_info, {}) order, _ = dependency_order(node_mgr) return order
def test_dependency(self): requested = ['P7', 'P8'] mgr = NodeManager({'Start Datetime': datetime.now()}, 10, self.lfl_params, requested, [], self.derived_nodes, {}, {}) order, _ = dependency_order(mgr) self.assertEqual(order, ['P4', 'P5', 'P6', 'P7', 'P8']) """
def test_graph_requesting_all_dependencies_links_root_to_all_requests(self): # build list of all nodes as required requested = self.lfl_params + list(self.derived_nodes.keys()) mgr = NodeManager({'Start Datetime': datetime.now()}, 1, self.lfl_params, requested, [], self.derived_nodes, {}, {}) gr = graph_nodes(mgr) # should be linked to all requested nodes self.assertEqual(sorted(gr.neighbors('root')), sorted(requested))
def test_graph_requesting_all_dependencies_links_root_to_end_leafs(self): # build list of all nodes as required requested = self.lfl_params + self.derived_nodes.keys() mgr = NodeManager(datetime.now(), 1, self.lfl_params, requested, [], self.derived_nodes, {}, {}) gr = graph_nodes(mgr) # should only be linked to end leafs self.assertEqual(gr.neighbors('root'), ['P8', 'P7'])
def test_required_available(self): nodes = ['a', 'b', 'c'] required = ['a', 'c'] mgr = NodeManager(datetime.now(), 10, nodes, nodes, required, {}, {}, {}) _graph = graph_nodes(mgr) gr_all, gr_st, order = process_order(_graph, mgr) self.assertEqual(set(required) - set(order), set())
def test_invalid_requirement_raises(self): lfl_params = [] requested = ['Moment of Takeoff'] node_modules = [import_module(Path('dummy_nodes') / 'derived_parameters')] # go through modules to get derived nodes derived_nodes = get_derived_nodes(node_modules) mgr = NodeManager({'Start Datetime': datetime.now()}, 10, lfl_params, requested, [], derived_nodes, {}, {}) self.assertRaises(ValueError, dependency_order, mgr)
def _generate_json(self, lfl_params): ''' Returns list of parameters used in the spanning tree. Note: LFL parameters not used will not be returned! ''' print "Establishing Node dependencies from Analysis Engine" # Ensure file is a valid HDF file before continuing: derived_nodes = get_derived_nodes(settings.NODE_MODULES) required_params = derived_nodes.keys() # TODO: Update ac_info with keys from provided fields: ac_info = { 'Family': u'B737 NG', 'Frame': u'737-3C', 'Identifier': u'15', 'Main Gear To Lowest Point Of Tail': None, 'Main Gear To Radio Altimeter Antenna': None, 'Manufacturer Serial Number': u'39009', 'Manufacturer': u'Boeing', 'Model': u'B737-8JP', 'Precise Positioning': True, 'Series': u'B737-800', 'Tail Number': 'G-ABCD', } # TODO: Option to populate an AFR: achieved_flight_record = {} # Generate the dependency tree: node_mgr = NodeManager( {}, 1000, lfl_params, required_params, [], derived_nodes, ac_info, achieved_flight_record, ) _graph = graph_nodes(node_mgr) gr_all, gr_st, order = process_order(_graph, node_mgr) # Save the dependency tree to tree.json: tree = os.path.join(AJAX_DIR, 'tree.json') with open(tree, 'w') as fh: json.dump(graph_adjacencies(gr_st), fh, indent=4) # Save the list of nodes to node_list.json: node_list = os.path.join(AJAX_DIR, 'node_list.json') spanning_tree_params = sorted(gr_st.nodes()) with open(node_list, 'w') as fh: json.dump(spanning_tree_params, fh, indent=4) return
def test_invalid_requirement_raises(self): lfl_params = [] requested = ['Smoothed Track', 'Moment of Takeoff'] #it's called Moment Of Takeoff try: # for test cmd line runners derived = get_derived_nodes(['tests.sample_derived_parameters']) except ImportError: # for IDE test runners derived = get_derived_nodes(['sample_derived_parameters']) mgr = NodeManager(datetime.now(), 10, lfl_params, requested, [], derived, {}, {}) self.assertRaises(nx.NetworkXError, dependency_order, mgr, draw=False)
def test_dependency_with_lowlevel_dependencies_requested(self): """ Simulate requesting a Raw Parameter as a dependency. This requires the requested node to be removed when it is not at the top of the dependency tree. """ requested = ['P7', 'P8', # top level nodes 'P4', 'P5', 'P6', # middle level node 'Raw3', # bottom level node ] mgr = NodeManager({'Start Datetime': datetime.now()}, 10, self.lfl_params + ['Floating'], requested, [], self.derived_nodes, {}, {}) order, _ = dependency_order(mgr) self.assertEqual(order, ['P4', 'P5', 'P6', 'P7', 'P8'])
def test_avoiding_possible_circular_dependency(self): # Possible circular dependency which can be avoided: # Gear Selected Down depends on Gear Down which depends on Gear Selected Down...! lfl_params = ['Airspeed', 'Gear (L) Down', 'Gear (L) Red Warning'] requested = ['Airspeed At Gear Down Selected'] derived = get_derived_nodes([import_module('sample_circular_dependency_nodes')]) mgr = NodeManager({'Start Datetime': datetime.now()}, 10, lfl_params, requested, [], derived, {}, {}) order, _ = dependency_order(mgr, draw=False) # As Gear Selected Down depends upon Gear Down self.assertEqual(order, ['Gear (L) Down', 'Gear Down', 'Gear (L) Red Warning', 'Gear Down Selected', 'Airspeed', 'Airspeed At Gear Down Selected'])
def test_dependency_one_path_circular(self): '''Test that if one Node can be created using its 2 dependencies, but one of them is optional and creates a circular dependency. ,-> P1 ~~~~~~~~~> P2 ~~~~~~~~> P3--、 | `---> Raw1 `---> Raw2 | | | `----------------------------------’ P1 requires Raw1 but tries also P2. P2 requires Raw 2 but also tries P3. P3 requires P1. This makes a circular dependency. But we don't want to give up here as we could still make P1 from Raw1 only, avoiding the circular path. ''' class P1(DerivedParameterNode): @classmethod def can_operate(self, avail): return 'Raw1' in avail def derive(self, P2=P('P2'), raw=P('Raw1')): pass class P2(DerivedParameterNode): @classmethod def can_operate(self, avail): return 'Raw2' in avail def derive(self, P3=P('P3'), raw=P('Raw2')): pass class P3(DerivedParameterNode): @classmethod def can_operate(self, avail): return 'P1' in avail def derive(self, P1=P('P1')): pass derived_nodes = { 'P0' : MockParam(dependencies=['P1']), 'P1' : P1, 'P2' : P2, 'P3' : P3, } requested = ['P0'] lfl_params = ['Raw1', 'Raw2'] segment_info = {'Start Datetime': datetime.now(), 'Segment Type': 'START_AND_STOP'} mgr = NodeManager(segment_info, 10, lfl_params, requested, [], derived_nodes, {}, {}) order, _ = dependency_order(mgr) self.assertEqual(order, ['P1', 'P3', 'P2', 'P0'])
def test_indent_tree(self): requested = ['P7', 'P8'] mgr2 = NodeManager({'Start Datetime': datetime.now()}, 10, self.lfl_params, requested, [], self.derived_nodes, {}, {}) gr = graph_nodes(mgr2) gr.node['Raw1']['active'] = True gr.node['Raw2']['active'] = False gr.node['P4']['active'] = False self.assertEqual( indent_tree(gr, 'P7', space='__', delim=' ', label=False), [' P7', '__ [P4]', '____ Raw1', '____ [Raw2]', '__ P5', '____ Raw3', '____ Raw4', '__ P6', '____ Raw3', ]) # don't recurse valid parameters... self.assertEqual( indent_tree(gr, 'P5', label=False, recurse_active=False), []) self.assertEqual( indent_tree(gr, 'P4', label=False, recurse_active=False), ['- [P4]', ' - [Raw2]', ]) self.assertEqual( indent_tree(gr, 'root'), ['- root', ' - P7 (DerivedParameterNode)', ' - [P4] (DerivedParameterNode)', ' - Raw1 (HDFNode)', ' - [Raw2] (HDFNode)', ' - P5 (DerivedParameterNode)', ' - Raw3 (HDFNode)', ' - Raw4 (HDFNode)', ' - P6 (DerivedParameterNode)', ' - Raw3 (HDFNode)', ' - P8 (DerivedParameterNode)', ' - Raw5 (HDFNode)', ])
def test_dependency(self): requested = ['P7', 'P8'] mgr = NodeManager(datetime.now(), 10, self.lfl_params, requested, [], self.derived_nodes, {}, {}) gr = graph_nodes(mgr) gr_all, gr_st, order = process_order(gr, mgr) self.assertEqual(len(gr_st), 11) pos = order.index self.assertTrue(pos('P8') > pos('Raw5')) self.assertTrue(pos('P7') > pos('P4')) self.assertTrue(pos('P7') > pos('P5')) self.assertTrue(pos('P7') > pos('P6')) self.assertTrue(pos('P6') > pos('Raw3')) self.assertTrue(pos('P5') > pos('Raw3')) self.assertTrue(pos('P5') > pos('Raw4')) self.assertTrue(pos('P4') > pos('Raw1')) self.assertTrue(pos('P4') > pos('Raw2')) self.assertFalse('root' in order) #don't include the root! """
def test_avoiding_possible_circular_dependency(self): # Possible circular dependency which can be avoided: # Gear Selected Down depends on Gear Down which depends on Gear Selected Down...! lfl_params = ['Airspeed', 'Gear (L) Down', 'Gear (L) Red Warning'] requested = ['Airspeed At Gear Down Selected'] try: # for test cmd line runners derived = get_derived_nodes( ['tests.sample_circular_dependency_nodes']) except ImportError: # for IDE test runners derived = get_derived_nodes(['sample_circular_dependency_nodes']) mgr = NodeManager(datetime.now(), 10, lfl_params, requested, [], derived, {}, {}) order, _ = dependency_order(mgr, draw=True) # As Gear Selected Down depends upon Gear Down self.assertEqual(order, [ 'Gear (L) Down', 'Gear Down', 'Gear (L) Red Warning', 'Gear Down Selected', 'Airspeed', 'Airspeed At Gear Down Selected' ])
def prep_order(flight, frame_dict, start_datetime, derived_nodes, required_params): ''' open example HDF to see recorded params and build process order''' derived_nodes_copy = derived_nodes #copy.deepcopy(derived_nodes) # precomputed_parameters = get_precomputed_parameters(flight) precomputed_keys = precomputed_parameters.keys() for k in flight.series.keys(): if k not in precomputed_keys: precomputed_keys.append(k) node_mgr = NodeManager( start_datetime, flight.duration, precomputed_keys, #flight.series.keys(), #from HDF. was hdf.valid_param_names(), #hdf_keys; should be from LFL required_params, #requested derived_nodes_copy, #methods that can be computed; equals profile + base nodes ???? flight.aircraft_info, achieved_flight_record=flight.achieved_flight_record) # calculate dependency tree process_order, gr_st = dependency_order(node_mgr, draw=False) logger.warning('process order: ' + str(process_order[:5]) + '...') #, gr_st return node_mgr, process_order # a list of node names
def derived_trimmer(hdf_path, node_names, dest): ''' Trims an HDF file of parameters which are not dependencies of nodes in node_names. :param hdf_path: file path of hdf file. :type hdf_path: str :param node_names: A list of Node names which are required. :type node_names: list of str :param dest: destination path for trimmed output file :type dest: str :return: parameters in stripped hdf file :rtype: [str] ''' with hdf_file(hdf_path) as hdf: duration = hdf.duration raw_nodes = hdf.valid_param_names() derived_nodes = get_derived_nodes(settings.NODE_MODULES) node_mgr = NodeManager({}, duration, raw_nodes, derived_nodes.keys(), [], derived_nodes, {}, {}) order, graph = dependency_order(node_mgr) node_names = set(node_names) queue = deque(i for i in order if i in node_names) params = {i for i in node_mgr.hdf_keys if i in node_names} visited = set() while queue: current = queue.popleft() visited.add(current) for name in graph[current]: if graph.nodes[name]['node_type'] == 'HDFNode': params.add(name) elif name not in visited: queue.append(name) return strip_hdf(hdf_path, params, dest)
def test_required_unavailable(self): nodes = ['a', 'b', 'c'] required = ['a', 'c', 'd'] mgr = NodeManager({'Start Datetime': datetime.now()}, 10, nodes, nodes, required, {}, {}, {}) self.assertRaises(ValueError, graph_nodes, mgr)
def process_flight(hdf_path, tail_number, aircraft_info={}, start_datetime=None, achieved_flight_record={}, requested=[], required=[], include_flight_attributes=True, additional_modules=[]): ''' Processes the HDF file (hdf_path) to derive the required_params (Nodes) within python modules (settings.NODE_MODULES). Note: For Flight Data Services, the definitive API is located here: "PolarisTaskManagement.test.tasks_mask.process_flight" :param hdf_path: Path to HDF File :type hdf_path: String :param aircraft: Aircraft specific attributes :type aircraft: dict :param start_datetime: Datetime of the origin of the data (at index 0) :type start_datetime: Datetime :param achieved_flight_record: See API Below :type achieved_flight_record: Dict :param requested: Derived nodes to process (dependencies will also be evaluated). :type requested: List of Strings :param required: Nodes which are required, otherwise an exception will be raised. :type required: List of Strings :param include_flight_attributes: Whether to include all flight attributes :type include_flight_attributes: Boolean :param additional_modules: List of module paths to import. :type additional_modules: List of Strings :returns: See below: :rtype: Dict Sample aircraft_info -------------------- { 'Tail Number': # Aircraft Registration 'Identifier': # Aircraft Ident 'Manufacturer': # e.g. Boeing 'Manufacturer Serial Number': #MSN 'Model': # e.g. 737-808-ER 'Series': # e.g. 737-800 'Family': # e.g. 737 'Frame': # e.g. 737-3C 'Main Gear To Altitude Radio': # Distance in metres 'Wing Span': # Distance in metres } Sample achieved_flight_record ----------------------------- { # Simple values first, e.g. string, int, float, etc. 'AFR Flight ID': # e.g. 1 'AFR Flight Number': # e.g. 1234 'AFR Type': # 'POSITIONING' 'AFR Off Blocks Datetime': # datetime(2015,01,01,13,00) 'AFR Takeoff Datetime': # datetime(2015,01,01,13,15) 'AFR Takeoff Pilot': # 'Joe Bloggs' 'AFR Takeoff Gross Weight': # weight in kg 'AFR Takeoff Fuel': # fuel in kg 'AFR Landing Datetime': # datetime(2015,01,01,18,45) 'AFR Landing Pilot': # 'Joe Bloggs' 'AFR Landing Gross Weight': # weight in kg 'AFR Landing Fuel': # weight in kg 'AFR On Blocks Datetime': # datetime(2015,01,01,19,00) 'AFR V2': # V2 used at takeoff in kts 'AFR Vapp': # Vapp used in kts 'AFR Vref': # Vref used in kts # More complex data that needs to be looked up next: 'AFR Takeoff Airport': { 'id': 4904, # unique id 'name': 'Athens Intl Airport Elefterios Venizel', 'code': {'iata': 'ATH', 'icao': 'LGAV'}, 'latitude': 37.9364, 'longitude': 23.9445, 'location': {'city': u'Athens', 'country': u'Greece'}, 'elevation': 266, # ft 'magnetic_variation': 'E003186 0106', } }, 'AFR Landing Aiport': { 'id': 1, # unique id 'name': 'Athens Intl Airport Elefterios Venizel', 'code': {'iata': 'ATH', 'icao': 'LGAV'}, 'latitude': 37.9364, 'longitude': 23.9445, 'location': {'city': u'Athens', 'country': u'Greece'}, 'elevation': 266, # ft 'magnetic_variation': 'E003186 0106', } }, 'AFR Destination Airport': None, # if not required, or exclude this key 'AFR Takeoff Runway': { 'id': 1, 'identifier': '21L', 'magnetic_heading': 212.6, 'strip': { 'id': 1, 'length': 13123, 'surface': 'ASP', 'width': 147}, 'start': { 'elevation': 308, 'latitude': 37.952425, 'longitude': 23.970422}, 'end': { 'elevation': 279, 'latitude': 37.923511, 'longitude': 23.943261}, 'glideslope': { 'angle': 3.0, 'elevation': 282, 'latitude': 37.9473, 'longitude': 23.9676, 'threshold_distance': 999}, 'localizer': { 'beam_width': 4.5, 'elevation': 256, 'frequency': 111100, 'heading': 213, 'latitude': 37.919281, 'longitude': 23.939294}, }, 'AFR Landing Runway': { 'id': 1, 'identifier': '21L', 'magnetic_heading': 212.6, 'strip': { 'id': 1, 'length': 13123, 'surface': 'ASP', 'width': 147}, 'start': { 'elevation': 308, 'latitude': 37.952425, 'longitude': 23.970422}, 'end': { 'elevation': 279, 'latitude': 37.923511, 'longitude': 23.943261}, 'glideslope': { 'angle': 3.0, 'elevation': 282, 'latitude': 37.9473, 'longitude': 23.9676, 'threshold_distance': 999}, 'localizer': { 'beam_width': 4.5, 'elevation': 256, 'frequency': 111100, 'heading': 213, 'latitude': 37.919281, 'longitude': 23.939294}, }, } Sample Return ------------- { 'flight':[Attribute('name value')], 'kti':[GeoKeyTimeInstance('index name latitude longitude')] if lat/long available else [KeyTimeInstance('index name')], 'kpv':[KeyPointValue('index value name slice')] } sample flight Attributes: [ Attribute('Takeoff Airport', {'id':1234, 'name':'Int. Airport'}, Attribute('Approaches', [4567,7890]), ... ], ''' if start_datetime is None: import pytz start_datetime = datetime.utcnow().replace(tzinfo=pytz.utc) logger.info("Processing: %s", hdf_path) if aircraft_info: # Aircraft info has already been provided. logger.info( "Using aircraft_info dictionary passed into process_flight '%s'." % aircraft_info) else: aircraft_info = get_aircraft_info(tail_number) aircraft_info['Tail Number'] = tail_number # go through modules to get derived nodes node_modules = additional_modules + settings.NODE_MODULES derived_nodes = get_derived_nodes(node_modules) if requested: requested = \ list(set(requested).intersection(set(derived_nodes))) else: # if requested isn't set, try using ALL derived_nodes! logger.info("No requested nodes declared, using all derived nodes") requested = derived_nodes.keys() # include all flight attributes as requested if include_flight_attributes: requested = list( set(requested + get_derived_nodes(['analysis_engine.flight_attribute']).keys()) ) # open HDF for reading with hdf_file(hdf_path) as hdf: hdf.start_datetime = start_datetime if hooks.PRE_FLIGHT_ANALYSIS: logger.info("Performing PRE_FLIGHT_ANALYSIS actions: %s", hooks.PRE_FLIGHT_ANALYSIS.func_name) hooks.PRE_FLIGHT_ANALYSIS(hdf, aircraft_info) else: logger.info("No PRE_FLIGHT_ANALYSIS actions to perform") # Track nodes. Assume that all params in HDF are from LFL(!) node_mgr = NodeManager(start_datetime, hdf.duration, hdf.valid_param_names(), requested, required, derived_nodes, aircraft_info, achieved_flight_record) # calculate dependency tree process_order, gr_st = dependency_order(node_mgr, draw=False) if settings.CACHE_PARAMETER_MIN_USAGE: # find params used more than for node in gr_st.nodes(): if node in node_mgr.derived_nodes: # this includes KPV/KTIs but they'll be ignored by HDF qty = len(gr_st.predecessors(node)) if qty > settings.CACHE_PARAMETER_MIN_USAGE: hdf.cache_param_list.append(node) logging.info("HDF set to cache parameters: %s", hdf.cache_param_list) # derive parameters kti_list, kpv_list, section_list, approach_list, flight_attrs = \ derive_parameters(hdf, node_mgr, process_order) # geo locate KTIs kti_list = geo_locate(hdf, kti_list) kti_list = _timestamp(start_datetime, kti_list) # geo locate KPVs kpv_list = geo_locate(hdf, kpv_list) kpv_list = _timestamp(start_datetime, kpv_list) # Store version of FlightDataAnalyser hdf.analysis_version = __version__ # Store dependency tree hdf.dependency_tree = json_graph.dumps(gr_st) # Store aircraft info hdf.set_attr('aircraft_info', aircraft_info) hdf.set_attr('achieved_flight_record', achieved_flight_record) return { 'flight': flight_attrs, 'kti': kti_list, 'kpv': kpv_list, 'approach': approach_list, 'phases': section_list, }
def process_flight(segment_info, tail_number, aircraft_info={}, achieved_flight_record={}, requested=[], required=[], include_flight_attributes=True, additional_modules=[], pre_flight_kwargs={}, force=False, initial={}, reprocess=False): ''' Processes the HDF file (segment_info['File']) to derive the required_params (Nodes) within python modules (settings.NODE_MODULES). Note: For Flight Data Services, the definitive API is located here: "PolarisTaskManagement.test.tasks_mask.process_flight" :param segment_info: Details of the segment to process :type segment_info: dict :param aircraft: Aircraft specific attributes :type aircraft: dict :param achieved_flight_record: See API Below :type achieved_flight_record: Dict :param requested: Derived nodes to process (dependencies will also be evaluated). :type requested: List of Strings :param required: Nodes which are required, otherwise an exception will be raised. :type required: List of Strings :param include_flight_attributes: Whether to include all flight attributes :type include_flight_attributes: Boolean :param additional_modules: List of module paths to import. :type additional_modules: List of Strings :param pre_flight_kwargs: Keyword arguments for the pre-flight analysis hook. :type pre_flight_kwargs: dict :param force: Ignore errors raised while deriving nodes. :type force: bool :param initial: Initial content for nodes to avoid reprocessing (excluding parameter nodes which are saved to the hdf). :type initial: dict :param reprocess: Force reprocessing of all Nodes (including derived Nodes already saved to the HDF file). :returns: See below: :rtype: Dict Sample segment_info -------------------- { 'File': # Path to HDF5 file to process 'Start Datetime': # Datetime of the origin of the data (at index 0) 'Segment Type': # segment type obtained from split segments e.g. START_AND_STOP } Sample aircraft_info -------------------- { 'Tail Number': # Aircraft Registration 'Identifier': # Aircraft Ident 'Manufacturer': # e.g. Boeing 'Manufacturer Serial Number': #MSN 'Model': # e.g. 737-808-ER 'Series': # e.g. 737-800 'Family': # e.g. 737 'Frame': # e.g. 737-3C 'Main Gear To Altitude Radio': # Distance in metres 'Wing Span': # Distance in metres } Sample achieved_flight_record ----------------------------- { # Simple values first, e.g. string, int, float, etc. 'AFR Flight ID': # e.g. 1 'AFR Flight Number': # e.g. 1234 'AFR Type': # 'POSITIONING' 'AFR Off Blocks Datetime': # datetime(2015,1,1,13,00) 'AFR Takeoff Datetime': # datetime(2015,1,1,13,15) 'AFR Takeoff Pilot': # 'Joe Bloggs' 'AFR Takeoff Gross Weight': # weight in kg 'AFR Takeoff Fuel': # fuel in kg 'AFR Landing Datetime': # datetime(2015,1,1,18,45) 'AFR Landing Pilot': # 'Joe Bloggs' 'AFR Landing Gross Weight': # weight in kg 'AFR Landing Fuel': # weight in kg 'AFR On Blocks Datetime': # datetime(2015,1,1,19,00) 'AFR V2': # V2 used at takeoff in kts 'AFR Vapp': # Vapp used in kts 'AFR Vref': # Vref used in kts # More complex data that needs to be looked up next: 'AFR Takeoff Airport': { 'id': 4904, # unique id 'name': 'Athens Intl Airport Elefterios Venizel', 'code': {'iata': 'ATH', 'icao': 'LGAV'}, 'latitude': 37.9364, 'longitude': 23.9445, 'location': {'city': u'Athens', 'country': u'Greece'}, 'elevation': 266, # ft 'magnetic_variation': 'E003186 0106', } }, 'AFR Landing Aiport': { 'id': 1, # unique id 'name': 'Athens Intl Airport Elefterios Venizel', 'code': {'iata': 'ATH', 'icao': 'LGAV'}, 'latitude': 37.9364, 'longitude': 23.9445, 'location': {'city': u'Athens', 'country': u'Greece'}, 'elevation': 266, # ft 'magnetic_variation': 'E003186 0106', } }, 'AFR Destination Airport': None, # if not required, or exclude this key 'AFR Takeoff Runway': { 'id': 1, 'identifier': '21L', 'magnetic_heading': 212.6, 'strip': { 'id': 1, 'length': 13123, 'surface': 'ASP', 'width': 147}, 'start': { 'elevation': 308, 'latitude': 37.952425, 'longitude': 23.970422}, 'end': { 'elevation': 279, 'latitude': 37.923511, 'longitude': 23.943261}, 'glideslope': { 'angle': 3.0, 'elevation': 282, 'latitude': 37.9473, 'longitude': 23.9676, 'threshold_distance': 999}, 'localizer': { 'beam_width': 4.5, 'elevation': 256, 'frequency': 111100, 'heading': 213, 'latitude': 37.919281, 'longitude': 23.939294}, }, 'AFR Landing Runway': { 'id': 1, 'identifier': '21L', 'magnetic_heading': 212.6, 'strip': { 'id': 1, 'length': 13123, 'surface': 'ASP', 'width': 147}, 'start': { 'elevation': 308, 'latitude': 37.952425, 'longitude': 23.970422}, 'end': { 'elevation': 279, 'latitude': 37.923511, 'longitude': 23.943261}, 'glideslope': { 'angle': 3.0, 'elevation': 282, 'latitude': 37.9473, 'longitude': 23.9676, 'threshold_distance': 999}, 'localizer': { 'beam_width': 4.5, 'elevation': 256, 'frequency': 111100, 'heading': 213, 'latitude': 37.919281, 'longitude': 23.939294}, }, } Sample Return ------------- { 'flight':[Attribute('name value')], 'kti':[GeoKeyTimeInstance('index name latitude longitude')] if lat/long available else [KeyTimeInstance('index name')], 'kpv':[KeyPointValue('index value name slice')] } sample flight Attributes: [ Attribute('Takeoff Airport', {'id':1234, 'name':'Int. Airport'}, Attribute('Approaches', [4567,7890]), ... ], ''' hdf_path = segment_info['File'] if 'Start Datetime' not in segment_info: import pytz segment_info['Start Datetime'] = datetime.utcnow().replace( tzinfo=pytz.utc) logger.debug("Processing: %s", hdf_path) if aircraft_info: # Aircraft info has already been provided. logger.info( "Using aircraft_info dictionary passed into process_flight '%s'." % aircraft_info) else: aircraft_info = get_aircraft_info(tail_number) aircraft_info['Tail Number'] = tail_number if aircraft_info['Aircraft Type'] == 'helicopter': node_modules = settings.NODE_MODULES + \ settings.NODE_HELICOPTER_MODULE_PATHS + additional_modules else: node_modules = settings.NODE_MODULES + additional_modules # go through modules to get derived nodes derived_nodes = get_derived_nodes(node_modules) if requested: requested = \ list(set(requested).intersection(set(derived_nodes))) else: # if requested isn't set, try using ALL derived_nodes! logger.debug("No requested nodes declared, using all derived nodes") # Enforce some sort of order in which the dependencies are traversed requested = sorted(list(derived_nodes.keys())) # include all flight attributes as requested if include_flight_attributes: requested = list( set(requested + list( get_derived_nodes(['analysis_engine.flight_attribute']).keys()) )) initial = process_flight_to_nodes(initial) for node_name in requested: initial.pop(node_name, None) # open HDF for reading with hdf_file(hdf_path) as hdf: hdf.start_datetime = segment_info['Start Datetime'] hook = hooks.PRE_FLIGHT_ANALYSIS if hook: logger.info( "Performing PRE_FLIGHT_ANALYSIS action '%s' with options: %s", getattr(hook, 'func_name', getattr(hook, '__name__')), pre_flight_kwargs) hook(hdf, aircraft_info, **pre_flight_kwargs) else: logger.info("No PRE_FLIGHT_ANALYSIS actions to perform") # Merge Params param_names = hdf.valid_lfl_param_names( ) if reprocess else hdf.valid_param_names() pre_process_parameters(hdf, segment_info, param_names, required, aircraft_info, achieved_flight_record, force=force) # Track nodes. node_mgr = NodeManager(segment_info, hdf.duration, param_names, requested, required, derived_nodes, aircraft_info, achieved_flight_record) # calculate dependency tree process_order, gr_st = dependency_order(node_mgr, draw=False) if settings.CACHE_PARAMETER_MIN_USAGE: # find params used more than for node in gr_st.nodes(): if node in node_mgr.derived_nodes: # this includes KPV/KTIs but they'll be ignored by HDF qty = len(gr_st.predecessors(node)) if qty > settings.CACHE_PARAMETER_MIN_USAGE: hdf.cache_param_list.append(node) logging.info("HDF set to cache parameters: %s", hdf.cache_param_list) # derive parameters ktis, kpvs, sections, approaches, flight_attrs = \ derive_parameters(hdf, node_mgr, process_order, params=initial, force=force) # geo locate KTIs ktis = geo_locate(hdf, ktis) ktis = _timestamp(segment_info['Start Datetime'], ktis) # geo locate KPVs kpvs = geo_locate(hdf, kpvs) kpvs = _timestamp(segment_info['Start Datetime'], kpvs) # Store version of FlightDataAnalyser hdf.analysis_version = __version__ # Store dependency tree hdf.dependency_tree = json.dumps(json_graph.node_link_data(gr_st)) # Store aircraft info hdf.set_attr('aircraft_info', aircraft_info) hdf.set_attr('achieved_flight_record', achieved_flight_record) return { 'flight': flight_attrs, 'kti': ktis, 'kpv': kpvs, 'approach': approaches, 'phases': sections, }
def test_required_unavailable(self): nodes = ['a', 'b', 'c'] required = ['a', 'c', 'd'] mgr = NodeManager(datetime.now(), 10, nodes, nodes, required, {}, {}, {}) gr = self.assertRaises(graph_nodes(mgr))