def test_sample_parameter_module(self): """Tests many options: can_operate on SmoothedTrack works with """ requested = ['Smoothed Track', 'Moment Of Takeoff', 'Vertical Speed', 'Slip On Runway'] lfl_params = ['Indicated Airspeed', 'Groundspeed', 'Pressure Altitude', 'Heading', 'TAT', 'Latitude', 'Longitude', 'Longitudinal g', 'Lateral g', 'Normal g', 'Pitch', 'Roll', ] try: # for test cmd line runners derived = get_derived_nodes(['tests.sample_derived_parameters']) except ImportError: # for IDE test runners derived = get_derived_nodes(['sample_derived_parameters']) nodes = NodeManager(datetime.now(), 10, lfl_params, requested, [], derived, {}, {}) order, _ = dependency_order(nodes, draw=False) pos = order.index self.assertTrue(len(order)) self.assertNotIn('Moment Of Takeoff', order) # not available self.assertTrue(pos('Vertical Speed') > pos('Pressure Altitude')) self.assertTrue(pos('Slip On Runway') > pos('Groundspeed')) self.assertTrue(pos('Slip On Runway') > pos('Horizontal g Across Track')) self.assertTrue(pos('Horizontal g Across Track') > pos('Roll')) self.assertFalse('Mach' in order) # Mach wasn't requested! self.assertFalse('Radio Altimeter' in order) self.assertEqual(len(nodes.hdf_keys), 12) self.assertEqual(len(nodes.requested), 4) self.assertEqual(len(nodes.derived_nodes), 13)
def test_avoiding_possible_circular_dependency(self): # Possible circular dependency which can be avoided: # Gear Selected Down depends on Gear Down which depends on Gear Selected Down...! lfl_params = ["Airspeed", "Gear (L) Down", "Gear (L) Red Warning"] requested = ["Airspeed At Gear Down Selected"] try: # for test cmd line runners derived = get_derived_nodes(["tests.sample_circular_dependency_nodes"]) except ImportError: # for IDE test runners derived = get_derived_nodes(["sample_circular_dependency_nodes"]) mgr = NodeManager({"Start Datetime": datetime.now()}, 10, lfl_params, requested, [], derived, {}, {}) order, _ = dependency_order(mgr, draw=True) # As Gear Selected Down depends upon Gear Down self.assertEqual( order, [ "Gear (L) Down", "Gear Down", "Gear (L) Red Warning", "Gear Down Selected", "Airspeed", "Airspeed At Gear Down Selected", ], )
def test_invalid_requirement_raises(self): lfl_params = [] requested = ["Smoothed Track", "Moment of Takeoff"] # it's called Moment Of Takeoff try: # for test cmd line runners derived = get_derived_nodes(["tests.sample_derived_parameters"]) except ImportError: # for IDE test runners derived = get_derived_nodes(["sample_derived_parameters"]) mgr = NodeManager({"Start Datetime": datetime.now()}, 10, lfl_params, requested, [], derived, {}, {}) self.assertRaises(nx.NetworkXError, dependency_order, mgr, draw=False)
def test_invalid_requirement_raises(self): lfl_params = [] requested = ['Smoothed Track', 'Moment of Takeoff'] #it's called Moment Of Takeoff try: # for test cmd line runners derived = get_derived_nodes(['tests.sample_derived_parameters']) except ImportError: # for IDE test runners derived = get_derived_nodes(['sample_derived_parameters']) mgr = NodeManager(datetime.now(), 10, lfl_params, requested, [], derived, {}, {}) self.assertRaises(nx.NetworkXError, dependency_order, mgr, draw=False)
def test_invalid_requirement_raises(self): lfl_params = [] requested = ['Smoothed Track', 'Moment of Takeoff'] #it's called Moment Of Takeoff derived = get_derived_nodes([import_module('sample_derived_parameters')]) mgr = NodeManager({'Start Datetime': datetime.now()}, 10, lfl_params, requested, [], derived, {}, {}) self.assertRaises(nx.NetworkXError, dependency_order, mgr, draw=False)
def parse_analyser_profiles(analyser_profiles, filter_modules=None): ''' Parse analyser profiles into additional_modules and required nodes as expected by process_flight. :param analyser_profiles: A list of analyser profile tuples containing semicolon separated module paths and whether or not the nodes are required e.g. [('package.module_one;package.module_two', True), ] :type analyser_profiles: [[str, bool], ] :param filter_paths: Optional list of analyser profiles to keep. :type filter_paths: [str] or None :returns: A list of additional module paths and a list of required node names. :rtype: [str], [str] ''' additional_modules = [] required_nodes = [] for import_paths, is_required in analyser_profiles: for import_path in import_paths.split(';'): if filter_modules is not None and import_path not in filter_modules: continue import_path = import_path.strip() if not import_path: continue additional_modules.append(import_path) if is_required: required_nodes.extend(get_derived_nodes([import_path])) return additional_modules, required_nodes
def test_sample_parameter_module(self): """Tests many options: can_operate on SmoothedTrack works with """ requested = ['Smoothed Track', 'Moment Of Takeoff', 'Vertical Speed', 'Slip On Runway'] lfl_params = ['Indicated Airspeed', 'Groundspeed', 'Pressure Altitude', 'Heading', 'TAT', 'Latitude', 'Longitude', 'Longitudinal g', 'Lateral g', 'Normal g', 'Pitch', 'Roll', ] derived = get_derived_nodes([import_module('sample_derived_parameters')]) nodes = NodeManager({'Start Datetime': datetime.now()}, 10, lfl_params, requested, [], derived, {}, {}) order, _ = dependency_order(nodes, draw=False) pos = order.index self.assertTrue(len(order)) self.assertNotIn('Moment Of Takeoff', order) # not available self.assertTrue(pos('Vertical Speed') > pos('Pressure Altitude')) self.assertTrue(pos('Slip On Runway') > pos('Groundspeed')) self.assertTrue(pos('Slip On Runway') > pos('Horizontal g Across Track')) self.assertTrue(pos('Horizontal g Across Track') > pos('Roll')) self.assertFalse('Mach' in order) # Mach wasn't requested! self.assertFalse('Radio Altimeter' in order) self.assertEqual(len(nodes.hdf_keys), 12) self.assertEqual(len(nodes.requested), 4) self.assertEqual(len(nodes.derived_nodes), 13)
def _get_dependency_order(self, requested, aircraft_info, lfl_params, draw=False, raise_cir_dep=True, segment_info={}): #derived_nodes = get_derived_nodes(settings.NODE_MODULES) if aircraft_info['Aircraft Type'] == 'helicopter': node_modules = settings.NODE_MODULES + settings.NODE_HELICOPTER_MODULE_PATHS else: node_modules = settings.NODE_MODULES # go through modules to get derived nodes derived_nodes = get_derived_nodes(node_modules) if requested == []: # Use all derived nodes if requested is empty requested = [ p for p in derived_nodes.keys() if p not in lfl_params ] if not segment_info: segment_info = { 'Start Datetime': datetime.now(), 'Segment Type': 'START_AND_STOP', } node_mgr = NodeManager(segment_info, 10, lfl_params, requested, [], derived_nodes, aircraft_info, {}) order, gr_st = dependency_order(node_mgr, draw=draw, raise_cir_dep=raise_cir_dep) return order, gr_st
def pre_process_parameters(hdf, segment_info, param_names, required, aircraft_info, achieved_flight_record, force=False): ''' Perform actions prior to main processing run. Actions such as merging parameters up front simplify processing paths by removing circular dependacies. ''' pre_processing_nodes = get_derived_nodes( settings.PRE_PROCESSING_MODULE_PATHS) requested = list(pre_processing_nodes.keys()) node_mgr = NodeManager(segment_info, hdf.duration, param_names, requested, required, pre_processing_nodes, aircraft_info, achieved_flight_record) process_order, gr_st = dependency_order(node_mgr, draw=False) ktis, kpvs, sections, approaches, flight_attrs = \ derive_parameters(hdf, node_mgr, process_order, force=force)
def process_flight_to_nodes(pf_results): ''' Load process flight results into Node objects. ''' from analysis_engine import node derived_nodes = get_derived_nodes(settings.NODE_MODULES) params = {} for node_type, nodes in pf_results.iteritems(): for node_name, items in nodes.iteritems(): try: node_cls = derived_nodes[node_name] except KeyError: #logger.warning('Derived node not found in code base: %s', node_name) continue if node_type == 'flight' and items: flight_attr = node.FlightAttributeNode(node_name) flight_attr.set_flight_attr(items[0].value) params[node_name] = flight_attr continue params[node_name] = node_cls(node_name, items=items) return params
def _get_dependency_order(self, requested, aircraft_info, lfl_params, segment_info={}): if not segment_info: segment_info = { 'Start Datetime': datetime.now(), 'Segment Type': 'START_AND_STOP', } rel_path = Path('dummy_nodes') pre_processing_modules = ['merge_multistate_parameters', 'merge_parameters'] node_modules = [ import_module(rel_path / 'pre_processing' / f'{mod}') for mod in pre_processing_modules ] pre_processing_nodes = get_derived_nodes(node_modules) pre_processing_requested = list(pre_processing_nodes.keys()) node_mgr = NodeManager( segment_info, 10, lfl_params, pre_processing_requested, [], pre_processing_nodes, aircraft_info, {}) order, _ = dependency_order(node_mgr) modules = { rel_path: [ 'derived_parameters', 'flight_phase', 'key_point_values', 'key_time_instances', 'approaches', 'multistate_parameters', 'flight_attribute' ] } if aircraft_info['Aircraft Type'] == 'helicopter': modules[rel_path / 'helicopter'] = [ 'derived_parameters', 'flight_phase', 'key_point_values', 'key_time_instances', 'multistate_parameters', ] node_modules = [ import_module(path / f'{mod}') for path, mods in modules.items() for mod in mods ] # go through modules to get derived nodes derived_nodes = get_derived_nodes(node_modules) if requested == []: # Use all derived nodes if requested is empty requested = [p for p in derived_nodes.keys() if p not in lfl_params] node_mgr= NodeManager(segment_info, 10, lfl_params + order, requested, [], derived_nodes, aircraft_info, {}) order, _ = dependency_order(node_mgr) return order
def test_invalid_requirement_raises(self): lfl_params = [] requested = ['Moment of Takeoff'] node_modules = [import_module(Path('dummy_nodes') / 'derived_parameters')] # go through modules to get derived nodes derived_nodes = get_derived_nodes(node_modules) mgr = NodeManager({'Start Datetime': datetime.now()}, 10, lfl_params, requested, [], derived_nodes, {}, {}) self.assertRaises(ValueError, dependency_order, mgr)
def test_avoiding_possible_circular_dependency(self): # Possible circular dependency which can be avoided: # Gear Selected Down depends on Gear Down which depends on Gear Selected Down...! lfl_params = ['Airspeed', 'Gear (L) Down', 'Gear (L) Red Warning'] requested = ['Airspeed At Gear Down Selected'] try: # for test cmd line runners derived = get_derived_nodes( ['tests.sample_circular_dependency_nodes']) except ImportError: # for IDE test runners derived = get_derived_nodes(['sample_circular_dependency_nodes']) mgr = NodeManager(datetime.now(), 10, lfl_params, requested, [], derived, {}, {}) order, _ = dependency_order(mgr, draw=True) # As Gear Selected Down depends upon Gear Down self.assertEqual(order, [ 'Gear (L) Down', 'Gear Down', 'Gear (L) Red Warning', 'Gear Down Selected', 'Airspeed', 'Airspeed At Gear Down Selected' ])
def test_sample_parameter_module(self): """Tests many options: can_operate on SmoothedTrack works with """ requested = ["Smoothed Track", "Moment Of Takeoff", "Vertical Speed", "Slip On Runway"] lfl_params = [ "Indicated Airspeed", "Groundspeed", "Pressure Altitude", "Heading", "TAT", "Latitude", "Longitude", "Longitudinal g", "Lateral g", "Normal g", "Pitch", "Roll", ] try: # for test cmd line runners derived = get_derived_nodes(["tests.sample_derived_parameters"]) except ImportError: # for IDE test runners derived = get_derived_nodes(["sample_derived_parameters"]) nodes = NodeManager({"Start Datetime": datetime.now()}, 10, lfl_params, requested, [], derived, {}, {}) order, _ = dependency_order(nodes, draw=False) pos = order.index self.assertTrue(len(order)) self.assertNotIn("Moment Of Takeoff", order) # not available self.assertTrue(pos("Vertical Speed") > pos("Pressure Altitude")) self.assertTrue(pos("Slip On Runway") > pos("Groundspeed")) self.assertTrue(pos("Slip On Runway") > pos("Horizontal g Across Track")) self.assertTrue(pos("Horizontal g Across Track") > pos("Roll")) self.assertFalse("Mach" in order) # Mach wasn't requested! self.assertFalse("Radio Altimeter" in order) self.assertEqual(len(nodes.hdf_keys), 12) self.assertEqual(len(nodes.requested), 4) self.assertEqual(len(nodes.derived_nodes), 13)
def test_avoiding_possible_circular_dependency(self): # Possible circular dependency which can be avoided: # Gear Selected Down depends on Gear Down which depends on Gear Selected Down...! lfl_params = ['Airspeed', 'Gear (L) Down', 'Gear (L) Red Warning'] requested = ['Airspeed At Gear Down Selected'] derived = get_derived_nodes([import_module('sample_circular_dependency_nodes')]) mgr = NodeManager({'Start Datetime': datetime.now()}, 10, lfl_params, requested, [], derived, {}, {}) order, _ = dependency_order(mgr, draw=False) # As Gear Selected Down depends upon Gear Down self.assertEqual(order, ['Gear (L) Down', 'Gear Down', 'Gear (L) Red Warning', 'Gear Down Selected', 'Airspeed', 'Airspeed At Gear Down Selected'])
def test_avoiding_possible_circular_dependency(self): # Possible circular dependency which can be avoided: # Gear Selected Down depends on Gear Down which depends on Gear Selected Down...! lfl_params = ['Airspeed', 'Gear (L) Down', 'Gear (L) Red Warning'] requested = ['Airspeed At Gear Down Selected'] derived = get_derived_nodes([import_module('sample_circular_dependency_nodes')]) mgr = NodeManager({'Start Datetime': datetime.now()}, 10, lfl_params, requested, [], derived, {}, {}) order, _ = dependency_order(mgr, draw=True) # As Gear Selected Down depends upon Gear Down self.assertEqual(order, ['Gear (L) Down', 'Gear Down', 'Gear (L) Red Warning', 'Gear Down Selected', 'Airspeed', 'Airspeed At Gear Down Selected'])
def pre_process_parameters(hdf, segment_info, param_names, required, aircraft_info, achieved_flight_record, force=False, dependency_tree_log=None): ''' Perform actions prior to main processing run. Actions such as merging parameters up front simplify processing paths by removing circular dependacies. ''' pre_processing_nodes = get_derived_nodes(settings.PRE_PROCESSING_MODULE_PATHS) requested = list(pre_processing_nodes.keys()) node_mgr = NodeManager( segment_info, hdf.duration, param_names, requested, required, pre_processing_nodes, aircraft_info, achieved_flight_record) process_order, gr_st = dependency_order(node_mgr, draw=False, dependency_tree_log=dependency_tree_log) ktis, kpvs, sections, approaches, flight_attrs = \ derive_parameters(hdf, node_mgr, process_order, force=force)
def _get_dependency_order(self, requested, aircraft_info, lfl_params, draw=False, raise_cir_dep=True, segment_info={}): #derived_nodes = get_derived_nodes(settings.NODE_MODULES) if aircraft_info['Aircraft Type'] == 'helicopter': node_modules = settings.NODE_MODULES + settings.NODE_HELICOPTER_MODULE_PATHS else: node_modules = settings.NODE_MODULES # go through modules to get derived nodes derived_nodes = get_derived_nodes(node_modules) if requested == []: # Use all derived nodes if requested is empty requested = [p for p in derived_nodes.keys() if p not in lfl_params] if not segment_info: segment_info = { 'Start Datetime': datetime.now(), 'Segment Type': 'START_AND_STOP', } node_mgr= NodeManager(segment_info, 10, lfl_params, requested, [], derived_nodes, aircraft_info, {}) order, gr_st = dependency_order(node_mgr, draw=draw, raise_cir_dep=raise_cir_dep) return order, gr_st
def process_flight(hdf_path, tail_number, aircraft_info={}, start_datetime=None, achieved_flight_record={}, requested=[], required=[], include_flight_attributes=True, additional_modules=[]): ''' Processes the HDF file (hdf_path) to derive the required_params (Nodes) within python modules (settings.NODE_MODULES). Note: For Flight Data Services, the definitive API is located here: "PolarisTaskManagement.test.tasks_mask.process_flight" :param hdf_path: Path to HDF File :type hdf_path: String :param aircraft: Aircraft specific attributes :type aircraft: dict :param start_datetime: Datetime of the origin of the data (at index 0) :type start_datetime: Datetime :param achieved_flight_record: See API Below :type achieved_flight_record: Dict :param requested: Derived nodes to process (dependencies will also be evaluated). :type requested: List of Strings :param required: Nodes which are required, otherwise an exception will be raised. :type required: List of Strings :param include_flight_attributes: Whether to include all flight attributes :type include_flight_attributes: Boolean :param additional_modules: List of module paths to import. :type additional_modules: List of Strings :returns: See below: :rtype: Dict Sample aircraft_info -------------------- { 'Tail Number': # Aircraft Registration 'Identifier': # Aircraft Ident 'Manufacturer': # e.g. Boeing 'Manufacturer Serial Number': #MSN 'Model': # e.g. 737-808-ER 'Series': # e.g. 737-800 'Family': # e.g. 737 'Frame': # e.g. 737-3C 'Main Gear To Altitude Radio': # Distance in metres 'Wing Span': # Distance in metres } Sample achieved_flight_record ----------------------------- { # Simple values first, e.g. string, int, float, etc. 'AFR Flight ID': # e.g. 1 'AFR Flight Number': # e.g. 1234 'AFR Type': # 'POSITIONING' 'AFR Off Blocks Datetime': # datetime(2015,01,01,13,00) 'AFR Takeoff Datetime': # datetime(2015,01,01,13,15) 'AFR Takeoff Pilot': # 'Joe Bloggs' 'AFR Takeoff Gross Weight': # weight in kg 'AFR Takeoff Fuel': # fuel in kg 'AFR Landing Datetime': # datetime(2015,01,01,18,45) 'AFR Landing Pilot': # 'Joe Bloggs' 'AFR Landing Gross Weight': # weight in kg 'AFR Landing Fuel': # weight in kg 'AFR On Blocks Datetime': # datetime(2015,01,01,19,00) 'AFR V2': # V2 used at takeoff in kts 'AFR Vapp': # Vapp used in kts 'AFR Vref': # Vref used in kts # More complex data that needs to be looked up next: 'AFR Takeoff Airport': { 'id': 4904, # unique id 'name': 'Athens Intl Airport Elefterios Venizel', 'code': {'iata': 'ATH', 'icao': 'LGAV'}, 'latitude': 37.9364, 'longitude': 23.9445, 'location': {'city': u'Athens', 'country': u'Greece'}, 'elevation': 266, # ft 'magnetic_variation': 'E003186 0106', } }, 'AFR Landing Aiport': { 'id': 1, # unique id 'name': 'Athens Intl Airport Elefterios Venizel', 'code': {'iata': 'ATH', 'icao': 'LGAV'}, 'latitude': 37.9364, 'longitude': 23.9445, 'location': {'city': u'Athens', 'country': u'Greece'}, 'elevation': 266, # ft 'magnetic_variation': 'E003186 0106', } }, 'AFR Destination Airport': None, # if not required, or exclude this key 'AFR Takeoff Runway': { 'id': 1, 'identifier': '21L', 'magnetic_heading': 212.6, 'strip': { 'id': 1, 'length': 13123, 'surface': 'ASP', 'width': 147}, 'start': { 'elevation': 308, 'latitude': 37.952425, 'longitude': 23.970422}, 'end': { 'elevation': 279, 'latitude': 37.923511, 'longitude': 23.943261}, 'glideslope': { 'angle': 3.0, 'elevation': 282, 'latitude': 37.9473, 'longitude': 23.9676, 'threshold_distance': 999}, 'localizer': { 'beam_width': 4.5, 'elevation': 256, 'frequency': 111100, 'heading': 213, 'latitude': 37.919281, 'longitude': 23.939294}, }, 'AFR Landing Runway': { 'id': 1, 'identifier': '21L', 'magnetic_heading': 212.6, 'strip': { 'id': 1, 'length': 13123, 'surface': 'ASP', 'width': 147}, 'start': { 'elevation': 308, 'latitude': 37.952425, 'longitude': 23.970422}, 'end': { 'elevation': 279, 'latitude': 37.923511, 'longitude': 23.943261}, 'glideslope': { 'angle': 3.0, 'elevation': 282, 'latitude': 37.9473, 'longitude': 23.9676, 'threshold_distance': 999}, 'localizer': { 'beam_width': 4.5, 'elevation': 256, 'frequency': 111100, 'heading': 213, 'latitude': 37.919281, 'longitude': 23.939294}, }, } Sample Return ------------- { 'flight':[Attribute('name value')], 'kti':[GeoKeyTimeInstance('index name latitude longitude')] if lat/long available else [KeyTimeInstance('index name')], 'kpv':[KeyPointValue('index value name slice')] } sample flight Attributes: [ Attribute('Takeoff Airport', {'id':1234, 'name':'Int. Airport'}, Attribute('Approaches', [4567,7890]), ... ], ''' if start_datetime is None: import pytz start_datetime = datetime.utcnow().replace(tzinfo=pytz.utc) logger.info("Processing: %s", hdf_path) if aircraft_info: # Aircraft info has already been provided. logger.info( "Using aircraft_info dictionary passed into process_flight '%s'." % aircraft_info) else: aircraft_info = get_aircraft_info(tail_number) aircraft_info['Tail Number'] = tail_number # go through modules to get derived nodes node_modules = additional_modules + settings.NODE_MODULES derived_nodes = get_derived_nodes(node_modules) if requested: requested = \ list(set(requested).intersection(set(derived_nodes))) else: # if requested isn't set, try using ALL derived_nodes! logger.info("No requested nodes declared, using all derived nodes") requested = derived_nodes.keys() # include all flight attributes as requested if include_flight_attributes: requested = list( set(requested + get_derived_nodes(['analysis_engine.flight_attribute']).keys()) ) # open HDF for reading with hdf_file(hdf_path) as hdf: hdf.start_datetime = start_datetime if hooks.PRE_FLIGHT_ANALYSIS: logger.info("Performing PRE_FLIGHT_ANALYSIS actions: %s", hooks.PRE_FLIGHT_ANALYSIS.func_name) hooks.PRE_FLIGHT_ANALYSIS(hdf, aircraft_info) else: logger.info("No PRE_FLIGHT_ANALYSIS actions to perform") # Track nodes. Assume that all params in HDF are from LFL(!) node_mgr = NodeManager(start_datetime, hdf.duration, hdf.valid_param_names(), requested, required, derived_nodes, aircraft_info, achieved_flight_record) # calculate dependency tree process_order, gr_st = dependency_order(node_mgr, draw=False) if settings.CACHE_PARAMETER_MIN_USAGE: # find params used more than for node in gr_st.nodes(): if node in node_mgr.derived_nodes: # this includes KPV/KTIs but they'll be ignored by HDF qty = len(gr_st.predecessors(node)) if qty > settings.CACHE_PARAMETER_MIN_USAGE: hdf.cache_param_list.append(node) logging.info("HDF set to cache parameters: %s", hdf.cache_param_list) # derive parameters kti_list, kpv_list, section_list, approach_list, flight_attrs = \ derive_parameters(hdf, node_mgr, process_order) # geo locate KTIs kti_list = geo_locate(hdf, kti_list) kti_list = _timestamp(start_datetime, kti_list) # geo locate KPVs kpv_list = geo_locate(hdf, kpv_list) kpv_list = _timestamp(start_datetime, kpv_list) # Store version of FlightDataAnalyser hdf.analysis_version = __version__ # Store dependency tree hdf.dependency_tree = json_graph.dumps(gr_st) # Store aircraft info hdf.set_attr('aircraft_info', aircraft_info) hdf.set_attr('achieved_flight_record', achieved_flight_record) return { 'flight': flight_attrs, 'kti': kti_list, 'kpv': kpv_list, 'approach': approach_list, 'phases': section_list, }
def process_flight(segment_info, tail_number, aircraft_info={}, achieved_flight_record={}, requested=[], required=[], include_flight_attributes=True, additional_modules=[], pre_flight_kwargs={}, force=False, initial={}, reprocess=False): ''' Processes the HDF file (segment_info['File']) to derive the required_params (Nodes) within python modules (settings.NODE_MODULES). Note: For Flight Data Services, the definitive API is located here: "PolarisTaskManagement.test.tasks_mask.process_flight" :param segment_info: Details of the segment to process :type segment_info: dict :param aircraft: Aircraft specific attributes :type aircraft: dict :param achieved_flight_record: See API Below :type achieved_flight_record: Dict :param requested: Derived nodes to process (dependencies will also be evaluated). :type requested: List of Strings :param required: Nodes which are required, otherwise an exception will be raised. :type required: List of Strings :param include_flight_attributes: Whether to include all flight attributes :type include_flight_attributes: Boolean :param additional_modules: List of module paths to import. :type additional_modules: List of Strings :param pre_flight_kwargs: Keyword arguments for the pre-flight analysis hook. :type pre_flight_kwargs: dict :param force: Ignore errors raised while deriving nodes. :type force: bool :param initial: Initial content for nodes to avoid reprocessing (excluding parameter nodes which are saved to the hdf). :type initial: dict :param reprocess: Force reprocessing of all Nodes (including derived Nodes already saved to the HDF file). :returns: See below: :rtype: Dict Sample segment_info -------------------- { 'File': # Path to HDF5 file to process 'Start Datetime': # Datetime of the origin of the data (at index 0) 'Segment Type': # segment type obtained from split segments e.g. START_AND_STOP } Sample aircraft_info -------------------- { 'Tail Number': # Aircraft Registration 'Identifier': # Aircraft Ident 'Manufacturer': # e.g. Boeing 'Manufacturer Serial Number': #MSN 'Model': # e.g. 737-808-ER 'Series': # e.g. 737-800 'Family': # e.g. 737 'Frame': # e.g. 737-3C 'Main Gear To Altitude Radio': # Distance in metres 'Wing Span': # Distance in metres } Sample achieved_flight_record ----------------------------- { # Simple values first, e.g. string, int, float, etc. 'AFR Flight ID': # e.g. 1 'AFR Flight Number': # e.g. 1234 'AFR Type': # 'POSITIONING' 'AFR Off Blocks Datetime': # datetime(2015,1,1,13,00) 'AFR Takeoff Datetime': # datetime(2015,1,1,13,15) 'AFR Takeoff Pilot': # 'Joe Bloggs' 'AFR Takeoff Gross Weight': # weight in kg 'AFR Takeoff Fuel': # fuel in kg 'AFR Landing Datetime': # datetime(2015,1,1,18,45) 'AFR Landing Pilot': # 'Joe Bloggs' 'AFR Landing Gross Weight': # weight in kg 'AFR Landing Fuel': # weight in kg 'AFR On Blocks Datetime': # datetime(2015,1,1,19,00) 'AFR V2': # V2 used at takeoff in kts 'AFR Vapp': # Vapp used in kts 'AFR Vref': # Vref used in kts # More complex data that needs to be looked up next: 'AFR Takeoff Airport': { 'id': 4904, # unique id 'name': 'Athens Intl Airport Elefterios Venizel', 'code': {'iata': 'ATH', 'icao': 'LGAV'}, 'latitude': 37.9364, 'longitude': 23.9445, 'location': {'city': u'Athens', 'country': u'Greece'}, 'elevation': 266, # ft 'magnetic_variation': 'E003186 0106', } }, 'AFR Landing Aiport': { 'id': 1, # unique id 'name': 'Athens Intl Airport Elefterios Venizel', 'code': {'iata': 'ATH', 'icao': 'LGAV'}, 'latitude': 37.9364, 'longitude': 23.9445, 'location': {'city': u'Athens', 'country': u'Greece'}, 'elevation': 266, # ft 'magnetic_variation': 'E003186 0106', } }, 'AFR Destination Airport': None, # if not required, or exclude this key 'AFR Takeoff Runway': { 'id': 1, 'identifier': '21L', 'magnetic_heading': 212.6, 'strip': { 'id': 1, 'length': 13123, 'surface': 'ASP', 'width': 147}, 'start': { 'elevation': 308, 'latitude': 37.952425, 'longitude': 23.970422}, 'end': { 'elevation': 279, 'latitude': 37.923511, 'longitude': 23.943261}, 'glideslope': { 'angle': 3.0, 'elevation': 282, 'latitude': 37.9473, 'longitude': 23.9676, 'threshold_distance': 999}, 'localizer': { 'beam_width': 4.5, 'elevation': 256, 'frequency': 111100, 'heading': 213, 'latitude': 37.919281, 'longitude': 23.939294}, }, 'AFR Landing Runway': { 'id': 1, 'identifier': '21L', 'magnetic_heading': 212.6, 'strip': { 'id': 1, 'length': 13123, 'surface': 'ASP', 'width': 147}, 'start': { 'elevation': 308, 'latitude': 37.952425, 'longitude': 23.970422}, 'end': { 'elevation': 279, 'latitude': 37.923511, 'longitude': 23.943261}, 'glideslope': { 'angle': 3.0, 'elevation': 282, 'latitude': 37.9473, 'longitude': 23.9676, 'threshold_distance': 999}, 'localizer': { 'beam_width': 4.5, 'elevation': 256, 'frequency': 111100, 'heading': 213, 'latitude': 37.919281, 'longitude': 23.939294}, }, } Sample Return ------------- { 'flight':[Attribute('name value')], 'kti':[GeoKeyTimeInstance('index name latitude longitude')] if lat/long available else [KeyTimeInstance('index name')], 'kpv':[KeyPointValue('index value name slice')] } sample flight Attributes: [ Attribute('Takeoff Airport', {'id':1234, 'name':'Int. Airport'}, Attribute('Approaches', [4567,7890]), ... ], ''' hdf_path = segment_info['File'] if 'Start Datetime' not in segment_info: import pytz segment_info['Start Datetime'] = datetime.utcnow().replace( tzinfo=pytz.utc) logger.debug("Processing: %s", hdf_path) if aircraft_info: # Aircraft info has already been provided. logger.info( "Using aircraft_info dictionary passed into process_flight '%s'." % aircraft_info) else: aircraft_info = get_aircraft_info(tail_number) aircraft_info['Tail Number'] = tail_number if aircraft_info['Aircraft Type'] == 'helicopter': node_modules = settings.NODE_MODULES + \ settings.NODE_HELICOPTER_MODULE_PATHS + additional_modules else: node_modules = settings.NODE_MODULES + additional_modules # go through modules to get derived nodes derived_nodes = get_derived_nodes(node_modules) if requested: requested = \ list(set(requested).intersection(set(derived_nodes))) else: # if requested isn't set, try using ALL derived_nodes! logger.debug("No requested nodes declared, using all derived nodes") # Enforce some sort of order in which the dependencies are traversed requested = sorted(list(derived_nodes.keys())) # include all flight attributes as requested if include_flight_attributes: requested = list( set(requested + list( get_derived_nodes(['analysis_engine.flight_attribute']).keys()) )) initial = process_flight_to_nodes(initial) for node_name in requested: initial.pop(node_name, None) # open HDF for reading with hdf_file(hdf_path) as hdf: hdf.start_datetime = segment_info['Start Datetime'] hook = hooks.PRE_FLIGHT_ANALYSIS if hook: logger.info( "Performing PRE_FLIGHT_ANALYSIS action '%s' with options: %s", getattr(hook, 'func_name', getattr(hook, '__name__')), pre_flight_kwargs) hook(hdf, aircraft_info, **pre_flight_kwargs) else: logger.info("No PRE_FLIGHT_ANALYSIS actions to perform") # Merge Params param_names = hdf.valid_lfl_param_names( ) if reprocess else hdf.valid_param_names() pre_process_parameters(hdf, segment_info, param_names, required, aircraft_info, achieved_flight_record, force=force) # Track nodes. node_mgr = NodeManager(segment_info, hdf.duration, param_names, requested, required, derived_nodes, aircraft_info, achieved_flight_record) # calculate dependency tree process_order, gr_st = dependency_order(node_mgr, draw=False) if settings.CACHE_PARAMETER_MIN_USAGE: # find params used more than for node in gr_st.nodes(): if node in node_mgr.derived_nodes: # this includes KPV/KTIs but they'll be ignored by HDF qty = len(gr_st.predecessors(node)) if qty > settings.CACHE_PARAMETER_MIN_USAGE: hdf.cache_param_list.append(node) logging.info("HDF set to cache parameters: %s", hdf.cache_param_list) # derive parameters ktis, kpvs, sections, approaches, flight_attrs = \ derive_parameters(hdf, node_mgr, process_order, params=initial, force=force) # geo locate KTIs ktis = geo_locate(hdf, ktis) ktis = _timestamp(segment_info['Start Datetime'], ktis) # geo locate KPVs kpvs = geo_locate(hdf, kpvs) kpvs = _timestamp(segment_info['Start Datetime'], kpvs) # Store version of FlightDataAnalyser hdf.analysis_version = __version__ # Store dependency tree hdf.dependency_tree = json.dumps(json_graph.node_link_data(gr_st)) # Store aircraft info hdf.set_attr('aircraft_info', aircraft_info) hdf.set_attr('achieved_flight_record', achieved_flight_record) return { 'flight': flight_attrs, 'kti': ktis, 'kpv': kpvs, 'approach': approaches, 'phases': sections, }
def process_flight(segment_info, tail_number, aircraft_info={}, achieved_flight_record={}, requested=[], required=[], include_flight_attributes=True, additional_modules=[], pre_flight_kwargs={}, force=False, initial={}, reprocess=False): ''' Processes the HDF file (segment_info['File']) to derive the required_params (Nodes) within python modules (settings.NODE_MODULES). Note: For Flight Data Services, the definitive API is located here: "PolarisTaskManagement.test.tasks_mask.process_flight" :param segment_info: Details of the segment to process :type segment_info: dict :param aircraft: Aircraft specific attributes :type aircraft: dict :param achieved_flight_record: See API Below :type achieved_flight_record: Dict :param requested: Derived nodes to process (dependencies will also be evaluated). :type requested: List of Strings :param required: Nodes which are required, otherwise an exception will be raised. :type required: List of Strings :param include_flight_attributes: Whether to include all flight attributes :type include_flight_attributes: Boolean :param additional_modules: List of module paths to import. :type additional_modules: List of Strings :param pre_flight_kwargs: Keyword arguments for the pre-flight analysis hook. :type pre_flight_kwargs: dict :param force: Ignore errors raised while deriving nodes. :type force: bool :param initial: Initial content for nodes to avoid reprocessing (excluding parameter nodes which are saved to the hdf). :type initial: dict :param reprocess: Force reprocessing of all Nodes (including derived Nodes already saved to the HDF file). :returns: See below: :rtype: Dict Sample segment_info -------------------- { 'File': # Path to HDF5 file to process 'Start Datetime': # Datetime of the origin of the data (at index 0) 'Segment Type': # segment type obtained from split segments e.g. START_AND_STOP } Sample aircraft_info -------------------- { 'Tail Number': # Aircraft Registration 'Identifier': # Aircraft Ident 'Manufacturer': # e.g. Boeing 'Manufacturer Serial Number': #MSN 'Model': # e.g. 737-808-ER 'Series': # e.g. 737-800 'Family': # e.g. 737 'Frame': # e.g. 737-3C 'Main Gear To Altitude Radio': # Distance in metres 'Wing Span': # Distance in metres } Sample achieved_flight_record ----------------------------- { # Simple values first, e.g. string, int, float, etc. 'AFR Flight ID': # e.g. 1 'AFR Flight Number': # e.g. 1234 'AFR Type': # 'POSITIONING' 'AFR Off Blocks Datetime': # datetime(2015,01,01,13,00) 'AFR Takeoff Datetime': # datetime(2015,01,01,13,15) 'AFR Takeoff Pilot': # 'Joe Bloggs' 'AFR Takeoff Gross Weight': # weight in kg 'AFR Takeoff Fuel': # fuel in kg 'AFR Landing Datetime': # datetime(2015,01,01,18,45) 'AFR Landing Pilot': # 'Joe Bloggs' 'AFR Landing Gross Weight': # weight in kg 'AFR Landing Fuel': # weight in kg 'AFR On Blocks Datetime': # datetime(2015,01,01,19,00) 'AFR V2': # V2 used at takeoff in kts 'AFR Vapp': # Vapp used in kts 'AFR Vref': # Vref used in kts # More complex data that needs to be looked up next: 'AFR Takeoff Airport': { 'id': 4904, # unique id 'name': 'Athens Intl Airport Elefterios Venizel', 'code': {'iata': 'ATH', 'icao': 'LGAV'}, 'latitude': 37.9364, 'longitude': 23.9445, 'location': {'city': u'Athens', 'country': u'Greece'}, 'elevation': 266, # ft 'magnetic_variation': 'E003186 0106', } }, 'AFR Landing Aiport': { 'id': 1, # unique id 'name': 'Athens Intl Airport Elefterios Venizel', 'code': {'iata': 'ATH', 'icao': 'LGAV'}, 'latitude': 37.9364, 'longitude': 23.9445, 'location': {'city': u'Athens', 'country': u'Greece'}, 'elevation': 266, # ft 'magnetic_variation': 'E003186 0106', } }, 'AFR Destination Airport': None, # if not required, or exclude this key 'AFR Takeoff Runway': { 'id': 1, 'identifier': '21L', 'magnetic_heading': 212.6, 'strip': { 'id': 1, 'length': 13123, 'surface': 'ASP', 'width': 147}, 'start': { 'elevation': 308, 'latitude': 37.952425, 'longitude': 23.970422}, 'end': { 'elevation': 279, 'latitude': 37.923511, 'longitude': 23.943261}, 'glideslope': { 'angle': 3.0, 'elevation': 282, 'latitude': 37.9473, 'longitude': 23.9676, 'threshold_distance': 999}, 'localizer': { 'beam_width': 4.5, 'elevation': 256, 'frequency': 111100, 'heading': 213, 'latitude': 37.919281, 'longitude': 23.939294}, }, 'AFR Landing Runway': { 'id': 1, 'identifier': '21L', 'magnetic_heading': 212.6, 'strip': { 'id': 1, 'length': 13123, 'surface': 'ASP', 'width': 147}, 'start': { 'elevation': 308, 'latitude': 37.952425, 'longitude': 23.970422}, 'end': { 'elevation': 279, 'latitude': 37.923511, 'longitude': 23.943261}, 'glideslope': { 'angle': 3.0, 'elevation': 282, 'latitude': 37.9473, 'longitude': 23.9676, 'threshold_distance': 999}, 'localizer': { 'beam_width': 4.5, 'elevation': 256, 'frequency': 111100, 'heading': 213, 'latitude': 37.919281, 'longitude': 23.939294}, }, } Sample Return ------------- { 'flight':[Attribute('name value')], 'kti':[GeoKeyTimeInstance('index name latitude longitude')] if lat/long available else [KeyTimeInstance('index name')], 'kpv':[KeyPointValue('index value name slice')] } sample flight Attributes: [ Attribute('Takeoff Airport', {'id':1234, 'name':'Int. Airport'}, Attribute('Approaches', [4567,7890]), ... ], ''' hdf_path = segment_info['File'] if 'Start Datetime' not in segment_info: import pytz segment_info['Start Datetime'] = datetime.utcnow().replace(tzinfo=pytz.utc) logger.info("Processing: %s", hdf_path) if aircraft_info: # Aircraft info has already been provided. logger.info( "Using aircraft_info dictionary passed into process_flight '%s'." % aircraft_info) else: aircraft_info = get_aircraft_info(tail_number) aircraft_info['Tail Number'] = tail_number # go through modules to get derived nodes node_modules = additional_modules + settings.NODE_MODULES derived_nodes = get_derived_nodes(node_modules) if requested: requested = \ list(set(requested).intersection(set(derived_nodes))) else: # if requested isn't set, try using ALL derived_nodes! logger.info("No requested nodes declared, using all derived nodes") requested = derived_nodes.keys() # include all flight attributes as requested if include_flight_attributes: requested = list(set( requested + get_derived_nodes( ['analysis_engine.flight_attribute']).keys())) initial = process_flight_to_nodes(initial) for node_name in requested: initial.pop(node_name, None) # open HDF for reading with hdf_file(hdf_path) as hdf: hdf.start_datetime = segment_info['Start Datetime'] if hooks.PRE_FLIGHT_ANALYSIS: logger.info("Performing PRE_FLIGHT_ANALYSIS action '%s' with options: %s", hooks.PRE_FLIGHT_ANALYSIS.func_name, pre_flight_kwargs) hooks.PRE_FLIGHT_ANALYSIS(hdf, aircraft_info, **pre_flight_kwargs) else: logger.info("No PRE_FLIGHT_ANALYSIS actions to perform") # Track nodes. param_names = hdf.valid_lfl_param_names() if reprocess else hdf.valid_param_names() node_mgr = NodeManager( segment_info, hdf.duration, param_names, requested, required, derived_nodes, aircraft_info, achieved_flight_record) # calculate dependency tree process_order, gr_st = dependency_order(node_mgr, draw=False) if settings.CACHE_PARAMETER_MIN_USAGE: # find params used more than for node in gr_st.nodes(): if node in node_mgr.derived_nodes: # this includes KPV/KTIs but they'll be ignored by HDF qty = len(gr_st.predecessors(node)) if qty > settings.CACHE_PARAMETER_MIN_USAGE: hdf.cache_param_list.append(node) logging.info("HDF set to cache parameters: %s", hdf.cache_param_list) # derive parameters ktis, kpvs, sections, approaches, flight_attrs = \ derive_parameters(hdf, node_mgr, process_order, params=initial, force=force) # geo locate KTIs ktis = geo_locate(hdf, ktis) ktis = _timestamp(segment_info['Start Datetime'], ktis) # geo locate KPVs kpvs = geo_locate(hdf, kpvs) kpvs = _timestamp(segment_info['Start Datetime'], kpvs) # Store version of FlightDataAnalyser hdf.analysis_version = __version__ # Store dependency tree hdf.dependency_tree = json_graph.dumps(gr_st) # Store aircraft info hdf.set_attr('aircraft_info', aircraft_info) hdf.set_attr('achieved_flight_record', achieved_flight_record) return { 'flight': flight_attrs, 'kti': ktis, 'kpv': kpvs, 'approach': approaches, 'phases': sections, }