def test_process_flight_to_nodes(self):
     nodes = process_flight_to_nodes(PROCESS_FLIGHT)
     self.assertEqual(len(nodes), 1)
     self.assertEqual(list(nodes.keys()), [KTI_NAME])
     self.assertIn(KTI_NAME, nodes)
     node = nodes[KTI_NAME]
     self.assertTrue(isinstance(node, KeyTimeInstanceNode))
     self.assertEqual(len(node), 1)
     for name in ('index', 'latitude', 'longitude', 'name'):
         self.assertEqual(getattr(node[0], name), KTI_JSONDICT[name])
 def test_process_flight_to_nodes(self):
     nodes = process_flight_to_nodes(PROCESS_FLIGHT)
     self.assertEqual(len(nodes), 1)
     self.assertEqual(nodes.keys(), [KTI_NAME])
     self.assertIn(KTI_NAME, nodes)
     node = nodes[KTI_NAME]
     self.assertTrue(isinstance(node, KeyTimeInstanceNode))
     self.assertEqual(len(node), 1)
     for name in ("index", "latitude", "longitude", "name"):
         self.assertEqual(getattr(node[0], name), KTI_JSONDICT[name])
示例#3
0
def process_flight(segment_info,
                   tail_number,
                   aircraft_info={},
                   achieved_flight_record={},
                   requested=[],
                   required=[],
                   include_flight_attributes=True,
                   additional_modules=[],
                   pre_flight_kwargs={},
                   force=False,
                   initial={},
                   reprocess=False):
    '''
    Processes the HDF file (segment_info['File']) to derive the required_params (Nodes)
    within python modules (settings.NODE_MODULES).

    Note: For Flight Data Services, the definitive API is located here:
        "PolarisTaskManagement.test.tasks_mask.process_flight"

    :param segment_info: Details of the segment to process
    :type segment_info: dict
    :param aircraft: Aircraft specific attributes
    :type aircraft: dict
    :param achieved_flight_record: See API Below
    :type achieved_flight_record: Dict
    :param requested: Derived nodes to process (dependencies will also be
        evaluated).
    :type requested: List of Strings
    :param required: Nodes which are required, otherwise an exception will be
        raised.
    :type required: List of Strings
    :param include_flight_attributes: Whether to include all flight attributes
    :type include_flight_attributes: Boolean
    :param additional_modules: List of module paths to import.
    :type additional_modules: List of Strings
    :param pre_flight_kwargs: Keyword arguments for the pre-flight analysis hook.
    :type pre_flight_kwargs: dict
    :param force: Ignore errors raised while deriving nodes.
    :type force: bool
    :param initial: Initial content for nodes to avoid reprocessing (excluding parameter nodes which are saved to the hdf).
    :type initial: dict
    :param reprocess: Force reprocessing of all Nodes (including derived Nodes already saved to the HDF file).

    :returns: See below:
    :rtype: Dict

    Sample segment_info
    --------------------
    {
        'File':  # Path to HDF5 file to process
        'Start Datetime':  # Datetime of the origin of the data (at index 0)
        'Segment Type': # segment type obtained from split segments e.g. START_AND_STOP
    }

    Sample aircraft_info
    --------------------
    {
        'Tail Number':  # Aircraft Registration
        'Identifier':  # Aircraft Ident
        'Manufacturer': # e.g. Boeing
        'Manufacturer Serial Number': #MSN
        'Model': # e.g. 737-808-ER
        'Series': # e.g. 737-800
        'Family': # e.g. 737
        'Frame': # e.g. 737-3C
        'Main Gear To Altitude Radio': # Distance in metres
        'Wing Span': # Distance in metres
    }

    Sample achieved_flight_record
    -----------------------------
    {
        # Simple values first, e.g. string, int, float, etc.
        'AFR Flight ID': # e.g. 1
        'AFR Flight Number': # e.g. 1234
        'AFR Type': # 'POSITIONING'
        'AFR Off Blocks Datetime': # datetime(2015,1,1,13,00)
        'AFR Takeoff Datetime': # datetime(2015,1,1,13,15)
        'AFR Takeoff Pilot': # 'Joe Bloggs'
        'AFR Takeoff Gross Weight': # weight in kg
        'AFR Takeoff Fuel': # fuel in kg
        'AFR Landing Datetime': # datetime(2015,1,1,18,45)
        'AFR Landing Pilot': # 'Joe Bloggs'
        'AFR Landing Gross Weight': # weight in kg
        'AFR Landing Fuel': # weight in kg
        'AFR On Blocks Datetime': # datetime(2015,1,1,19,00)
        'AFR V2': # V2 used at takeoff in kts
        'AFR Vapp': # Vapp used in kts
        'AFR Vref': # Vref used in kts
        # More complex data that needs to be looked up next:
        'AFR Takeoff Airport':  {
            'id': 4904, # unique id
            'name': 'Athens Intl Airport Elefterios Venizel',
            'code': {'iata': 'ATH', 'icao': 'LGAV'},
            'latitude': 37.9364,
            'longitude': 23.9445,
            'location': {'city': u'Athens', 'country': u'Greece'},
            'elevation': 266, # ft
            'magnetic_variation': 'E003186 0106',
            }
           },
        'AFR Landing Aiport': {
            'id': 1, # unique id
            'name': 'Athens Intl Airport Elefterios Venizel',
            'code': {'iata': 'ATH', 'icao': 'LGAV'},
            'latitude': 37.9364,
            'longitude': 23.9445,
            'location': {'city': u'Athens', 'country': u'Greece'},
            'elevation': 266, # ft
            'magnetic_variation': 'E003186 0106',
            }
           },
        'AFR Destination Airport': None, # if not required, or exclude this key
        'AFR Takeoff Runway': {
            'id': 1,
            'identifier': '21L',
            'magnetic_heading': 212.6,
            'strip': {
                'id': 1,
                'length': 13123,
                'surface': 'ASP',
                'width': 147},
            'start': {
                'elevation': 308,
                'latitude': 37.952425,
                'longitude': 23.970422},
            'end': {
                'elevation': 279,
                'latitude': 37.923511,
                'longitude': 23.943261},
            'glideslope': {
                'angle': 3.0,
                'elevation': 282,
                'latitude': 37.9473,
                'longitude': 23.9676,
                'threshold_distance': 999},
            'localizer': {
                'beam_width': 4.5,
                'elevation': 256,
                'frequency': 111100,
                'heading': 213,
                'latitude': 37.919281,
                'longitude': 23.939294},
            },
        'AFR Landing Runway': {
            'id': 1,
            'identifier': '21L',
            'magnetic_heading': 212.6,
            'strip': {
                'id': 1,
                'length': 13123,
                'surface': 'ASP',
                'width': 147},
            'start': {
                'elevation': 308,
                'latitude': 37.952425,
                'longitude': 23.970422},
            'end': {
                'elevation': 279,
                'latitude': 37.923511,
                'longitude': 23.943261},
            'glideslope': {
                'angle': 3.0,
                'elevation': 282,
                'latitude': 37.9473,
                'longitude': 23.9676,
                'threshold_distance': 999},
            'localizer': {
                'beam_width': 4.5,
                'elevation': 256,
                'frequency': 111100,
                'heading': 213,
                'latitude': 37.919281,
                'longitude': 23.939294},
            },
    }

    Sample Return
    -------------
    {
        'flight':[Attribute('name value')],
        'kti':[GeoKeyTimeInstance('index name latitude longitude')]
            if lat/long available
            else [KeyTimeInstance('index name')],
        'kpv':[KeyPointValue('index value name slice')]
    }

    sample flight Attributes:

    [
        Attribute('Takeoff Airport', {'id':1234, 'name':'Int. Airport'},
        Attribute('Approaches', [4567,7890]),
        ...
    ],

    '''

    hdf_path = segment_info['File']
    if 'Start Datetime' not in segment_info:
        import pytz
        segment_info['Start Datetime'] = datetime.utcnow().replace(
            tzinfo=pytz.utc)
    logger.debug("Processing: %s", hdf_path)

    if aircraft_info:
        # Aircraft info has already been provided.
        logger.info(
            "Using aircraft_info dictionary passed into process_flight '%s'." %
            aircraft_info)
    else:
        aircraft_info = get_aircraft_info(tail_number)

    aircraft_info['Tail Number'] = tail_number

    if aircraft_info['Aircraft Type'] == 'helicopter':
        node_modules = settings.NODE_MODULES + \
            settings.NODE_HELICOPTER_MODULE_PATHS + additional_modules
    else:
        node_modules = settings.NODE_MODULES + additional_modules
    # go through modules to get derived nodes
    derived_nodes = get_derived_nodes(node_modules)

    if requested:
        requested = \
            list(set(requested).intersection(set(derived_nodes)))
    else:
        # if requested isn't set, try using ALL derived_nodes!
        logger.debug("No requested nodes declared, using all derived nodes")
        # Enforce some sort of order in which the dependencies are traversed
        requested = sorted(list(derived_nodes.keys()))

    # include all flight attributes as requested
    if include_flight_attributes:
        requested = list(
            set(requested + list(
                get_derived_nodes(['analysis_engine.flight_attribute']).keys())
                ))

    initial = process_flight_to_nodes(initial)
    for node_name in requested:
        initial.pop(node_name, None)

    # open HDF for reading
    with hdf_file(hdf_path) as hdf:
        hdf.start_datetime = segment_info['Start Datetime']
        hook = hooks.PRE_FLIGHT_ANALYSIS
        if hook:
            logger.info(
                "Performing PRE_FLIGHT_ANALYSIS action '%s' with options: %s",
                getattr(hook, 'func_name', getattr(hook, '__name__')),
                pre_flight_kwargs)
            hook(hdf, aircraft_info, **pre_flight_kwargs)
        else:
            logger.info("No PRE_FLIGHT_ANALYSIS actions to perform")

        # Merge Params
        param_names = hdf.valid_lfl_param_names(
        ) if reprocess else hdf.valid_param_names()
        pre_process_parameters(hdf,
                               segment_info,
                               param_names,
                               required,
                               aircraft_info,
                               achieved_flight_record,
                               force=force)

        # Track nodes.
        node_mgr = NodeManager(segment_info, hdf.duration, param_names,
                               requested, required, derived_nodes,
                               aircraft_info, achieved_flight_record)
        # calculate dependency tree
        process_order, gr_st = dependency_order(node_mgr, draw=False)
        if settings.CACHE_PARAMETER_MIN_USAGE:
            # find params used more than
            for node in gr_st.nodes():
                if node in node_mgr.derived_nodes:
                    # this includes KPV/KTIs but they'll be ignored by HDF
                    qty = len(gr_st.predecessors(node))
                    if qty > settings.CACHE_PARAMETER_MIN_USAGE:
                        hdf.cache_param_list.append(node)
            logging.info("HDF set to cache parameters: %s",
                         hdf.cache_param_list)

        # derive parameters
        ktis, kpvs, sections, approaches, flight_attrs = \
            derive_parameters(hdf, node_mgr, process_order, params=initial, force=force)

        # geo locate KTIs
        ktis = geo_locate(hdf, ktis)
        ktis = _timestamp(segment_info['Start Datetime'], ktis)

        # geo locate KPVs
        kpvs = geo_locate(hdf, kpvs)
        kpvs = _timestamp(segment_info['Start Datetime'], kpvs)

        # Store version of FlightDataAnalyser
        hdf.analysis_version = __version__
        # Store dependency tree
        hdf.dependency_tree = json.dumps(json_graph.node_link_data(gr_st))
        # Store aircraft info
        hdf.set_attr('aircraft_info', aircraft_info)
        hdf.set_attr('achieved_flight_record', achieved_flight_record)

    return {
        'flight': flight_attrs,
        'kti': ktis,
        'kpv': kpvs,
        'approach': approaches,
        'phases': sections,
    }
def process_flight(segment_info, tail_number, aircraft_info={}, achieved_flight_record={},
                   requested=[], required=[], include_flight_attributes=True,
                   additional_modules=[], pre_flight_kwargs={}, force=False,
                   initial={}, reprocess=False):
    '''
    Processes the HDF file (segment_info['File']) to derive the required_params (Nodes)
    within python modules (settings.NODE_MODULES).

    Note: For Flight Data Services, the definitive API is located here:
        "PolarisTaskManagement.test.tasks_mask.process_flight"

    :param segment_info: Details of the segment to process
    :type segment_info: dict
    :param aircraft: Aircraft specific attributes
    :type aircraft: dict
    :param achieved_flight_record: See API Below
    :type achieved_flight_record: Dict
    :param requested: Derived nodes to process (dependencies will also be
        evaluated).
    :type requested: List of Strings
    :param required: Nodes which are required, otherwise an exception will be
        raised.
    :type required: List of Strings
    :param include_flight_attributes: Whether to include all flight attributes
    :type include_flight_attributes: Boolean
    :param additional_modules: List of module paths to import.
    :type additional_modules: List of Strings
    :param pre_flight_kwargs: Keyword arguments for the pre-flight analysis hook.
    :type pre_flight_kwargs: dict
    :param force: Ignore errors raised while deriving nodes.
    :type force: bool
    :param initial: Initial content for nodes to avoid reprocessing (excluding parameter nodes which are saved to the hdf).
    :type initial: dict
    :param reprocess: Force reprocessing of all Nodes (including derived Nodes already saved to the HDF file).

    :returns: See below:
    :rtype: Dict

    Sample segment_info
    --------------------
    {
        'File':  # Path to HDF5 file to process
        'Start Datetime':  # Datetime of the origin of the data (at index 0)
        'Segment Type': # segment type obtained from split segments e.g. START_AND_STOP
    }

    Sample aircraft_info
    --------------------
    {
        'Tail Number':  # Aircraft Registration
        'Identifier':  # Aircraft Ident
        'Manufacturer': # e.g. Boeing
        'Manufacturer Serial Number': #MSN
        'Model': # e.g. 737-808-ER
        'Series': # e.g. 737-800
        'Family': # e.g. 737
        'Frame': # e.g. 737-3C
        'Main Gear To Altitude Radio': # Distance in metres
        'Wing Span': # Distance in metres
    }

    Sample achieved_flight_record
    -----------------------------
    {
        # Simple values first, e.g. string, int, float, etc.
        'AFR Flight ID': # e.g. 1
        'AFR Flight Number': # e.g. 1234
        'AFR Type': # 'POSITIONING'
        'AFR Off Blocks Datetime': # datetime(2015,01,01,13,00)
        'AFR Takeoff Datetime': # datetime(2015,01,01,13,15)
        'AFR Takeoff Pilot': # 'Joe Bloggs'
        'AFR Takeoff Gross Weight': # weight in kg
        'AFR Takeoff Fuel': # fuel in kg
        'AFR Landing Datetime': # datetime(2015,01,01,18,45)
        'AFR Landing Pilot': # 'Joe Bloggs'
        'AFR Landing Gross Weight': # weight in kg
        'AFR Landing Fuel': # weight in kg
        'AFR On Blocks Datetime': # datetime(2015,01,01,19,00)
        'AFR V2': # V2 used at takeoff in kts
        'AFR Vapp': # Vapp used in kts
        'AFR Vref': # Vref used in kts
        # More complex data that needs to be looked up next:
        'AFR Takeoff Airport':  {
            'id': 4904, # unique id
            'name': 'Athens Intl Airport Elefterios Venizel',
            'code': {'iata': 'ATH', 'icao': 'LGAV'},
            'latitude': 37.9364,
            'longitude': 23.9445,
            'location': {'city': u'Athens', 'country': u'Greece'},
            'elevation': 266, # ft
            'magnetic_variation': 'E003186 0106',
            }
           },
        'AFR Landing Aiport': {
            'id': 1, # unique id
            'name': 'Athens Intl Airport Elefterios Venizel',
            'code': {'iata': 'ATH', 'icao': 'LGAV'},
            'latitude': 37.9364,
            'longitude': 23.9445,
            'location': {'city': u'Athens', 'country': u'Greece'},
            'elevation': 266, # ft
            'magnetic_variation': 'E003186 0106',
            }
           },
        'AFR Destination Airport': None, # if not required, or exclude this key
        'AFR Takeoff Runway': {
            'id': 1,
            'identifier': '21L',
            'magnetic_heading': 212.6,
            'strip': {
                'id': 1,
                'length': 13123,
                'surface': 'ASP',
                'width': 147},
            'start': {
                'elevation': 308,
                'latitude': 37.952425,
                'longitude': 23.970422},
            'end': {
                'elevation': 279,
                'latitude': 37.923511,
                'longitude': 23.943261},
            'glideslope': {
                'angle': 3.0,
                'elevation': 282,
                'latitude': 37.9473,
                'longitude': 23.9676,
                'threshold_distance': 999},
            'localizer': {
                'beam_width': 4.5,
                'elevation': 256,
                'frequency': 111100,
                'heading': 213,
                'latitude': 37.919281,
                'longitude': 23.939294},
            },
        'AFR Landing Runway': {
            'id': 1,
            'identifier': '21L',
            'magnetic_heading': 212.6,
            'strip': {
                'id': 1,
                'length': 13123,
                'surface': 'ASP',
                'width': 147},
            'start': {
                'elevation': 308,
                'latitude': 37.952425,
                'longitude': 23.970422},
            'end': {
                'elevation': 279,
                'latitude': 37.923511,
                'longitude': 23.943261},
            'glideslope': {
                'angle': 3.0,
                'elevation': 282,
                'latitude': 37.9473,
                'longitude': 23.9676,
                'threshold_distance': 999},
            'localizer': {
                'beam_width': 4.5,
                'elevation': 256,
                'frequency': 111100,
                'heading': 213,
                'latitude': 37.919281,
                'longitude': 23.939294},
            },
    }

    Sample Return
    -------------
    {
        'flight':[Attribute('name value')],
        'kti':[GeoKeyTimeInstance('index name latitude longitude')]
            if lat/long available
            else [KeyTimeInstance('index name')],
        'kpv':[KeyPointValue('index value name slice')]
    }

    sample flight Attributes:

    [
        Attribute('Takeoff Airport', {'id':1234, 'name':'Int. Airport'},
        Attribute('Approaches', [4567,7890]),
        ...
    ],

    '''
    
    hdf_path = segment_info['File']
    if 'Start Datetime' not in segment_info:
        import pytz
        segment_info['Start Datetime'] = datetime.utcnow().replace(tzinfo=pytz.utc)
    logger.info("Processing: %s", hdf_path)

    if aircraft_info:
        # Aircraft info has already been provided.
        logger.info(
            "Using aircraft_info dictionary passed into process_flight '%s'." %
            aircraft_info)
    else:
        aircraft_info = get_aircraft_info(tail_number)

    aircraft_info['Tail Number'] = tail_number

    # go through modules to get derived nodes
    node_modules = additional_modules + settings.NODE_MODULES
    derived_nodes = get_derived_nodes(node_modules)

    if requested:
        requested = \
            list(set(requested).intersection(set(derived_nodes)))
    else:
        # if requested isn't set, try using ALL derived_nodes!
        logger.info("No requested nodes declared, using all derived nodes")
        requested = derived_nodes.keys()

    # include all flight attributes as requested
    if include_flight_attributes:
        requested = list(set(
            requested + get_derived_nodes(
                ['analysis_engine.flight_attribute']).keys()))
    
    initial = process_flight_to_nodes(initial)
    for node_name in requested:
        initial.pop(node_name, None)

    # open HDF for reading
    with hdf_file(hdf_path) as hdf:
        hdf.start_datetime = segment_info['Start Datetime']
        if hooks.PRE_FLIGHT_ANALYSIS:
            logger.info("Performing PRE_FLIGHT_ANALYSIS action '%s' with options: %s",
                        hooks.PRE_FLIGHT_ANALYSIS.func_name, pre_flight_kwargs)
            hooks.PRE_FLIGHT_ANALYSIS(hdf, aircraft_info, **pre_flight_kwargs)
        else:
            logger.info("No PRE_FLIGHT_ANALYSIS actions to perform")
        # Track nodes.
        param_names = hdf.valid_lfl_param_names() if reprocess else hdf.valid_param_names()
        node_mgr = NodeManager(
            segment_info, hdf.duration, param_names,
            requested, required, derived_nodes, aircraft_info,
            achieved_flight_record)
        # calculate dependency tree
        process_order, gr_st = dependency_order(node_mgr, draw=False)
        if settings.CACHE_PARAMETER_MIN_USAGE:
            # find params used more than
            for node in gr_st.nodes():
                if node in node_mgr.derived_nodes:
                    # this includes KPV/KTIs but they'll be ignored by HDF
                    qty = len(gr_st.predecessors(node))
                    if qty > settings.CACHE_PARAMETER_MIN_USAGE:
                        hdf.cache_param_list.append(node)
            logging.info("HDF set to cache parameters: %s",
                         hdf.cache_param_list)

        # derive parameters
        ktis, kpvs, sections, approaches, flight_attrs = \
            derive_parameters(hdf, node_mgr, process_order, params=initial, force=force)

        # geo locate KTIs
        ktis = geo_locate(hdf, ktis)
        ktis = _timestamp(segment_info['Start Datetime'], ktis)

        # geo locate KPVs
        kpvs = geo_locate(hdf, kpvs)
        kpvs = _timestamp(segment_info['Start Datetime'], kpvs)

        # Store version of FlightDataAnalyser
        hdf.analysis_version = __version__
        # Store dependency tree
        hdf.dependency_tree = json_graph.dumps(gr_st)
        # Store aircraft info
        hdf.set_attr('aircraft_info', aircraft_info)
        hdf.set_attr('achieved_flight_record', achieved_flight_record)

    return {
        'flight': flight_attrs,
        'kti': ktis,
        'kpv': kpvs,
        'approach': approaches,
        'phases': sections,
    }