def geo_locate(hdf, items):
    '''
    Translate KeyTimeInstance into GeoKeyTimeInstance namedtuples
    '''
    if 'Latitude Smoothed' not in hdf.valid_param_names() \
       or 'Longitude Smoothed' not in hdf.valid_param_names():
        logger.warning("Could not geo-locate as either 'Latitude Smoothed' or "
                       "'Longitude Smoothed' were not found within the hdf.")
        return items
    
    lat_hdf = hdf['Latitude Smoothed']
    lon_hdf = hdf['Longitude Smoothed']
    
    if (not lat_hdf.array.count()) or (not lon_hdf.array.count()):
        logger.warning("Could not geo-locate as either 'Latitude Smoothed' or "
                       "'Longitude Smoothed' have no unmasked values.")
        return items
    
    lat_pos = derived_param_from_hdf(lat_hdf)
    lon_pos = derived_param_from_hdf(lon_hdf)
    
    # We want to place start of flight and end of flight markers at the ends
    # of the data which may extend more than REPAIR_DURATION seconds beyond
    # the end of the valid data. Hence by setting this to None and
    # extrapolate=True we achieve this goal.
    lat_pos.array = repair_mask(lat_pos.array, repair_duration=None, extrapolate=True)
    lon_pos.array = repair_mask(lon_pos.array, repair_duration=None, extrapolate=True)
    
    for item in itertools.chain.from_iterable(six.itervalues(items)):
        item.latitude = lat_pos.at(item.index) or None
        item.longitude = lon_pos.at(item.index) or None
    return items
def geo_locate(hdf, items):
    '''
    Translate KeyTimeInstance into GeoKeyTimeInstance namedtuples
    '''
    if 'Latitude Smoothed' not in hdf.valid_param_names() \
       or 'Longitude Smoothed' not in hdf.valid_param_names():
        logger.warning("Could not geo-locate as either 'Latitude Smoothed' or "
                       "'Longitude Smoothed' were not found within the hdf.")
        return items
    
    lat_hdf = hdf['Latitude Smoothed']
    lon_hdf = hdf['Longitude Smoothed']
    
    if (not lat_hdf.array.count()) or (not lon_hdf.array.count()):
        logger.warning("Could not geo-locate as either 'Latitude Smoothed' or "
                       "'Longitude Smoothed' have no unmasked values.")
        return items
    
    lat_pos = derived_param_from_hdf(lat_hdf)
    lon_pos = derived_param_from_hdf(lon_hdf)
    
    # We want to place start of flight and end of flight markers at the ends
    # of the data which may extend more than REPAIR_DURATION seconds beyond
    # the end of the valid data. Hence by setting this to None and
    # extrapolate=True we achieve this goal.
    lat_pos.array = repair_mask(lat_pos.array, repair_duration=None, extrapolate=True)
    lon_pos.array = repair_mask(lon_pos.array, repair_duration=None, extrapolate=True)
    
    for item in itertools.chain.from_iterable(items.itervalues()):
        item.latitude = lat_pos.at(item.index) or None
        item.longitude = lon_pos.at(item.index) or None
    return items
 def load_from_hdf5(self, flight_dict, required_series=[]): #all_hdfseries, 
     '''load data from an hdf flight data file
         flight_series is a dictionary with fields:  'filepath', 'aircraft_info', 'repo'        
     '''
     # look up aircraft info by tail number
     self.filepath = flight_dict['filepath']
     self.aircraft_info  = flight_dict['aircraft_info']
             
     with hdfaccess.file.hdf_file(self.filepath) as ff:
         self.duration = ff.duration
         self.start_datetime = ff.start_datetime
         self.start_epoch = ff.hdf.attrs.get('start_timestamp') #keep as epoch
         self.superframe_present = ff.superframe_present 
         self.hdfaccess_version =  ff.hdfaccess_version 
         self.reliable_frame_counter = ff.reliable_frame_counter 
         
         # for profiles we can choose to load only selected series   
         if required_series==[]:
              series_to_load = ff.keys() #all_hdfseries
         else:
             series_to_load = frozenset(required_series).intersection( frozenset(ff.keys()) )
                         
         for k in series_to_load:
             self.series[k] =  ff.get_param(k, valid_only=False)
             if  self.series[k].lfl==True:  
                 self.lfl_params.append(k)                     
             is_invalid =self.series[k].invalid
             if is_invalid is None or is_invalid==False:
                 param_node = node.derived_param_from_hdf( self.series[k] )
                 param_node.units = self.series[k].units #better
                 param_node.lfl = self.series[k].lfl
                 param_node.data_type =self.series[k].data_type 
                 self.parameters[k] = param_node
def geo_locate(hdf, items):
    '''
    Translate KeyTimeInstance into GeoKeyTimeInstance namedtuples
    '''
    if 'Latitude Smoothed' not in hdf.valid_param_names() \
       or 'Longitude Smoothed' not in hdf.valid_param_names():
        logger.warning("Could not geo-locate as either 'Latitude Smoothed' or "
                       "'Longitude Smoothed' were not found within the hdf.")
        return items

    lat_pos = derived_param_from_hdf(hdf['Latitude Smoothed'])
    lon_pos = derived_param_from_hdf(hdf['Longitude Smoothed'])
    lat_rep = repair_mask(lat_pos, extrapolate=True)
    lon_rep = repair_mask(lon_pos, extrapolate=True)
    for item in items:
        item.latitude = lat_rep.at(item.index) or None
        item.longitude = lon_rep.at(item.index) or None
    return items
def geo_locate(hdf, items):
    '''
    Translate KeyTimeInstance into GeoKeyTimeInstance namedtuples
    '''
    if 'Latitude Smoothed' not in hdf.valid_param_names() \
       or 'Longitude Smoothed' not in hdf.valid_param_names():
        logger.warning("Could not geo-locate as either 'Latitude Smoothed' or "
                       "'Longitude Smoothed' were not found within the hdf.")
        return items

    lat_pos = derived_param_from_hdf(hdf['Latitude Smoothed'])
    lon_pos = derived_param_from_hdf(hdf['Longitude Smoothed'])
    lat_pos.array = repair_mask(lat_pos.array, extrapolate=True)
    lon_pos.array = repair_mask(lon_pos.array, extrapolate=True)
    for item in items:
        item.latitude = lat_pos.at(item.index) or None
        item.longitude = lon_pos.at(item.index) or None
    return items
Esempio n. 6
0
def build_ordered_dependencies(node_class, node_mgr, params, hdf):
    # build ordered dependencies  
    deps = []
    node_deps = node_class.get_dependency_names()
    for dep_name in node_deps:
        if dep_name in params:  # already calculated KPV/KTI/Phase
            deps.append(params[dep_name])
        elif node_mgr.get_attribute(dep_name) is not None:
            deps.append(node_mgr.get_attribute(dep_name))
        elif dep_name in node_mgr.hdf_keys:  # KC: duplicating work here? <<<<<<<<<<<<<<<<
            # LFL/Derived parameter. Cast LFL param as derived param so we have get_aligned()
            try:
                dp = derived_param_from_hdf(hdf.get_param(dep_name, valid_only=True))
            except KeyError:  # Parameter is invalid.
                dp = None
            deps.append(dp)
        else:  # dependency not available
            deps.append(None)
    if all([d is None for d in deps]):
        raise RuntimeError("No dependencies available - Node: %s" % node_class.__name__)
    return deps
    def load_from_hdf5(self, flight_dict, required_series=[]):  #all_hdfseries,
        '''load data from an hdf flight data file
            flight_series is a dictionary with fields:  'filepath', 'aircraft_info', 'repo'        
        '''
        # look up aircraft info by tail number
        self.filepath = flight_dict['filepath']
        self.aircraft_info = flight_dict['aircraft_info']
        if not os.path.exists(self.filepath):
            logger.warning('cannot find: ' + self.filepath)
            #pdb.set_trace()

        with hdfaccess.file.hdf_file(self.filepath) as ff:
            self.duration = ff.duration
            self.start_datetime = ff.start_datetime
            self.start_epoch = ff.hdf.attrs.get(
                'start_timestamp')  #keep as epoch
            self.superframe_present = ff.superframe_present
            self.hdfaccess_version = ff.hdfaccess_version
            self.reliable_frame_counter = ff.reliable_frame_counter

            # for profiles we can choose to load only selected series
            if required_series == []:
                series_to_load = ff.keys()  #all_hdfseries
            else:
                series_to_load = frozenset(required_series).intersection(
                    frozenset(ff.keys()))

            for k in series_to_load:
                self.series[k] = ff.get_param(k, valid_only=False)
                if self.series[k].lfl == True:
                    self.lfl_params.append(k)
                is_invalid = self.series[k].invalid
                if is_invalid is None or is_invalid == False:
                    param_node = node.derived_param_from_hdf(self.series[k])
                    param_node.units = self.series[k].units  #better
                    param_node.lfl = self.series[k].lfl
                    param_node.data_type = self.series[k].data_type
                    self.parameters[k] = param_node
Esempio n. 8
0
def derive_parameters(hdf, node_mgr, process_order, params=None, force=False):
    '''
    Derives parameters in process_order. Dependencies are sourced via the
    node_mgr.

    :param hdf: Data file accessor used to get and save parameter data and
        attributes
    :type hdf: hdf_file
    :param node_mgr: Used to determine the type of node in the process_order
    :type node_mgr: NodeManager
    :param process_order: Parameter / Node class names in the required order to
        be processed
    :type process_order: list of strings
    '''
    if not params:
        params = {}
    # OPT: local lookup is faster than module-level (small).
    node_subclasses = NODE_SUBCLASSES

    # store all derived params that aren't masked arrays
    approaches = {}
    # duplicate storage, but maintaining types
    kpvs = {}
    ktis = {}
    # 'Node Name' : node()  pass in node.get_accessor()
    sections = {}
    flight_attrs = {}
    # cache of nodes to avoid repeated array alignment
    cache = {} if NODE_CACHE else None
    duration = hdf.duration

    for param_name in process_order:
        if param_name in node_mgr.hdf_keys:
            continue

        elif param_name in params:
            node = params[param_name]
            # populate output already at 1Hz
            if node.node_type is KeyPointValueNode:
                kpvs[param_name] = list(node)
            elif node.node_type is KeyTimeInstanceNode:
                ktis[param_name] = list(node)
            elif node.node_type is FlightAttributeNode:
                flight_attrs[param_name] = [Attribute(node.name, node.value)]
            elif node.node_type is SectionNode:
                sections[param_name] = list(node)
            # DerivedParameterNodes are not supported in initial data.
            continue

        elif node_mgr.get_attribute(param_name) is not None:
            # add attribute to dictionary of available params
            ###params[param_name] = node_mgr.get_attribute(param_name)
            #TODO: optimise with only one call to get_attribute
            continue

        #NB raises KeyError if Node is "unknown"
        node_class = node_mgr.derived_nodes[param_name]

        # build ordered dependencies
        deps = []
        node_deps = node_class.get_dependency_names()
        for dep_name in node_deps:
            if dep_name in params:  # already calculated KPV/KTI/Phase
                deps.append(params[dep_name])
            elif node_mgr.get_attribute(dep_name) is not None:
                deps.append(node_mgr.get_attribute(dep_name))
            elif dep_name in node_mgr.hdf_keys:
                # LFL/Derived parameter
                # all parameters (LFL or other) need get_aligned which is
                # available on DerivedParameterNode
                try:
                    dp = derived_param_from_hdf(hdf.get_param(dep_name,
                                                              valid_only=True),
                                                cache=cache)
                except KeyError:
                    # Parameter is invalid.
                    dp = None
                deps.append(dp)
            else:  # dependency not available
                deps.append(None)
        if all([d is None for d in deps]):
            raise RuntimeError("No dependencies available - Nodes cannot "
                               "operate without ANY dependencies available! "
                               "Node: %s" % node_class.__name__)

        # initialise node
        node = node_class(cache=cache)
        # shhh, secret accessors for developing nodes in debug mode
        node._p = params
        node._h = hdf
        node._n = node_mgr
        logger.debug("Processing %s `%s`",
                     get_node_type(node, node_subclasses), param_name)
        # Derive the resulting value

        try:
            node = node.get_derived(deps)
        except:
            if not force:
                raise

        del node._p
        del node._h
        del node._n

        if node.node_type is KeyPointValueNode:
            params[param_name] = node

            aligned_kpvs = []
            for one_hz in node.get_aligned(P(frequency=1, offset=0)):
                if not (0 <= one_hz.index <= duration + 4):
                    raise IndexError(
                        "KPV '%s' index %.2f is not between 0 and %d" %
                        (one_hz.name, one_hz.index, duration))
                aligned_kpvs.append(one_hz)
            kpvs[param_name] = aligned_kpvs
        elif node.node_type is KeyTimeInstanceNode:
            params[param_name] = node

            aligned_ktis = []
            for one_hz in node.get_aligned(P(frequency=1, offset=0)):
                if not (0 <= one_hz.index <= duration + 4):
                    raise IndexError(
                        "KTI '%s' index %.2f is not between 0 and %d" %
                        (one_hz.name, one_hz.index, duration))
                aligned_ktis.append(one_hz)
            ktis[param_name] = aligned_ktis
        elif node.node_type is FlightAttributeNode:
            params[param_name] = node
            try:
                # only has one Attribute node, store as a list for consistency
                flight_attrs[param_name] = [Attribute(node.name, node.value)]
            except:
                logger.warning(
                    "Flight Attribute Node '%s' returned empty "
                    "handed.", param_name)
        elif issubclass(node.node_type, SectionNode):
            aligned_section = node.get_aligned(P(frequency=1, offset=0))
            for index, one_hz in enumerate(aligned_section):
                # SectionNodes allow slice starts and stops being None which
                # signifies the beginning and end of the data. To avoid
                # TypeErrors in subsequent derive methods which perform
                # arithmetic on section slice start and stops, replace with 0
                # or hdf.duration.
                fallback = lambda x, y: x if x is not None else y

                duration = fallback(duration, 0)

                start = fallback(one_hz.slice.start, 0)
                stop = fallback(one_hz.slice.stop, duration)
                start_edge = fallback(one_hz.start_edge, 0)
                stop_edge = fallback(one_hz.stop_edge, duration)

                slice_ = slice(start, stop)
                one_hz = Section(one_hz.name, slice_, start_edge, stop_edge)
                aligned_section[index] = one_hz

                if not (0 <= start <= duration and 0 <= stop <= duration + 4):
                    msg = "Section '%s' (%.2f, %.2f) not between 0 and %d"
                    raise IndexError(msg %
                                     (one_hz.name, start, stop, duration))
                if not 0 <= start_edge <= duration:
                    msg = "Section '%s' start_edge (%.2f) not between 0 and %d"
                    raise IndexError(msg % (one_hz.name, start_edge, duration))
                if not 0 <= stop_edge <= duration + 4:
                    msg = "Section '%s' stop_edge (%.2f) not between 0 and %d"
                    raise IndexError(msg % (one_hz.name, stop_edge, duration))
                #section_list.append(one_hz)
            params[param_name] = aligned_section
            sections[param_name] = list(aligned_section)
        elif issubclass(node.node_type, DerivedParameterNode):
            if duration:
                # check that the right number of nodes were returned Allow a
                # small tolerance. For example if duration in seconds is 2822,
                # then there will be an array length of  1411 at 0.5Hz and 706
                # at 0.25Hz (rounded upwards). If we combine two 0.25Hz
                # parameters then we will have an array length of 1412.
                expected_length = duration * node.frequency
                if node.array is None or (force and len(node.array) == 0):
                    logger.warning(
                        "No array set; creating a fully masked "
                        "array for %s", param_name)
                    array_length = expected_length
                    # Where a parameter is wholly masked, we fill the HDF
                    # file with masked zeros to maintain structure.
                    node.array = \
                        np_ma_masked_zeros(expected_length)
                else:
                    array_length = len(node.array)
                length_diff = array_length - expected_length
                if length_diff == 0:
                    pass
                elif 0 < length_diff < 5:
                    logger.warning(
                        "Cutting excess data for parameter '%s'. "
                        "Expected length was '%s' while resulting "
                        "array length was '%s'.", param_name, expected_length,
                        len(node.array))
                    node.array = node.array[:expected_length]
                else:
                    raise ValueError(
                        "Array length mismatch for parameter "
                        "'%s'. Expected '%s', resulting array "
                        "length '%s'." %
                        (param_name, expected_length, array_length))

            hdf.set_param(node)
            # Keep hdf_keys up to date.
            node_mgr.hdf_keys.append(param_name)
        elif issubclass(node.node_type, ApproachNode):
            aligned_approach = node.get_aligned(P(frequency=1, offset=0))
            for approach in aligned_approach:
                # Does not allow slice start or stops to be None.
                valid_turnoff = (not approach.turnoff
                                 or (0 <= approach.turnoff <= duration))
                valid_slice = ((0 <= approach.slice.start <= duration)
                               and (0 <= approach.slice.stop <= duration))
                valid_gs_est = (not approach.gs_est or
                                ((0 <= approach.gs_est.start <= duration) and
                                 (0 <= approach.gs_est.stop <= duration)))
                valid_loc_est = (not approach.loc_est or
                                 ((0 <= approach.loc_est.start <= duration) and
                                  (0 <= approach.loc_est.stop <= duration)))
                if not all(
                    [valid_turnoff, valid_slice, valid_gs_est, valid_loc_est]):
                    raise ValueError('ApproachItem contains index outside of '
                                     'flight data: %s' % approach)
            params[param_name] = aligned_approach
            approaches[param_name] = list(aligned_approach)
        else:
            raise NotImplementedError("Unknown Type %s" % node.__class__)
        continue
    return ktis, kpvs, sections, approaches, flight_attrs
def derive_parameters(hdf, node_mgr, process_order, params=None, force=False):
    '''
    Derives parameters in process_order. Dependencies are sourced via the
    node_mgr.

    :param hdf: Data file accessor used to get and save parameter data and
        attributes
    :type hdf: hdf_file
    :param node_mgr: Used to determine the type of node in the process_order
    :type node_mgr: NodeManager
    :param process_order: Parameter / Node class names in the required order to
        be processed
    :type process_order: list of strings
    '''
    if not params:
        params = {}
    # OPT: local lookup is faster than module-level (small).
    node_subclasses = NODE_SUBCLASSES
    
    # store all derived params that aren't masked arrays
    approaches = {}
    # duplicate storage, but maintaining types
    kpvs = {}
    ktis = {}
    # 'Node Name' : node()  pass in node.get_accessor()
    sections = {}
    flight_attrs = {}
    duration = hdf.duration

    for param_name in process_order:
        if param_name in node_mgr.hdf_keys:
            continue
        
        elif param_name in params:
            node = params[param_name]
            # populate output already at 1Hz
            if node.node_type is KeyPointValueNode:
                kpvs[param_name] = list(node)
            elif node.node_type is KeyTimeInstanceNode:
                ktis[param_name] = list(node)
            elif node.node_type is FlightAttributeNode:
                flight_attrs[param_name] = [Attribute(node.name, node.value)]
            elif node.node_type is SectionNode:
                sections[param_name] = list(node)
            # DerivedParameterNodes are not supported in initial data.
            continue

        elif node_mgr.get_attribute(param_name) is not None:
            # add attribute to dictionary of available params
            ###params[param_name] = node_mgr.get_attribute(param_name)
            #TODO: optimise with only one call to get_attribute
            continue

        #NB raises KeyError if Node is "unknown"
        node_class = node_mgr.derived_nodes[param_name]

        # build ordered dependencies
        deps = []
        node_deps = node_class.get_dependency_names()
        for dep_name in node_deps:
            if dep_name in params:  # already calculated KPV/KTI/Phase
                deps.append(params[dep_name])
            elif node_mgr.get_attribute(dep_name) is not None:
                deps.append(node_mgr.get_attribute(dep_name))
            elif dep_name in node_mgr.hdf_keys:
                # LFL/Derived parameter
                # all parameters (LFL or other) need get_aligned which is
                # available on DerivedParameterNode
                try:
                    dp = derived_param_from_hdf(hdf.get_param(
                        dep_name, valid_only=True))
                except KeyError:
                    # Parameter is invalid.
                    dp = None
                deps.append(dp)
            else:  # dependency not available
                deps.append(None)
        if all([d is None for d in deps]):
            raise RuntimeError(
                "No dependencies available - Nodes cannot "
                "operate without ANY dependencies available! "
                "Node: %s" % node_class.__name__)

        # initialise node
        node = node_class()
        # shhh, secret accessors for developing nodes in debug mode
        node._p = params
        node._h = hdf
        node._n = node_mgr
        logger.info("Processing %s `%s`", get_node_type(node, node_subclasses), param_name)
        # Derive the resulting value
        
        try:
            node = node.get_derived(deps)
        except:
            if not force:
                raise
        
        del node._p
        del node._h
        del node._n

        if node.node_type is KeyPointValueNode:
            params[param_name] = node
            
            aligned_kpvs = []
            for one_hz in node.get_aligned(P(frequency=1, offset=0)):
                if not (0 <= one_hz.index <= duration+4):
                    raise IndexError(
                        "KPV '%s' index %.2f is not between 0 and %d" %
                        (one_hz.name, one_hz.index, duration))
                aligned_kpvs.append(one_hz)
            kpvs[param_name] = aligned_kpvs
        elif node.node_type is KeyTimeInstanceNode:
            params[param_name] = node
            
            aligned_ktis = []
            for one_hz in node.get_aligned(P(frequency=1, offset=0)):
                if not (0 <= one_hz.index <= duration+4):
                    raise IndexError(
                        "KTI '%s' index %.2f is not between 0 and %d" %
                        (one_hz.name, one_hz.index, duration))
                aligned_ktis.append(one_hz)
            ktis[param_name] = aligned_ktis
        elif node.node_type is FlightAttributeNode:
            params[param_name] = node
            try:
                # only has one Attribute node, store as a list for consistency
                flight_attrs[param_name] = [Attribute(node.name, node.value)]
            except:
                logger.warning("Flight Attribute Node '%s' returned empty "
                               "handed.", param_name)
        elif issubclass(node.node_type, SectionNode):
            aligned_section = node.get_aligned(P(frequency=1, offset=0))
            for index, one_hz in enumerate(aligned_section):
                # SectionNodes allow slice starts and stops being None which
                # signifies the beginning and end of the data. To avoid
                # TypeErrors in subsequent derive methods which perform
                # arithmetic on section slice start and stops, replace with 0
                # or hdf.duration.
                fallback = lambda x, y: x if x is not None else y

                duration = fallback(duration, 0)

                start = fallback(one_hz.slice.start, 0)
                stop = fallback(one_hz.slice.stop, duration)
                start_edge = fallback(one_hz.start_edge, 0)
                stop_edge = fallback(one_hz.stop_edge, duration)

                slice_ = slice(start, stop)
                one_hz = Section(one_hz.name, slice_, start_edge, stop_edge)
                aligned_section[index] = one_hz

                if not (0 <= start <= duration and 0 <= stop <= duration + 4):
                    msg = "Section '%s' (%.2f, %.2f) not between 0 and %d"
                    raise IndexError(
                        msg % (one_hz.name, start, stop, duration))
                if not 0 <= start_edge <= duration:
                    msg = "Section '%s' start_edge (%.2f) not between 0 and %d"
                    raise IndexError(msg % (one_hz.name, start_edge, duration))
                if not 0 <= stop_edge <= duration + 4:
                    msg = "Section '%s' stop_edge (%.2f) not between 0 and %d"
                    raise IndexError(msg % (one_hz.name, stop_edge, duration))
                #section_list.append(one_hz)
            params[param_name] = aligned_section
            sections[param_name] = list(aligned_section)
        elif issubclass(node.node_type, DerivedParameterNode):
            if duration:
                # check that the right number of nodes were returned Allow a
                # small tolerance. For example if duration in seconds is 2822,
                # then there will be an array length of  1411 at 0.5Hz and 706
                # at 0.25Hz (rounded upwards). If we combine two 0.25Hz
                # parameters then we will have an array length of 1412.
                expected_length = duration * node.frequency
                if node.array is None or (force and len(node.array) == 0):
                    logger.warning("No array set; creating a fully masked "
                                   "array for %s", param_name)
                    array_length = expected_length
                    # Where a parameter is wholly masked, we fill the HDF
                    # file with masked zeros to maintain structure.
                    node.array = \
                        np_ma_masked_zeros(expected_length)
                else:
                    array_length = len(node.array)
                length_diff = array_length - expected_length
                if length_diff == 0:
                    pass
                elif 0 < length_diff < 5:
                    logger.warning("Cutting excess data for parameter '%s'. "
                                   "Expected length was '%s' while resulting "
                                   "array length was '%s'.", param_name,
                                   expected_length, len(node.array))
                    node.array = node.array[:expected_length]
                else:
                    raise ValueError("Array length mismatch for parameter "
                                     "'%s'. Expected '%s', resulting array "
                                     "length '%s'." % (param_name,
                                                       expected_length,
                                                       array_length))

            hdf.set_param(node)
            # Keep hdf_keys up to date.
            node_mgr.hdf_keys.append(param_name)
        elif issubclass(node.node_type, ApproachNode):
            aligned_approach = node.get_aligned(P(frequency=1, offset=0))
            for approach in aligned_approach:
                # Does not allow slice start or stops to be None.
                valid_turnoff = (not approach.turnoff or
                                 (0 <= approach.turnoff <= duration))
                valid_slice = ((0 <= approach.slice.start <= duration) and
                               (0 <= approach.slice.stop <= duration))
                valid_gs_est = (not approach.gs_est or
                                ((0 <= approach.gs_est.start <= duration) and
                                 (0 <= approach.gs_est.stop <= duration)))
                valid_loc_est = (not approach.loc_est or
                                 ((0 <= approach.loc_est.start <= duration) and
                                  (0 <= approach.loc_est.stop <= duration)))
                if not all([valid_turnoff, valid_slice, valid_gs_est,
                            valid_loc_est]):
                    raise ValueError('ApproachItem contains index outside of '
                                     'flight data: %s' % approach)
            params[param_name] = aligned_approach
            approaches[param_name] = list(aligned_approach)
        else:
            raise NotImplementedError("Unknown Type %s" % node.__class__)
        continue
    return ktis, kpvs, sections, approaches, flight_attrs
Esempio n. 10
0
def derive_parameters_mitre(hdf, node_mgr, process_order, precomputed_parameters={}):
    '''
    Derives the parameter values and if limits are available, applies
    parameter validation upon each param before storing the resulting masked
    array back into the hdf file.
    
    :param hdf: Data file accessor used to get and save parameter data and attributes
    :type hdf: hdf_file
    :param node_mgr: Used to determine the type of node in the process_order
    :type node_mgr: NodeManager
    :param process_order: Parameter / Node class names in the required order to be processed
    :type process_order: list of strings
    '''
    params    = precomputed_parameters   # dictionary of derived params that aren't masked arrays

    approach_list = ApproachNode(restrict_names=False)
    kpv_list = KeyPointValueNode(restrict_names=False) # duplicate storage, but maintaining types
    kti_list = KeyTimeInstanceNode(restrict_names=False)
    section_list = SectionNode()  # 'Node Name' : node()  pass in node.get_accessor()
    flight_attrs = []
    duration = hdf.duration
    
    for param_name in process_order:
        if param_name in node_mgr.hdf_keys:
            logger.debug('  derive_: hdf '+param_name)            
            continue        
        elif node_mgr.get_attribute(param_name) is not None:
            logger.debug('  derive_: get_attribute '+param_name)
            continue
        elif param_name in params:  # already calculated KPV/KTI/Phase ***********************NEW
            logger.debug('  derive_parameters: re-using '+param_name)
            continue

        logger.debug('  derive_: computing '+param_name)        
        node_class = node_mgr.derived_nodes[param_name]  #NB raises KeyError if Node is "unknown"
        
        # build ordered dependencies
        deps = []
        node_deps = node_class.get_dependency_names()
        for dep_name in node_deps:
            if dep_name in params:  # already calculated KPV/KTI/Phase
                deps.append(params[dep_name])
            elif node_mgr.get_attribute(dep_name) is not None:
                deps.append(node_mgr.get_attribute(dep_name))
            elif dep_name in node_mgr.hdf_keys:  
                # LFL/Derived parameter
                # all parameters (LFL or other) need get_aligned which is
                # available on DerivedParameterNode
                try:
                    dp = derived_param_from_hdf(hdf.get_param(dep_name,
                                                              valid_only=True))
                except KeyError:
                    # Parameter is invalid.
                    dp = None
                deps.append(dp)
            else:  # dependency not available
                deps.append(None)
        if all([d is None for d in deps]):
            raise RuntimeError("No dependencies available - Nodes cannot "
                               "operate without ANY dependencies available! "
                               "Node: %s" % node_class.__name__)

        # initialise node
        node = node_class()
        logger.info("Processing parameter %s", param_name)
        # Derive the resulting value

        result = node.get_derived(deps)

        if node.node_type is KeyPointValueNode:
            #Q: track node instead of result here??
            params[param_name] = result
            for one_hz in result.get_aligned(P(frequency=1, offset=0)):
                if not (0 <= one_hz.index <= duration):
                    raise IndexError(
                        "KPV '%s' index %.2f is not between 0 and %d" %
                        (one_hz.name, one_hz.index, duration))
                kpv_list.append(one_hz)
        elif node.node_type is KeyTimeInstanceNode:
            params[param_name] = result
            for one_hz in result.get_aligned(P(frequency=1, offset=0)):
                if not (0 <= one_hz.index <= duration):
                    raise IndexError(
                        "KTI '%s' index %.2f is not between 0 and %d" %
                        (one_hz.name, one_hz.index, duration))
                kti_list.append(one_hz)
        elif node.node_type is FlightAttributeNode:
            params[param_name] = result
            try:
                flight_attrs.append(Attribute(result.name, result.value)) # only has one Attribute result
            except:
                logger.warning("Flight Attribute Node '%s' returned empty "
                               "handed.", param_name)
        elif issubclass(node.node_type, SectionNode):
            aligned_section = result.get_aligned(P(frequency=1, offset=0))
            for index, one_hz in enumerate(aligned_section):
                # SectionNodes allow slice starts and stops being None which
                # signifies the beginning and end of the data. To avoid TypeErrors
                # in subsequent derive methods which perform arithmetic on section
                # slice start and stops, replace with 0 or hdf.duration.
                fallback = lambda x, y: x if x is not None else y

                duration = fallback(duration, 0)

                start = fallback(one_hz.slice.start, 0)
                stop = fallback(one_hz.slice.stop, duration)
                start_edge = fallback(one_hz.start_edge, 0)
                stop_edge = fallback(one_hz.stop_edge, duration)

                slice_ = slice(start, stop)
                one_hz = Section(one_hz.name, slice_, start_edge, stop_edge)
                aligned_section[index] = one_hz
                
                if not (0 <= start <= duration and 0 <= stop <= duration + 1):
                    msg = "Section '%s' (%.2f, %.2f) not between 0 and %d"
                    raise IndexError(msg % (one_hz.name, start, stop, duration))
                if not 0 <= start_edge <= duration:
                    msg = "Section '%s' start_edge (%.2f) not between 0 and %d"
                    raise IndexError(msg % (one_hz.name, start_edge, duration))
                if not 0 <= stop_edge <= duration + 1:
                    msg = "Section '%s' stop_edge (%.2f) not between 0 and %d"
                    raise IndexError(msg % (one_hz.name, stop_edge, duration))
                section_list.append(one_hz)
            params[param_name] = aligned_section
        elif issubclass(node.node_type, DerivedParameterNode):
            if duration:
                # check that the right number of results were returned
                # Allow a small tolerance. For example if duration in seconds
                # is 2822, then there will be an array length of  1411 at 0.5Hz and 706
                # at 0.25Hz (rounded upwards). If we combine two 0.25Hz
                # parameters then we will have an array length of 1412.
                expected_length = duration * result.frequency
                if result.array is None:
                    logger.warning("No array set; creating a fully masked array for %s", param_name)
                    array_length = expected_length
                    # Where a parameter is wholly masked, we fill the HDF
                    # file with masked zeros to maintain structure.
                    result.array = \
                        np_ma_masked_zeros_like(np.ma.arange(expected_length))
                else:
                    array_length = len(result.array)
                length_diff = array_length - expected_length
                if length_diff == 0:
                    pass
                elif 0 < length_diff < 5:
                    logger.warning("Cutting excess data for parameter '%s'. "
                                   "Expected length was '%s' while resulting "
                                   "array length was '%s'.", param_name,
                                   expected_length, len(result.array))
                    result.array = result.array[:expected_length]
                else:
                    raise ValueError("Array length mismatch for parameter "
                                     "'%s'. Expected '%s', resulting array "
                                     "length '%s'." % (param_name,
                                                       expected_length,
                                                       array_length))
            
            hdf.set_param(result)
            # Keep hdf_keys up to date.
            node_mgr.hdf_keys.append(param_name)
        elif issubclass(node.node_type, ApproachNode):
            aligned_approach = result.get_aligned(P(frequency=1, offset=0))
            for approach in aligned_approach:
                # Does not allow slice start or stops to be None.
                valid_turnoff = (not approach.turnoff or
                                 (0 <= approach.turnoff <= duration))
                valid_slice = ((0 <= approach.slice.start <= duration) and
                               (0 <= approach.slice.stop <= duration))
                valid_gs_est = (not approach.gs_est or
                                ((0 <= approach.gs_est.start <= duration) and
                                 (0 <= approach.gs_est.stop <= duration)))
                valid_loc_est = (not approach.loc_est or
                                 ((0 <= approach.loc_est.start <= duration) and
                                  (0 <= approach.loc_est.stop <= duration)))
                if not all([valid_turnoff, valid_slice, valid_gs_est,
                            valid_loc_est]):
                    raise ValueError('ApproachItem contains index outside of '
                                     'flight data: %s' % approach)
                approach_list.append(approach)
            params[param_name] = aligned_approach
        else:
            raise NotImplementedError("Unknown Type %s" % node.__class__)
        continue
    return kti_list, kpv_list, section_list, approach_list, flight_attrs, params
Esempio n. 11
0
def track_to_kml(hdf_path,
                 kti_list,
                 kpv_list,
                 approach_list,
                 plot_altitude=None,
                 dest_path=None):
    '''
    Plot results of process_flight onto a KML track.
    
    :param flight_attrs: List of Flight Attributes
    :type flight_attrs: list
    :param plot_altitude: Name of Altitude parameter to use in KML
    :type plot_altitude: String
    '''
    one_hz = Parameter()
    kml = simplekml.Kml()
    with hdf_file(hdf_path) as hdf:
        # Latitude param, Longitude param, track name, colour
        coord_params = (
            {
                'lat': 'Latitude Smoothed',
                'lon': 'Longitude Smoothed',
                'track': 'Smoothed',
                'colour': 'ff7fff7f'
            },
            {
                'lat': 'Latitude Prepared',
                'lon': 'Longitude Prepared',
                'track': 'Prepared',
                'colour': 'A11EB3'
            },
            {
                'lat': 'Latitude',
                'lon': 'Longitude',
                'track': 'Recorded',
                'colour': 'ff0000ff'
            },
            {
                'lat': 'Latitude (Coarse)',
                'lon': 'Longitude (Coarse)',
                'track': 'Coarse',
                'colour': 'ff0000ff'
            },
        )
        altitude_absolute_params = ('Altitude QNH', 'Altitude STD',
                                    'Altitude AAL')
        altitude_relative_params = ('Altitude Radio', )
        # Check latitude and longitude pair exist.
        if not any(c['lat'] in hdf and c['lon'] in hdf for c in coord_params):
            logger.error(
                "Cannot write track as coordinate paarmeters not in hdf")
            return False
        # Choose best altitude parameter if not specified.
        if not plot_altitude:
            altitude_params = itertools.chain(altitude_absolute_params,
                                              altitude_relative_params)
            try:
                plot_altitude = next(p for p in altitude_params if p in hdf)
            except StopIteration:
                logger.warning("Disabling altitude on KML plot as it is "
                               "unavailable.")
        # Get altitude param from hdf.
        if plot_altitude:
            alt = derived_param_from_hdf(
                hdf[plot_altitude]).get_aligned(one_hz)
            alt.array = repair_mask(alt.array,
                                    frequency=alt.frequency,
                                    repair_duration=None) / METRES_TO_FEET
        else:
            alt = None

        if plot_altitude in altitude_absolute_params:
            altitude_mode = simplekml.constants.AltitudeMode.absolute
        elif plot_altitude in altitude_relative_params:
            altitude_mode = simplekml.constants.AltitudeMode.relativetoground
        else:
            altitude_mode = simplekml.constants.AltitudeMode.clamptoground

        ## Get best latitude and longitude parameters.
        best_lat = None
        best_lon = None

        for coord_config in coord_params:
            lat_name = coord_config['lat']
            lon_name = coord_config['lon']
            if not lat_name in hdf or not lon_name in hdf:
                continue
            lat = hdf[lat_name]
            lon = hdf[lon_name]
            best = not best_lat or not best_lon
            add_track(kml,
                      coord_config['track'],
                      lat,
                      lon,
                      coord_config['colour'],
                      alt_param=alt,
                      alt_mode=altitude_mode,
                      visible=best)
            add_track(kml,
                      coord_config['track'] + ' On Ground',
                      lat,
                      lon,
                      coord_config['colour'],
                      visible=best)
            if best:
                best_lat = derived_param_from_hdf(lat).get_aligned(one_hz)
                best_lon = derived_param_from_hdf(lon).get_aligned(one_hz)

    # Add KTIs.
    for kti in kti_list:
        kti_point_values = {'name': kti.name}
        if kti.name in SKIP_KTIS:
            continue

        altitude = alt.at(kti.index) if plot_altitude else None
        kti_point_values['altitudemode'] = altitude_mode
        if altitude:
            kti_point_values['coords'] = ((kti.longitude, kti.latitude,
                                           altitude), )
        else:
            kti_point_values['coords'] = ((kti.longitude, kti.latitude), )
        kml.newpoint(**kti_point_values)

    # Add KPVs.
    for kpv in kpv_list:

        # Trap kpvs with invalid latitude or longitude data (normally happens
        # at the start of the data where accelerometer offsets are declared,
        # and this avoids casting kpvs into the Atlantic.
        kpv_lat = best_lat.at(kpv.index)
        kpv_lon = best_lon.at(kpv.index)
        if kpv_lat == None or kpv_lon == None or \
           (kpv_lat == 0.0 and kpv_lon == 0.0):
            continue

        if kpv.name in SKIP_KPVS:
            continue

        style = simplekml.Style()
        style.iconstyle.color = simplekml.Color.red
        kpv_point_values = {'name': '%s (%.3f)' % (kpv.name, kpv.value)}
        altitude = alt.at(kpv.index) if plot_altitude else None
        kpv_point_values['altitudemode'] = altitude_mode
        if altitude:
            kpv_point_values['coords'] = ((kpv_lon, kpv_lat, altitude), )
        else:
            kpv_point_values['coords'] = ((kpv_lon, kpv_lat), )

        pnt = kml.newpoint(**kpv_point_values)
        pnt.style = style

    # Add approach centre lines.
    for app in approach_list:
        try:
            draw_centreline(kml, app.runway)
        except:
            pass

    if not dest_path:
        dest_path = hdf_path + ".kml"
    kml.save(dest_path)
    return dest_path
def track_to_kml(hdf_path, kti_list, kpv_list, approach_list,
                 plot_altitude=None, dest_path=None):
    '''
    Plot results of process_flight onto a KML track.

    :param flight_attrs: List of Flight Attributes
    :type flight_attrs: list
    :param plot_altitude: Name of Altitude parameter to use in KML
    :type plot_altitude: String
    '''
    one_hz = Parameter()
    kml = simplekml.Kml()
    with hdf_file(hdf_path) as hdf:
        # Latitude param, Longitude param, track name, colour
        coord_params = (
            {'lat': 'Latitude Smoothed',
             'lon': 'Longitude Smoothed',
             'track': 'Smoothed',
             'colour': 'ff7fff7f'},
            {'lat': 'Latitude Prepared',
             'lon': 'Longitude Prepared',
             'track': 'Prepared',
             'colour': 'ffA11EB3'},
            {'lat': 'Latitude',
             'lon': 'Longitude',
             'track': 'Recorded',
             'colour': 'ff0000ff'},
            {'lat': 'Latitude (Coarse)',
             'lon': 'Longitude (Coarse)',
             'track': 'Coarse',
             'colour': 'ff0000ff'},
        )
        altitude_absolute_params = ('Altitude Visualization With Ground Offset',
                                    'Altitude QNH', 'Altitude STD',
                                    'Altitude AAL')
        altitude_relative_params = ('Altitude Radio',)
        # Check latitude and longitude pair exist.
        if not any(c['lat'] in hdf and c['lon'] in hdf for c in coord_params):
            logger.error("Cannot write track as coordinate paarmeters not in hdf")
            return False
        # Choose best altitude parameter if not specified.
        if not plot_altitude:
            altitude_params = itertools.chain(altitude_absolute_params,
                                              altitude_relative_params)
            try:
                plot_altitude = next(p for p in altitude_params if p in hdf)
            except StopIteration:
                logger.warning("Disabling altitude on KML plot as it is "
                               "unavailable.")
        # Get altitude param from hdf.
        if plot_altitude and plot_altitude in hdf.keys():
            alt = derived_param_from_hdf(hdf[plot_altitude]).get_aligned(one_hz)
            alt.array = repair_mask(alt.array, frequency=alt.frequency, repair_duration=None)
            alt.array = ut.convert(alt.array, ut.FT, ut.METER)
        else:
            alt = None

        if plot_altitude in altitude_absolute_params:
            altitude_mode = simplekml.constants.AltitudeMode.absolute
        elif plot_altitude in altitude_relative_params:
            altitude_mode = simplekml.constants.AltitudeMode.relativetoground
        else:
            altitude_mode = simplekml.constants.AltitudeMode.clamptoground

        ## Get best latitude and longitude parameters.
        best_lat = None
        best_lon = None

        for coord_config in coord_params:
            lat_name = coord_config['lat']
            lon_name = coord_config['lon']
            if not lat_name in hdf or not lon_name in hdf:
                continue
            lat = hdf[lat_name]
            lon = hdf[lon_name]
            best = not best_lat or not best_lon
            add_track(kml, coord_config['track'], lat, lon,
                      coord_config['colour'], alt_param=alt,
                      alt_mode=altitude_mode,
                      visible=best)
            add_track(kml, coord_config['track'] + ' On Ground', lat, lon,
                      coord_config['colour'], visible=best)
            if best:
                best_lat = derived_param_from_hdf(lat).get_aligned(one_hz)
                best_lon = derived_param_from_hdf(lon).get_aligned(one_hz)

    # Add KTIs.
    for kti in kti_list:
        kti_point_values = {'name': kti.name}

        if not KEEP_KTIS and kti.name in SKIP_KTIS:
            continue
        elif len(KEEP_KTIS)>0 and (kti.name not in KEEP_KTIS):
            continue

        altitude = alt.at(kti.index) if alt else None
        kti_point_values['altitudemode'] = altitude_mode
        if altitude:
            kti_point_values['coords'] = ((kti.longitude, kti.latitude, altitude),)
        else:
            kti_point_values['coords'] = ((kti.longitude, kti.latitude),)
        kml.newpoint(**kti_point_values)

    # Add KPVs.
    for kpv in kpv_list:

        # Trap kpvs with invalid latitude or longitude data (normally happens
        # at the start of the data where accelerometer offsets are declared,
        # and this avoids casting kpvs into the Atlantic.
        kpv_lat = best_lat.at(kpv.index)
        kpv_lon = best_lon.at(kpv.index)
        if kpv_lat is None or kpv_lon is None or \
           (kpv_lat == 0.0 and kpv_lon == 0.0):
            continue

        if not KEEP_KPVS and kpv.name in SKIP_KPVS:
            continue
        elif len(KEEP_KPVS)>0 and (kpv.name not in KEEP_KPVS):
            continue

        style = simplekml.Style()
        style.iconstyle.color = simplekml.Color.red
        kpv_point_values = {'name': '%s (%.3f)' % (kpv.name, kpv.value)}
        altitude = alt.at(kpv.index) if alt else None
        kpv_point_values['altitudemode'] = altitude_mode
        if altitude:
            kpv_point_values['coords'] = ((kpv_lon, kpv_lat, altitude),)
        else:
            kpv_point_values['coords'] = ((kpv_lon, kpv_lat),)

        pnt = kml.newpoint(**kpv_point_values)
        pnt.style = style

    # Add approach centre lines.
    for app in approach_list:
        try:
            draw_centreline(kml, app.runway)
        except:
            pass

    if not dest_path:
        dest_path = hdf_path + ".kml"
    kml.save(dest_path)
    return dest_path
Esempio n. 13
0
def track_to_kml(hdf_path, kti_list, kpv_list, approach_list,
                 plot_altitude='Altitude QNH', dest_path=None):
    '''
    Plot results of process_flight onto a KML track.
    
    :param flight_attrs: List of Flight Attributes
    :type flight_attrs: list
    :param plot_altitude: Name of Altitude parameter to use in KML
    :type plot_altitude: String
    '''
    one_hz = Parameter()
    kml = simplekml.Kml()
    with hdf_file(hdf_path) as hdf:
        if 'Latitude Smoothed' not in hdf:
            return False
        if plot_altitude not in hdf:
            logger.warning("Disabling altitude on KML plot as it is unavailable.")
            plot_altitude = False
        if plot_altitude:
            alt = derived_param_from_hdf(hdf[plot_altitude]).get_aligned(one_hz)
            alt.array = repair_mask(alt.array, frequency=alt.frequency, repair_duration=None) / METRES_TO_FEET
        else:
            alt = None
            
        if plot_altitude in ['Altitude QNH', 'Altitude AAL', 'Altitude STD']:
            altitude_mode = simplekml.constants.AltitudeMode.absolute
        elif plot_altitude in ['Altitude Radio']:
            altitude_mode = simplekml.constants.AltitudeMode.relativetoground
        else:
            altitude_mode = simplekml.constants.AltitudeMode.clamptoground
        
        # TODO: align everything to 0 offset
        smooth_lat = derived_param_from_hdf(hdf['Latitude Smoothed']).get_aligned(one_hz)
        smooth_lon = derived_param_from_hdf(hdf['Longitude Smoothed']).get_aligned(one_hz)
        add_track(kml, 'Smoothed', smooth_lat, smooth_lon, 'ff7fff7f', 
                  alt_param=alt, alt_mode=altitude_mode)
        add_track(kml, 'Smoothed On Ground', smooth_lat, smooth_lon, 'ff7fff7f')        
    
        if 'Latitude Prepared' in hdf and 'Longitude Prepared' in hdf:
            lat = hdf['Latitude Prepared']
            lon = hdf['Longitude Prepared']
            add_track(kml, 'Prepared Track', lat, lon, 'A11EB3', visible=False)
        
        if 'Latitude' in hdf and 'Longitude' in hdf:
            lat_r = hdf['Latitude']
            lon_r = hdf['Longitude']
            # add RAW track default invisible
            add_track(kml, 'Recorded Track', lat_r, lon_r, 'ff0000ff', visible=False)

    for kti in kti_list:
        kti_point_values = {'name': kti.name}
        if kti.name in SKIP_KTIS:
            continue
        
        altitude = alt.at(kti.index) if plot_altitude else None
        kti_point_values['altitudemode'] = altitude_mode
        if altitude:
            kti_point_values['coords'] = ((kti.longitude, kti.latitude, altitude),)
        else:
            kti_point_values['coords'] = ((kti.longitude, kti.latitude),)
        kml.newpoint(**kti_point_values)
        
    
    for kpv in kpv_list:

        # Trap kpvs with invalid latitude or longitude data (normally happens
        # at the start of the data where accelerometer offsets are declared,
        # and this avoids casting kpvs into the Atlantic.
        kpv_lat = smooth_lat.at(kpv.index)
        kpv_lon = smooth_lon.at(kpv.index)
        if kpv_lat == None or kpv_lon == None or \
           (kpv_lat == 0.0 and kpv_lon == 0.0):
            continue

        if kpv.name in SKIP_KPVS:
            continue
        
        style = simplekml.Style()
        style.iconstyle.color = simplekml.Color.red
        kpv_point_values = {'name': '%s (%.3f)' % (kpv.name, kpv.value)}
        altitude = alt.at(kpv.index) if plot_altitude else None
        kpv_point_values['altitudemode'] = altitude_mode
        if altitude:
            kpv_point_values['coords'] = ((kpv_lon, kpv_lat, altitude),)
        else:
            kpv_point_values['coords'] = ((kpv_lon, kpv_lat),)
        
        pnt = kml.newpoint(**kpv_point_values)
        pnt.style = style
    
    for app in approach_list:
        try:
            draw_centreline(kml, app.runway)
        except:
            pass
                

    if not dest_path:
        dest_path = hdf_path + ".kml"
    kml.save(dest_path)
    return