def geo_locate(hdf, items): ''' Translate KeyTimeInstance into GeoKeyTimeInstance namedtuples ''' if 'Latitude Smoothed' not in hdf.valid_param_names() \ or 'Longitude Smoothed' not in hdf.valid_param_names(): logger.warning("Could not geo-locate as either 'Latitude Smoothed' or " "'Longitude Smoothed' were not found within the hdf.") return items lat_hdf = hdf['Latitude Smoothed'] lon_hdf = hdf['Longitude Smoothed'] if (not lat_hdf.array.count()) or (not lon_hdf.array.count()): logger.warning("Could not geo-locate as either 'Latitude Smoothed' or " "'Longitude Smoothed' have no unmasked values.") return items lat_pos = derived_param_from_hdf(lat_hdf) lon_pos = derived_param_from_hdf(lon_hdf) # We want to place start of flight and end of flight markers at the ends # of the data which may extend more than REPAIR_DURATION seconds beyond # the end of the valid data. Hence by setting this to None and # extrapolate=True we achieve this goal. lat_pos.array = repair_mask(lat_pos.array, repair_duration=None, extrapolate=True) lon_pos.array = repair_mask(lon_pos.array, repair_duration=None, extrapolate=True) for item in itertools.chain.from_iterable(six.itervalues(items)): item.latitude = lat_pos.at(item.index) or None item.longitude = lon_pos.at(item.index) or None return items
def geo_locate(hdf, items): ''' Translate KeyTimeInstance into GeoKeyTimeInstance namedtuples ''' if 'Latitude Smoothed' not in hdf.valid_param_names() \ or 'Longitude Smoothed' not in hdf.valid_param_names(): logger.warning("Could not geo-locate as either 'Latitude Smoothed' or " "'Longitude Smoothed' were not found within the hdf.") return items lat_hdf = hdf['Latitude Smoothed'] lon_hdf = hdf['Longitude Smoothed'] if (not lat_hdf.array.count()) or (not lon_hdf.array.count()): logger.warning("Could not geo-locate as either 'Latitude Smoothed' or " "'Longitude Smoothed' have no unmasked values.") return items lat_pos = derived_param_from_hdf(lat_hdf) lon_pos = derived_param_from_hdf(lon_hdf) # We want to place start of flight and end of flight markers at the ends # of the data which may extend more than REPAIR_DURATION seconds beyond # the end of the valid data. Hence by setting this to None and # extrapolate=True we achieve this goal. lat_pos.array = repair_mask(lat_pos.array, repair_duration=None, extrapolate=True) lon_pos.array = repair_mask(lon_pos.array, repair_duration=None, extrapolate=True) for item in itertools.chain.from_iterable(items.itervalues()): item.latitude = lat_pos.at(item.index) or None item.longitude = lon_pos.at(item.index) or None return items
def load_from_hdf5(self, flight_dict, required_series=[]): #all_hdfseries, '''load data from an hdf flight data file flight_series is a dictionary with fields: 'filepath', 'aircraft_info', 'repo' ''' # look up aircraft info by tail number self.filepath = flight_dict['filepath'] self.aircraft_info = flight_dict['aircraft_info'] with hdfaccess.file.hdf_file(self.filepath) as ff: self.duration = ff.duration self.start_datetime = ff.start_datetime self.start_epoch = ff.hdf.attrs.get('start_timestamp') #keep as epoch self.superframe_present = ff.superframe_present self.hdfaccess_version = ff.hdfaccess_version self.reliable_frame_counter = ff.reliable_frame_counter # for profiles we can choose to load only selected series if required_series==[]: series_to_load = ff.keys() #all_hdfseries else: series_to_load = frozenset(required_series).intersection( frozenset(ff.keys()) ) for k in series_to_load: self.series[k] = ff.get_param(k, valid_only=False) if self.series[k].lfl==True: self.lfl_params.append(k) is_invalid =self.series[k].invalid if is_invalid is None or is_invalid==False: param_node = node.derived_param_from_hdf( self.series[k] ) param_node.units = self.series[k].units #better param_node.lfl = self.series[k].lfl param_node.data_type =self.series[k].data_type self.parameters[k] = param_node
def geo_locate(hdf, items): ''' Translate KeyTimeInstance into GeoKeyTimeInstance namedtuples ''' if 'Latitude Smoothed' not in hdf.valid_param_names() \ or 'Longitude Smoothed' not in hdf.valid_param_names(): logger.warning("Could not geo-locate as either 'Latitude Smoothed' or " "'Longitude Smoothed' were not found within the hdf.") return items lat_pos = derived_param_from_hdf(hdf['Latitude Smoothed']) lon_pos = derived_param_from_hdf(hdf['Longitude Smoothed']) lat_rep = repair_mask(lat_pos, extrapolate=True) lon_rep = repair_mask(lon_pos, extrapolate=True) for item in items: item.latitude = lat_rep.at(item.index) or None item.longitude = lon_rep.at(item.index) or None return items
def geo_locate(hdf, items): ''' Translate KeyTimeInstance into GeoKeyTimeInstance namedtuples ''' if 'Latitude Smoothed' not in hdf.valid_param_names() \ or 'Longitude Smoothed' not in hdf.valid_param_names(): logger.warning("Could not geo-locate as either 'Latitude Smoothed' or " "'Longitude Smoothed' were not found within the hdf.") return items lat_pos = derived_param_from_hdf(hdf['Latitude Smoothed']) lon_pos = derived_param_from_hdf(hdf['Longitude Smoothed']) lat_pos.array = repair_mask(lat_pos.array, extrapolate=True) lon_pos.array = repair_mask(lon_pos.array, extrapolate=True) for item in items: item.latitude = lat_pos.at(item.index) or None item.longitude = lon_pos.at(item.index) or None return items
def build_ordered_dependencies(node_class, node_mgr, params, hdf): # build ordered dependencies deps = [] node_deps = node_class.get_dependency_names() for dep_name in node_deps: if dep_name in params: # already calculated KPV/KTI/Phase deps.append(params[dep_name]) elif node_mgr.get_attribute(dep_name) is not None: deps.append(node_mgr.get_attribute(dep_name)) elif dep_name in node_mgr.hdf_keys: # KC: duplicating work here? <<<<<<<<<<<<<<<< # LFL/Derived parameter. Cast LFL param as derived param so we have get_aligned() try: dp = derived_param_from_hdf(hdf.get_param(dep_name, valid_only=True)) except KeyError: # Parameter is invalid. dp = None deps.append(dp) else: # dependency not available deps.append(None) if all([d is None for d in deps]): raise RuntimeError("No dependencies available - Node: %s" % node_class.__name__) return deps
def load_from_hdf5(self, flight_dict, required_series=[]): #all_hdfseries, '''load data from an hdf flight data file flight_series is a dictionary with fields: 'filepath', 'aircraft_info', 'repo' ''' # look up aircraft info by tail number self.filepath = flight_dict['filepath'] self.aircraft_info = flight_dict['aircraft_info'] if not os.path.exists(self.filepath): logger.warning('cannot find: ' + self.filepath) #pdb.set_trace() with hdfaccess.file.hdf_file(self.filepath) as ff: self.duration = ff.duration self.start_datetime = ff.start_datetime self.start_epoch = ff.hdf.attrs.get( 'start_timestamp') #keep as epoch self.superframe_present = ff.superframe_present self.hdfaccess_version = ff.hdfaccess_version self.reliable_frame_counter = ff.reliable_frame_counter # for profiles we can choose to load only selected series if required_series == []: series_to_load = ff.keys() #all_hdfseries else: series_to_load = frozenset(required_series).intersection( frozenset(ff.keys())) for k in series_to_load: self.series[k] = ff.get_param(k, valid_only=False) if self.series[k].lfl == True: self.lfl_params.append(k) is_invalid = self.series[k].invalid if is_invalid is None or is_invalid == False: param_node = node.derived_param_from_hdf(self.series[k]) param_node.units = self.series[k].units #better param_node.lfl = self.series[k].lfl param_node.data_type = self.series[k].data_type self.parameters[k] = param_node
def derive_parameters(hdf, node_mgr, process_order, params=None, force=False): ''' Derives parameters in process_order. Dependencies are sourced via the node_mgr. :param hdf: Data file accessor used to get and save parameter data and attributes :type hdf: hdf_file :param node_mgr: Used to determine the type of node in the process_order :type node_mgr: NodeManager :param process_order: Parameter / Node class names in the required order to be processed :type process_order: list of strings ''' if not params: params = {} # OPT: local lookup is faster than module-level (small). node_subclasses = NODE_SUBCLASSES # store all derived params that aren't masked arrays approaches = {} # duplicate storage, but maintaining types kpvs = {} ktis = {} # 'Node Name' : node() pass in node.get_accessor() sections = {} flight_attrs = {} # cache of nodes to avoid repeated array alignment cache = {} if NODE_CACHE else None duration = hdf.duration for param_name in process_order: if param_name in node_mgr.hdf_keys: continue elif param_name in params: node = params[param_name] # populate output already at 1Hz if node.node_type is KeyPointValueNode: kpvs[param_name] = list(node) elif node.node_type is KeyTimeInstanceNode: ktis[param_name] = list(node) elif node.node_type is FlightAttributeNode: flight_attrs[param_name] = [Attribute(node.name, node.value)] elif node.node_type is SectionNode: sections[param_name] = list(node) # DerivedParameterNodes are not supported in initial data. continue elif node_mgr.get_attribute(param_name) is not None: # add attribute to dictionary of available params ###params[param_name] = node_mgr.get_attribute(param_name) #TODO: optimise with only one call to get_attribute continue #NB raises KeyError if Node is "unknown" node_class = node_mgr.derived_nodes[param_name] # build ordered dependencies deps = [] node_deps = node_class.get_dependency_names() for dep_name in node_deps: if dep_name in params: # already calculated KPV/KTI/Phase deps.append(params[dep_name]) elif node_mgr.get_attribute(dep_name) is not None: deps.append(node_mgr.get_attribute(dep_name)) elif dep_name in node_mgr.hdf_keys: # LFL/Derived parameter # all parameters (LFL or other) need get_aligned which is # available on DerivedParameterNode try: dp = derived_param_from_hdf(hdf.get_param(dep_name, valid_only=True), cache=cache) except KeyError: # Parameter is invalid. dp = None deps.append(dp) else: # dependency not available deps.append(None) if all([d is None for d in deps]): raise RuntimeError("No dependencies available - Nodes cannot " "operate without ANY dependencies available! " "Node: %s" % node_class.__name__) # initialise node node = node_class(cache=cache) # shhh, secret accessors for developing nodes in debug mode node._p = params node._h = hdf node._n = node_mgr logger.debug("Processing %s `%s`", get_node_type(node, node_subclasses), param_name) # Derive the resulting value try: node = node.get_derived(deps) except: if not force: raise del node._p del node._h del node._n if node.node_type is KeyPointValueNode: params[param_name] = node aligned_kpvs = [] for one_hz in node.get_aligned(P(frequency=1, offset=0)): if not (0 <= one_hz.index <= duration + 4): raise IndexError( "KPV '%s' index %.2f is not between 0 and %d" % (one_hz.name, one_hz.index, duration)) aligned_kpvs.append(one_hz) kpvs[param_name] = aligned_kpvs elif node.node_type is KeyTimeInstanceNode: params[param_name] = node aligned_ktis = [] for one_hz in node.get_aligned(P(frequency=1, offset=0)): if not (0 <= one_hz.index <= duration + 4): raise IndexError( "KTI '%s' index %.2f is not between 0 and %d" % (one_hz.name, one_hz.index, duration)) aligned_ktis.append(one_hz) ktis[param_name] = aligned_ktis elif node.node_type is FlightAttributeNode: params[param_name] = node try: # only has one Attribute node, store as a list for consistency flight_attrs[param_name] = [Attribute(node.name, node.value)] except: logger.warning( "Flight Attribute Node '%s' returned empty " "handed.", param_name) elif issubclass(node.node_type, SectionNode): aligned_section = node.get_aligned(P(frequency=1, offset=0)) for index, one_hz in enumerate(aligned_section): # SectionNodes allow slice starts and stops being None which # signifies the beginning and end of the data. To avoid # TypeErrors in subsequent derive methods which perform # arithmetic on section slice start and stops, replace with 0 # or hdf.duration. fallback = lambda x, y: x if x is not None else y duration = fallback(duration, 0) start = fallback(one_hz.slice.start, 0) stop = fallback(one_hz.slice.stop, duration) start_edge = fallback(one_hz.start_edge, 0) stop_edge = fallback(one_hz.stop_edge, duration) slice_ = slice(start, stop) one_hz = Section(one_hz.name, slice_, start_edge, stop_edge) aligned_section[index] = one_hz if not (0 <= start <= duration and 0 <= stop <= duration + 4): msg = "Section '%s' (%.2f, %.2f) not between 0 and %d" raise IndexError(msg % (one_hz.name, start, stop, duration)) if not 0 <= start_edge <= duration: msg = "Section '%s' start_edge (%.2f) not between 0 and %d" raise IndexError(msg % (one_hz.name, start_edge, duration)) if not 0 <= stop_edge <= duration + 4: msg = "Section '%s' stop_edge (%.2f) not between 0 and %d" raise IndexError(msg % (one_hz.name, stop_edge, duration)) #section_list.append(one_hz) params[param_name] = aligned_section sections[param_name] = list(aligned_section) elif issubclass(node.node_type, DerivedParameterNode): if duration: # check that the right number of nodes were returned Allow a # small tolerance. For example if duration in seconds is 2822, # then there will be an array length of 1411 at 0.5Hz and 706 # at 0.25Hz (rounded upwards). If we combine two 0.25Hz # parameters then we will have an array length of 1412. expected_length = duration * node.frequency if node.array is None or (force and len(node.array) == 0): logger.warning( "No array set; creating a fully masked " "array for %s", param_name) array_length = expected_length # Where a parameter is wholly masked, we fill the HDF # file with masked zeros to maintain structure. node.array = \ np_ma_masked_zeros(expected_length) else: array_length = len(node.array) length_diff = array_length - expected_length if length_diff == 0: pass elif 0 < length_diff < 5: logger.warning( "Cutting excess data for parameter '%s'. " "Expected length was '%s' while resulting " "array length was '%s'.", param_name, expected_length, len(node.array)) node.array = node.array[:expected_length] else: raise ValueError( "Array length mismatch for parameter " "'%s'. Expected '%s', resulting array " "length '%s'." % (param_name, expected_length, array_length)) hdf.set_param(node) # Keep hdf_keys up to date. node_mgr.hdf_keys.append(param_name) elif issubclass(node.node_type, ApproachNode): aligned_approach = node.get_aligned(P(frequency=1, offset=0)) for approach in aligned_approach: # Does not allow slice start or stops to be None. valid_turnoff = (not approach.turnoff or (0 <= approach.turnoff <= duration)) valid_slice = ((0 <= approach.slice.start <= duration) and (0 <= approach.slice.stop <= duration)) valid_gs_est = (not approach.gs_est or ((0 <= approach.gs_est.start <= duration) and (0 <= approach.gs_est.stop <= duration))) valid_loc_est = (not approach.loc_est or ((0 <= approach.loc_est.start <= duration) and (0 <= approach.loc_est.stop <= duration))) if not all( [valid_turnoff, valid_slice, valid_gs_est, valid_loc_est]): raise ValueError('ApproachItem contains index outside of ' 'flight data: %s' % approach) params[param_name] = aligned_approach approaches[param_name] = list(aligned_approach) else: raise NotImplementedError("Unknown Type %s" % node.__class__) continue return ktis, kpvs, sections, approaches, flight_attrs
def derive_parameters(hdf, node_mgr, process_order, params=None, force=False): ''' Derives parameters in process_order. Dependencies are sourced via the node_mgr. :param hdf: Data file accessor used to get and save parameter data and attributes :type hdf: hdf_file :param node_mgr: Used to determine the type of node in the process_order :type node_mgr: NodeManager :param process_order: Parameter / Node class names in the required order to be processed :type process_order: list of strings ''' if not params: params = {} # OPT: local lookup is faster than module-level (small). node_subclasses = NODE_SUBCLASSES # store all derived params that aren't masked arrays approaches = {} # duplicate storage, but maintaining types kpvs = {} ktis = {} # 'Node Name' : node() pass in node.get_accessor() sections = {} flight_attrs = {} duration = hdf.duration for param_name in process_order: if param_name in node_mgr.hdf_keys: continue elif param_name in params: node = params[param_name] # populate output already at 1Hz if node.node_type is KeyPointValueNode: kpvs[param_name] = list(node) elif node.node_type is KeyTimeInstanceNode: ktis[param_name] = list(node) elif node.node_type is FlightAttributeNode: flight_attrs[param_name] = [Attribute(node.name, node.value)] elif node.node_type is SectionNode: sections[param_name] = list(node) # DerivedParameterNodes are not supported in initial data. continue elif node_mgr.get_attribute(param_name) is not None: # add attribute to dictionary of available params ###params[param_name] = node_mgr.get_attribute(param_name) #TODO: optimise with only one call to get_attribute continue #NB raises KeyError if Node is "unknown" node_class = node_mgr.derived_nodes[param_name] # build ordered dependencies deps = [] node_deps = node_class.get_dependency_names() for dep_name in node_deps: if dep_name in params: # already calculated KPV/KTI/Phase deps.append(params[dep_name]) elif node_mgr.get_attribute(dep_name) is not None: deps.append(node_mgr.get_attribute(dep_name)) elif dep_name in node_mgr.hdf_keys: # LFL/Derived parameter # all parameters (LFL or other) need get_aligned which is # available on DerivedParameterNode try: dp = derived_param_from_hdf(hdf.get_param( dep_name, valid_only=True)) except KeyError: # Parameter is invalid. dp = None deps.append(dp) else: # dependency not available deps.append(None) if all([d is None for d in deps]): raise RuntimeError( "No dependencies available - Nodes cannot " "operate without ANY dependencies available! " "Node: %s" % node_class.__name__) # initialise node node = node_class() # shhh, secret accessors for developing nodes in debug mode node._p = params node._h = hdf node._n = node_mgr logger.info("Processing %s `%s`", get_node_type(node, node_subclasses), param_name) # Derive the resulting value try: node = node.get_derived(deps) except: if not force: raise del node._p del node._h del node._n if node.node_type is KeyPointValueNode: params[param_name] = node aligned_kpvs = [] for one_hz in node.get_aligned(P(frequency=1, offset=0)): if not (0 <= one_hz.index <= duration+4): raise IndexError( "KPV '%s' index %.2f is not between 0 and %d" % (one_hz.name, one_hz.index, duration)) aligned_kpvs.append(one_hz) kpvs[param_name] = aligned_kpvs elif node.node_type is KeyTimeInstanceNode: params[param_name] = node aligned_ktis = [] for one_hz in node.get_aligned(P(frequency=1, offset=0)): if not (0 <= one_hz.index <= duration+4): raise IndexError( "KTI '%s' index %.2f is not between 0 and %d" % (one_hz.name, one_hz.index, duration)) aligned_ktis.append(one_hz) ktis[param_name] = aligned_ktis elif node.node_type is FlightAttributeNode: params[param_name] = node try: # only has one Attribute node, store as a list for consistency flight_attrs[param_name] = [Attribute(node.name, node.value)] except: logger.warning("Flight Attribute Node '%s' returned empty " "handed.", param_name) elif issubclass(node.node_type, SectionNode): aligned_section = node.get_aligned(P(frequency=1, offset=0)) for index, one_hz in enumerate(aligned_section): # SectionNodes allow slice starts and stops being None which # signifies the beginning and end of the data. To avoid # TypeErrors in subsequent derive methods which perform # arithmetic on section slice start and stops, replace with 0 # or hdf.duration. fallback = lambda x, y: x if x is not None else y duration = fallback(duration, 0) start = fallback(one_hz.slice.start, 0) stop = fallback(one_hz.slice.stop, duration) start_edge = fallback(one_hz.start_edge, 0) stop_edge = fallback(one_hz.stop_edge, duration) slice_ = slice(start, stop) one_hz = Section(one_hz.name, slice_, start_edge, stop_edge) aligned_section[index] = one_hz if not (0 <= start <= duration and 0 <= stop <= duration + 4): msg = "Section '%s' (%.2f, %.2f) not between 0 and %d" raise IndexError( msg % (one_hz.name, start, stop, duration)) if not 0 <= start_edge <= duration: msg = "Section '%s' start_edge (%.2f) not between 0 and %d" raise IndexError(msg % (one_hz.name, start_edge, duration)) if not 0 <= stop_edge <= duration + 4: msg = "Section '%s' stop_edge (%.2f) not between 0 and %d" raise IndexError(msg % (one_hz.name, stop_edge, duration)) #section_list.append(one_hz) params[param_name] = aligned_section sections[param_name] = list(aligned_section) elif issubclass(node.node_type, DerivedParameterNode): if duration: # check that the right number of nodes were returned Allow a # small tolerance. For example if duration in seconds is 2822, # then there will be an array length of 1411 at 0.5Hz and 706 # at 0.25Hz (rounded upwards). If we combine two 0.25Hz # parameters then we will have an array length of 1412. expected_length = duration * node.frequency if node.array is None or (force and len(node.array) == 0): logger.warning("No array set; creating a fully masked " "array for %s", param_name) array_length = expected_length # Where a parameter is wholly masked, we fill the HDF # file with masked zeros to maintain structure. node.array = \ np_ma_masked_zeros(expected_length) else: array_length = len(node.array) length_diff = array_length - expected_length if length_diff == 0: pass elif 0 < length_diff < 5: logger.warning("Cutting excess data for parameter '%s'. " "Expected length was '%s' while resulting " "array length was '%s'.", param_name, expected_length, len(node.array)) node.array = node.array[:expected_length] else: raise ValueError("Array length mismatch for parameter " "'%s'. Expected '%s', resulting array " "length '%s'." % (param_name, expected_length, array_length)) hdf.set_param(node) # Keep hdf_keys up to date. node_mgr.hdf_keys.append(param_name) elif issubclass(node.node_type, ApproachNode): aligned_approach = node.get_aligned(P(frequency=1, offset=0)) for approach in aligned_approach: # Does not allow slice start or stops to be None. valid_turnoff = (not approach.turnoff or (0 <= approach.turnoff <= duration)) valid_slice = ((0 <= approach.slice.start <= duration) and (0 <= approach.slice.stop <= duration)) valid_gs_est = (not approach.gs_est or ((0 <= approach.gs_est.start <= duration) and (0 <= approach.gs_est.stop <= duration))) valid_loc_est = (not approach.loc_est or ((0 <= approach.loc_est.start <= duration) and (0 <= approach.loc_est.stop <= duration))) if not all([valid_turnoff, valid_slice, valid_gs_est, valid_loc_est]): raise ValueError('ApproachItem contains index outside of ' 'flight data: %s' % approach) params[param_name] = aligned_approach approaches[param_name] = list(aligned_approach) else: raise NotImplementedError("Unknown Type %s" % node.__class__) continue return ktis, kpvs, sections, approaches, flight_attrs
def derive_parameters_mitre(hdf, node_mgr, process_order, precomputed_parameters={}): ''' Derives the parameter values and if limits are available, applies parameter validation upon each param before storing the resulting masked array back into the hdf file. :param hdf: Data file accessor used to get and save parameter data and attributes :type hdf: hdf_file :param node_mgr: Used to determine the type of node in the process_order :type node_mgr: NodeManager :param process_order: Parameter / Node class names in the required order to be processed :type process_order: list of strings ''' params = precomputed_parameters # dictionary of derived params that aren't masked arrays approach_list = ApproachNode(restrict_names=False) kpv_list = KeyPointValueNode(restrict_names=False) # duplicate storage, but maintaining types kti_list = KeyTimeInstanceNode(restrict_names=False) section_list = SectionNode() # 'Node Name' : node() pass in node.get_accessor() flight_attrs = [] duration = hdf.duration for param_name in process_order: if param_name in node_mgr.hdf_keys: logger.debug(' derive_: hdf '+param_name) continue elif node_mgr.get_attribute(param_name) is not None: logger.debug(' derive_: get_attribute '+param_name) continue elif param_name in params: # already calculated KPV/KTI/Phase ***********************NEW logger.debug(' derive_parameters: re-using '+param_name) continue logger.debug(' derive_: computing '+param_name) node_class = node_mgr.derived_nodes[param_name] #NB raises KeyError if Node is "unknown" # build ordered dependencies deps = [] node_deps = node_class.get_dependency_names() for dep_name in node_deps: if dep_name in params: # already calculated KPV/KTI/Phase deps.append(params[dep_name]) elif node_mgr.get_attribute(dep_name) is not None: deps.append(node_mgr.get_attribute(dep_name)) elif dep_name in node_mgr.hdf_keys: # LFL/Derived parameter # all parameters (LFL or other) need get_aligned which is # available on DerivedParameterNode try: dp = derived_param_from_hdf(hdf.get_param(dep_name, valid_only=True)) except KeyError: # Parameter is invalid. dp = None deps.append(dp) else: # dependency not available deps.append(None) if all([d is None for d in deps]): raise RuntimeError("No dependencies available - Nodes cannot " "operate without ANY dependencies available! " "Node: %s" % node_class.__name__) # initialise node node = node_class() logger.info("Processing parameter %s", param_name) # Derive the resulting value result = node.get_derived(deps) if node.node_type is KeyPointValueNode: #Q: track node instead of result here?? params[param_name] = result for one_hz in result.get_aligned(P(frequency=1, offset=0)): if not (0 <= one_hz.index <= duration): raise IndexError( "KPV '%s' index %.2f is not between 0 and %d" % (one_hz.name, one_hz.index, duration)) kpv_list.append(one_hz) elif node.node_type is KeyTimeInstanceNode: params[param_name] = result for one_hz in result.get_aligned(P(frequency=1, offset=0)): if not (0 <= one_hz.index <= duration): raise IndexError( "KTI '%s' index %.2f is not between 0 and %d" % (one_hz.name, one_hz.index, duration)) kti_list.append(one_hz) elif node.node_type is FlightAttributeNode: params[param_name] = result try: flight_attrs.append(Attribute(result.name, result.value)) # only has one Attribute result except: logger.warning("Flight Attribute Node '%s' returned empty " "handed.", param_name) elif issubclass(node.node_type, SectionNode): aligned_section = result.get_aligned(P(frequency=1, offset=0)) for index, one_hz in enumerate(aligned_section): # SectionNodes allow slice starts and stops being None which # signifies the beginning and end of the data. To avoid TypeErrors # in subsequent derive methods which perform arithmetic on section # slice start and stops, replace with 0 or hdf.duration. fallback = lambda x, y: x if x is not None else y duration = fallback(duration, 0) start = fallback(one_hz.slice.start, 0) stop = fallback(one_hz.slice.stop, duration) start_edge = fallback(one_hz.start_edge, 0) stop_edge = fallback(one_hz.stop_edge, duration) slice_ = slice(start, stop) one_hz = Section(one_hz.name, slice_, start_edge, stop_edge) aligned_section[index] = one_hz if not (0 <= start <= duration and 0 <= stop <= duration + 1): msg = "Section '%s' (%.2f, %.2f) not between 0 and %d" raise IndexError(msg % (one_hz.name, start, stop, duration)) if not 0 <= start_edge <= duration: msg = "Section '%s' start_edge (%.2f) not between 0 and %d" raise IndexError(msg % (one_hz.name, start_edge, duration)) if not 0 <= stop_edge <= duration + 1: msg = "Section '%s' stop_edge (%.2f) not between 0 and %d" raise IndexError(msg % (one_hz.name, stop_edge, duration)) section_list.append(one_hz) params[param_name] = aligned_section elif issubclass(node.node_type, DerivedParameterNode): if duration: # check that the right number of results were returned # Allow a small tolerance. For example if duration in seconds # is 2822, then there will be an array length of 1411 at 0.5Hz and 706 # at 0.25Hz (rounded upwards). If we combine two 0.25Hz # parameters then we will have an array length of 1412. expected_length = duration * result.frequency if result.array is None: logger.warning("No array set; creating a fully masked array for %s", param_name) array_length = expected_length # Where a parameter is wholly masked, we fill the HDF # file with masked zeros to maintain structure. result.array = \ np_ma_masked_zeros_like(np.ma.arange(expected_length)) else: array_length = len(result.array) length_diff = array_length - expected_length if length_diff == 0: pass elif 0 < length_diff < 5: logger.warning("Cutting excess data for parameter '%s'. " "Expected length was '%s' while resulting " "array length was '%s'.", param_name, expected_length, len(result.array)) result.array = result.array[:expected_length] else: raise ValueError("Array length mismatch for parameter " "'%s'. Expected '%s', resulting array " "length '%s'." % (param_name, expected_length, array_length)) hdf.set_param(result) # Keep hdf_keys up to date. node_mgr.hdf_keys.append(param_name) elif issubclass(node.node_type, ApproachNode): aligned_approach = result.get_aligned(P(frequency=1, offset=0)) for approach in aligned_approach: # Does not allow slice start or stops to be None. valid_turnoff = (not approach.turnoff or (0 <= approach.turnoff <= duration)) valid_slice = ((0 <= approach.slice.start <= duration) and (0 <= approach.slice.stop <= duration)) valid_gs_est = (not approach.gs_est or ((0 <= approach.gs_est.start <= duration) and (0 <= approach.gs_est.stop <= duration))) valid_loc_est = (not approach.loc_est or ((0 <= approach.loc_est.start <= duration) and (0 <= approach.loc_est.stop <= duration))) if not all([valid_turnoff, valid_slice, valid_gs_est, valid_loc_est]): raise ValueError('ApproachItem contains index outside of ' 'flight data: %s' % approach) approach_list.append(approach) params[param_name] = aligned_approach else: raise NotImplementedError("Unknown Type %s" % node.__class__) continue return kti_list, kpv_list, section_list, approach_list, flight_attrs, params
def track_to_kml(hdf_path, kti_list, kpv_list, approach_list, plot_altitude=None, dest_path=None): ''' Plot results of process_flight onto a KML track. :param flight_attrs: List of Flight Attributes :type flight_attrs: list :param plot_altitude: Name of Altitude parameter to use in KML :type plot_altitude: String ''' one_hz = Parameter() kml = simplekml.Kml() with hdf_file(hdf_path) as hdf: # Latitude param, Longitude param, track name, colour coord_params = ( { 'lat': 'Latitude Smoothed', 'lon': 'Longitude Smoothed', 'track': 'Smoothed', 'colour': 'ff7fff7f' }, { 'lat': 'Latitude Prepared', 'lon': 'Longitude Prepared', 'track': 'Prepared', 'colour': 'A11EB3' }, { 'lat': 'Latitude', 'lon': 'Longitude', 'track': 'Recorded', 'colour': 'ff0000ff' }, { 'lat': 'Latitude (Coarse)', 'lon': 'Longitude (Coarse)', 'track': 'Coarse', 'colour': 'ff0000ff' }, ) altitude_absolute_params = ('Altitude QNH', 'Altitude STD', 'Altitude AAL') altitude_relative_params = ('Altitude Radio', ) # Check latitude and longitude pair exist. if not any(c['lat'] in hdf and c['lon'] in hdf for c in coord_params): logger.error( "Cannot write track as coordinate paarmeters not in hdf") return False # Choose best altitude parameter if not specified. if not plot_altitude: altitude_params = itertools.chain(altitude_absolute_params, altitude_relative_params) try: plot_altitude = next(p for p in altitude_params if p in hdf) except StopIteration: logger.warning("Disabling altitude on KML plot as it is " "unavailable.") # Get altitude param from hdf. if plot_altitude: alt = derived_param_from_hdf( hdf[plot_altitude]).get_aligned(one_hz) alt.array = repair_mask(alt.array, frequency=alt.frequency, repair_duration=None) / METRES_TO_FEET else: alt = None if plot_altitude in altitude_absolute_params: altitude_mode = simplekml.constants.AltitudeMode.absolute elif plot_altitude in altitude_relative_params: altitude_mode = simplekml.constants.AltitudeMode.relativetoground else: altitude_mode = simplekml.constants.AltitudeMode.clamptoground ## Get best latitude and longitude parameters. best_lat = None best_lon = None for coord_config in coord_params: lat_name = coord_config['lat'] lon_name = coord_config['lon'] if not lat_name in hdf or not lon_name in hdf: continue lat = hdf[lat_name] lon = hdf[lon_name] best = not best_lat or not best_lon add_track(kml, coord_config['track'], lat, lon, coord_config['colour'], alt_param=alt, alt_mode=altitude_mode, visible=best) add_track(kml, coord_config['track'] + ' On Ground', lat, lon, coord_config['colour'], visible=best) if best: best_lat = derived_param_from_hdf(lat).get_aligned(one_hz) best_lon = derived_param_from_hdf(lon).get_aligned(one_hz) # Add KTIs. for kti in kti_list: kti_point_values = {'name': kti.name} if kti.name in SKIP_KTIS: continue altitude = alt.at(kti.index) if plot_altitude else None kti_point_values['altitudemode'] = altitude_mode if altitude: kti_point_values['coords'] = ((kti.longitude, kti.latitude, altitude), ) else: kti_point_values['coords'] = ((kti.longitude, kti.latitude), ) kml.newpoint(**kti_point_values) # Add KPVs. for kpv in kpv_list: # Trap kpvs with invalid latitude or longitude data (normally happens # at the start of the data where accelerometer offsets are declared, # and this avoids casting kpvs into the Atlantic. kpv_lat = best_lat.at(kpv.index) kpv_lon = best_lon.at(kpv.index) if kpv_lat == None or kpv_lon == None or \ (kpv_lat == 0.0 and kpv_lon == 0.0): continue if kpv.name in SKIP_KPVS: continue style = simplekml.Style() style.iconstyle.color = simplekml.Color.red kpv_point_values = {'name': '%s (%.3f)' % (kpv.name, kpv.value)} altitude = alt.at(kpv.index) if plot_altitude else None kpv_point_values['altitudemode'] = altitude_mode if altitude: kpv_point_values['coords'] = ((kpv_lon, kpv_lat, altitude), ) else: kpv_point_values['coords'] = ((kpv_lon, kpv_lat), ) pnt = kml.newpoint(**kpv_point_values) pnt.style = style # Add approach centre lines. for app in approach_list: try: draw_centreline(kml, app.runway) except: pass if not dest_path: dest_path = hdf_path + ".kml" kml.save(dest_path) return dest_path
def track_to_kml(hdf_path, kti_list, kpv_list, approach_list, plot_altitude=None, dest_path=None): ''' Plot results of process_flight onto a KML track. :param flight_attrs: List of Flight Attributes :type flight_attrs: list :param plot_altitude: Name of Altitude parameter to use in KML :type plot_altitude: String ''' one_hz = Parameter() kml = simplekml.Kml() with hdf_file(hdf_path) as hdf: # Latitude param, Longitude param, track name, colour coord_params = ( {'lat': 'Latitude Smoothed', 'lon': 'Longitude Smoothed', 'track': 'Smoothed', 'colour': 'ff7fff7f'}, {'lat': 'Latitude Prepared', 'lon': 'Longitude Prepared', 'track': 'Prepared', 'colour': 'ffA11EB3'}, {'lat': 'Latitude', 'lon': 'Longitude', 'track': 'Recorded', 'colour': 'ff0000ff'}, {'lat': 'Latitude (Coarse)', 'lon': 'Longitude (Coarse)', 'track': 'Coarse', 'colour': 'ff0000ff'}, ) altitude_absolute_params = ('Altitude Visualization With Ground Offset', 'Altitude QNH', 'Altitude STD', 'Altitude AAL') altitude_relative_params = ('Altitude Radio',) # Check latitude and longitude pair exist. if not any(c['lat'] in hdf and c['lon'] in hdf for c in coord_params): logger.error("Cannot write track as coordinate paarmeters not in hdf") return False # Choose best altitude parameter if not specified. if not plot_altitude: altitude_params = itertools.chain(altitude_absolute_params, altitude_relative_params) try: plot_altitude = next(p for p in altitude_params if p in hdf) except StopIteration: logger.warning("Disabling altitude on KML plot as it is " "unavailable.") # Get altitude param from hdf. if plot_altitude and plot_altitude in hdf.keys(): alt = derived_param_from_hdf(hdf[plot_altitude]).get_aligned(one_hz) alt.array = repair_mask(alt.array, frequency=alt.frequency, repair_duration=None) alt.array = ut.convert(alt.array, ut.FT, ut.METER) else: alt = None if plot_altitude in altitude_absolute_params: altitude_mode = simplekml.constants.AltitudeMode.absolute elif plot_altitude in altitude_relative_params: altitude_mode = simplekml.constants.AltitudeMode.relativetoground else: altitude_mode = simplekml.constants.AltitudeMode.clamptoground ## Get best latitude and longitude parameters. best_lat = None best_lon = None for coord_config in coord_params: lat_name = coord_config['lat'] lon_name = coord_config['lon'] if not lat_name in hdf or not lon_name in hdf: continue lat = hdf[lat_name] lon = hdf[lon_name] best = not best_lat or not best_lon add_track(kml, coord_config['track'], lat, lon, coord_config['colour'], alt_param=alt, alt_mode=altitude_mode, visible=best) add_track(kml, coord_config['track'] + ' On Ground', lat, lon, coord_config['colour'], visible=best) if best: best_lat = derived_param_from_hdf(lat).get_aligned(one_hz) best_lon = derived_param_from_hdf(lon).get_aligned(one_hz) # Add KTIs. for kti in kti_list: kti_point_values = {'name': kti.name} if not KEEP_KTIS and kti.name in SKIP_KTIS: continue elif len(KEEP_KTIS)>0 and (kti.name not in KEEP_KTIS): continue altitude = alt.at(kti.index) if alt else None kti_point_values['altitudemode'] = altitude_mode if altitude: kti_point_values['coords'] = ((kti.longitude, kti.latitude, altitude),) else: kti_point_values['coords'] = ((kti.longitude, kti.latitude),) kml.newpoint(**kti_point_values) # Add KPVs. for kpv in kpv_list: # Trap kpvs with invalid latitude or longitude data (normally happens # at the start of the data where accelerometer offsets are declared, # and this avoids casting kpvs into the Atlantic. kpv_lat = best_lat.at(kpv.index) kpv_lon = best_lon.at(kpv.index) if kpv_lat is None or kpv_lon is None or \ (kpv_lat == 0.0 and kpv_lon == 0.0): continue if not KEEP_KPVS and kpv.name in SKIP_KPVS: continue elif len(KEEP_KPVS)>0 and (kpv.name not in KEEP_KPVS): continue style = simplekml.Style() style.iconstyle.color = simplekml.Color.red kpv_point_values = {'name': '%s (%.3f)' % (kpv.name, kpv.value)} altitude = alt.at(kpv.index) if alt else None kpv_point_values['altitudemode'] = altitude_mode if altitude: kpv_point_values['coords'] = ((kpv_lon, kpv_lat, altitude),) else: kpv_point_values['coords'] = ((kpv_lon, kpv_lat),) pnt = kml.newpoint(**kpv_point_values) pnt.style = style # Add approach centre lines. for app in approach_list: try: draw_centreline(kml, app.runway) except: pass if not dest_path: dest_path = hdf_path + ".kml" kml.save(dest_path) return dest_path
def track_to_kml(hdf_path, kti_list, kpv_list, approach_list, plot_altitude='Altitude QNH', dest_path=None): ''' Plot results of process_flight onto a KML track. :param flight_attrs: List of Flight Attributes :type flight_attrs: list :param plot_altitude: Name of Altitude parameter to use in KML :type plot_altitude: String ''' one_hz = Parameter() kml = simplekml.Kml() with hdf_file(hdf_path) as hdf: if 'Latitude Smoothed' not in hdf: return False if plot_altitude not in hdf: logger.warning("Disabling altitude on KML plot as it is unavailable.") plot_altitude = False if plot_altitude: alt = derived_param_from_hdf(hdf[plot_altitude]).get_aligned(one_hz) alt.array = repair_mask(alt.array, frequency=alt.frequency, repair_duration=None) / METRES_TO_FEET else: alt = None if plot_altitude in ['Altitude QNH', 'Altitude AAL', 'Altitude STD']: altitude_mode = simplekml.constants.AltitudeMode.absolute elif plot_altitude in ['Altitude Radio']: altitude_mode = simplekml.constants.AltitudeMode.relativetoground else: altitude_mode = simplekml.constants.AltitudeMode.clamptoground # TODO: align everything to 0 offset smooth_lat = derived_param_from_hdf(hdf['Latitude Smoothed']).get_aligned(one_hz) smooth_lon = derived_param_from_hdf(hdf['Longitude Smoothed']).get_aligned(one_hz) add_track(kml, 'Smoothed', smooth_lat, smooth_lon, 'ff7fff7f', alt_param=alt, alt_mode=altitude_mode) add_track(kml, 'Smoothed On Ground', smooth_lat, smooth_lon, 'ff7fff7f') if 'Latitude Prepared' in hdf and 'Longitude Prepared' in hdf: lat = hdf['Latitude Prepared'] lon = hdf['Longitude Prepared'] add_track(kml, 'Prepared Track', lat, lon, 'A11EB3', visible=False) if 'Latitude' in hdf and 'Longitude' in hdf: lat_r = hdf['Latitude'] lon_r = hdf['Longitude'] # add RAW track default invisible add_track(kml, 'Recorded Track', lat_r, lon_r, 'ff0000ff', visible=False) for kti in kti_list: kti_point_values = {'name': kti.name} if kti.name in SKIP_KTIS: continue altitude = alt.at(kti.index) if plot_altitude else None kti_point_values['altitudemode'] = altitude_mode if altitude: kti_point_values['coords'] = ((kti.longitude, kti.latitude, altitude),) else: kti_point_values['coords'] = ((kti.longitude, kti.latitude),) kml.newpoint(**kti_point_values) for kpv in kpv_list: # Trap kpvs with invalid latitude or longitude data (normally happens # at the start of the data where accelerometer offsets are declared, # and this avoids casting kpvs into the Atlantic. kpv_lat = smooth_lat.at(kpv.index) kpv_lon = smooth_lon.at(kpv.index) if kpv_lat == None or kpv_lon == None or \ (kpv_lat == 0.0 and kpv_lon == 0.0): continue if kpv.name in SKIP_KPVS: continue style = simplekml.Style() style.iconstyle.color = simplekml.Color.red kpv_point_values = {'name': '%s (%.3f)' % (kpv.name, kpv.value)} altitude = alt.at(kpv.index) if plot_altitude else None kpv_point_values['altitudemode'] = altitude_mode if altitude: kpv_point_values['coords'] = ((kpv_lon, kpv_lat, altitude),) else: kpv_point_values['coords'] = ((kpv_lon, kpv_lat),) pnt = kml.newpoint(**kpv_point_values) pnt.style = style for app in approach_list: try: draw_centreline(kml, app.runway) except: pass if not dest_path: dest_path = hdf_path + ".kml" kml.save(dest_path) return