def geo_locate(hdf, items):
'''
Translate KeyTimeInstance into GeoKeyTimeInstance namedtuples
'''
if 'Latitude Smoothed' not in hdf.valid_param_names() \
or 'Longitude Smoothed' not in hdf.valid_param_names():
logger.warning("Could not geo-locate as either 'Latitude Smoothed' or "
"'Longitude Smoothed' were not found within the hdf.")
return items
lat_hdf = hdf['Latitude Smoothed']
lon_hdf = hdf['Longitude Smoothed']
if (not lat_hdf.array.count()) or (not lon_hdf.array.count()):
logger.warning("Could not geo-locate as either 'Latitude Smoothed' or "
"'Longitude Smoothed' have no unmasked values.")
return items
lat_pos = derived_param_from_hdf(lat_hdf)
lon_pos = derived_param_from_hdf(lon_hdf)
# We want to place start of flight and end of flight markers at the ends
# of the data which may extend more than REPAIR_DURATION seconds beyond
# the end of the valid data. Hence by setting this to None and
# extrapolate=True we achieve this goal.
lat_pos.array = repair_mask(lat_pos.array, repair_duration=None, extrapolate=True)
lon_pos.array = repair_mask(lon_pos.array, repair_duration=None, extrapolate=True)
for item in itertools.chain.from_iterable(six.itervalues(items)):
item.latitude = lat_pos.at(item.index) or None
item.longitude = lon_pos.at(item.index) or None
return items
def geo_locate(hdf, items):
'''
Translate KeyTimeInstance into GeoKeyTimeInstance namedtuples
'''
if 'Latitude Smoothed' not in hdf.valid_param_names() \
or 'Longitude Smoothed' not in hdf.valid_param_names():
logger.warning("Could not geo-locate as either 'Latitude Smoothed' or "
"'Longitude Smoothed' were not found within the hdf.")
return items
lat_hdf = hdf['Latitude Smoothed']
lon_hdf = hdf['Longitude Smoothed']
if (not lat_hdf.array.count()) or (not lon_hdf.array.count()):
logger.warning("Could not geo-locate as either 'Latitude Smoothed' or "
"'Longitude Smoothed' have no unmasked values.")
return items
lat_pos = derived_param_from_hdf(lat_hdf)
lon_pos = derived_param_from_hdf(lon_hdf)
# We want to place start of flight and end of flight markers at the ends
# of the data which may extend more than REPAIR_DURATION seconds beyond
# the end of the valid data. Hence by setting this to None and
# extrapolate=True we achieve this goal.
lat_pos.array = repair_mask(lat_pos.array, repair_duration=None, extrapolate=True)
lon_pos.array = repair_mask(lon_pos.array, repair_duration=None, extrapolate=True)
for item in itertools.chain.from_iterable(items.itervalues()):
item.latitude = lat_pos.at(item.index) or None
item.longitude = lon_pos.at(item.index) or None
return items
def load_from_hdf5(self, flight_dict, required_series=[]): #all_hdfseries,
'''load data from an hdf flight data file
flight_series is a dictionary with fields: 'filepath', 'aircraft_info', 'repo'
'''
# look up aircraft info by tail number
self.filepath = flight_dict['filepath']
self.aircraft_info = flight_dict['aircraft_info']
with hdfaccess.file.hdf_file(self.filepath) as ff:
self.duration = ff.duration
self.start_datetime = ff.start_datetime
self.start_epoch = ff.hdf.attrs.get('start_timestamp') #keep as epoch
self.superframe_present = ff.superframe_present
self.hdfaccess_version = ff.hdfaccess_version
self.reliable_frame_counter = ff.reliable_frame_counter
# for profiles we can choose to load only selected series
if required_series==[]:
series_to_load = ff.keys() #all_hdfseries
else:
series_to_load = frozenset(required_series).intersection( frozenset(ff.keys()) )
for k in series_to_load:
self.series[k] = ff.get_param(k, valid_only=False)
if self.series[k].lfl==True:
self.lfl_params.append(k)
is_invalid =self.series[k].invalid
if is_invalid is None or is_invalid==False:
param_node = node.derived_param_from_hdf( self.series[k] )
param_node.units = self.series[k].units #better
param_node.lfl = self.series[k].lfl
param_node.data_type =self.series[k].data_type
self.parameters[k] = param_node
def geo_locate(hdf, items):
'''
Translate KeyTimeInstance into GeoKeyTimeInstance namedtuples
'''
if 'Latitude Smoothed' not in hdf.valid_param_names() \
or 'Longitude Smoothed' not in hdf.valid_param_names():
logger.warning("Could not geo-locate as either 'Latitude Smoothed' or "
"'Longitude Smoothed' were not found within the hdf.")
return items
lat_pos = derived_param_from_hdf(hdf['Latitude Smoothed'])
lon_pos = derived_param_from_hdf(hdf['Longitude Smoothed'])
lat_rep = repair_mask(lat_pos, extrapolate=True)
lon_rep = repair_mask(lon_pos, extrapolate=True)
for item in items:
item.latitude = lat_rep.at(item.index) or None
item.longitude = lon_rep.at(item.index) or None
return items
def geo_locate(hdf, items):
'''
Translate KeyTimeInstance into GeoKeyTimeInstance namedtuples
'''
if 'Latitude Smoothed' not in hdf.valid_param_names() \
or 'Longitude Smoothed' not in hdf.valid_param_names():
logger.warning("Could not geo-locate as either 'Latitude Smoothed' or "
"'Longitude Smoothed' were not found within the hdf.")
return items
lat_pos = derived_param_from_hdf(hdf['Latitude Smoothed'])
lon_pos = derived_param_from_hdf(hdf['Longitude Smoothed'])
lat_pos.array = repair_mask(lat_pos.array, extrapolate=True)
lon_pos.array = repair_mask(lon_pos.array, extrapolate=True)
for item in items:
item.latitude = lat_pos.at(item.index) or None
item.longitude = lon_pos.at(item.index) or None
return items
def build_ordered_dependencies(node_class, node_mgr, params, hdf):
# build ordered dependencies
deps = []
node_deps = node_class.get_dependency_names()
for dep_name in node_deps:
if dep_name in params: # already calculated KPV/KTI/Phase
deps.append(params[dep_name])
elif node_mgr.get_attribute(dep_name) is not None:
deps.append(node_mgr.get_attribute(dep_name))
elif dep_name in node_mgr.hdf_keys: # KC: duplicating work here? <<<<<<<<<<<<<<<<
# LFL/Derived parameter. Cast LFL param as derived param so we have get_aligned()
try:
dp = derived_param_from_hdf(hdf.get_param(dep_name, valid_only=True))
except KeyError: # Parameter is invalid.
dp = None
deps.append(dp)
else: # dependency not available
deps.append(None)
if all([d is None for d in deps]):
raise RuntimeError("No dependencies available - Node: %s" % node_class.__name__)
return deps
def load_from_hdf5(self, flight_dict, required_series=[]): #all_hdfseries,
'''load data from an hdf flight data file
flight_series is a dictionary with fields: 'filepath', 'aircraft_info', 'repo'
'''
# look up aircraft info by tail number
self.filepath = flight_dict['filepath']
self.aircraft_info = flight_dict['aircraft_info']
if not os.path.exists(self.filepath):
logger.warning('cannot find: ' + self.filepath)
#pdb.set_trace()
with hdfaccess.file.hdf_file(self.filepath) as ff:
self.duration = ff.duration
self.start_datetime = ff.start_datetime
self.start_epoch = ff.hdf.attrs.get(
'start_timestamp') #keep as epoch
self.superframe_present = ff.superframe_present
self.hdfaccess_version = ff.hdfaccess_version
self.reliable_frame_counter = ff.reliable_frame_counter
# for profiles we can choose to load only selected series
if required_series == []:
series_to_load = ff.keys() #all_hdfseries
else:
series_to_load = frozenset(required_series).intersection(
frozenset(ff.keys()))
for k in series_to_load:
self.series[k] = ff.get_param(k, valid_only=False)
if self.series[k].lfl == True:
self.lfl_params.append(k)
is_invalid = self.series[k].invalid
if is_invalid is None or is_invalid == False:
param_node = node.derived_param_from_hdf(self.series[k])
param_node.units = self.series[k].units #better
param_node.lfl = self.series[k].lfl
param_node.data_type = self.series[k].data_type
self.parameters[k] = param_node
def derive_parameters(hdf, node_mgr, process_order, params=None, force=False):
'''
Derives parameters in process_order. Dependencies are sourced via the
node_mgr.
:param hdf: Data file accessor used to get and save parameter data and
attributes
:type hdf: hdf_file
:param node_mgr: Used to determine the type of node in the process_order
:type node_mgr: NodeManager
:param process_order: Parameter / Node class names in the required order to
be processed
:type process_order: list of strings
'''
if not params:
params = {}
# OPT: local lookup is faster than module-level (small).
node_subclasses = NODE_SUBCLASSES
# store all derived params that aren't masked arrays
approaches = {}
# duplicate storage, but maintaining types
kpvs = {}
ktis = {}
# 'Node Name' : node() pass in node.get_accessor()
sections = {}
flight_attrs = {}
# cache of nodes to avoid repeated array alignment
cache = {} if NODE_CACHE else None
duration = hdf.duration
for param_name in process_order:
if param_name in node_mgr.hdf_keys:
continue
elif param_name in params:
node = params[param_name]
# populate output already at 1Hz
if node.node_type is KeyPointValueNode:
kpvs[param_name] = list(node)
elif node.node_type is KeyTimeInstanceNode:
ktis[param_name] = list(node)
elif node.node_type is FlightAttributeNode:
flight_attrs[param_name] = [Attribute(node.name, node.value)]
elif node.node_type is SectionNode:
sections[param_name] = list(node)
# DerivedParameterNodes are not supported in initial data.
continue
elif node_mgr.get_attribute(param_name) is not None:
# add attribute to dictionary of available params
###params[param_name] = node_mgr.get_attribute(param_name)
#TODO: optimise with only one call to get_attribute
continue
#NB raises KeyError if Node is "unknown"
node_class = node_mgr.derived_nodes[param_name]
# build ordered dependencies
deps = []
node_deps = node_class.get_dependency_names()
for dep_name in node_deps:
if dep_name in params: # already calculated KPV/KTI/Phase
deps.append(params[dep_name])
elif node_mgr.get_attribute(dep_name) is not None:
deps.append(node_mgr.get_attribute(dep_name))
elif dep_name in node_mgr.hdf_keys:
# LFL/Derived parameter
# all parameters (LFL or other) need get_aligned which is
# available on DerivedParameterNode
try:
dp = derived_param_from_hdf(hdf.get_param(dep_name,
valid_only=True),
cache=cache)
except KeyError:
# Parameter is invalid.
dp = None
deps.append(dp)
else: # dependency not available
deps.append(None)
if all([d is None for d in deps]):
raise RuntimeError("No dependencies available - Nodes cannot "
"operate without ANY dependencies available! "
"Node: %s" % node_class.__name__)
# initialise node
node = node_class(cache=cache)
# shhh, secret accessors for developing nodes in debug mode
node._p = params
node._h = hdf
node._n = node_mgr
logger.debug("Processing %s `%s`",
get_node_type(node, node_subclasses), param_name)
# Derive the resulting value
try:
node = node.get_derived(deps)
except:
if not force:
raise
del node._p
del node._h
del node._n
if node.node_type is KeyPointValueNode:
params[param_name] = node
aligned_kpvs = []
for one_hz in node.get_aligned(P(frequency=1, offset=0)):
if not (0 <= one_hz.index <= duration + 4):
raise IndexError(
"KPV '%s' index %.2f is not between 0 and %d" %
(one_hz.name, one_hz.index, duration))
aligned_kpvs.append(one_hz)
kpvs[param_name] = aligned_kpvs
elif node.node_type is KeyTimeInstanceNode:
params[param_name] = node
aligned_ktis = []
for one_hz in node.get_aligned(P(frequency=1, offset=0)):
if not (0 <= one_hz.index <= duration + 4):
raise IndexError(
"KTI '%s' index %.2f is not between 0 and %d" %
(one_hz.name, one_hz.index, duration))
aligned_ktis.append(one_hz)
ktis[param_name] = aligned_ktis
elif node.node_type is FlightAttributeNode:
params[param_name] = node
try:
# only has one Attribute node, store as a list for consistency
flight_attrs[param_name] = [Attribute(node.name, node.value)]
except:
logger.warning(
"Flight Attribute Node '%s' returned empty "
"handed.", param_name)
elif issubclass(node.node_type, SectionNode):
aligned_section = node.get_aligned(P(frequency=1, offset=0))
for index, one_hz in enumerate(aligned_section):
# SectionNodes allow slice starts and stops being None which
# signifies the beginning and end of the data. To avoid
# TypeErrors in subsequent derive methods which perform
# arithmetic on section slice start and stops, replace with 0
# or hdf.duration.
fallback = lambda x, y: x if x is not None else y
duration = fallback(duration, 0)
start = fallback(one_hz.slice.start, 0)
stop = fallback(one_hz.slice.stop, duration)
start_edge = fallback(one_hz.start_edge, 0)
stop_edge = fallback(one_hz.stop_edge, duration)
slice_ = slice(start, stop)
one_hz = Section(one_hz.name, slice_, start_edge, stop_edge)
aligned_section[index] = one_hz
if not (0 <= start <= duration and 0 <= stop <= duration + 4):
msg = "Section '%s' (%.2f, %.2f) not between 0 and %d"
raise IndexError(msg %
(one_hz.name, start, stop, duration))
if not 0 <= start_edge <= duration:
msg = "Section '%s' start_edge (%.2f) not between 0 and %d"
raise IndexError(msg % (one_hz.name, start_edge, duration))
if not 0 <= stop_edge <= duration + 4:
msg = "Section '%s' stop_edge (%.2f) not between 0 and %d"
raise IndexError(msg % (one_hz.name, stop_edge, duration))
#section_list.append(one_hz)
params[param_name] = aligned_section
sections[param_name] = list(aligned_section)
elif issubclass(node.node_type, DerivedParameterNode):
if duration:
# check that the right number of nodes were returned Allow a
# small tolerance. For example if duration in seconds is 2822,
# then there will be an array length of 1411 at 0.5Hz and 706
# at 0.25Hz (rounded upwards). If we combine two 0.25Hz
# parameters then we will have an array length of 1412.
expected_length = duration * node.frequency
if node.array is None or (force and len(node.array) == 0):
logger.warning(
"No array set; creating a fully masked "
"array for %s", param_name)
array_length = expected_length
# Where a parameter is wholly masked, we fill the HDF
# file with masked zeros to maintain structure.
node.array = \
np_ma_masked_zeros(expected_length)
else:
array_length = len(node.array)
length_diff = array_length - expected_length
if length_diff == 0:
pass
elif 0 < length_diff < 5:
logger.warning(
"Cutting excess data for parameter '%s'. "
"Expected length was '%s' while resulting "
"array length was '%s'.", param_name, expected_length,
len(node.array))
node.array = node.array[:expected_length]
else:
raise ValueError(
"Array length mismatch for parameter "
"'%s'. Expected '%s', resulting array "
"length '%s'." %
(param_name, expected_length, array_length))
hdf.set_param(node)
# Keep hdf_keys up to date.
node_mgr.hdf_keys.append(param_name)
elif issubclass(node.node_type, ApproachNode):
aligned_approach = node.get_aligned(P(frequency=1, offset=0))
for approach in aligned_approach:
# Does not allow slice start or stops to be None.
valid_turnoff = (not approach.turnoff
or (0 <= approach.turnoff <= duration))
valid_slice = ((0 <= approach.slice.start <= duration)
and (0 <= approach.slice.stop <= duration))
valid_gs_est = (not approach.gs_est or
((0 <= approach.gs_est.start <= duration) and
(0 <= approach.gs_est.stop <= duration)))
valid_loc_est = (not approach.loc_est or
((0 <= approach.loc_est.start <= duration) and
(0 <= approach.loc_est.stop <= duration)))
if not all(
[valid_turnoff, valid_slice, valid_gs_est, valid_loc_est]):
raise ValueError('ApproachItem contains index outside of '
'flight data: %s' % approach)
params[param_name] = aligned_approach
approaches[param_name] = list(aligned_approach)
else:
raise NotImplementedError("Unknown Type %s" % node.__class__)
continue
return ktis, kpvs, sections, approaches, flight_attrs
def derive_parameters(hdf, node_mgr, process_order, params=None, force=False):
'''
Derives parameters in process_order. Dependencies are sourced via the
node_mgr.
:param hdf: Data file accessor used to get and save parameter data and
attributes
:type hdf: hdf_file
:param node_mgr: Used to determine the type of node in the process_order
:type node_mgr: NodeManager
:param process_order: Parameter / Node class names in the required order to
be processed
:type process_order: list of strings
'''
if not params:
params = {}
# OPT: local lookup is faster than module-level (small).
node_subclasses = NODE_SUBCLASSES
# store all derived params that aren't masked arrays
approaches = {}
# duplicate storage, but maintaining types
kpvs = {}
ktis = {}
# 'Node Name' : node() pass in node.get_accessor()
sections = {}
flight_attrs = {}
duration = hdf.duration
for param_name in process_order:
if param_name in node_mgr.hdf_keys:
continue
elif param_name in params:
node = params[param_name]
# populate output already at 1Hz
if node.node_type is KeyPointValueNode:
kpvs[param_name] = list(node)
elif node.node_type is KeyTimeInstanceNode:
ktis[param_name] = list(node)
elif node.node_type is FlightAttributeNode:
flight_attrs[param_name] = [Attribute(node.name, node.value)]
elif node.node_type is SectionNode:
sections[param_name] = list(node)
# DerivedParameterNodes are not supported in initial data.
continue
elif node_mgr.get_attribute(param_name) is not None:
# add attribute to dictionary of available params
###params[param_name] = node_mgr.get_attribute(param_name)
#TODO: optimise with only one call to get_attribute
continue
#NB raises KeyError if Node is "unknown"
node_class = node_mgr.derived_nodes[param_name]
# build ordered dependencies
deps = []
node_deps = node_class.get_dependency_names()
for dep_name in node_deps:
if dep_name in params: # already calculated KPV/KTI/Phase
deps.append(params[dep_name])
elif node_mgr.get_attribute(dep_name) is not None:
deps.append(node_mgr.get_attribute(dep_name))
elif dep_name in node_mgr.hdf_keys:
# LFL/Derived parameter
# all parameters (LFL or other) need get_aligned which is
# available on DerivedParameterNode
try:
dp = derived_param_from_hdf(hdf.get_param(
dep_name, valid_only=True))
except KeyError:
# Parameter is invalid.
dp = None
deps.append(dp)
else: # dependency not available
deps.append(None)
if all([d is None for d in deps]):
raise RuntimeError(
"No dependencies available - Nodes cannot "
"operate without ANY dependencies available! "
"Node: %s" % node_class.__name__)
# initialise node
node = node_class()
# shhh, secret accessors for developing nodes in debug mode
node._p = params
node._h = hdf
node._n = node_mgr
logger.info("Processing %s `%s`", get_node_type(node, node_subclasses), param_name)
# Derive the resulting value
try:
node = node.get_derived(deps)
except:
if not force:
raise
del node._p
del node._h
del node._n
if node.node_type is KeyPointValueNode:
params[param_name] = node
aligned_kpvs = []
for one_hz in node.get_aligned(P(frequency=1, offset=0)):
if not (0 <= one_hz.index <= duration+4):
raise IndexError(
"KPV '%s' index %.2f is not between 0 and %d" %
(one_hz.name, one_hz.index, duration))
aligned_kpvs.append(one_hz)
kpvs[param_name] = aligned_kpvs
elif node.node_type is KeyTimeInstanceNode:
params[param_name] = node
aligned_ktis = []
for one_hz in node.get_aligned(P(frequency=1, offset=0)):
if not (0 <= one_hz.index <= duration+4):
raise IndexError(
"KTI '%s' index %.2f is not between 0 and %d" %
(one_hz.name, one_hz.index, duration))
aligned_ktis.append(one_hz)
ktis[param_name] = aligned_ktis
elif node.node_type is FlightAttributeNode:
params[param_name] = node
try:
# only has one Attribute node, store as a list for consistency
flight_attrs[param_name] = [Attribute(node.name, node.value)]
except:
logger.warning("Flight Attribute Node '%s' returned empty "
"handed.", param_name)
elif issubclass(node.node_type, SectionNode):
aligned_section = node.get_aligned(P(frequency=1, offset=0))
for index, one_hz in enumerate(aligned_section):
# SectionNodes allow slice starts and stops being None which
# signifies the beginning and end of the data. To avoid
# TypeErrors in subsequent derive methods which perform
# arithmetic on section slice start and stops, replace with 0
# or hdf.duration.
fallback = lambda x, y: x if x is not None else y
duration = fallback(duration, 0)
start = fallback(one_hz.slice.start, 0)
stop = fallback(one_hz.slice.stop, duration)
start_edge = fallback(one_hz.start_edge, 0)
stop_edge = fallback(one_hz.stop_edge, duration)
slice_ = slice(start, stop)
one_hz = Section(one_hz.name, slice_, start_edge, stop_edge)
aligned_section[index] = one_hz
if not (0 <= start <= duration and 0 <= stop <= duration + 4):
msg = "Section '%s' (%.2f, %.2f) not between 0 and %d"
raise IndexError(
msg % (one_hz.name, start, stop, duration))
if not 0 <= start_edge <= duration:
msg = "Section '%s' start_edge (%.2f) not between 0 and %d"
raise IndexError(msg % (one_hz.name, start_edge, duration))
if not 0 <= stop_edge <= duration + 4:
msg = "Section '%s' stop_edge (%.2f) not between 0 and %d"
raise IndexError(msg % (one_hz.name, stop_edge, duration))
#section_list.append(one_hz)
params[param_name] = aligned_section
sections[param_name] = list(aligned_section)
elif issubclass(node.node_type, DerivedParameterNode):
if duration:
# check that the right number of nodes were returned Allow a
# small tolerance. For example if duration in seconds is 2822,
# then there will be an array length of 1411 at 0.5Hz and 706
# at 0.25Hz (rounded upwards). If we combine two 0.25Hz
# parameters then we will have an array length of 1412.
expected_length = duration * node.frequency
if node.array is None or (force and len(node.array) == 0):
logger.warning("No array set; creating a fully masked "
"array for %s", param_name)
array_length = expected_length
# Where a parameter is wholly masked, we fill the HDF
# file with masked zeros to maintain structure.
node.array = \
np_ma_masked_zeros(expected_length)
else:
array_length = len(node.array)
length_diff = array_length - expected_length
if length_diff == 0:
pass
elif 0 < length_diff < 5:
logger.warning("Cutting excess data for parameter '%s'. "
"Expected length was '%s' while resulting "
"array length was '%s'.", param_name,
expected_length, len(node.array))
node.array = node.array[:expected_length]
else:
raise ValueError("Array length mismatch for parameter "
"'%s'. Expected '%s', resulting array "
"length '%s'." % (param_name,
expected_length,
array_length))
hdf.set_param(node)
# Keep hdf_keys up to date.
node_mgr.hdf_keys.append(param_name)
elif issubclass(node.node_type, ApproachNode):
aligned_approach = node.get_aligned(P(frequency=1, offset=0))
for approach in aligned_approach:
# Does not allow slice start or stops to be None.
valid_turnoff = (not approach.turnoff or
(0 <= approach.turnoff <= duration))
valid_slice = ((0 <= approach.slice.start <= duration) and
(0 <= approach.slice.stop <= duration))
valid_gs_est = (not approach.gs_est or
((0 <= approach.gs_est.start <= duration) and
(0 <= approach.gs_est.stop <= duration)))
valid_loc_est = (not approach.loc_est or
((0 <= approach.loc_est.start <= duration) and
(0 <= approach.loc_est.stop <= duration)))
if not all([valid_turnoff, valid_slice, valid_gs_est,
valid_loc_est]):
raise ValueError('ApproachItem contains index outside of '
'flight data: %s' % approach)
params[param_name] = aligned_approach
approaches[param_name] = list(aligned_approach)
else:
raise NotImplementedError("Unknown Type %s" % node.__class__)
continue
return ktis, kpvs, sections, approaches, flight_attrs
def derive_parameters_mitre(hdf, node_mgr, process_order, precomputed_parameters={}):
'''
Derives the parameter values and if limits are available, applies
parameter validation upon each param before storing the resulting masked
array back into the hdf file.
:param hdf: Data file accessor used to get and save parameter data and attributes
:type hdf: hdf_file
:param node_mgr: Used to determine the type of node in the process_order
:type node_mgr: NodeManager
:param process_order: Parameter / Node class names in the required order to be processed
:type process_order: list of strings
'''
params = precomputed_parameters # dictionary of derived params that aren't masked arrays
approach_list = ApproachNode(restrict_names=False)
kpv_list = KeyPointValueNode(restrict_names=False) # duplicate storage, but maintaining types
kti_list = KeyTimeInstanceNode(restrict_names=False)
section_list = SectionNode() # 'Node Name' : node() pass in node.get_accessor()
flight_attrs = []
duration = hdf.duration
for param_name in process_order:
if param_name in node_mgr.hdf_keys:
logger.debug(' derive_: hdf '+param_name)
continue
elif node_mgr.get_attribute(param_name) is not None:
logger.debug(' derive_: get_attribute '+param_name)
continue
elif param_name in params: # already calculated KPV/KTI/Phase ***********************NEW
logger.debug(' derive_parameters: re-using '+param_name)
continue
logger.debug(' derive_: computing '+param_name)
node_class = node_mgr.derived_nodes[param_name] #NB raises KeyError if Node is "unknown"
# build ordered dependencies
deps = []
node_deps = node_class.get_dependency_names()
for dep_name in node_deps:
if dep_name in params: # already calculated KPV/KTI/Phase
deps.append(params[dep_name])
elif node_mgr.get_attribute(dep_name) is not None:
deps.append(node_mgr.get_attribute(dep_name))
elif dep_name in node_mgr.hdf_keys:
# LFL/Derived parameter
# all parameters (LFL or other) need get_aligned which is
# available on DerivedParameterNode
try:
dp = derived_param_from_hdf(hdf.get_param(dep_name,
valid_only=True))
except KeyError:
# Parameter is invalid.
dp = None
deps.append(dp)
else: # dependency not available
deps.append(None)
if all([d is None for d in deps]):
raise RuntimeError("No dependencies available - Nodes cannot "
"operate without ANY dependencies available! "
"Node: %s" % node_class.__name__)
# initialise node
node = node_class()
logger.info("Processing parameter %s", param_name)
# Derive the resulting value
result = node.get_derived(deps)
if node.node_type is KeyPointValueNode:
#Q: track node instead of result here??
params[param_name] = result
for one_hz in result.get_aligned(P(frequency=1, offset=0)):
if not (0 <= one_hz.index <= duration):
raise IndexError(
"KPV '%s' index %.2f is not between 0 and %d" %
(one_hz.name, one_hz.index, duration))
kpv_list.append(one_hz)
elif node.node_type is KeyTimeInstanceNode:
params[param_name] = result
for one_hz in result.get_aligned(P(frequency=1, offset=0)):
if not (0 <= one_hz.index <= duration):
raise IndexError(
"KTI '%s' index %.2f is not between 0 and %d" %
(one_hz.name, one_hz.index, duration))
kti_list.append(one_hz)
elif node.node_type is FlightAttributeNode:
params[param_name] = result
try:
flight_attrs.append(Attribute(result.name, result.value)) # only has one Attribute result
except:
logger.warning("Flight Attribute Node '%s' returned empty "
"handed.", param_name)
elif issubclass(node.node_type, SectionNode):
aligned_section = result.get_aligned(P(frequency=1, offset=0))
for index, one_hz in enumerate(aligned_section):
# SectionNodes allow slice starts and stops being None which
# signifies the beginning and end of the data. To avoid TypeErrors
# in subsequent derive methods which perform arithmetic on section
# slice start and stops, replace with 0 or hdf.duration.
fallback = lambda x, y: x if x is not None else y
duration = fallback(duration, 0)
start = fallback(one_hz.slice.start, 0)
stop = fallback(one_hz.slice.stop, duration)
start_edge = fallback(one_hz.start_edge, 0)
stop_edge = fallback(one_hz.stop_edge, duration)
slice_ = slice(start, stop)
one_hz = Section(one_hz.name, slice_, start_edge, stop_edge)
aligned_section[index] = one_hz
if not (0 <= start <= duration and 0 <= stop <= duration + 1):
msg = "Section '%s' (%.2f, %.2f) not between 0 and %d"
raise IndexError(msg % (one_hz.name, start, stop, duration))
if not 0 <= start_edge <= duration:
msg = "Section '%s' start_edge (%.2f) not between 0 and %d"
raise IndexError(msg % (one_hz.name, start_edge, duration))
if not 0 <= stop_edge <= duration + 1:
msg = "Section '%s' stop_edge (%.2f) not between 0 and %d"
raise IndexError(msg % (one_hz.name, stop_edge, duration))
section_list.append(one_hz)
params[param_name] = aligned_section
elif issubclass(node.node_type, DerivedParameterNode):
if duration:
# check that the right number of results were returned
# Allow a small tolerance. For example if duration in seconds
# is 2822, then there will be an array length of 1411 at 0.5Hz and 706
# at 0.25Hz (rounded upwards). If we combine two 0.25Hz
# parameters then we will have an array length of 1412.
expected_length = duration * result.frequency
if result.array is None:
logger.warning("No array set; creating a fully masked array for %s", param_name)
array_length = expected_length
# Where a parameter is wholly masked, we fill the HDF
# file with masked zeros to maintain structure.
result.array = \
np_ma_masked_zeros_like(np.ma.arange(expected_length))
else:
array_length = len(result.array)
length_diff = array_length - expected_length
if length_diff == 0:
pass
elif 0 < length_diff < 5:
logger.warning("Cutting excess data for parameter '%s'. "
"Expected length was '%s' while resulting "
"array length was '%s'.", param_name,
expected_length, len(result.array))
result.array = result.array[:expected_length]
else:
raise ValueError("Array length mismatch for parameter "
"'%s'. Expected '%s', resulting array "
"length '%s'." % (param_name,
expected_length,
array_length))
hdf.set_param(result)
# Keep hdf_keys up to date.
node_mgr.hdf_keys.append(param_name)
elif issubclass(node.node_type, ApproachNode):
aligned_approach = result.get_aligned(P(frequency=1, offset=0))
for approach in aligned_approach:
# Does not allow slice start or stops to be None.
valid_turnoff = (not approach.turnoff or
(0 <= approach.turnoff <= duration))
valid_slice = ((0 <= approach.slice.start <= duration) and
(0 <= approach.slice.stop <= duration))
valid_gs_est = (not approach.gs_est or
((0 <= approach.gs_est.start <= duration) and
(0 <= approach.gs_est.stop <= duration)))
valid_loc_est = (not approach.loc_est or
((0 <= approach.loc_est.start <= duration) and
(0 <= approach.loc_est.stop <= duration)))
if not all([valid_turnoff, valid_slice, valid_gs_est,
valid_loc_est]):
raise ValueError('ApproachItem contains index outside of '
'flight data: %s' % approach)
approach_list.append(approach)
params[param_name] = aligned_approach
else:
raise NotImplementedError("Unknown Type %s" % node.__class__)
continue
return kti_list, kpv_list, section_list, approach_list, flight_attrs, params
def track_to_kml(hdf_path,
kti_list,
kpv_list,
approach_list,
plot_altitude=None,
dest_path=None):
'''
Plot results of process_flight onto a KML track.
:param flight_attrs: List of Flight Attributes
:type flight_attrs: list
:param plot_altitude: Name of Altitude parameter to use in KML
:type plot_altitude: String
'''
one_hz = Parameter()
kml = simplekml.Kml()
with hdf_file(hdf_path) as hdf:
# Latitude param, Longitude param, track name, colour
coord_params = (
{
'lat': 'Latitude Smoothed',
'lon': 'Longitude Smoothed',
'track': 'Smoothed',
'colour': 'ff7fff7f'
},
{
'lat': 'Latitude Prepared',
'lon': 'Longitude Prepared',
'track': 'Prepared',
'colour': 'A11EB3'
},
{
'lat': 'Latitude',
'lon': 'Longitude',
'track': 'Recorded',
'colour': 'ff0000ff'
},
{
'lat': 'Latitude (Coarse)',
'lon': 'Longitude (Coarse)',
'track': 'Coarse',
'colour': 'ff0000ff'
},
)
altitude_absolute_params = ('Altitude QNH', 'Altitude STD',
'Altitude AAL')
altitude_relative_params = ('Altitude Radio', )
# Check latitude and longitude pair exist.
if not any(c['lat'] in hdf and c['lon'] in hdf for c in coord_params):
logger.error(
"Cannot write track as coordinate paarmeters not in hdf")
return False
# Choose best altitude parameter if not specified.
if not plot_altitude:
altitude_params = itertools.chain(altitude_absolute_params,
altitude_relative_params)
try:
plot_altitude = next(p for p in altitude_params if p in hdf)
except StopIteration:
logger.warning("Disabling altitude on KML plot as it is "
"unavailable.")
# Get altitude param from hdf.
if plot_altitude:
alt = derived_param_from_hdf(
hdf[plot_altitude]).get_aligned(one_hz)
alt.array = repair_mask(alt.array,
frequency=alt.frequency,
repair_duration=None) / METRES_TO_FEET
else:
alt = None
if plot_altitude in altitude_absolute_params:
altitude_mode = simplekml.constants.AltitudeMode.absolute
elif plot_altitude in altitude_relative_params:
altitude_mode = simplekml.constants.AltitudeMode.relativetoground
else:
altitude_mode = simplekml.constants.AltitudeMode.clamptoground
## Get best latitude and longitude parameters.
best_lat = None
best_lon = None
for coord_config in coord_params:
lat_name = coord_config['lat']
lon_name = coord_config['lon']
if not lat_name in hdf or not lon_name in hdf:
continue
lat = hdf[lat_name]
lon = hdf[lon_name]
best = not best_lat or not best_lon
add_track(kml,
coord_config['track'],
lat,
lon,
coord_config['colour'],
alt_param=alt,
alt_mode=altitude_mode,
visible=best)
add_track(kml,
coord_config['track'] + ' On Ground',
lat,
lon,
coord_config['colour'],
visible=best)
if best:
best_lat = derived_param_from_hdf(lat).get_aligned(one_hz)
best_lon = derived_param_from_hdf(lon).get_aligned(one_hz)
# Add KTIs.
for kti in kti_list:
kti_point_values = {'name': kti.name}
if kti.name in SKIP_KTIS:
continue
altitude = alt.at(kti.index) if plot_altitude else None
kti_point_values['altitudemode'] = altitude_mode
if altitude:
kti_point_values['coords'] = ((kti.longitude, kti.latitude,
altitude), )
else:
kti_point_values['coords'] = ((kti.longitude, kti.latitude), )
kml.newpoint(**kti_point_values)
# Add KPVs.
for kpv in kpv_list:
# Trap kpvs with invalid latitude or longitude data (normally happens
# at the start of the data where accelerometer offsets are declared,
# and this avoids casting kpvs into the Atlantic.
kpv_lat = best_lat.at(kpv.index)
kpv_lon = best_lon.at(kpv.index)
if kpv_lat == None or kpv_lon == None or \
(kpv_lat == 0.0 and kpv_lon == 0.0):
continue
if kpv.name in SKIP_KPVS:
continue
style = simplekml.Style()
style.iconstyle.color = simplekml.Color.red
kpv_point_values = {'name': '%s (%.3f)' % (kpv.name, kpv.value)}
altitude = alt.at(kpv.index) if plot_altitude else None
kpv_point_values['altitudemode'] = altitude_mode
if altitude:
kpv_point_values['coords'] = ((kpv_lon, kpv_lat, altitude), )
else:
kpv_point_values['coords'] = ((kpv_lon, kpv_lat), )
pnt = kml.newpoint(**kpv_point_values)
pnt.style = style
# Add approach centre lines.
for app in approach_list:
try:
draw_centreline(kml, app.runway)
except:
pass
if not dest_path:
dest_path = hdf_path + ".kml"
kml.save(dest_path)
return dest_path
def track_to_kml(hdf_path, kti_list, kpv_list, approach_list,
plot_altitude=None, dest_path=None):
'''
Plot results of process_flight onto a KML track.
:param flight_attrs: List of Flight Attributes
:type flight_attrs: list
:param plot_altitude: Name of Altitude parameter to use in KML
:type plot_altitude: String
'''
one_hz = Parameter()
kml = simplekml.Kml()
with hdf_file(hdf_path) as hdf:
# Latitude param, Longitude param, track name, colour
coord_params = (
{'lat': 'Latitude Smoothed',
'lon': 'Longitude Smoothed',
'track': 'Smoothed',
'colour': 'ff7fff7f'},
{'lat': 'Latitude Prepared',
'lon': 'Longitude Prepared',
'track': 'Prepared',
'colour': 'ffA11EB3'},
{'lat': 'Latitude',
'lon': 'Longitude',
'track': 'Recorded',
'colour': 'ff0000ff'},
{'lat': 'Latitude (Coarse)',
'lon': 'Longitude (Coarse)',
'track': 'Coarse',
'colour': 'ff0000ff'},
)
altitude_absolute_params = ('Altitude Visualization With Ground Offset',
'Altitude QNH', 'Altitude STD',
'Altitude AAL')
altitude_relative_params = ('Altitude Radio',)
# Check latitude and longitude pair exist.
if not any(c['lat'] in hdf and c['lon'] in hdf for c in coord_params):
logger.error("Cannot write track as coordinate paarmeters not in hdf")
return False
# Choose best altitude parameter if not specified.
if not plot_altitude:
altitude_params = itertools.chain(altitude_absolute_params,
altitude_relative_params)
try:
plot_altitude = next(p for p in altitude_params if p in hdf)
except StopIteration:
logger.warning("Disabling altitude on KML plot as it is "
"unavailable.")
# Get altitude param from hdf.
if plot_altitude and plot_altitude in hdf.keys():
alt = derived_param_from_hdf(hdf[plot_altitude]).get_aligned(one_hz)
alt.array = repair_mask(alt.array, frequency=alt.frequency, repair_duration=None)
alt.array = ut.convert(alt.array, ut.FT, ut.METER)
else:
alt = None
if plot_altitude in altitude_absolute_params:
altitude_mode = simplekml.constants.AltitudeMode.absolute
elif plot_altitude in altitude_relative_params:
altitude_mode = simplekml.constants.AltitudeMode.relativetoground
else:
altitude_mode = simplekml.constants.AltitudeMode.clamptoground
## Get best latitude and longitude parameters.
best_lat = None
best_lon = None
for coord_config in coord_params:
lat_name = coord_config['lat']
lon_name = coord_config['lon']
if not lat_name in hdf or not lon_name in hdf:
continue
lat = hdf[lat_name]
lon = hdf[lon_name]
best = not best_lat or not best_lon
add_track(kml, coord_config['track'], lat, lon,
coord_config['colour'], alt_param=alt,
alt_mode=altitude_mode,
visible=best)
add_track(kml, coord_config['track'] + ' On Ground', lat, lon,
coord_config['colour'], visible=best)
if best:
best_lat = derived_param_from_hdf(lat).get_aligned(one_hz)
best_lon = derived_param_from_hdf(lon).get_aligned(one_hz)
# Add KTIs.
for kti in kti_list:
kti_point_values = {'name': kti.name}
if not KEEP_KTIS and kti.name in SKIP_KTIS:
continue
elif len(KEEP_KTIS)>0 and (kti.name not in KEEP_KTIS):
continue
altitude = alt.at(kti.index) if alt else None
kti_point_values['altitudemode'] = altitude_mode
if altitude:
kti_point_values['coords'] = ((kti.longitude, kti.latitude, altitude),)
else:
kti_point_values['coords'] = ((kti.longitude, kti.latitude),)
kml.newpoint(**kti_point_values)
# Add KPVs.
for kpv in kpv_list:
# Trap kpvs with invalid latitude or longitude data (normally happens
# at the start of the data where accelerometer offsets are declared,
# and this avoids casting kpvs into the Atlantic.
kpv_lat = best_lat.at(kpv.index)
kpv_lon = best_lon.at(kpv.index)
if kpv_lat is None or kpv_lon is None or \
(kpv_lat == 0.0 and kpv_lon == 0.0):
continue
if not KEEP_KPVS and kpv.name in SKIP_KPVS:
continue
elif len(KEEP_KPVS)>0 and (kpv.name not in KEEP_KPVS):
continue
style = simplekml.Style()
style.iconstyle.color = simplekml.Color.red
kpv_point_values = {'name': '%s (%.3f)' % (kpv.name, kpv.value)}
altitude = alt.at(kpv.index) if alt else None
kpv_point_values['altitudemode'] = altitude_mode
if altitude:
kpv_point_values['coords'] = ((kpv_lon, kpv_lat, altitude),)
else:
kpv_point_values['coords'] = ((kpv_lon, kpv_lat),)
pnt = kml.newpoint(**kpv_point_values)
pnt.style = style
# Add approach centre lines.
for app in approach_list:
try:
draw_centreline(kml, app.runway)
except:
pass
if not dest_path:
dest_path = hdf_path + ".kml"
kml.save(dest_path)
return dest_path
def track_to_kml(hdf_path, kti_list, kpv_list, approach_list,
plot_altitude='Altitude QNH', dest_path=None):
'''
Plot results of process_flight onto a KML track.
:param flight_attrs: List of Flight Attributes
:type flight_attrs: list
:param plot_altitude: Name of Altitude parameter to use in KML
:type plot_altitude: String
'''
one_hz = Parameter()
kml = simplekml.Kml()
with hdf_file(hdf_path) as hdf:
if 'Latitude Smoothed' not in hdf:
return False
if plot_altitude not in hdf:
logger.warning("Disabling altitude on KML plot as it is unavailable.")
plot_altitude = False
if plot_altitude:
alt = derived_param_from_hdf(hdf[plot_altitude]).get_aligned(one_hz)
alt.array = repair_mask(alt.array, frequency=alt.frequency, repair_duration=None) / METRES_TO_FEET
else:
alt = None
if plot_altitude in ['Altitude QNH', 'Altitude AAL', 'Altitude STD']:
altitude_mode = simplekml.constants.AltitudeMode.absolute
elif plot_altitude in ['Altitude Radio']:
altitude_mode = simplekml.constants.AltitudeMode.relativetoground
else:
altitude_mode = simplekml.constants.AltitudeMode.clamptoground
# TODO: align everything to 0 offset
smooth_lat = derived_param_from_hdf(hdf['Latitude Smoothed']).get_aligned(one_hz)
smooth_lon = derived_param_from_hdf(hdf['Longitude Smoothed']).get_aligned(one_hz)
add_track(kml, 'Smoothed', smooth_lat, smooth_lon, 'ff7fff7f',
alt_param=alt, alt_mode=altitude_mode)
add_track(kml, 'Smoothed On Ground', smooth_lat, smooth_lon, 'ff7fff7f')
if 'Latitude Prepared' in hdf and 'Longitude Prepared' in hdf:
lat = hdf['Latitude Prepared']
lon = hdf['Longitude Prepared']
add_track(kml, 'Prepared Track', lat, lon, 'A11EB3', visible=False)
if 'Latitude' in hdf and 'Longitude' in hdf:
lat_r = hdf['Latitude']
lon_r = hdf['Longitude']
# add RAW track default invisible
add_track(kml, 'Recorded Track', lat_r, lon_r, 'ff0000ff', visible=False)
for kti in kti_list:
kti_point_values = {'name': kti.name}
if kti.name in SKIP_KTIS:
continue
altitude = alt.at(kti.index) if plot_altitude else None
kti_point_values['altitudemode'] = altitude_mode
if altitude:
kti_point_values['coords'] = ((kti.longitude, kti.latitude, altitude),)
else:
kti_point_values['coords'] = ((kti.longitude, kti.latitude),)
kml.newpoint(**kti_point_values)
for kpv in kpv_list:
# Trap kpvs with invalid latitude or longitude data (normally happens
# at the start of the data where accelerometer offsets are declared,
# and this avoids casting kpvs into the Atlantic.
kpv_lat = smooth_lat.at(kpv.index)
kpv_lon = smooth_lon.at(kpv.index)
if kpv_lat == None or kpv_lon == None or \
(kpv_lat == 0.0 and kpv_lon == 0.0):
continue
if kpv.name in SKIP_KPVS:
continue
style = simplekml.Style()
style.iconstyle.color = simplekml.Color.red
kpv_point_values = {'name': '%s (%.3f)' % (kpv.name, kpv.value)}
altitude = alt.at(kpv.index) if plot_altitude else None
kpv_point_values['altitudemode'] = altitude_mode
if altitude:
kpv_point_values['coords'] = ((kpv_lon, kpv_lat, altitude),)
else:
kpv_point_values['coords'] = ((kpv_lon, kpv_lat),)
pnt = kml.newpoint(**kpv_point_values)
pnt.style = style
for app in approach_list:
try:
draw_centreline(kml, app.runway)
except:
pass
if not dest_path:
dest_path = hdf_path + ".kml"
kml.save(dest_path)
return