def derive_parameters(hdf, node_mgr, process_order, params=None, force=False):
'''
Derives parameters in process_order. Dependencies are sourced via the
node_mgr.
:param hdf: Data file accessor used to get and save parameter data and
attributes
:type hdf: hdf_file
:param node_mgr: Used to determine the type of node in the process_order
:type node_mgr: NodeManager
:param process_order: Parameter / Node class names in the required order to
be processed
:type process_order: list of strings
'''
if not params:
params = {}
# OPT: local lookup is faster than module-level (small).
node_subclasses = NODE_SUBCLASSES
# store all derived params that aren't masked arrays
approaches = {}
# duplicate storage, but maintaining types
kpvs = {}
ktis = {}
# 'Node Name' : node() pass in node.get_accessor()
sections = {}
flight_attrs = {}
# cache of nodes to avoid repeated array alignment
cache = {} if NODE_CACHE else None
duration = hdf.duration
for param_name in process_order:
if param_name in node_mgr.hdf_keys:
continue
elif param_name in params:
node = params[param_name]
# populate output already at 1Hz
if node.node_type is KeyPointValueNode:
kpvs[param_name] = list(node)
elif node.node_type is KeyTimeInstanceNode:
ktis[param_name] = list(node)
elif node.node_type is FlightAttributeNode:
flight_attrs[param_name] = [Attribute(node.name, node.value)]
elif node.node_type is SectionNode:
sections[param_name] = list(node)
# DerivedParameterNodes are not supported in initial data.
continue
elif node_mgr.get_attribute(param_name) is not None:
# add attribute to dictionary of available params
###params[param_name] = node_mgr.get_attribute(param_name)
#TODO: optimise with only one call to get_attribute
continue
#NB raises KeyError if Node is "unknown"
node_class = node_mgr.derived_nodes[param_name]
# build ordered dependencies
deps = []
node_deps = node_class.get_dependency_names()
for dep_name in node_deps:
if dep_name in params: # already calculated KPV/KTI/Phase
deps.append(params[dep_name])
elif node_mgr.get_attribute(dep_name) is not None:
deps.append(node_mgr.get_attribute(dep_name))
elif dep_name in node_mgr.hdf_keys:
# LFL/Derived parameter
# all parameters (LFL or other) need get_aligned which is
# available on DerivedParameterNode
try:
dp = derived_param_from_hdf(hdf.get_param(dep_name,
valid_only=True),
cache=cache)
except KeyError:
# Parameter is invalid.
dp = None
deps.append(dp)
else: # dependency not available
deps.append(None)
if all([d is None for d in deps]):
raise RuntimeError("No dependencies available - Nodes cannot "
"operate without ANY dependencies available! "
"Node: %s" % node_class.__name__)
# initialise node
node = node_class(cache=cache)
# shhh, secret accessors for developing nodes in debug mode
node._p = params
node._h = hdf
node._n = node_mgr
logger.debug("Processing %s `%s`",
get_node_type(node, node_subclasses), param_name)
# Derive the resulting value
try:
node = node.get_derived(deps)
except:
if not force:
raise
del node._p
del node._h
del node._n
if node.node_type is KeyPointValueNode:
params[param_name] = node
aligned_kpvs = []
for one_hz in node.get_aligned(P(frequency=1, offset=0)):
if not (0 <= one_hz.index <= duration + 4):
raise IndexError(
"KPV '%s' index %.2f is not between 0 and %d" %
(one_hz.name, one_hz.index, duration))
aligned_kpvs.append(one_hz)
kpvs[param_name] = aligned_kpvs
elif node.node_type is KeyTimeInstanceNode:
params[param_name] = node
aligned_ktis = []
for one_hz in node.get_aligned(P(frequency=1, offset=0)):
if not (0 <= one_hz.index <= duration + 4):
raise IndexError(
"KTI '%s' index %.2f is not between 0 and %d" %
(one_hz.name, one_hz.index, duration))
aligned_ktis.append(one_hz)
ktis[param_name] = aligned_ktis
elif node.node_type is FlightAttributeNode:
params[param_name] = node
try:
# only has one Attribute node, store as a list for consistency
flight_attrs[param_name] = [Attribute(node.name, node.value)]
except:
logger.warning(
"Flight Attribute Node '%s' returned empty "
"handed.", param_name)
elif issubclass(node.node_type, SectionNode):
aligned_section = node.get_aligned(P(frequency=1, offset=0))
for index, one_hz in enumerate(aligned_section):
# SectionNodes allow slice starts and stops being None which
# signifies the beginning and end of the data. To avoid
# TypeErrors in subsequent derive methods which perform
# arithmetic on section slice start and stops, replace with 0
# or hdf.duration.
fallback = lambda x, y: x if x is not None else y
duration = fallback(duration, 0)
start = fallback(one_hz.slice.start, 0)
stop = fallback(one_hz.slice.stop, duration)
start_edge = fallback(one_hz.start_edge, 0)
stop_edge = fallback(one_hz.stop_edge, duration)
slice_ = slice(start, stop)
one_hz = Section(one_hz.name, slice_, start_edge, stop_edge)
aligned_section[index] = one_hz
if not (0 <= start <= duration and 0 <= stop <= duration + 4):
msg = "Section '%s' (%.2f, %.2f) not between 0 and %d"
raise IndexError(msg %
(one_hz.name, start, stop, duration))
if not 0 <= start_edge <= duration:
msg = "Section '%s' start_edge (%.2f) not between 0 and %d"
raise IndexError(msg % (one_hz.name, start_edge, duration))
if not 0 <= stop_edge <= duration + 4:
msg = "Section '%s' stop_edge (%.2f) not between 0 and %d"
raise IndexError(msg % (one_hz.name, stop_edge, duration))
#section_list.append(one_hz)
params[param_name] = aligned_section
sections[param_name] = list(aligned_section)
elif issubclass(node.node_type, DerivedParameterNode):
if duration:
# check that the right number of nodes were returned Allow a
# small tolerance. For example if duration in seconds is 2822,
# then there will be an array length of 1411 at 0.5Hz and 706
# at 0.25Hz (rounded upwards). If we combine two 0.25Hz
# parameters then we will have an array length of 1412.
expected_length = duration * node.frequency
if node.array is None or (force and len(node.array) == 0):
logger.warning(
"No array set; creating a fully masked "
"array for %s", param_name)
array_length = expected_length
# Where a parameter is wholly masked, we fill the HDF
# file with masked zeros to maintain structure.
node.array = \
np_ma_masked_zeros(expected_length)
else:
array_length = len(node.array)
length_diff = array_length - expected_length
if length_diff == 0:
pass
elif 0 < length_diff < 5:
logger.warning(
"Cutting excess data for parameter '%s'. "
"Expected length was '%s' while resulting "
"array length was '%s'.", param_name, expected_length,
len(node.array))
node.array = node.array[:expected_length]
else:
raise ValueError(
"Array length mismatch for parameter "
"'%s'. Expected '%s', resulting array "
"length '%s'." %
(param_name, expected_length, array_length))
hdf.set_param(node)
# Keep hdf_keys up to date.
node_mgr.hdf_keys.append(param_name)
elif issubclass(node.node_type, ApproachNode):
aligned_approach = node.get_aligned(P(frequency=1, offset=0))
for approach in aligned_approach:
# Does not allow slice start or stops to be None.
valid_turnoff = (not approach.turnoff
or (0 <= approach.turnoff <= duration))
valid_slice = ((0 <= approach.slice.start <= duration)
and (0 <= approach.slice.stop <= duration))
valid_gs_est = (not approach.gs_est or
((0 <= approach.gs_est.start <= duration) and
(0 <= approach.gs_est.stop <= duration)))
valid_loc_est = (not approach.loc_est or
((0 <= approach.loc_est.start <= duration) and
(0 <= approach.loc_est.stop <= duration)))
if not all(
[valid_turnoff, valid_slice, valid_gs_est, valid_loc_est]):
raise ValueError('ApproachItem contains index outside of '
'flight data: %s' % approach)
params[param_name] = aligned_approach
approaches[param_name] = list(aligned_approach)
else:
raise NotImplementedError("Unknown Type %s" % node.__class__)
continue
return ktis, kpvs, sections, approaches, flight_attrs
def derive_parameters(hdf, node_mgr, process_order, params=None, force=False):
'''
Derives parameters in process_order. Dependencies are sourced via the
node_mgr.
:param hdf: Data file accessor used to get and save parameter data and
attributes
:type hdf: hdf_file
:param node_mgr: Used to determine the type of node in the process_order
:type node_mgr: NodeManager
:param process_order: Parameter / Node class names in the required order to
be processed
:type process_order: list of strings
'''
if not params:
params = {}
# OPT: local lookup is faster than module-level (small).
node_subclasses = NODE_SUBCLASSES
# store all derived params that aren't masked arrays
approaches = {}
# duplicate storage, but maintaining types
kpvs = {}
ktis = {}
# 'Node Name' : node() pass in node.get_accessor()
sections = {}
flight_attrs = {}
duration = hdf.duration
for param_name in process_order:
if param_name in node_mgr.hdf_keys:
continue
elif param_name in params:
node = params[param_name]
# populate output already at 1Hz
if node.node_type is KeyPointValueNode:
kpvs[param_name] = list(node)
elif node.node_type is KeyTimeInstanceNode:
ktis[param_name] = list(node)
elif node.node_type is FlightAttributeNode:
flight_attrs[param_name] = [Attribute(node.name, node.value)]
elif node.node_type is SectionNode:
sections[param_name] = list(node)
# DerivedParameterNodes are not supported in initial data.
continue
elif node_mgr.get_attribute(param_name) is not None:
# add attribute to dictionary of available params
###params[param_name] = node_mgr.get_attribute(param_name)
#TODO: optimise with only one call to get_attribute
continue
#NB raises KeyError if Node is "unknown"
node_class = node_mgr.derived_nodes[param_name]
# build ordered dependencies
deps = []
node_deps = node_class.get_dependency_names()
for dep_name in node_deps:
if dep_name in params: # already calculated KPV/KTI/Phase
deps.append(params[dep_name])
elif node_mgr.get_attribute(dep_name) is not None:
deps.append(node_mgr.get_attribute(dep_name))
elif dep_name in node_mgr.hdf_keys:
# LFL/Derived parameter
# all parameters (LFL or other) need get_aligned which is
# available on DerivedParameterNode
try:
dp = derived_param_from_hdf(hdf.get_param(
dep_name, valid_only=True))
except KeyError:
# Parameter is invalid.
dp = None
deps.append(dp)
else: # dependency not available
deps.append(None)
if all([d is None for d in deps]):
raise RuntimeError(
"No dependencies available - Nodes cannot "
"operate without ANY dependencies available! "
"Node: %s" % node_class.__name__)
# initialise node
node = node_class()
# shhh, secret accessors for developing nodes in debug mode
node._p = params
node._h = hdf
node._n = node_mgr
logger.info("Processing %s `%s`", get_node_type(node, node_subclasses), param_name)
# Derive the resulting value
try:
node = node.get_derived(deps)
except:
if not force:
raise
del node._p
del node._h
del node._n
if node.node_type is KeyPointValueNode:
params[param_name] = node
aligned_kpvs = []
for one_hz in node.get_aligned(P(frequency=1, offset=0)):
if not (0 <= one_hz.index <= duration+4):
raise IndexError(
"KPV '%s' index %.2f is not between 0 and %d" %
(one_hz.name, one_hz.index, duration))
aligned_kpvs.append(one_hz)
kpvs[param_name] = aligned_kpvs
elif node.node_type is KeyTimeInstanceNode:
params[param_name] = node
aligned_ktis = []
for one_hz in node.get_aligned(P(frequency=1, offset=0)):
if not (0 <= one_hz.index <= duration+4):
raise IndexError(
"KTI '%s' index %.2f is not between 0 and %d" %
(one_hz.name, one_hz.index, duration))
aligned_ktis.append(one_hz)
ktis[param_name] = aligned_ktis
elif node.node_type is FlightAttributeNode:
params[param_name] = node
try:
# only has one Attribute node, store as a list for consistency
flight_attrs[param_name] = [Attribute(node.name, node.value)]
except:
logger.warning("Flight Attribute Node '%s' returned empty "
"handed.", param_name)
elif issubclass(node.node_type, SectionNode):
aligned_section = node.get_aligned(P(frequency=1, offset=0))
for index, one_hz in enumerate(aligned_section):
# SectionNodes allow slice starts and stops being None which
# signifies the beginning and end of the data. To avoid
# TypeErrors in subsequent derive methods which perform
# arithmetic on section slice start and stops, replace with 0
# or hdf.duration.
fallback = lambda x, y: x if x is not None else y
duration = fallback(duration, 0)
start = fallback(one_hz.slice.start, 0)
stop = fallback(one_hz.slice.stop, duration)
start_edge = fallback(one_hz.start_edge, 0)
stop_edge = fallback(one_hz.stop_edge, duration)
slice_ = slice(start, stop)
one_hz = Section(one_hz.name, slice_, start_edge, stop_edge)
aligned_section[index] = one_hz
if not (0 <= start <= duration and 0 <= stop <= duration + 4):
msg = "Section '%s' (%.2f, %.2f) not between 0 and %d"
raise IndexError(
msg % (one_hz.name, start, stop, duration))
if not 0 <= start_edge <= duration:
msg = "Section '%s' start_edge (%.2f) not between 0 and %d"
raise IndexError(msg % (one_hz.name, start_edge, duration))
if not 0 <= stop_edge <= duration + 4:
msg = "Section '%s' stop_edge (%.2f) not between 0 and %d"
raise IndexError(msg % (one_hz.name, stop_edge, duration))
#section_list.append(one_hz)
params[param_name] = aligned_section
sections[param_name] = list(aligned_section)
elif issubclass(node.node_type, DerivedParameterNode):
if duration:
# check that the right number of nodes were returned Allow a
# small tolerance. For example if duration in seconds is 2822,
# then there will be an array length of 1411 at 0.5Hz and 706
# at 0.25Hz (rounded upwards). If we combine two 0.25Hz
# parameters then we will have an array length of 1412.
expected_length = duration * node.frequency
if node.array is None or (force and len(node.array) == 0):
logger.warning("No array set; creating a fully masked "
"array for %s", param_name)
array_length = expected_length
# Where a parameter is wholly masked, we fill the HDF
# file with masked zeros to maintain structure.
node.array = \
np_ma_masked_zeros(expected_length)
else:
array_length = len(node.array)
length_diff = array_length - expected_length
if length_diff == 0:
pass
elif 0 < length_diff < 5:
logger.warning("Cutting excess data for parameter '%s'. "
"Expected length was '%s' while resulting "
"array length was '%s'.", param_name,
expected_length, len(node.array))
node.array = node.array[:expected_length]
else:
raise ValueError("Array length mismatch for parameter "
"'%s'. Expected '%s', resulting array "
"length '%s'." % (param_name,
expected_length,
array_length))
hdf.set_param(node)
# Keep hdf_keys up to date.
node_mgr.hdf_keys.append(param_name)
elif issubclass(node.node_type, ApproachNode):
aligned_approach = node.get_aligned(P(frequency=1, offset=0))
for approach in aligned_approach:
# Does not allow slice start or stops to be None.
valid_turnoff = (not approach.turnoff or
(0 <= approach.turnoff <= duration))
valid_slice = ((0 <= approach.slice.start <= duration) and
(0 <= approach.slice.stop <= duration))
valid_gs_est = (not approach.gs_est or
((0 <= approach.gs_est.start <= duration) and
(0 <= approach.gs_est.stop <= duration)))
valid_loc_est = (not approach.loc_est or
((0 <= approach.loc_est.start <= duration) and
(0 <= approach.loc_est.stop <= duration)))
if not all([valid_turnoff, valid_slice, valid_gs_est,
valid_loc_est]):
raise ValueError('ApproachItem contains index outside of '
'flight data: %s' % approach)
params[param_name] = aligned_approach
approaches[param_name] = list(aligned_approach)
else:
raise NotImplementedError("Unknown Type %s" % node.__class__)
continue
return ktis, kpvs, sections, approaches, flight_attrs