def buildsection(name, begin, end):
'''from FlightDataAnalyzer tests
A little routine to make building Sections for testing easier.
Example: land = buildsection('Landing', 100, 120)
'''
result = Section(name, slice(begin, end, None), begin, end)
return SectionNode(name, items=[result])
def buildsections(name, *args):
'''
Like buildsection, this is used to build SectionNodes for test purposes.
lands = buildsections('name',[from1,to1],[from2,to2])
Example of use:
approach = buildsections('Approach', [80,90], [100,110])None
'''
return SectionNode(name, items=[builditem(name, *a) for a in args])
def buildsection(name, *args):
'''
A little routine to make building Sections for testing easier.
:param name: name for a test Section
:returns: a SectionNode populated correctly.
Example: land = buildsection('Landing', 100, 120)
'''
return SectionNode(name, items=[builditem(name, *args)])
def buildsections(*args):
'''from FlightDataAnalyzer tests
Example: approach = buildsections('Approach', [80,90], [100,110])
'''
built_list = []
name = args[0]
for a in args[1:]:
begin, end = a[0], a[1]
new_section = Section(name, slice(begin, end, None), begin, end)
built_list.append(new_section)
return SectionNode(name, items=built_list)
def derive_parameters_mitre(hdf, node_mgr, process_order, precomputed_parameters={}):
'''
Derives the parameter values and if limits are available, applies
parameter validation upon each param before storing the resulting masked
array back into the hdf file.
:param hdf: Data file accessor used to get and save parameter data and attributes
:type hdf: hdf_file
:param node_mgr: Used to determine the type of node in the process_order
:type node_mgr: NodeManager
:param process_order: Parameter / Node class names in the required order to be processed
:type process_order: list of strings
'''
params = precomputed_parameters # dictionary of derived params that aren't masked arrays
approach_list = ApproachNode(restrict_names=False)
kpv_list = KeyPointValueNode(restrict_names=False) # duplicate storage, but maintaining types
kti_list = KeyTimeInstanceNode(restrict_names=False)
section_list = SectionNode() # 'Node Name' : node() pass in node.get_accessor()
flight_attrs = []
duration = hdf.duration
for param_name in process_order:
if param_name in node_mgr.hdf_keys:
logger.debug(' derive_: hdf '+param_name)
continue
elif node_mgr.get_attribute(param_name) is not None:
logger.debug(' derive_: get_attribute '+param_name)
continue
elif param_name in params: # already calculated KPV/KTI/Phase ***********************NEW
logger.debug(' derive_parameters: re-using '+param_name)
continue
logger.debug(' derive_: computing '+param_name)
node_class = node_mgr.derived_nodes[param_name] #NB raises KeyError if Node is "unknown"
# build ordered dependencies
deps = []
node_deps = node_class.get_dependency_names()
for dep_name in node_deps:
if dep_name in params: # already calculated KPV/KTI/Phase
deps.append(params[dep_name])
elif node_mgr.get_attribute(dep_name) is not None:
deps.append(node_mgr.get_attribute(dep_name))
elif dep_name in node_mgr.hdf_keys:
# LFL/Derived parameter
# all parameters (LFL or other) need get_aligned which is
# available on DerivedParameterNode
try:
dp = derived_param_from_hdf(hdf.get_param(dep_name,
valid_only=True))
except KeyError:
# Parameter is invalid.
dp = None
deps.append(dp)
else: # dependency not available
deps.append(None)
if all([d is None for d in deps]):
raise RuntimeError("No dependencies available - Nodes cannot "
"operate without ANY dependencies available! "
"Node: %s" % node_class.__name__)
# initialise node
node = node_class()
logger.info("Processing parameter %s", param_name)
# Derive the resulting value
result = node.get_derived(deps)
if node.node_type is KeyPointValueNode:
#Q: track node instead of result here??
params[param_name] = result
for one_hz in result.get_aligned(P(frequency=1, offset=0)):
if not (0 <= one_hz.index <= duration):
raise IndexError(
"KPV '%s' index %.2f is not between 0 and %d" %
(one_hz.name, one_hz.index, duration))
kpv_list.append(one_hz)
elif node.node_type is KeyTimeInstanceNode:
params[param_name] = result
for one_hz in result.get_aligned(P(frequency=1, offset=0)):
if not (0 <= one_hz.index <= duration):
raise IndexError(
"KTI '%s' index %.2f is not between 0 and %d" %
(one_hz.name, one_hz.index, duration))
kti_list.append(one_hz)
elif node.node_type is FlightAttributeNode:
params[param_name] = result
try:
flight_attrs.append(Attribute(result.name, result.value)) # only has one Attribute result
except:
logger.warning("Flight Attribute Node '%s' returned empty "
"handed.", param_name)
elif issubclass(node.node_type, SectionNode):
aligned_section = result.get_aligned(P(frequency=1, offset=0))
for index, one_hz in enumerate(aligned_section):
# SectionNodes allow slice starts and stops being None which
# signifies the beginning and end of the data. To avoid TypeErrors
# in subsequent derive methods which perform arithmetic on section
# slice start and stops, replace with 0 or hdf.duration.
fallback = lambda x, y: x if x is not None else y
duration = fallback(duration, 0)
start = fallback(one_hz.slice.start, 0)
stop = fallback(one_hz.slice.stop, duration)
start_edge = fallback(one_hz.start_edge, 0)
stop_edge = fallback(one_hz.stop_edge, duration)
slice_ = slice(start, stop)
one_hz = Section(one_hz.name, slice_, start_edge, stop_edge)
aligned_section[index] = one_hz
if not (0 <= start <= duration and 0 <= stop <= duration + 1):
msg = "Section '%s' (%.2f, %.2f) not between 0 and %d"
raise IndexError(msg % (one_hz.name, start, stop, duration))
if not 0 <= start_edge <= duration:
msg = "Section '%s' start_edge (%.2f) not between 0 and %d"
raise IndexError(msg % (one_hz.name, start_edge, duration))
if not 0 <= stop_edge <= duration + 1:
msg = "Section '%s' stop_edge (%.2f) not between 0 and %d"
raise IndexError(msg % (one_hz.name, stop_edge, duration))
section_list.append(one_hz)
params[param_name] = aligned_section
elif issubclass(node.node_type, DerivedParameterNode):
if duration:
# check that the right number of results were returned
# Allow a small tolerance. For example if duration in seconds
# is 2822, then there will be an array length of 1411 at 0.5Hz and 706
# at 0.25Hz (rounded upwards). If we combine two 0.25Hz
# parameters then we will have an array length of 1412.
expected_length = duration * result.frequency
if result.array is None:
logger.warning("No array set; creating a fully masked array for %s", param_name)
array_length = expected_length
# Where a parameter is wholly masked, we fill the HDF
# file with masked zeros to maintain structure.
result.array = \
np_ma_masked_zeros_like(np.ma.arange(expected_length))
else:
array_length = len(result.array)
length_diff = array_length - expected_length
if length_diff == 0:
pass
elif 0 < length_diff < 5:
logger.warning("Cutting excess data for parameter '%s'. "
"Expected length was '%s' while resulting "
"array length was '%s'.", param_name,
expected_length, len(result.array))
result.array = result.array[:expected_length]
else:
raise ValueError("Array length mismatch for parameter "
"'%s'. Expected '%s', resulting array "
"length '%s'." % (param_name,
expected_length,
array_length))
hdf.set_param(result)
# Keep hdf_keys up to date.
node_mgr.hdf_keys.append(param_name)
elif issubclass(node.node_type, ApproachNode):
aligned_approach = result.get_aligned(P(frequency=1, offset=0))
for approach in aligned_approach:
# Does not allow slice start or stops to be None.
valid_turnoff = (not approach.turnoff or
(0 <= approach.turnoff <= duration))
valid_slice = ((0 <= approach.slice.start <= duration) and
(0 <= approach.slice.stop <= duration))
valid_gs_est = (not approach.gs_est or
((0 <= approach.gs_est.start <= duration) and
(0 <= approach.gs_est.stop <= duration)))
valid_loc_est = (not approach.loc_est or
((0 <= approach.loc_est.start <= duration) and
(0 <= approach.loc_est.stop <= duration)))
if not all([valid_turnoff, valid_slice, valid_gs_est,
valid_loc_est]):
raise ValueError('ApproachItem contains index outside of '
'flight data: %s' % approach)
approach_list.append(approach)
params[param_name] = aligned_approach
else:
raise NotImplementedError("Unknown Type %s" % node.__class__)
continue
return kti_list, kpv_list, section_list, approach_list, flight_attrs, params
def derive_parameters_series(flight, node_mgr, process_order, precomputed={}):
'''
Non HDF5 version. Suitable for FFD and Notebook profile development.
Derives the parameter values and if limits are available, applies
parameter validation upon each param before storing the resulting masked
array back into the hdf file.
:param series: Data file accessor used to get and save parameter data and attributes
:type series: dictionary of ParameterNode objects
:param node_mgr: Used to determine the type of node in the process_order
:type node_mgr: NodeManager
:param process_order: Parameter / Node class names in the required order to be processed
:type process_order: list of strings
'''
duration = flight.duration
params = precomputed #{} # dictionary of derived params that aren't masked arrays
pre_aligned = {} #key = (name, frequency, offset)
res = {
'series': {},
'approach': ApproachNode(restrict_names=False),
'kpv': KeyPointValueNode(restrict_names=False),
'kti': KeyTimeInstanceNode(restrict_names=False),
'phase': SectionNode(),
'attr': []
} #results by node type
#print 'Process Order'
#for p in process_order:
# print ' ',p
for param_name in process_order:
#if param_name in node_mgr.hdf_keys:
# logger.info('_derive_: hdf '+param_name)
# continue
#elif
if node_mgr.get_attribute(param_name) is not None:
logger.info('_derive_: get_attribute ' + param_name)
continue
elif param_name in params.keys(): # already calculated
logger.info('_derive_: re-using precomputed' + param_name)
continue
elif not node_mgr.derived_nodes.has_key(param_name):
logger.info('_derive_: in process_order but not derived_nodes: ' +
param_name)
continue
elif param_name in res['series'].keys():
print 'derive_parameters: in series but not params:', param_name
#logger.info('_derive_: series'+param_name)
#continue
####compute###########################################################
logger.info('_derive_: computing ' + param_name)
node_class = node_mgr.derived_nodes[
param_name] #NB raises KeyError if Node is "unknown"
try:
deps, pre_aligned, node = get_deps_series(node_class, params,
node_mgr, pre_aligned)
except Exception, e:
logger.exception('ERROR ' + param_name + ' get_deps')
raise
try:
node.derive(*deps) #node.get_derived(deps)
result = node
except Exception, e:
logger.exception('ERROR ' + param_name + ' get_derived')
raise
def derive_parameters(hdf, node_mgr, process_order):
'''
Derives parameters in process_order. Dependencies are sourced via the
node_mgr.
:param hdf: Data file accessor used to get and save parameter data and
attributes
:type hdf: hdf_file
:param node_mgr: Used to determine the type of node in the process_order
:type node_mgr: NodeManager
:param process_order: Parameter / Node class names in the required order to
be processed
:type process_order: list of strings
'''
# store all derived params that aren't masked arrays
params = {}
approach_list = ApproachNode(restrict_names=False)
# duplicate storage, but maintaining types
kpv_list = KeyPointValueNode(restrict_names=False)
kti_list = KeyTimeInstanceNode(restrict_names=False)
# 'Node Name' : node() pass in node.get_accessor()
section_list = SectionNode()
flight_attrs = []
duration = hdf.duration
for param_name in process_order:
if param_name in node_mgr.hdf_keys:
continue
elif node_mgr.get_attribute(param_name) is not None:
# add attribute to dictionary of available params
###params[param_name] = node_mgr.get_attribute(param_name)
#TODO: optimise with only one call to get_attribute
continue
#NB raises KeyError if Node is "unknown"
node_class = node_mgr.derived_nodes[param_name]
# build ordered dependencies
deps = []
node_deps = node_class.get_dependency_names()
for dep_name in node_deps:
if dep_name in params: # already calculated KPV/KTI/Phase
deps.append(params[dep_name])
elif node_mgr.get_attribute(dep_name) is not None:
deps.append(node_mgr.get_attribute(dep_name))
elif dep_name in node_mgr.hdf_keys:
# LFL/Derived parameter
# all parameters (LFL or other) need get_aligned which is
# available on DerivedParameterNode
try:
dp = derived_param_from_hdf(
hdf.get_param(dep_name, valid_only=True))
except KeyError:
# Parameter is invalid.
dp = None
deps.append(dp)
else: # dependency not available
deps.append(None)
if all([d is None for d in deps]):
raise RuntimeError("No dependencies available - Nodes cannot "
"operate without ANY dependencies available! "
"Node: %s" % node_class.__name__)
# initialise node
node = node_class()
# shhh, secret accessors for developing nodes in debug mode
node._p = params
node._h = hdf
node._n = node_mgr
logger.info("Processing %s `%s`", get_node_type(node), param_name)
# Derive the resulting value
result = node.get_derived(deps)
del node._p
del node._h
del node._n
if node.node_type is KeyPointValueNode:
#Q: track node instead of result here??
params[param_name] = result
for one_hz in result.get_aligned(P(frequency=1, offset=0)):
if not (0 <= one_hz.index <= duration + 4):
raise IndexError(
"KPV '%s' index %.2f is not between 0 and %d" %
(one_hz.name, one_hz.index, duration))
kpv_list.append(one_hz)
elif node.node_type is KeyTimeInstanceNode:
params[param_name] = result
for one_hz in result.get_aligned(P(frequency=1, offset=0)):
if not (0 <= one_hz.index <= duration + 4):
raise IndexError(
"KTI '%s' index %.2f is not between 0 and %d" %
(one_hz.name, one_hz.index, duration))
kti_list.append(one_hz)
elif node.node_type is FlightAttributeNode:
params[param_name] = result
try:
# only has one Attribute result
flight_attrs.append(Attribute(result.name, result.value))
except:
logger.warning(
"Flight Attribute Node '%s' returned empty "
"handed.", param_name)
elif issubclass(node.node_type, SectionNode):
aligned_section = result.get_aligned(P(frequency=1, offset=0))
for index, one_hz in enumerate(aligned_section):
# SectionNodes allow slice starts and stops being None which
# signifies the beginning and end of the data. To avoid
# TypeErrors in subsequent derive methods which perform
# arithmetic on section slice start and stops, replace with 0
# or hdf.duration.
fallback = lambda x, y: x if x is not None else y
duration = fallback(duration, 0)
start = fallback(one_hz.slice.start, 0)
stop = fallback(one_hz.slice.stop, duration)
start_edge = fallback(one_hz.start_edge, 0)
stop_edge = fallback(one_hz.stop_edge, duration)
slice_ = slice(start, stop)
one_hz = Section(one_hz.name, slice_, start_edge, stop_edge)
aligned_section[index] = one_hz
if not (0 <= start <= duration and 0 <= stop <= duration + 4):
msg = "Section '%s' (%.2f, %.2f) not between 0 and %d"
raise IndexError(msg %
(one_hz.name, start, stop, duration))
if not 0 <= start_edge <= duration:
msg = "Section '%s' start_edge (%.2f) not between 0 and %d"
raise IndexError(msg % (one_hz.name, start_edge, duration))
if not 0 <= stop_edge <= duration + 4:
msg = "Section '%s' stop_edge (%.2f) not between 0 and %d"
raise IndexError(msg % (one_hz.name, stop_edge, duration))
section_list.append(one_hz)
params[param_name] = aligned_section
elif issubclass(node.node_type, DerivedParameterNode):
if duration:
# check that the right number of results were returned Allow a
# small tolerance. For example if duration in seconds is 2822,
# then there will be an array length of 1411 at 0.5Hz and 706
# at 0.25Hz (rounded upwards). If we combine two 0.25Hz
# parameters then we will have an array length of 1412.
expected_length = duration * result.frequency
if result.array is None:
logger.warning(
"No array set; creating a fully masked "
"array for %s", param_name)
array_length = expected_length
# Where a parameter is wholly masked, we fill the HDF
# file with masked zeros to maintain structure.
result.array = \
np_ma_masked_zeros_like(np.ma.arange(expected_length))
else:
array_length = len(result.array)
length_diff = array_length - expected_length
if length_diff == 0:
pass
elif 0 < length_diff < 5:
logger.warning(
"Cutting excess data for parameter '%s'. "
"Expected length was '%s' while resulting "
"array length was '%s'.", param_name, expected_length,
len(result.array))
result.array = result.array[:expected_length]
else:
raise ValueError(
"Array length mismatch for parameter "
"'%s'. Expected '%s', resulting array "
"length '%s'." %
(param_name, expected_length, array_length))
hdf.set_param(result)
# Keep hdf_keys up to date.
node_mgr.hdf_keys.append(param_name)
elif issubclass(node.node_type, ApproachNode):
aligned_approach = result.get_aligned(P(frequency=1, offset=0))
for approach in aligned_approach:
# Does not allow slice start or stops to be None.
valid_turnoff = (not approach.turnoff
or (0 <= approach.turnoff <= duration))
valid_slice = ((0 <= approach.slice.start <= duration)
and (0 <= approach.slice.stop <= duration))
valid_gs_est = (not approach.gs_est or
((0 <= approach.gs_est.start <= duration) and
(0 <= approach.gs_est.stop <= duration)))
valid_loc_est = (not approach.loc_est or
((0 <= approach.loc_est.start <= duration) and
(0 <= approach.loc_est.stop <= duration)))
if not all(
[valid_turnoff, valid_slice, valid_gs_est, valid_loc_est]):
raise ValueError('ApproachItem contains index outside of '
'flight data: %s' % approach)
approach_list.append(approach)
params[param_name] = aligned_approach
else:
raise NotImplementedError("Unknown Type %s" % node.__class__)
continue
return kti_list, kpv_list, section_list, approach_list, flight_attrs