def test_graph_nodes_using_sample_tree(self):
requested = ['P7', 'P8']
mgr2 = NodeManager(datetime.now(), 10, self.lfl_params, requested, [],
self.derived_nodes, {}, {})
gr = graph_nodes(mgr2)
self.assertEqual(len(gr), 11)
self.assertEqual(gr.neighbors('root'), ['P8', 'P7'])
def derived_trimmer(hdf_path, node_names, dest):
'''
Trims an HDF file of parameters which are not dependencies of nodes in
node_names.
:param hdf_path: file path of hdf file.
:type hdf_path: str
:param node_names: A list of Node names which are required.
:type node_names: list of str
:param dest: destination path for trimmed output file
:type dest: str
:return: parameters in stripped hdf file
:rtype: [str]
'''
params = []
with hdf_file(hdf_path) as hdf:
derived_nodes = get_derived_nodes(settings.NODE_MODULES)
node_mgr = NodeManager(
datetime.now(), hdf.duration, hdf.valid_param_names(), [], [],
derived_nodes, {}, {})
_graph = graph_nodes(node_mgr)
for node_name in node_names:
deps = dependencies3(_graph, node_name, node_mgr)
params.extend(filter(lambda d: d in node_mgr.hdf_keys, deps))
return strip_hdf(hdf_path, params, dest)
def test_invalid_requirement_raises(self):
lfl_params = []
requested = ['Smoothed Track', 'Moment of Takeoff'] #it's called Moment Of Takeoff
derived = get_derived_nodes([import_module('sample_derived_parameters')])
mgr = NodeManager({'Start Datetime': datetime.now()}, 10, lfl_params, requested, [],
derived, {}, {})
self.assertRaises(nx.NetworkXError, dependency_order, mgr, draw=False)
def test_graph_middle_level_depenency_builds_partial_tree(self):
requested = ['P5']
mgr = NodeManager(datetime.now(), 1, self.lfl_params, requested, [],
self.derived_nodes, {}, {})
gr = graph_nodes(mgr)
# should only be linked to P5
self.assertEqual(gr.neighbors('root'), ['P5'])
def test_graph_nodes_with_duplicate_key_in_lfl_and_derived(self):
# Test that LFL nodes are used in place of Derived where available.
# Tests a few of the colours
class One(DerivedParameterNode):
# Hack to allow objects rather than classes to be added to the tree.
__base__ = DerivedParameterNode
__bases__ = [__base__]
def derive(self, dep=P('DepOne')):
pass
class Four(DerivedParameterNode):
# Hack to allow objects rather than classes to be added to the tree.
__base__ = DerivedParameterNode
__bases__ = [__base__]
def derive(self, dep=P('DepFour')):
pass
one = One('overridden')
four = Four('used')
mgr1 = NodeManager({'Start Datetime': datetime.now()}, 10, [1, 2], [2, 4], [],
{1:one, 4:four},{}, {})
gr = graph_nodes(mgr1)
self.assertEqual(len(gr), 5)
# LFL
self.assertEqual(gr.edges(1), []) # as it's in LFL, it shouldn't have any edges
self.assertEqual(gr.node[1], {'color': '#72f4eb', 'node_type': 'HDFNode'})
# Derived
self.assertEqual(gr.edges(4), [(4,'DepFour')])
self.assertEqual(gr.node[4], {'color': '#72cdf4', 'node_type': 'DerivedParameterNode'})
# Root
from analysis_engine.dependency_graph import draw_graph
draw_graph(gr, 'test_graph_nodes_with_duplicate_key_in_lfl_and_derived')
self.assertEqual(gr.successors('root'), [2,4]) # only the two requested are linked
self.assertEqual(gr.node['root'], {'color': '#ffffff'})
def test_sample_parameter_module(self):
"""Tests many options:
can_operate on SmoothedTrack works with
"""
requested = ['Smoothed Track', 'Moment Of Takeoff', 'Vertical Speed',
'Slip On Runway']
lfl_params = ['Indicated Airspeed',
'Groundspeed',
'Pressure Altitude',
'Heading', 'TAT',
'Latitude', 'Longitude',
'Longitudinal g', 'Lateral g', 'Normal g',
'Pitch', 'Roll',
]
derived = get_derived_nodes([import_module('sample_derived_parameters')])
nodes = NodeManager({'Start Datetime': datetime.now()}, 10, lfl_params,
requested, [], derived, {}, {})
order, _ = dependency_order(nodes, draw=False)
pos = order.index
self.assertTrue(len(order))
self.assertNotIn('Moment Of Takeoff', order) # not available
self.assertTrue(pos('Vertical Speed') > pos('Pressure Altitude'))
self.assertTrue(pos('Slip On Runway') > pos('Groundspeed'))
self.assertTrue(pos('Slip On Runway') > pos('Horizontal g Across Track'))
self.assertTrue(pos('Horizontal g Across Track') > pos('Roll'))
self.assertFalse('Mach' in order) # Mach wasn't requested!
self.assertFalse('Radio Altimeter' in order)
self.assertEqual(len(nodes.hdf_keys), 12)
self.assertEqual(len(nodes.requested), 4)
self.assertEqual(len(nodes.derived_nodes), 13)
def test_dependency_with_lowlevel_dependencies_requested(self):
""" Simulate requesting a Raw Parameter as a dependency. This requires
the requested node to be removed when it is not at the top of the
dependency tree.
"""
requested = [
'P7',
'P8', # top level nodes
'P4',
'P5',
'P6', # middle level node
'Raw3', # bottom level node
]
mgr = NodeManager(datetime.now(), 10, self.lfl_params + ['Floating'],
requested, [], self.derived_nodes, {}, {})
gr = graph_nodes(mgr)
gr_all, gr_st, order = process_order(gr, mgr)
self.assertEqual(len(gr_st), 11)
pos = order.index
self.assertTrue(pos('P8') > pos('Raw5'))
self.assertTrue(pos('P7') > pos('P4'))
self.assertTrue(pos('P7') > pos('P5'))
self.assertTrue(pos('P7') > pos('P6'))
self.assertTrue(pos('P6') > pos('Raw3'))
self.assertTrue(pos('P5') > pos('Raw3'))
self.assertTrue(pos('P5') > pos('Raw4'))
self.assertTrue(pos('P4') > pos('Raw1'))
self.assertTrue(pos('P4') > pos('Raw2'))
self.assertFalse('Floating' in order)
self.assertFalse('root' in order) #don't include the root!
def _get_dependency_order(self,
requested,
aircraft_info,
lfl_params,
draw=False,
raise_cir_dep=True,
segment_info={}):
#derived_nodes = get_derived_nodes(settings.NODE_MODULES)
if aircraft_info['Aircraft Type'] == 'helicopter':
node_modules = settings.NODE_MODULES + settings.NODE_HELICOPTER_MODULE_PATHS
else:
node_modules = settings.NODE_MODULES
# go through modules to get derived nodes
derived_nodes = get_derived_nodes(node_modules)
if requested == []:
# Use all derived nodes if requested is empty
requested = [
p for p in derived_nodes.keys() if p not in lfl_params
]
if not segment_info:
segment_info = {
'Start Datetime': datetime.now(),
'Segment Type': 'START_AND_STOP',
}
node_mgr = NodeManager(segment_info, 10, lfl_params, requested, [],
derived_nodes, aircraft_info, {})
order, gr_st = dependency_order(node_mgr,
draw=draw,
raise_cir_dep=raise_cir_dep)
return order, gr_st
def pre_process_parameters(hdf,
segment_info,
param_names,
required,
aircraft_info,
achieved_flight_record,
force=False):
'''
Perform actions prior to main processing run.
Actions such as merging parameters up front simplify processing paths by
removing circular dependacies.
'''
pre_processing_nodes = get_derived_nodes(
settings.PRE_PROCESSING_MODULE_PATHS)
requested = list(pre_processing_nodes.keys())
node_mgr = NodeManager(segment_info, hdf.duration, param_names, requested,
required, pre_processing_nodes, aircraft_info,
achieved_flight_record)
process_order, gr_st = dependency_order(node_mgr, draw=False)
ktis, kpvs, sections, approaches, flight_attrs = \
derive_parameters(hdf, node_mgr, process_order, force=force)
def test_graph_nodes_with_duplicate_key_in_lfl_and_derived(self):
# Test that LFL nodes are used in place of Derived where available.
# Tests a few of the colours
class One(DerivedParameterNode):
# Hack to allow objects rather than classes to be added to the tree.
__base__ = DerivedParameterNode
__bases__ = [__base__]
def derive(self, dep=P('DepOne')):
pass
class Four(DerivedParameterNode):
# Hack to allow objects rather than classes to be added to the tree.
__base__ = DerivedParameterNode
__bases__ = [__base__]
def derive(self, dep=P('DepFour')):
pass
one = One('overridden')
four = Four('used')
mgr1 = NodeManager({'Start Datetime': datetime.now()}, 10, ['1', '2'], ['2', '4'], [],
{'1':one, '4':four},{}, {})
gr = graph_nodes(mgr1)
self.assertEqual(len(gr), 5)
# LFL
self.assertEqual(list(gr.edges('1')), []) # as it's in LFL, it shouldn't have any edges
self.assertEqual(gr.node['1'], {'node_type': 'HDFNode'})
# Derived
self.assertEqual(list(gr.edges('4')), [('4','DepFour')])
self.assertEqual(gr.node['4'], {'node_type': 'DerivedParameterNode'})
# Root
self.assertEqual(list(gr.successors('root')), ['2','4']) # only the two requested are linked
self.assertEqual(gr.node['root'], {})
def _get_dependency_order(self, requested, aircraft_info, lfl_params, segment_info={}):
if not segment_info:
segment_info = {
'Start Datetime': datetime.now(),
'Segment Type': 'START_AND_STOP',
}
rel_path = Path('dummy_nodes')
pre_processing_modules = ['merge_multistate_parameters', 'merge_parameters']
node_modules = [
import_module(rel_path / 'pre_processing' / f'{mod}')
for mod in pre_processing_modules
]
pre_processing_nodes = get_derived_nodes(node_modules)
pre_processing_requested = list(pre_processing_nodes.keys())
node_mgr = NodeManager(
segment_info, 10, lfl_params,
pre_processing_requested, [], pre_processing_nodes, aircraft_info, {})
order, _ = dependency_order(node_mgr)
modules = {
rel_path: [
'derived_parameters', 'flight_phase', 'key_point_values',
'key_time_instances', 'approaches', 'multistate_parameters',
'flight_attribute'
]
}
if aircraft_info['Aircraft Type'] == 'helicopter':
modules[rel_path / 'helicopter'] = [
'derived_parameters', 'flight_phase', 'key_point_values',
'key_time_instances', 'multistate_parameters',
]
node_modules = [
import_module(path / f'{mod}') for path, mods in modules.items()
for mod in mods
]
# go through modules to get derived nodes
derived_nodes = get_derived_nodes(node_modules)
if requested == []:
# Use all derived nodes if requested is empty
requested = [p for p in derived_nodes.keys() if p not in lfl_params]
node_mgr= NodeManager(segment_info, 10, lfl_params + order,
requested, [], derived_nodes, aircraft_info, {})
order, _ = dependency_order(node_mgr)
return order
def test_dependency(self):
requested = ['P7', 'P8']
mgr = NodeManager({'Start Datetime': datetime.now()}, 10, self.lfl_params, requested, [],
self.derived_nodes, {}, {})
order, _ = dependency_order(mgr)
self.assertEqual(order, ['P4', 'P5', 'P6', 'P7', 'P8'])
"""
def test_graph_requesting_all_dependencies_links_root_to_all_requests(self):
# build list of all nodes as required
requested = self.lfl_params + list(self.derived_nodes.keys())
mgr = NodeManager({'Start Datetime': datetime.now()}, 1, self.lfl_params, requested, [],
self.derived_nodes, {}, {})
gr = graph_nodes(mgr)
# should be linked to all requested nodes
self.assertEqual(sorted(gr.neighbors('root')), sorted(requested))
def test_graph_requesting_all_dependencies_links_root_to_end_leafs(self):
# build list of all nodes as required
requested = self.lfl_params + self.derived_nodes.keys()
mgr = NodeManager(datetime.now(), 1, self.lfl_params, requested, [],
self.derived_nodes, {}, {})
gr = graph_nodes(mgr)
# should only be linked to end leafs
self.assertEqual(gr.neighbors('root'), ['P8', 'P7'])
def test_required_available(self):
nodes = ['a', 'b', 'c']
required = ['a', 'c']
mgr = NodeManager(datetime.now(), 10, nodes, nodes, required, {}, {},
{})
_graph = graph_nodes(mgr)
gr_all, gr_st, order = process_order(_graph, mgr)
self.assertEqual(set(required) - set(order), set())
def test_invalid_requirement_raises(self):
lfl_params = []
requested = ['Moment of Takeoff']
node_modules = [import_module(Path('dummy_nodes') / 'derived_parameters')]
# go through modules to get derived nodes
derived_nodes = get_derived_nodes(node_modules)
mgr = NodeManager({'Start Datetime': datetime.now()}, 10, lfl_params, requested, [],
derived_nodes, {}, {})
self.assertRaises(ValueError, dependency_order, mgr)
def _generate_json(self, lfl_params):
'''
Returns list of parameters used in the spanning tree.
Note: LFL parameters not used will not be returned!
'''
print "Establishing Node dependencies from Analysis Engine"
# Ensure file is a valid HDF file before continuing:
derived_nodes = get_derived_nodes(settings.NODE_MODULES)
required_params = derived_nodes.keys()
# TODO: Update ac_info with keys from provided fields:
ac_info = {
'Family': u'B737 NG',
'Frame': u'737-3C',
'Identifier': u'15',
'Main Gear To Lowest Point Of Tail': None,
'Main Gear To Radio Altimeter Antenna': None,
'Manufacturer Serial Number': u'39009',
'Manufacturer': u'Boeing',
'Model': u'B737-8JP',
'Precise Positioning': True,
'Series': u'B737-800',
'Tail Number': 'G-ABCD',
}
# TODO: Option to populate an AFR:
achieved_flight_record = {}
# Generate the dependency tree:
node_mgr = NodeManager(
{},
1000,
lfl_params,
required_params,
[],
derived_nodes,
ac_info,
achieved_flight_record,
)
_graph = graph_nodes(node_mgr)
gr_all, gr_st, order = process_order(_graph, node_mgr)
# Save the dependency tree to tree.json:
tree = os.path.join(AJAX_DIR, 'tree.json')
with open(tree, 'w') as fh:
json.dump(graph_adjacencies(gr_st), fh, indent=4)
# Save the list of nodes to node_list.json:
node_list = os.path.join(AJAX_DIR, 'node_list.json')
spanning_tree_params = sorted(gr_st.nodes())
with open(node_list, 'w') as fh:
json.dump(spanning_tree_params, fh, indent=4)
return
def test_invalid_requirement_raises(self):
lfl_params = []
requested = ['Smoothed Track',
'Moment of Takeoff'] #it's called Moment Of Takeoff
try:
# for test cmd line runners
derived = get_derived_nodes(['tests.sample_derived_parameters'])
except ImportError:
# for IDE test runners
derived = get_derived_nodes(['sample_derived_parameters'])
mgr = NodeManager(datetime.now(), 10, lfl_params, requested, [],
derived, {}, {})
self.assertRaises(nx.NetworkXError, dependency_order, mgr, draw=False)
def test_dependency_with_lowlevel_dependencies_requested(self):
""" Simulate requesting a Raw Parameter as a dependency. This requires
the requested node to be removed when it is not at the top of the
dependency tree.
"""
requested = ['P7', 'P8', # top level nodes
'P4', 'P5', 'P6', # middle level node
'Raw3', # bottom level node
]
mgr = NodeManager({'Start Datetime': datetime.now()}, 10, self.lfl_params + ['Floating'],
requested, [], self.derived_nodes, {}, {})
order, _ = dependency_order(mgr)
self.assertEqual(order, ['P4', 'P5', 'P6', 'P7', 'P8'])
def test_avoiding_possible_circular_dependency(self):
# Possible circular dependency which can be avoided:
# Gear Selected Down depends on Gear Down which depends on Gear Selected Down...!
lfl_params = ['Airspeed', 'Gear (L) Down', 'Gear (L) Red Warning']
requested = ['Airspeed At Gear Down Selected']
derived = get_derived_nodes([import_module('sample_circular_dependency_nodes')])
mgr = NodeManager({'Start Datetime': datetime.now()}, 10, lfl_params, requested, [],
derived, {}, {})
order, _ = dependency_order(mgr, draw=False)
# As Gear Selected Down depends upon Gear Down
self.assertEqual(order,
['Gear (L) Down', 'Gear Down', 'Gear (L) Red Warning',
'Gear Down Selected', 'Airspeed', 'Airspeed At Gear Down Selected'])
def test_dependency_one_path_circular(self):
'''Test that if one Node can be created using its 2 dependencies, but one of
them is optional and creates a circular dependency.
,-> P1 ~~~~~~~~~> P2 ~~~~~~~~> P3--、
| `---> Raw1 `---> Raw2 |
| |
`----------------------------------’
P1 requires Raw1 but tries also P2. P2 requires Raw 2 but also tries P3.
P3 requires P1. This makes a circular dependency. But we don't want to give up
here as we could still make P1 from Raw1 only, avoiding the circular path.
'''
class P1(DerivedParameterNode):
@classmethod
def can_operate(self, avail):
return 'Raw1' in avail
def derive(self, P2=P('P2'), raw=P('Raw1')):
pass
class P2(DerivedParameterNode):
@classmethod
def can_operate(self, avail):
return 'Raw2' in avail
def derive(self, P3=P('P3'), raw=P('Raw2')):
pass
class P3(DerivedParameterNode):
@classmethod
def can_operate(self, avail):
return 'P1' in avail
def derive(self, P1=P('P1')):
pass
derived_nodes = {
'P0' : MockParam(dependencies=['P1']),
'P1' : P1,
'P2' : P2,
'P3' : P3,
}
requested = ['P0']
lfl_params = ['Raw1', 'Raw2']
segment_info = {'Start Datetime': datetime.now(), 'Segment Type': 'START_AND_STOP'}
mgr = NodeManager(segment_info, 10, lfl_params, requested, [], derived_nodes, {}, {})
order, _ = dependency_order(mgr)
self.assertEqual(order, ['P1', 'P3', 'P2', 'P0'])
def test_indent_tree(self):
requested = ['P7', 'P8']
mgr2 = NodeManager({'Start Datetime': datetime.now()}, 10, self.lfl_params,
requested, [], self.derived_nodes, {}, {})
gr = graph_nodes(mgr2)
gr.node['Raw1']['active'] = True
gr.node['Raw2']['active'] = False
gr.node['P4']['active'] = False
self.assertEqual(
indent_tree(gr, 'P7', space='__', delim=' ', label=False),
[' P7',
'__ [P4]',
'____ Raw1',
'____ [Raw2]',
'__ P5',
'____ Raw3',
'____ Raw4',
'__ P6',
'____ Raw3',
])
# don't recurse valid parameters...
self.assertEqual(
indent_tree(gr, 'P5', label=False, recurse_active=False),
[])
self.assertEqual(
indent_tree(gr, 'P4', label=False, recurse_active=False),
['- [P4]',
' - [Raw2]',
])
self.assertEqual(
indent_tree(gr, 'root'),
['- root',
' - P7 (DerivedParameterNode)',
' - [P4] (DerivedParameterNode)',
' - Raw1 (HDFNode)',
' - [Raw2] (HDFNode)',
' - P5 (DerivedParameterNode)',
' - Raw3 (HDFNode)',
' - Raw4 (HDFNode)',
' - P6 (DerivedParameterNode)',
' - Raw3 (HDFNode)',
' - P8 (DerivedParameterNode)',
' - Raw5 (HDFNode)',
])
def test_dependency(self):
requested = ['P7', 'P8']
mgr = NodeManager(datetime.now(), 10, self.lfl_params, requested, [],
self.derived_nodes, {}, {})
gr = graph_nodes(mgr)
gr_all, gr_st, order = process_order(gr, mgr)
self.assertEqual(len(gr_st), 11)
pos = order.index
self.assertTrue(pos('P8') > pos('Raw5'))
self.assertTrue(pos('P7') > pos('P4'))
self.assertTrue(pos('P7') > pos('P5'))
self.assertTrue(pos('P7') > pos('P6'))
self.assertTrue(pos('P6') > pos('Raw3'))
self.assertTrue(pos('P5') > pos('Raw3'))
self.assertTrue(pos('P5') > pos('Raw4'))
self.assertTrue(pos('P4') > pos('Raw1'))
self.assertTrue(pos('P4') > pos('Raw2'))
self.assertFalse('root' in order) #don't include the root!
"""
def test_avoiding_possible_circular_dependency(self):
# Possible circular dependency which can be avoided:
# Gear Selected Down depends on Gear Down which depends on Gear Selected Down...!
lfl_params = ['Airspeed', 'Gear (L) Down', 'Gear (L) Red Warning']
requested = ['Airspeed At Gear Down Selected']
try:
# for test cmd line runners
derived = get_derived_nodes(
['tests.sample_circular_dependency_nodes'])
except ImportError:
# for IDE test runners
derived = get_derived_nodes(['sample_circular_dependency_nodes'])
mgr = NodeManager(datetime.now(), 10, lfl_params, requested, [],
derived, {}, {})
order, _ = dependency_order(mgr, draw=True)
# As Gear Selected Down depends upon Gear Down
self.assertEqual(order, [
'Gear (L) Down', 'Gear Down', 'Gear (L) Red Warning',
'Gear Down Selected', 'Airspeed', 'Airspeed At Gear Down Selected'
])
def prep_order(flight, frame_dict, start_datetime, derived_nodes,
required_params):
''' open example HDF to see recorded params and build process order'''
derived_nodes_copy = derived_nodes #copy.deepcopy(derived_nodes) #
precomputed_parameters = get_precomputed_parameters(flight)
precomputed_keys = precomputed_parameters.keys()
for k in flight.series.keys():
if k not in precomputed_keys:
precomputed_keys.append(k)
node_mgr = NodeManager(
start_datetime,
flight.duration,
precomputed_keys, #flight.series.keys(), #from HDF. was hdf.valid_param_names(), #hdf_keys; should be from LFL
required_params, #requested
derived_nodes_copy, #methods that can be computed; equals profile + base nodes ????
flight.aircraft_info,
achieved_flight_record=flight.achieved_flight_record)
# calculate dependency tree
process_order, gr_st = dependency_order(node_mgr, draw=False)
logger.warning('process order: ' + str(process_order[:5]) +
'...') #, gr_st
return node_mgr, process_order # a list of node names
def derived_trimmer(hdf_path, node_names, dest):
'''
Trims an HDF file of parameters which are not dependencies of nodes in
node_names.
:param hdf_path: file path of hdf file.
:type hdf_path: str
:param node_names: A list of Node names which are required.
:type node_names: list of str
:param dest: destination path for trimmed output file
:type dest: str
:return: parameters in stripped hdf file
:rtype: [str]
'''
with hdf_file(hdf_path) as hdf:
duration = hdf.duration
raw_nodes = hdf.valid_param_names()
derived_nodes = get_derived_nodes(settings.NODE_MODULES)
node_mgr = NodeManager({}, duration, raw_nodes, derived_nodes.keys(), [],
derived_nodes, {}, {})
order, graph = dependency_order(node_mgr)
node_names = set(node_names)
queue = deque(i for i in order if i in node_names)
params = {i for i in node_mgr.hdf_keys if i in node_names}
visited = set()
while queue:
current = queue.popleft()
visited.add(current)
for name in graph[current]:
if graph.nodes[name]['node_type'] == 'HDFNode':
params.add(name)
elif name not in visited:
queue.append(name)
return strip_hdf(hdf_path, params, dest)
def test_required_unavailable(self):
nodes = ['a', 'b', 'c']
required = ['a', 'c', 'd']
mgr = NodeManager({'Start Datetime': datetime.now()}, 10, nodes, nodes,
required, {}, {}, {})
self.assertRaises(ValueError, graph_nodes, mgr)
def process_flight(hdf_path,
tail_number,
aircraft_info={},
start_datetime=None,
achieved_flight_record={},
requested=[],
required=[],
include_flight_attributes=True,
additional_modules=[]):
'''
Processes the HDF file (hdf_path) to derive the required_params (Nodes)
within python modules (settings.NODE_MODULES).
Note: For Flight Data Services, the definitive API is located here:
"PolarisTaskManagement.test.tasks_mask.process_flight"
:param hdf_path: Path to HDF File
:type hdf_path: String
:param aircraft: Aircraft specific attributes
:type aircraft: dict
:param start_datetime: Datetime of the origin of the data (at index 0)
:type start_datetime: Datetime
:param achieved_flight_record: See API Below
:type achieved_flight_record: Dict
:param requested: Derived nodes to process (dependencies will also be
evaluated).
:type requested: List of Strings
:param required: Nodes which are required, otherwise an exception will be
raised.
:type required: List of Strings
:param include_flight_attributes: Whether to include all flight attributes
:type include_flight_attributes: Boolean
:param additional_modules: List of module paths to import.
:type additional_modules: List of Strings
:returns: See below:
:rtype: Dict
Sample aircraft_info
--------------------
{
'Tail Number': # Aircraft Registration
'Identifier': # Aircraft Ident
'Manufacturer': # e.g. Boeing
'Manufacturer Serial Number': #MSN
'Model': # e.g. 737-808-ER
'Series': # e.g. 737-800
'Family': # e.g. 737
'Frame': # e.g. 737-3C
'Main Gear To Altitude Radio': # Distance in metres
'Wing Span': # Distance in metres
}
Sample achieved_flight_record
-----------------------------
{
# Simple values first, e.g. string, int, float, etc.
'AFR Flight ID': # e.g. 1
'AFR Flight Number': # e.g. 1234
'AFR Type': # 'POSITIONING'
'AFR Off Blocks Datetime': # datetime(2015,01,01,13,00)
'AFR Takeoff Datetime': # datetime(2015,01,01,13,15)
'AFR Takeoff Pilot': # 'Joe Bloggs'
'AFR Takeoff Gross Weight': # weight in kg
'AFR Takeoff Fuel': # fuel in kg
'AFR Landing Datetime': # datetime(2015,01,01,18,45)
'AFR Landing Pilot': # 'Joe Bloggs'
'AFR Landing Gross Weight': # weight in kg
'AFR Landing Fuel': # weight in kg
'AFR On Blocks Datetime': # datetime(2015,01,01,19,00)
'AFR V2': # V2 used at takeoff in kts
'AFR Vapp': # Vapp used in kts
'AFR Vref': # Vref used in kts
# More complex data that needs to be looked up next:
'AFR Takeoff Airport': {
'id': 4904, # unique id
'name': 'Athens Intl Airport Elefterios Venizel',
'code': {'iata': 'ATH', 'icao': 'LGAV'},
'latitude': 37.9364,
'longitude': 23.9445,
'location': {'city': u'Athens', 'country': u'Greece'},
'elevation': 266, # ft
'magnetic_variation': 'E003186 0106',
}
},
'AFR Landing Aiport': {
'id': 1, # unique id
'name': 'Athens Intl Airport Elefterios Venizel',
'code': {'iata': 'ATH', 'icao': 'LGAV'},
'latitude': 37.9364,
'longitude': 23.9445,
'location': {'city': u'Athens', 'country': u'Greece'},
'elevation': 266, # ft
'magnetic_variation': 'E003186 0106',
}
},
'AFR Destination Airport': None, # if not required, or exclude this key
'AFR Takeoff Runway': {
'id': 1,
'identifier': '21L',
'magnetic_heading': 212.6,
'strip': {
'id': 1,
'length': 13123,
'surface': 'ASP',
'width': 147},
'start': {
'elevation': 308,
'latitude': 37.952425,
'longitude': 23.970422},
'end': {
'elevation': 279,
'latitude': 37.923511,
'longitude': 23.943261},
'glideslope': {
'angle': 3.0,
'elevation': 282,
'latitude': 37.9473,
'longitude': 23.9676,
'threshold_distance': 999},
'localizer': {
'beam_width': 4.5,
'elevation': 256,
'frequency': 111100,
'heading': 213,
'latitude': 37.919281,
'longitude': 23.939294},
},
'AFR Landing Runway': {
'id': 1,
'identifier': '21L',
'magnetic_heading': 212.6,
'strip': {
'id': 1,
'length': 13123,
'surface': 'ASP',
'width': 147},
'start': {
'elevation': 308,
'latitude': 37.952425,
'longitude': 23.970422},
'end': {
'elevation': 279,
'latitude': 37.923511,
'longitude': 23.943261},
'glideslope': {
'angle': 3.0,
'elevation': 282,
'latitude': 37.9473,
'longitude': 23.9676,
'threshold_distance': 999},
'localizer': {
'beam_width': 4.5,
'elevation': 256,
'frequency': 111100,
'heading': 213,
'latitude': 37.919281,
'longitude': 23.939294},
},
}
Sample Return
-------------
{
'flight':[Attribute('name value')],
'kti':[GeoKeyTimeInstance('index name latitude longitude')]
if lat/long available
else [KeyTimeInstance('index name')],
'kpv':[KeyPointValue('index value name slice')]
}
sample flight Attributes:
[
Attribute('Takeoff Airport', {'id':1234, 'name':'Int. Airport'},
Attribute('Approaches', [4567,7890]),
...
],
'''
if start_datetime is None:
import pytz
start_datetime = datetime.utcnow().replace(tzinfo=pytz.utc)
logger.info("Processing: %s", hdf_path)
if aircraft_info:
# Aircraft info has already been provided.
logger.info(
"Using aircraft_info dictionary passed into process_flight '%s'." %
aircraft_info)
else:
aircraft_info = get_aircraft_info(tail_number)
aircraft_info['Tail Number'] = tail_number
# go through modules to get derived nodes
node_modules = additional_modules + settings.NODE_MODULES
derived_nodes = get_derived_nodes(node_modules)
if requested:
requested = \
list(set(requested).intersection(set(derived_nodes)))
else:
# if requested isn't set, try using ALL derived_nodes!
logger.info("No requested nodes declared, using all derived nodes")
requested = derived_nodes.keys()
# include all flight attributes as requested
if include_flight_attributes:
requested = list(
set(requested +
get_derived_nodes(['analysis_engine.flight_attribute']).keys())
)
# open HDF for reading
with hdf_file(hdf_path) as hdf:
hdf.start_datetime = start_datetime
if hooks.PRE_FLIGHT_ANALYSIS:
logger.info("Performing PRE_FLIGHT_ANALYSIS actions: %s",
hooks.PRE_FLIGHT_ANALYSIS.func_name)
hooks.PRE_FLIGHT_ANALYSIS(hdf, aircraft_info)
else:
logger.info("No PRE_FLIGHT_ANALYSIS actions to perform")
# Track nodes. Assume that all params in HDF are from LFL(!)
node_mgr = NodeManager(start_datetime, hdf.duration,
hdf.valid_param_names(), requested, required,
derived_nodes, aircraft_info,
achieved_flight_record)
# calculate dependency tree
process_order, gr_st = dependency_order(node_mgr, draw=False)
if settings.CACHE_PARAMETER_MIN_USAGE:
# find params used more than
for node in gr_st.nodes():
if node in node_mgr.derived_nodes:
# this includes KPV/KTIs but they'll be ignored by HDF
qty = len(gr_st.predecessors(node))
if qty > settings.CACHE_PARAMETER_MIN_USAGE:
hdf.cache_param_list.append(node)
logging.info("HDF set to cache parameters: %s",
hdf.cache_param_list)
# derive parameters
kti_list, kpv_list, section_list, approach_list, flight_attrs = \
derive_parameters(hdf, node_mgr, process_order)
# geo locate KTIs
kti_list = geo_locate(hdf, kti_list)
kti_list = _timestamp(start_datetime, kti_list)
# geo locate KPVs
kpv_list = geo_locate(hdf, kpv_list)
kpv_list = _timestamp(start_datetime, kpv_list)
# Store version of FlightDataAnalyser
hdf.analysis_version = __version__
# Store dependency tree
hdf.dependency_tree = json_graph.dumps(gr_st)
# Store aircraft info
hdf.set_attr('aircraft_info', aircraft_info)
hdf.set_attr('achieved_flight_record', achieved_flight_record)
return {
'flight': flight_attrs,
'kti': kti_list,
'kpv': kpv_list,
'approach': approach_list,
'phases': section_list,
}
def process_flight(segment_info,
tail_number,
aircraft_info={},
achieved_flight_record={},
requested=[],
required=[],
include_flight_attributes=True,
additional_modules=[],
pre_flight_kwargs={},
force=False,
initial={},
reprocess=False):
'''
Processes the HDF file (segment_info['File']) to derive the required_params (Nodes)
within python modules (settings.NODE_MODULES).
Note: For Flight Data Services, the definitive API is located here:
"PolarisTaskManagement.test.tasks_mask.process_flight"
:param segment_info: Details of the segment to process
:type segment_info: dict
:param aircraft: Aircraft specific attributes
:type aircraft: dict
:param achieved_flight_record: See API Below
:type achieved_flight_record: Dict
:param requested: Derived nodes to process (dependencies will also be
evaluated).
:type requested: List of Strings
:param required: Nodes which are required, otherwise an exception will be
raised.
:type required: List of Strings
:param include_flight_attributes: Whether to include all flight attributes
:type include_flight_attributes: Boolean
:param additional_modules: List of module paths to import.
:type additional_modules: List of Strings
:param pre_flight_kwargs: Keyword arguments for the pre-flight analysis hook.
:type pre_flight_kwargs: dict
:param force: Ignore errors raised while deriving nodes.
:type force: bool
:param initial: Initial content for nodes to avoid reprocessing (excluding parameter nodes which are saved to the hdf).
:type initial: dict
:param reprocess: Force reprocessing of all Nodes (including derived Nodes already saved to the HDF file).
:returns: See below:
:rtype: Dict
Sample segment_info
--------------------
{
'File': # Path to HDF5 file to process
'Start Datetime': # Datetime of the origin of the data (at index 0)
'Segment Type': # segment type obtained from split segments e.g. START_AND_STOP
}
Sample aircraft_info
--------------------
{
'Tail Number': # Aircraft Registration
'Identifier': # Aircraft Ident
'Manufacturer': # e.g. Boeing
'Manufacturer Serial Number': #MSN
'Model': # e.g. 737-808-ER
'Series': # e.g. 737-800
'Family': # e.g. 737
'Frame': # e.g. 737-3C
'Main Gear To Altitude Radio': # Distance in metres
'Wing Span': # Distance in metres
}
Sample achieved_flight_record
-----------------------------
{
# Simple values first, e.g. string, int, float, etc.
'AFR Flight ID': # e.g. 1
'AFR Flight Number': # e.g. 1234
'AFR Type': # 'POSITIONING'
'AFR Off Blocks Datetime': # datetime(2015,1,1,13,00)
'AFR Takeoff Datetime': # datetime(2015,1,1,13,15)
'AFR Takeoff Pilot': # 'Joe Bloggs'
'AFR Takeoff Gross Weight': # weight in kg
'AFR Takeoff Fuel': # fuel in kg
'AFR Landing Datetime': # datetime(2015,1,1,18,45)
'AFR Landing Pilot': # 'Joe Bloggs'
'AFR Landing Gross Weight': # weight in kg
'AFR Landing Fuel': # weight in kg
'AFR On Blocks Datetime': # datetime(2015,1,1,19,00)
'AFR V2': # V2 used at takeoff in kts
'AFR Vapp': # Vapp used in kts
'AFR Vref': # Vref used in kts
# More complex data that needs to be looked up next:
'AFR Takeoff Airport': {
'id': 4904, # unique id
'name': 'Athens Intl Airport Elefterios Venizel',
'code': {'iata': 'ATH', 'icao': 'LGAV'},
'latitude': 37.9364,
'longitude': 23.9445,
'location': {'city': u'Athens', 'country': u'Greece'},
'elevation': 266, # ft
'magnetic_variation': 'E003186 0106',
}
},
'AFR Landing Aiport': {
'id': 1, # unique id
'name': 'Athens Intl Airport Elefterios Venizel',
'code': {'iata': 'ATH', 'icao': 'LGAV'},
'latitude': 37.9364,
'longitude': 23.9445,
'location': {'city': u'Athens', 'country': u'Greece'},
'elevation': 266, # ft
'magnetic_variation': 'E003186 0106',
}
},
'AFR Destination Airport': None, # if not required, or exclude this key
'AFR Takeoff Runway': {
'id': 1,
'identifier': '21L',
'magnetic_heading': 212.6,
'strip': {
'id': 1,
'length': 13123,
'surface': 'ASP',
'width': 147},
'start': {
'elevation': 308,
'latitude': 37.952425,
'longitude': 23.970422},
'end': {
'elevation': 279,
'latitude': 37.923511,
'longitude': 23.943261},
'glideslope': {
'angle': 3.0,
'elevation': 282,
'latitude': 37.9473,
'longitude': 23.9676,
'threshold_distance': 999},
'localizer': {
'beam_width': 4.5,
'elevation': 256,
'frequency': 111100,
'heading': 213,
'latitude': 37.919281,
'longitude': 23.939294},
},
'AFR Landing Runway': {
'id': 1,
'identifier': '21L',
'magnetic_heading': 212.6,
'strip': {
'id': 1,
'length': 13123,
'surface': 'ASP',
'width': 147},
'start': {
'elevation': 308,
'latitude': 37.952425,
'longitude': 23.970422},
'end': {
'elevation': 279,
'latitude': 37.923511,
'longitude': 23.943261},
'glideslope': {
'angle': 3.0,
'elevation': 282,
'latitude': 37.9473,
'longitude': 23.9676,
'threshold_distance': 999},
'localizer': {
'beam_width': 4.5,
'elevation': 256,
'frequency': 111100,
'heading': 213,
'latitude': 37.919281,
'longitude': 23.939294},
},
}
Sample Return
-------------
{
'flight':[Attribute('name value')],
'kti':[GeoKeyTimeInstance('index name latitude longitude')]
if lat/long available
else [KeyTimeInstance('index name')],
'kpv':[KeyPointValue('index value name slice')]
}
sample flight Attributes:
[
Attribute('Takeoff Airport', {'id':1234, 'name':'Int. Airport'},
Attribute('Approaches', [4567,7890]),
...
],
'''
hdf_path = segment_info['File']
if 'Start Datetime' not in segment_info:
import pytz
segment_info['Start Datetime'] = datetime.utcnow().replace(
tzinfo=pytz.utc)
logger.debug("Processing: %s", hdf_path)
if aircraft_info:
# Aircraft info has already been provided.
logger.info(
"Using aircraft_info dictionary passed into process_flight '%s'." %
aircraft_info)
else:
aircraft_info = get_aircraft_info(tail_number)
aircraft_info['Tail Number'] = tail_number
if aircraft_info['Aircraft Type'] == 'helicopter':
node_modules = settings.NODE_MODULES + \
settings.NODE_HELICOPTER_MODULE_PATHS + additional_modules
else:
node_modules = settings.NODE_MODULES + additional_modules
# go through modules to get derived nodes
derived_nodes = get_derived_nodes(node_modules)
if requested:
requested = \
list(set(requested).intersection(set(derived_nodes)))
else:
# if requested isn't set, try using ALL derived_nodes!
logger.debug("No requested nodes declared, using all derived nodes")
# Enforce some sort of order in which the dependencies are traversed
requested = sorted(list(derived_nodes.keys()))
# include all flight attributes as requested
if include_flight_attributes:
requested = list(
set(requested + list(
get_derived_nodes(['analysis_engine.flight_attribute']).keys())
))
initial = process_flight_to_nodes(initial)
for node_name in requested:
initial.pop(node_name, None)
# open HDF for reading
with hdf_file(hdf_path) as hdf:
hdf.start_datetime = segment_info['Start Datetime']
hook = hooks.PRE_FLIGHT_ANALYSIS
if hook:
logger.info(
"Performing PRE_FLIGHT_ANALYSIS action '%s' with options: %s",
getattr(hook, 'func_name', getattr(hook, '__name__')),
pre_flight_kwargs)
hook(hdf, aircraft_info, **pre_flight_kwargs)
else:
logger.info("No PRE_FLIGHT_ANALYSIS actions to perform")
# Merge Params
param_names = hdf.valid_lfl_param_names(
) if reprocess else hdf.valid_param_names()
pre_process_parameters(hdf,
segment_info,
param_names,
required,
aircraft_info,
achieved_flight_record,
force=force)
# Track nodes.
node_mgr = NodeManager(segment_info, hdf.duration, param_names,
requested, required, derived_nodes,
aircraft_info, achieved_flight_record)
# calculate dependency tree
process_order, gr_st = dependency_order(node_mgr, draw=False)
if settings.CACHE_PARAMETER_MIN_USAGE:
# find params used more than
for node in gr_st.nodes():
if node in node_mgr.derived_nodes:
# this includes KPV/KTIs but they'll be ignored by HDF
qty = len(gr_st.predecessors(node))
if qty > settings.CACHE_PARAMETER_MIN_USAGE:
hdf.cache_param_list.append(node)
logging.info("HDF set to cache parameters: %s",
hdf.cache_param_list)
# derive parameters
ktis, kpvs, sections, approaches, flight_attrs = \
derive_parameters(hdf, node_mgr, process_order, params=initial, force=force)
# geo locate KTIs
ktis = geo_locate(hdf, ktis)
ktis = _timestamp(segment_info['Start Datetime'], ktis)
# geo locate KPVs
kpvs = geo_locate(hdf, kpvs)
kpvs = _timestamp(segment_info['Start Datetime'], kpvs)
# Store version of FlightDataAnalyser
hdf.analysis_version = __version__
# Store dependency tree
hdf.dependency_tree = json.dumps(json_graph.node_link_data(gr_st))
# Store aircraft info
hdf.set_attr('aircraft_info', aircraft_info)
hdf.set_attr('achieved_flight_record', achieved_flight_record)
return {
'flight': flight_attrs,
'kti': ktis,
'kpv': kpvs,
'approach': approaches,
'phases': sections,
}
def test_required_unavailable(self):
nodes = ['a', 'b', 'c']
required = ['a', 'c', 'd']
mgr = NodeManager(datetime.now(), 10, nodes, nodes, required, {}, {},
{})
gr = self.assertRaises(graph_nodes(mgr))
