def test_sample_parameter_module(self):
"""Tests many options:
can_operate on SmoothedTrack works with
"""
requested = ['Smoothed Track', 'Moment Of Takeoff', 'Vertical Speed',
'Slip On Runway']
lfl_params = ['Indicated Airspeed',
'Groundspeed',
'Pressure Altitude',
'Heading', 'TAT',
'Latitude', 'Longitude',
'Longitudinal g', 'Lateral g', 'Normal g',
'Pitch', 'Roll',
]
try:
# for test cmd line runners
derived = get_derived_nodes(['tests.sample_derived_parameters'])
except ImportError:
# for IDE test runners
derived = get_derived_nodes(['sample_derived_parameters'])
nodes = NodeManager(datetime.now(), 10, lfl_params, requested, [],
derived, {}, {})
order, _ = dependency_order(nodes, draw=False)
pos = order.index
self.assertTrue(len(order))
self.assertNotIn('Moment Of Takeoff', order) # not available
self.assertTrue(pos('Vertical Speed') > pos('Pressure Altitude'))
self.assertTrue(pos('Slip On Runway') > pos('Groundspeed'))
self.assertTrue(pos('Slip On Runway') > pos('Horizontal g Across Track'))
self.assertTrue(pos('Horizontal g Across Track') > pos('Roll'))
self.assertFalse('Mach' in order) # Mach wasn't requested!
self.assertFalse('Radio Altimeter' in order)
self.assertEqual(len(nodes.hdf_keys), 12)
self.assertEqual(len(nodes.requested), 4)
self.assertEqual(len(nodes.derived_nodes), 13)
def test_avoiding_possible_circular_dependency(self):
# Possible circular dependency which can be avoided:
# Gear Selected Down depends on Gear Down which depends on Gear Selected Down...!
lfl_params = ["Airspeed", "Gear (L) Down", "Gear (L) Red Warning"]
requested = ["Airspeed At Gear Down Selected"]
try:
# for test cmd line runners
derived = get_derived_nodes(["tests.sample_circular_dependency_nodes"])
except ImportError:
# for IDE test runners
derived = get_derived_nodes(["sample_circular_dependency_nodes"])
mgr = NodeManager({"Start Datetime": datetime.now()}, 10, lfl_params, requested, [], derived, {}, {})
order, _ = dependency_order(mgr, draw=True)
# As Gear Selected Down depends upon Gear Down
self.assertEqual(
order,
[
"Gear (L) Down",
"Gear Down",
"Gear (L) Red Warning",
"Gear Down Selected",
"Airspeed",
"Airspeed At Gear Down Selected",
],
)
def test_invalid_requirement_raises(self):
lfl_params = []
requested = ["Smoothed Track", "Moment of Takeoff"] # it's called Moment Of Takeoff
try:
# for test cmd line runners
derived = get_derived_nodes(["tests.sample_derived_parameters"])
except ImportError:
# for IDE test runners
derived = get_derived_nodes(["sample_derived_parameters"])
mgr = NodeManager({"Start Datetime": datetime.now()}, 10, lfl_params, requested, [], derived, {}, {})
self.assertRaises(nx.NetworkXError, dependency_order, mgr, draw=False)
def test_invalid_requirement_raises(self):
lfl_params = []
requested = ['Smoothed Track',
'Moment of Takeoff'] #it's called Moment Of Takeoff
try:
# for test cmd line runners
derived = get_derived_nodes(['tests.sample_derived_parameters'])
except ImportError:
# for IDE test runners
derived = get_derived_nodes(['sample_derived_parameters'])
mgr = NodeManager(datetime.now(), 10, lfl_params, requested, [],
derived, {}, {})
self.assertRaises(nx.NetworkXError, dependency_order, mgr, draw=False)
def test_invalid_requirement_raises(self):
lfl_params = []
requested = ['Smoothed Track', 'Moment of Takeoff'] #it's called Moment Of Takeoff
derived = get_derived_nodes([import_module('sample_derived_parameters')])
mgr = NodeManager({'Start Datetime': datetime.now()}, 10, lfl_params, requested, [],
derived, {}, {})
self.assertRaises(nx.NetworkXError, dependency_order, mgr, draw=False)
def parse_analyser_profiles(analyser_profiles, filter_modules=None):
'''
Parse analyser profiles into additional_modules and required nodes as
expected by process_flight.
:param analyser_profiles: A list of analyser profile tuples containing
semicolon separated module paths and whether or not the nodes are
required e.g. [('package.module_one;package.module_two', True), ]
:type analyser_profiles: [[str, bool], ]
:param filter_paths: Optional list of analyser profiles to keep.
:type filter_paths: [str] or None
:returns: A list of additional module paths and a list of required node
names.
:rtype: [str], [str]
'''
additional_modules = []
required_nodes = []
for import_paths, is_required in analyser_profiles:
for import_path in import_paths.split(';'):
if filter_modules is not None and import_path not in filter_modules:
continue
import_path = import_path.strip()
if not import_path:
continue
additional_modules.append(import_path)
if is_required:
required_nodes.extend(get_derived_nodes([import_path]))
return additional_modules, required_nodes
def test_invalid_requirement_raises(self):
lfl_params = []
requested = ['Smoothed Track', 'Moment of Takeoff'] #it's called Moment Of Takeoff
derived = get_derived_nodes([import_module('sample_derived_parameters')])
mgr = NodeManager({'Start Datetime': datetime.now()}, 10, lfl_params, requested, [],
derived, {}, {})
self.assertRaises(nx.NetworkXError, dependency_order, mgr, draw=False)
def test_sample_parameter_module(self):
"""Tests many options:
can_operate on SmoothedTrack works with
"""
requested = ['Smoothed Track', 'Moment Of Takeoff', 'Vertical Speed',
'Slip On Runway']
lfl_params = ['Indicated Airspeed',
'Groundspeed',
'Pressure Altitude',
'Heading', 'TAT',
'Latitude', 'Longitude',
'Longitudinal g', 'Lateral g', 'Normal g',
'Pitch', 'Roll',
]
derived = get_derived_nodes([import_module('sample_derived_parameters')])
nodes = NodeManager({'Start Datetime': datetime.now()}, 10, lfl_params,
requested, [], derived, {}, {})
order, _ = dependency_order(nodes, draw=False)
pos = order.index
self.assertTrue(len(order))
self.assertNotIn('Moment Of Takeoff', order) # not available
self.assertTrue(pos('Vertical Speed') > pos('Pressure Altitude'))
self.assertTrue(pos('Slip On Runway') > pos('Groundspeed'))
self.assertTrue(pos('Slip On Runway') > pos('Horizontal g Across Track'))
self.assertTrue(pos('Horizontal g Across Track') > pos('Roll'))
self.assertFalse('Mach' in order) # Mach wasn't requested!
self.assertFalse('Radio Altimeter' in order)
self.assertEqual(len(nodes.hdf_keys), 12)
self.assertEqual(len(nodes.requested), 4)
self.assertEqual(len(nodes.derived_nodes), 13)
def parse_analyser_profiles(analyser_profiles, filter_modules=None):
'''
Parse analyser profiles into additional_modules and required nodes as
expected by process_flight.
:param analyser_profiles: A list of analyser profile tuples containing
semicolon separated module paths and whether or not the nodes are
required e.g. [('package.module_one;package.module_two', True), ]
:type analyser_profiles: [[str, bool], ]
:param filter_paths: Optional list of analyser profiles to keep.
:type filter_paths: [str] or None
:returns: A list of additional module paths and a list of required node
names.
:rtype: [str], [str]
'''
additional_modules = []
required_nodes = []
for import_paths, is_required in analyser_profiles:
for import_path in import_paths.split(';'):
if filter_modules is not None and import_path not in filter_modules:
continue
import_path = import_path.strip()
if not import_path:
continue
additional_modules.append(import_path)
if is_required:
required_nodes.extend(get_derived_nodes([import_path]))
return additional_modules, required_nodes
def _get_dependency_order(self,
requested,
aircraft_info,
lfl_params,
draw=False,
raise_cir_dep=True,
segment_info={}):
#derived_nodes = get_derived_nodes(settings.NODE_MODULES)
if aircraft_info['Aircraft Type'] == 'helicopter':
node_modules = settings.NODE_MODULES + settings.NODE_HELICOPTER_MODULE_PATHS
else:
node_modules = settings.NODE_MODULES
# go through modules to get derived nodes
derived_nodes = get_derived_nodes(node_modules)
if requested == []:
# Use all derived nodes if requested is empty
requested = [
p for p in derived_nodes.keys() if p not in lfl_params
]
if not segment_info:
segment_info = {
'Start Datetime': datetime.now(),
'Segment Type': 'START_AND_STOP',
}
node_mgr = NodeManager(segment_info, 10, lfl_params, requested, [],
derived_nodes, aircraft_info, {})
order, gr_st = dependency_order(node_mgr,
draw=draw,
raise_cir_dep=raise_cir_dep)
return order, gr_st
def pre_process_parameters(hdf,
segment_info,
param_names,
required,
aircraft_info,
achieved_flight_record,
force=False):
'''
Perform actions prior to main processing run.
Actions such as merging parameters up front simplify processing paths by
removing circular dependacies.
'''
pre_processing_nodes = get_derived_nodes(
settings.PRE_PROCESSING_MODULE_PATHS)
requested = list(pre_processing_nodes.keys())
node_mgr = NodeManager(segment_info, hdf.duration, param_names, requested,
required, pre_processing_nodes, aircraft_info,
achieved_flight_record)
process_order, gr_st = dependency_order(node_mgr, draw=False)
ktis, kpvs, sections, approaches, flight_attrs = \
derive_parameters(hdf, node_mgr, process_order, force=force)
def process_flight_to_nodes(pf_results):
'''
Load process flight results into Node objects.
'''
from analysis_engine import node
derived_nodes = get_derived_nodes(settings.NODE_MODULES)
params = {}
for node_type, nodes in pf_results.iteritems():
for node_name, items in nodes.iteritems():
try:
node_cls = derived_nodes[node_name]
except KeyError:
#logger.warning('Derived node not found in code base: %s', node_name)
continue
if node_type == 'flight' and items:
flight_attr = node.FlightAttributeNode(node_name)
flight_attr.set_flight_attr(items[0].value)
params[node_name] = flight_attr
continue
params[node_name] = node_cls(node_name, items=items)
return params
def process_flight_to_nodes(pf_results):
'''
Load process flight results into Node objects.
'''
from analysis_engine import node
derived_nodes = get_derived_nodes(settings.NODE_MODULES)
params = {}
for node_type, nodes in pf_results.iteritems():
for node_name, items in nodes.iteritems():
try:
node_cls = derived_nodes[node_name]
except KeyError:
#logger.warning('Derived node not found in code base: %s', node_name)
continue
if node_type == 'flight' and items:
flight_attr = node.FlightAttributeNode(node_name)
flight_attr.set_flight_attr(items[0].value)
params[node_name] = flight_attr
continue
params[node_name] = node_cls(node_name, items=items)
return params
def _get_dependency_order(self, requested, aircraft_info, lfl_params, segment_info={}):
if not segment_info:
segment_info = {
'Start Datetime': datetime.now(),
'Segment Type': 'START_AND_STOP',
}
rel_path = Path('dummy_nodes')
pre_processing_modules = ['merge_multistate_parameters', 'merge_parameters']
node_modules = [
import_module(rel_path / 'pre_processing' / f'{mod}')
for mod in pre_processing_modules
]
pre_processing_nodes = get_derived_nodes(node_modules)
pre_processing_requested = list(pre_processing_nodes.keys())
node_mgr = NodeManager(
segment_info, 10, lfl_params,
pre_processing_requested, [], pre_processing_nodes, aircraft_info, {})
order, _ = dependency_order(node_mgr)
modules = {
rel_path: [
'derived_parameters', 'flight_phase', 'key_point_values',
'key_time_instances', 'approaches', 'multistate_parameters',
'flight_attribute'
]
}
if aircraft_info['Aircraft Type'] == 'helicopter':
modules[rel_path / 'helicopter'] = [
'derived_parameters', 'flight_phase', 'key_point_values',
'key_time_instances', 'multistate_parameters',
]
node_modules = [
import_module(path / f'{mod}') for path, mods in modules.items()
for mod in mods
]
# go through modules to get derived nodes
derived_nodes = get_derived_nodes(node_modules)
if requested == []:
# Use all derived nodes if requested is empty
requested = [p for p in derived_nodes.keys() if p not in lfl_params]
node_mgr= NodeManager(segment_info, 10, lfl_params + order,
requested, [], derived_nodes, aircraft_info, {})
order, _ = dependency_order(node_mgr)
return order
def test_invalid_requirement_raises(self):
lfl_params = []
requested = ['Moment of Takeoff']
node_modules = [import_module(Path('dummy_nodes') / 'derived_parameters')]
# go through modules to get derived nodes
derived_nodes = get_derived_nodes(node_modules)
mgr = NodeManager({'Start Datetime': datetime.now()}, 10, lfl_params, requested, [],
derived_nodes, {}, {})
self.assertRaises(ValueError, dependency_order, mgr)
def test_avoiding_possible_circular_dependency(self):
# Possible circular dependency which can be avoided:
# Gear Selected Down depends on Gear Down which depends on Gear Selected Down...!
lfl_params = ['Airspeed', 'Gear (L) Down', 'Gear (L) Red Warning']
requested = ['Airspeed At Gear Down Selected']
try:
# for test cmd line runners
derived = get_derived_nodes(
['tests.sample_circular_dependency_nodes'])
except ImportError:
# for IDE test runners
derived = get_derived_nodes(['sample_circular_dependency_nodes'])
mgr = NodeManager(datetime.now(), 10, lfl_params, requested, [],
derived, {}, {})
order, _ = dependency_order(mgr, draw=True)
# As Gear Selected Down depends upon Gear Down
self.assertEqual(order, [
'Gear (L) Down', 'Gear Down', 'Gear (L) Red Warning',
'Gear Down Selected', 'Airspeed', 'Airspeed At Gear Down Selected'
])
def test_sample_parameter_module(self):
"""Tests many options:
can_operate on SmoothedTrack works with
"""
requested = ["Smoothed Track", "Moment Of Takeoff", "Vertical Speed", "Slip On Runway"]
lfl_params = [
"Indicated Airspeed",
"Groundspeed",
"Pressure Altitude",
"Heading",
"TAT",
"Latitude",
"Longitude",
"Longitudinal g",
"Lateral g",
"Normal g",
"Pitch",
"Roll",
]
try:
# for test cmd line runners
derived = get_derived_nodes(["tests.sample_derived_parameters"])
except ImportError:
# for IDE test runners
derived = get_derived_nodes(["sample_derived_parameters"])
nodes = NodeManager({"Start Datetime": datetime.now()}, 10, lfl_params, requested, [], derived, {}, {})
order, _ = dependency_order(nodes, draw=False)
pos = order.index
self.assertTrue(len(order))
self.assertNotIn("Moment Of Takeoff", order) # not available
self.assertTrue(pos("Vertical Speed") > pos("Pressure Altitude"))
self.assertTrue(pos("Slip On Runway") > pos("Groundspeed"))
self.assertTrue(pos("Slip On Runway") > pos("Horizontal g Across Track"))
self.assertTrue(pos("Horizontal g Across Track") > pos("Roll"))
self.assertFalse("Mach" in order) # Mach wasn't requested!
self.assertFalse("Radio Altimeter" in order)
self.assertEqual(len(nodes.hdf_keys), 12)
self.assertEqual(len(nodes.requested), 4)
self.assertEqual(len(nodes.derived_nodes), 13)
def test_avoiding_possible_circular_dependency(self):
# Possible circular dependency which can be avoided:
# Gear Selected Down depends on Gear Down which depends on Gear Selected Down...!
lfl_params = ['Airspeed', 'Gear (L) Down', 'Gear (L) Red Warning']
requested = ['Airspeed At Gear Down Selected']
derived = get_derived_nodes([import_module('sample_circular_dependency_nodes')])
mgr = NodeManager({'Start Datetime': datetime.now()}, 10, lfl_params, requested, [],
derived, {}, {})
order, _ = dependency_order(mgr, draw=False)
# As Gear Selected Down depends upon Gear Down
self.assertEqual(order,
['Gear (L) Down', 'Gear Down', 'Gear (L) Red Warning',
'Gear Down Selected', 'Airspeed', 'Airspeed At Gear Down Selected'])
def test_avoiding_possible_circular_dependency(self):
# Possible circular dependency which can be avoided:
# Gear Selected Down depends on Gear Down which depends on Gear Selected Down...!
lfl_params = ['Airspeed', 'Gear (L) Down', 'Gear (L) Red Warning']
requested = ['Airspeed At Gear Down Selected']
derived = get_derived_nodes([import_module('sample_circular_dependency_nodes')])
mgr = NodeManager({'Start Datetime': datetime.now()}, 10, lfl_params, requested, [],
derived, {}, {})
order, _ = dependency_order(mgr, draw=True)
# As Gear Selected Down depends upon Gear Down
self.assertEqual(order,
['Gear (L) Down', 'Gear Down', 'Gear (L) Red Warning',
'Gear Down Selected', 'Airspeed', 'Airspeed At Gear Down Selected'])
def pre_process_parameters(hdf, segment_info, param_names, required,
aircraft_info, achieved_flight_record, force=False,
dependency_tree_log=None):
'''
Perform actions prior to main processing run.
Actions such as merging parameters up front simplify processing paths by
removing circular dependacies.
'''
pre_processing_nodes = get_derived_nodes(settings.PRE_PROCESSING_MODULE_PATHS)
requested = list(pre_processing_nodes.keys())
node_mgr = NodeManager(
segment_info, hdf.duration, param_names,
requested, required, pre_processing_nodes, aircraft_info,
achieved_flight_record)
process_order, gr_st = dependency_order(node_mgr, draw=False, dependency_tree_log=dependency_tree_log)
ktis, kpvs, sections, approaches, flight_attrs = \
derive_parameters(hdf, node_mgr, process_order, force=force)
def _get_dependency_order(self, requested, aircraft_info, lfl_params,
draw=False, raise_cir_dep=True, segment_info={}):
#derived_nodes = get_derived_nodes(settings.NODE_MODULES)
if aircraft_info['Aircraft Type'] == 'helicopter':
node_modules = settings.NODE_MODULES + settings.NODE_HELICOPTER_MODULE_PATHS
else:
node_modules = settings.NODE_MODULES
# go through modules to get derived nodes
derived_nodes = get_derived_nodes(node_modules)
if requested == []:
# Use all derived nodes if requested is empty
requested = [p for p in derived_nodes.keys() if p not in lfl_params]
if not segment_info:
segment_info = {
'Start Datetime': datetime.now(),
'Segment Type': 'START_AND_STOP',
}
node_mgr= NodeManager(segment_info, 10, lfl_params,
requested, [], derived_nodes, aircraft_info, {})
order, gr_st = dependency_order(node_mgr, draw=draw,
raise_cir_dep=raise_cir_dep)
return order, gr_st
def process_flight(hdf_path,
tail_number,
aircraft_info={},
start_datetime=None,
achieved_flight_record={},
requested=[],
required=[],
include_flight_attributes=True,
additional_modules=[]):
'''
Processes the HDF file (hdf_path) to derive the required_params (Nodes)
within python modules (settings.NODE_MODULES).
Note: For Flight Data Services, the definitive API is located here:
"PolarisTaskManagement.test.tasks_mask.process_flight"
:param hdf_path: Path to HDF File
:type hdf_path: String
:param aircraft: Aircraft specific attributes
:type aircraft: dict
:param start_datetime: Datetime of the origin of the data (at index 0)
:type start_datetime: Datetime
:param achieved_flight_record: See API Below
:type achieved_flight_record: Dict
:param requested: Derived nodes to process (dependencies will also be
evaluated).
:type requested: List of Strings
:param required: Nodes which are required, otherwise an exception will be
raised.
:type required: List of Strings
:param include_flight_attributes: Whether to include all flight attributes
:type include_flight_attributes: Boolean
:param additional_modules: List of module paths to import.
:type additional_modules: List of Strings
:returns: See below:
:rtype: Dict
Sample aircraft_info
--------------------
{
'Tail Number': # Aircraft Registration
'Identifier': # Aircraft Ident
'Manufacturer': # e.g. Boeing
'Manufacturer Serial Number': #MSN
'Model': # e.g. 737-808-ER
'Series': # e.g. 737-800
'Family': # e.g. 737
'Frame': # e.g. 737-3C
'Main Gear To Altitude Radio': # Distance in metres
'Wing Span': # Distance in metres
}
Sample achieved_flight_record
-----------------------------
{
# Simple values first, e.g. string, int, float, etc.
'AFR Flight ID': # e.g. 1
'AFR Flight Number': # e.g. 1234
'AFR Type': # 'POSITIONING'
'AFR Off Blocks Datetime': # datetime(2015,01,01,13,00)
'AFR Takeoff Datetime': # datetime(2015,01,01,13,15)
'AFR Takeoff Pilot': # 'Joe Bloggs'
'AFR Takeoff Gross Weight': # weight in kg
'AFR Takeoff Fuel': # fuel in kg
'AFR Landing Datetime': # datetime(2015,01,01,18,45)
'AFR Landing Pilot': # 'Joe Bloggs'
'AFR Landing Gross Weight': # weight in kg
'AFR Landing Fuel': # weight in kg
'AFR On Blocks Datetime': # datetime(2015,01,01,19,00)
'AFR V2': # V2 used at takeoff in kts
'AFR Vapp': # Vapp used in kts
'AFR Vref': # Vref used in kts
# More complex data that needs to be looked up next:
'AFR Takeoff Airport': {
'id': 4904, # unique id
'name': 'Athens Intl Airport Elefterios Venizel',
'code': {'iata': 'ATH', 'icao': 'LGAV'},
'latitude': 37.9364,
'longitude': 23.9445,
'location': {'city': u'Athens', 'country': u'Greece'},
'elevation': 266, # ft
'magnetic_variation': 'E003186 0106',
}
},
'AFR Landing Aiport': {
'id': 1, # unique id
'name': 'Athens Intl Airport Elefterios Venizel',
'code': {'iata': 'ATH', 'icao': 'LGAV'},
'latitude': 37.9364,
'longitude': 23.9445,
'location': {'city': u'Athens', 'country': u'Greece'},
'elevation': 266, # ft
'magnetic_variation': 'E003186 0106',
}
},
'AFR Destination Airport': None, # if not required, or exclude this key
'AFR Takeoff Runway': {
'id': 1,
'identifier': '21L',
'magnetic_heading': 212.6,
'strip': {
'id': 1,
'length': 13123,
'surface': 'ASP',
'width': 147},
'start': {
'elevation': 308,
'latitude': 37.952425,
'longitude': 23.970422},
'end': {
'elevation': 279,
'latitude': 37.923511,
'longitude': 23.943261},
'glideslope': {
'angle': 3.0,
'elevation': 282,
'latitude': 37.9473,
'longitude': 23.9676,
'threshold_distance': 999},
'localizer': {
'beam_width': 4.5,
'elevation': 256,
'frequency': 111100,
'heading': 213,
'latitude': 37.919281,
'longitude': 23.939294},
},
'AFR Landing Runway': {
'id': 1,
'identifier': '21L',
'magnetic_heading': 212.6,
'strip': {
'id': 1,
'length': 13123,
'surface': 'ASP',
'width': 147},
'start': {
'elevation': 308,
'latitude': 37.952425,
'longitude': 23.970422},
'end': {
'elevation': 279,
'latitude': 37.923511,
'longitude': 23.943261},
'glideslope': {
'angle': 3.0,
'elevation': 282,
'latitude': 37.9473,
'longitude': 23.9676,
'threshold_distance': 999},
'localizer': {
'beam_width': 4.5,
'elevation': 256,
'frequency': 111100,
'heading': 213,
'latitude': 37.919281,
'longitude': 23.939294},
},
}
Sample Return
-------------
{
'flight':[Attribute('name value')],
'kti':[GeoKeyTimeInstance('index name latitude longitude')]
if lat/long available
else [KeyTimeInstance('index name')],
'kpv':[KeyPointValue('index value name slice')]
}
sample flight Attributes:
[
Attribute('Takeoff Airport', {'id':1234, 'name':'Int. Airport'},
Attribute('Approaches', [4567,7890]),
...
],
'''
if start_datetime is None:
import pytz
start_datetime = datetime.utcnow().replace(tzinfo=pytz.utc)
logger.info("Processing: %s", hdf_path)
if aircraft_info:
# Aircraft info has already been provided.
logger.info(
"Using aircraft_info dictionary passed into process_flight '%s'." %
aircraft_info)
else:
aircraft_info = get_aircraft_info(tail_number)
aircraft_info['Tail Number'] = tail_number
# go through modules to get derived nodes
node_modules = additional_modules + settings.NODE_MODULES
derived_nodes = get_derived_nodes(node_modules)
if requested:
requested = \
list(set(requested).intersection(set(derived_nodes)))
else:
# if requested isn't set, try using ALL derived_nodes!
logger.info("No requested nodes declared, using all derived nodes")
requested = derived_nodes.keys()
# include all flight attributes as requested
if include_flight_attributes:
requested = list(
set(requested +
get_derived_nodes(['analysis_engine.flight_attribute']).keys())
)
# open HDF for reading
with hdf_file(hdf_path) as hdf:
hdf.start_datetime = start_datetime
if hooks.PRE_FLIGHT_ANALYSIS:
logger.info("Performing PRE_FLIGHT_ANALYSIS actions: %s",
hooks.PRE_FLIGHT_ANALYSIS.func_name)
hooks.PRE_FLIGHT_ANALYSIS(hdf, aircraft_info)
else:
logger.info("No PRE_FLIGHT_ANALYSIS actions to perform")
# Track nodes. Assume that all params in HDF are from LFL(!)
node_mgr = NodeManager(start_datetime, hdf.duration,
hdf.valid_param_names(), requested, required,
derived_nodes, aircraft_info,
achieved_flight_record)
# calculate dependency tree
process_order, gr_st = dependency_order(node_mgr, draw=False)
if settings.CACHE_PARAMETER_MIN_USAGE:
# find params used more than
for node in gr_st.nodes():
if node in node_mgr.derived_nodes:
# this includes KPV/KTIs but they'll be ignored by HDF
qty = len(gr_st.predecessors(node))
if qty > settings.CACHE_PARAMETER_MIN_USAGE:
hdf.cache_param_list.append(node)
logging.info("HDF set to cache parameters: %s",
hdf.cache_param_list)
# derive parameters
kti_list, kpv_list, section_list, approach_list, flight_attrs = \
derive_parameters(hdf, node_mgr, process_order)
# geo locate KTIs
kti_list = geo_locate(hdf, kti_list)
kti_list = _timestamp(start_datetime, kti_list)
# geo locate KPVs
kpv_list = geo_locate(hdf, kpv_list)
kpv_list = _timestamp(start_datetime, kpv_list)
# Store version of FlightDataAnalyser
hdf.analysis_version = __version__
# Store dependency tree
hdf.dependency_tree = json_graph.dumps(gr_st)
# Store aircraft info
hdf.set_attr('aircraft_info', aircraft_info)
hdf.set_attr('achieved_flight_record', achieved_flight_record)
return {
'flight': flight_attrs,
'kti': kti_list,
'kpv': kpv_list,
'approach': approach_list,
'phases': section_list,
}
def process_flight(segment_info,
tail_number,
aircraft_info={},
achieved_flight_record={},
requested=[],
required=[],
include_flight_attributes=True,
additional_modules=[],
pre_flight_kwargs={},
force=False,
initial={},
reprocess=False):
'''
Processes the HDF file (segment_info['File']) to derive the required_params (Nodes)
within python modules (settings.NODE_MODULES).
Note: For Flight Data Services, the definitive API is located here:
"PolarisTaskManagement.test.tasks_mask.process_flight"
:param segment_info: Details of the segment to process
:type segment_info: dict
:param aircraft: Aircraft specific attributes
:type aircraft: dict
:param achieved_flight_record: See API Below
:type achieved_flight_record: Dict
:param requested: Derived nodes to process (dependencies will also be
evaluated).
:type requested: List of Strings
:param required: Nodes which are required, otherwise an exception will be
raised.
:type required: List of Strings
:param include_flight_attributes: Whether to include all flight attributes
:type include_flight_attributes: Boolean
:param additional_modules: List of module paths to import.
:type additional_modules: List of Strings
:param pre_flight_kwargs: Keyword arguments for the pre-flight analysis hook.
:type pre_flight_kwargs: dict
:param force: Ignore errors raised while deriving nodes.
:type force: bool
:param initial: Initial content for nodes to avoid reprocessing (excluding parameter nodes which are saved to the hdf).
:type initial: dict
:param reprocess: Force reprocessing of all Nodes (including derived Nodes already saved to the HDF file).
:returns: See below:
:rtype: Dict
Sample segment_info
--------------------
{
'File': # Path to HDF5 file to process
'Start Datetime': # Datetime of the origin of the data (at index 0)
'Segment Type': # segment type obtained from split segments e.g. START_AND_STOP
}
Sample aircraft_info
--------------------
{
'Tail Number': # Aircraft Registration
'Identifier': # Aircraft Ident
'Manufacturer': # e.g. Boeing
'Manufacturer Serial Number': #MSN
'Model': # e.g. 737-808-ER
'Series': # e.g. 737-800
'Family': # e.g. 737
'Frame': # e.g. 737-3C
'Main Gear To Altitude Radio': # Distance in metres
'Wing Span': # Distance in metres
}
Sample achieved_flight_record
-----------------------------
{
# Simple values first, e.g. string, int, float, etc.
'AFR Flight ID': # e.g. 1
'AFR Flight Number': # e.g. 1234
'AFR Type': # 'POSITIONING'
'AFR Off Blocks Datetime': # datetime(2015,1,1,13,00)
'AFR Takeoff Datetime': # datetime(2015,1,1,13,15)
'AFR Takeoff Pilot': # 'Joe Bloggs'
'AFR Takeoff Gross Weight': # weight in kg
'AFR Takeoff Fuel': # fuel in kg
'AFR Landing Datetime': # datetime(2015,1,1,18,45)
'AFR Landing Pilot': # 'Joe Bloggs'
'AFR Landing Gross Weight': # weight in kg
'AFR Landing Fuel': # weight in kg
'AFR On Blocks Datetime': # datetime(2015,1,1,19,00)
'AFR V2': # V2 used at takeoff in kts
'AFR Vapp': # Vapp used in kts
'AFR Vref': # Vref used in kts
# More complex data that needs to be looked up next:
'AFR Takeoff Airport': {
'id': 4904, # unique id
'name': 'Athens Intl Airport Elefterios Venizel',
'code': {'iata': 'ATH', 'icao': 'LGAV'},
'latitude': 37.9364,
'longitude': 23.9445,
'location': {'city': u'Athens', 'country': u'Greece'},
'elevation': 266, # ft
'magnetic_variation': 'E003186 0106',
}
},
'AFR Landing Aiport': {
'id': 1, # unique id
'name': 'Athens Intl Airport Elefterios Venizel',
'code': {'iata': 'ATH', 'icao': 'LGAV'},
'latitude': 37.9364,
'longitude': 23.9445,
'location': {'city': u'Athens', 'country': u'Greece'},
'elevation': 266, # ft
'magnetic_variation': 'E003186 0106',
}
},
'AFR Destination Airport': None, # if not required, or exclude this key
'AFR Takeoff Runway': {
'id': 1,
'identifier': '21L',
'magnetic_heading': 212.6,
'strip': {
'id': 1,
'length': 13123,
'surface': 'ASP',
'width': 147},
'start': {
'elevation': 308,
'latitude': 37.952425,
'longitude': 23.970422},
'end': {
'elevation': 279,
'latitude': 37.923511,
'longitude': 23.943261},
'glideslope': {
'angle': 3.0,
'elevation': 282,
'latitude': 37.9473,
'longitude': 23.9676,
'threshold_distance': 999},
'localizer': {
'beam_width': 4.5,
'elevation': 256,
'frequency': 111100,
'heading': 213,
'latitude': 37.919281,
'longitude': 23.939294},
},
'AFR Landing Runway': {
'id': 1,
'identifier': '21L',
'magnetic_heading': 212.6,
'strip': {
'id': 1,
'length': 13123,
'surface': 'ASP',
'width': 147},
'start': {
'elevation': 308,
'latitude': 37.952425,
'longitude': 23.970422},
'end': {
'elevation': 279,
'latitude': 37.923511,
'longitude': 23.943261},
'glideslope': {
'angle': 3.0,
'elevation': 282,
'latitude': 37.9473,
'longitude': 23.9676,
'threshold_distance': 999},
'localizer': {
'beam_width': 4.5,
'elevation': 256,
'frequency': 111100,
'heading': 213,
'latitude': 37.919281,
'longitude': 23.939294},
},
}
Sample Return
-------------
{
'flight':[Attribute('name value')],
'kti':[GeoKeyTimeInstance('index name latitude longitude')]
if lat/long available
else [KeyTimeInstance('index name')],
'kpv':[KeyPointValue('index value name slice')]
}
sample flight Attributes:
[
Attribute('Takeoff Airport', {'id':1234, 'name':'Int. Airport'},
Attribute('Approaches', [4567,7890]),
...
],
'''
hdf_path = segment_info['File']
if 'Start Datetime' not in segment_info:
import pytz
segment_info['Start Datetime'] = datetime.utcnow().replace(
tzinfo=pytz.utc)
logger.debug("Processing: %s", hdf_path)
if aircraft_info:
# Aircraft info has already been provided.
logger.info(
"Using aircraft_info dictionary passed into process_flight '%s'." %
aircraft_info)
else:
aircraft_info = get_aircraft_info(tail_number)
aircraft_info['Tail Number'] = tail_number
if aircraft_info['Aircraft Type'] == 'helicopter':
node_modules = settings.NODE_MODULES + \
settings.NODE_HELICOPTER_MODULE_PATHS + additional_modules
else:
node_modules = settings.NODE_MODULES + additional_modules
# go through modules to get derived nodes
derived_nodes = get_derived_nodes(node_modules)
if requested:
requested = \
list(set(requested).intersection(set(derived_nodes)))
else:
# if requested isn't set, try using ALL derived_nodes!
logger.debug("No requested nodes declared, using all derived nodes")
# Enforce some sort of order in which the dependencies are traversed
requested = sorted(list(derived_nodes.keys()))
# include all flight attributes as requested
if include_flight_attributes:
requested = list(
set(requested + list(
get_derived_nodes(['analysis_engine.flight_attribute']).keys())
))
initial = process_flight_to_nodes(initial)
for node_name in requested:
initial.pop(node_name, None)
# open HDF for reading
with hdf_file(hdf_path) as hdf:
hdf.start_datetime = segment_info['Start Datetime']
hook = hooks.PRE_FLIGHT_ANALYSIS
if hook:
logger.info(
"Performing PRE_FLIGHT_ANALYSIS action '%s' with options: %s",
getattr(hook, 'func_name', getattr(hook, '__name__')),
pre_flight_kwargs)
hook(hdf, aircraft_info, **pre_flight_kwargs)
else:
logger.info("No PRE_FLIGHT_ANALYSIS actions to perform")
# Merge Params
param_names = hdf.valid_lfl_param_names(
) if reprocess else hdf.valid_param_names()
pre_process_parameters(hdf,
segment_info,
param_names,
required,
aircraft_info,
achieved_flight_record,
force=force)
# Track nodes.
node_mgr = NodeManager(segment_info, hdf.duration, param_names,
requested, required, derived_nodes,
aircraft_info, achieved_flight_record)
# calculate dependency tree
process_order, gr_st = dependency_order(node_mgr, draw=False)
if settings.CACHE_PARAMETER_MIN_USAGE:
# find params used more than
for node in gr_st.nodes():
if node in node_mgr.derived_nodes:
# this includes KPV/KTIs but they'll be ignored by HDF
qty = len(gr_st.predecessors(node))
if qty > settings.CACHE_PARAMETER_MIN_USAGE:
hdf.cache_param_list.append(node)
logging.info("HDF set to cache parameters: %s",
hdf.cache_param_list)
# derive parameters
ktis, kpvs, sections, approaches, flight_attrs = \
derive_parameters(hdf, node_mgr, process_order, params=initial, force=force)
# geo locate KTIs
ktis = geo_locate(hdf, ktis)
ktis = _timestamp(segment_info['Start Datetime'], ktis)
# geo locate KPVs
kpvs = geo_locate(hdf, kpvs)
kpvs = _timestamp(segment_info['Start Datetime'], kpvs)
# Store version of FlightDataAnalyser
hdf.analysis_version = __version__
# Store dependency tree
hdf.dependency_tree = json.dumps(json_graph.node_link_data(gr_st))
# Store aircraft info
hdf.set_attr('aircraft_info', aircraft_info)
hdf.set_attr('achieved_flight_record', achieved_flight_record)
return {
'flight': flight_attrs,
'kti': ktis,
'kpv': kpvs,
'approach': approaches,
'phases': sections,
}
def process_flight(segment_info, tail_number, aircraft_info={}, achieved_flight_record={},
requested=[], required=[], include_flight_attributes=True,
additional_modules=[], pre_flight_kwargs={}, force=False,
initial={}, reprocess=False):
'''
Processes the HDF file (segment_info['File']) to derive the required_params (Nodes)
within python modules (settings.NODE_MODULES).
Note: For Flight Data Services, the definitive API is located here:
"PolarisTaskManagement.test.tasks_mask.process_flight"
:param segment_info: Details of the segment to process
:type segment_info: dict
:param aircraft: Aircraft specific attributes
:type aircraft: dict
:param achieved_flight_record: See API Below
:type achieved_flight_record: Dict
:param requested: Derived nodes to process (dependencies will also be
evaluated).
:type requested: List of Strings
:param required: Nodes which are required, otherwise an exception will be
raised.
:type required: List of Strings
:param include_flight_attributes: Whether to include all flight attributes
:type include_flight_attributes: Boolean
:param additional_modules: List of module paths to import.
:type additional_modules: List of Strings
:param pre_flight_kwargs: Keyword arguments for the pre-flight analysis hook.
:type pre_flight_kwargs: dict
:param force: Ignore errors raised while deriving nodes.
:type force: bool
:param initial: Initial content for nodes to avoid reprocessing (excluding parameter nodes which are saved to the hdf).
:type initial: dict
:param reprocess: Force reprocessing of all Nodes (including derived Nodes already saved to the HDF file).
:returns: See below:
:rtype: Dict
Sample segment_info
--------------------
{
'File': # Path to HDF5 file to process
'Start Datetime': # Datetime of the origin of the data (at index 0)
'Segment Type': # segment type obtained from split segments e.g. START_AND_STOP
}
Sample aircraft_info
--------------------
{
'Tail Number': # Aircraft Registration
'Identifier': # Aircraft Ident
'Manufacturer': # e.g. Boeing
'Manufacturer Serial Number': #MSN
'Model': # e.g. 737-808-ER
'Series': # e.g. 737-800
'Family': # e.g. 737
'Frame': # e.g. 737-3C
'Main Gear To Altitude Radio': # Distance in metres
'Wing Span': # Distance in metres
}
Sample achieved_flight_record
-----------------------------
{
# Simple values first, e.g. string, int, float, etc.
'AFR Flight ID': # e.g. 1
'AFR Flight Number': # e.g. 1234
'AFR Type': # 'POSITIONING'
'AFR Off Blocks Datetime': # datetime(2015,01,01,13,00)
'AFR Takeoff Datetime': # datetime(2015,01,01,13,15)
'AFR Takeoff Pilot': # 'Joe Bloggs'
'AFR Takeoff Gross Weight': # weight in kg
'AFR Takeoff Fuel': # fuel in kg
'AFR Landing Datetime': # datetime(2015,01,01,18,45)
'AFR Landing Pilot': # 'Joe Bloggs'
'AFR Landing Gross Weight': # weight in kg
'AFR Landing Fuel': # weight in kg
'AFR On Blocks Datetime': # datetime(2015,01,01,19,00)
'AFR V2': # V2 used at takeoff in kts
'AFR Vapp': # Vapp used in kts
'AFR Vref': # Vref used in kts
# More complex data that needs to be looked up next:
'AFR Takeoff Airport': {
'id': 4904, # unique id
'name': 'Athens Intl Airport Elefterios Venizel',
'code': {'iata': 'ATH', 'icao': 'LGAV'},
'latitude': 37.9364,
'longitude': 23.9445,
'location': {'city': u'Athens', 'country': u'Greece'},
'elevation': 266, # ft
'magnetic_variation': 'E003186 0106',
}
},
'AFR Landing Aiport': {
'id': 1, # unique id
'name': 'Athens Intl Airport Elefterios Venizel',
'code': {'iata': 'ATH', 'icao': 'LGAV'},
'latitude': 37.9364,
'longitude': 23.9445,
'location': {'city': u'Athens', 'country': u'Greece'},
'elevation': 266, # ft
'magnetic_variation': 'E003186 0106',
}
},
'AFR Destination Airport': None, # if not required, or exclude this key
'AFR Takeoff Runway': {
'id': 1,
'identifier': '21L',
'magnetic_heading': 212.6,
'strip': {
'id': 1,
'length': 13123,
'surface': 'ASP',
'width': 147},
'start': {
'elevation': 308,
'latitude': 37.952425,
'longitude': 23.970422},
'end': {
'elevation': 279,
'latitude': 37.923511,
'longitude': 23.943261},
'glideslope': {
'angle': 3.0,
'elevation': 282,
'latitude': 37.9473,
'longitude': 23.9676,
'threshold_distance': 999},
'localizer': {
'beam_width': 4.5,
'elevation': 256,
'frequency': 111100,
'heading': 213,
'latitude': 37.919281,
'longitude': 23.939294},
},
'AFR Landing Runway': {
'id': 1,
'identifier': '21L',
'magnetic_heading': 212.6,
'strip': {
'id': 1,
'length': 13123,
'surface': 'ASP',
'width': 147},
'start': {
'elevation': 308,
'latitude': 37.952425,
'longitude': 23.970422},
'end': {
'elevation': 279,
'latitude': 37.923511,
'longitude': 23.943261},
'glideslope': {
'angle': 3.0,
'elevation': 282,
'latitude': 37.9473,
'longitude': 23.9676,
'threshold_distance': 999},
'localizer': {
'beam_width': 4.5,
'elevation': 256,
'frequency': 111100,
'heading': 213,
'latitude': 37.919281,
'longitude': 23.939294},
},
}
Sample Return
-------------
{
'flight':[Attribute('name value')],
'kti':[GeoKeyTimeInstance('index name latitude longitude')]
if lat/long available
else [KeyTimeInstance('index name')],
'kpv':[KeyPointValue('index value name slice')]
}
sample flight Attributes:
[
Attribute('Takeoff Airport', {'id':1234, 'name':'Int. Airport'},
Attribute('Approaches', [4567,7890]),
...
],
'''
hdf_path = segment_info['File']
if 'Start Datetime' not in segment_info:
import pytz
segment_info['Start Datetime'] = datetime.utcnow().replace(tzinfo=pytz.utc)
logger.info("Processing: %s", hdf_path)
if aircraft_info:
# Aircraft info has already been provided.
logger.info(
"Using aircraft_info dictionary passed into process_flight '%s'." %
aircraft_info)
else:
aircraft_info = get_aircraft_info(tail_number)
aircraft_info['Tail Number'] = tail_number
# go through modules to get derived nodes
node_modules = additional_modules + settings.NODE_MODULES
derived_nodes = get_derived_nodes(node_modules)
if requested:
requested = \
list(set(requested).intersection(set(derived_nodes)))
else:
# if requested isn't set, try using ALL derived_nodes!
logger.info("No requested nodes declared, using all derived nodes")
requested = derived_nodes.keys()
# include all flight attributes as requested
if include_flight_attributes:
requested = list(set(
requested + get_derived_nodes(
['analysis_engine.flight_attribute']).keys()))
initial = process_flight_to_nodes(initial)
for node_name in requested:
initial.pop(node_name, None)
# open HDF for reading
with hdf_file(hdf_path) as hdf:
hdf.start_datetime = segment_info['Start Datetime']
if hooks.PRE_FLIGHT_ANALYSIS:
logger.info("Performing PRE_FLIGHT_ANALYSIS action '%s' with options: %s",
hooks.PRE_FLIGHT_ANALYSIS.func_name, pre_flight_kwargs)
hooks.PRE_FLIGHT_ANALYSIS(hdf, aircraft_info, **pre_flight_kwargs)
else:
logger.info("No PRE_FLIGHT_ANALYSIS actions to perform")
# Track nodes.
param_names = hdf.valid_lfl_param_names() if reprocess else hdf.valid_param_names()
node_mgr = NodeManager(
segment_info, hdf.duration, param_names,
requested, required, derived_nodes, aircraft_info,
achieved_flight_record)
# calculate dependency tree
process_order, gr_st = dependency_order(node_mgr, draw=False)
if settings.CACHE_PARAMETER_MIN_USAGE:
# find params used more than
for node in gr_st.nodes():
if node in node_mgr.derived_nodes:
# this includes KPV/KTIs but they'll be ignored by HDF
qty = len(gr_st.predecessors(node))
if qty > settings.CACHE_PARAMETER_MIN_USAGE:
hdf.cache_param_list.append(node)
logging.info("HDF set to cache parameters: %s",
hdf.cache_param_list)
# derive parameters
ktis, kpvs, sections, approaches, flight_attrs = \
derive_parameters(hdf, node_mgr, process_order, params=initial, force=force)
# geo locate KTIs
ktis = geo_locate(hdf, ktis)
ktis = _timestamp(segment_info['Start Datetime'], ktis)
# geo locate KPVs
kpvs = geo_locate(hdf, kpvs)
kpvs = _timestamp(segment_info['Start Datetime'], kpvs)
# Store version of FlightDataAnalyser
hdf.analysis_version = __version__
# Store dependency tree
hdf.dependency_tree = json_graph.dumps(gr_st)
# Store aircraft info
hdf.set_attr('aircraft_info', aircraft_info)
hdf.set_attr('achieved_flight_record', achieved_flight_record)
return {
'flight': flight_attrs,
'kti': ktis,
'kpv': kpvs,
'approach': approaches,
'phases': sections,
}
