def test_graph_nodes_with_duplicate_key_in_lfl_and_derived(self): """ Test that LFL nodes are used in place of Derived where available. Tests a few of the colours """ class One(DerivedParameterNode): # Hack to allow objects rather than classes to be added to the tree. __base__ = DerivedParameterNode __bases__ = [__base__] def derive(self, dep=P('DepOne')): pass class Four(DerivedParameterNode): # Hack to allow objects rather than classes to be added to the tree. __base__ = DerivedParameterNode __bases__ = [__base__] def derive(self, dep=P('DepFour')): pass one = One('overridden') four = Four('used') mgr1 = NodeManager({'Start Datetime': datetime.now()}, 10, [1, 2], [2, 4], [], {1:one, 4:four},{}, {}) gr = graph_nodes(mgr1) self.assertEqual(len(gr), 5) # LFL self.assertEqual(gr.edges(1), []) # as it's in LFL, it shouldn't have any edges self.assertEqual(gr.node[1], {'color': '#72f4eb', 'node_type': 'HDFNode'}) # Derived self.assertEqual(gr.edges(4), [(4,'DepFour')]) self.assertEqual(gr.node[4], {'color': '#72cdf4', 'node_type': 'DerivedParameterNode'}) # Root from analysis_engine.dependency_graph import draw_graph draw_graph(gr, 'test_graph_nodes_with_duplicate_key_in_lfl_and_derived') self.assertEqual(gr.successors('root'), [2,4]) # only the two requested are linked self.assertEqual(gr.node['root'], {'color': '#ffffff'})
def test_dependency_with_lowlevel_dependencies_requested(self): """ Simulate requesting a Raw Parameter as a dependency. This requires the requested node to be removed when it is not at the top of the dependency tree. """ requested = [ 'P7', 'P8', # top level nodes 'P4', 'P5', 'P6', # middle level node 'Raw3', # bottom level node ] mgr = NodeManager(datetime.now(), 10, self.lfl_params + ['Floating'], requested, [], self.derived_nodes, {}, {}) gr = graph_nodes(mgr) gr_all, gr_st, order = process_order(gr, mgr) self.assertEqual(len(gr_st), 11) pos = order.index self.assertTrue(pos('P8') > pos('Raw5')) self.assertTrue(pos('P7') > pos('P4')) self.assertTrue(pos('P7') > pos('P5')) self.assertTrue(pos('P7') > pos('P6')) self.assertTrue(pos('P6') > pos('Raw3')) self.assertTrue(pos('P5') > pos('Raw3')) self.assertTrue(pos('P5') > pos('Raw4')) self.assertTrue(pos('P4') > pos('Raw1')) self.assertTrue(pos('P4') > pos('Raw2')) self.assertFalse('Floating' in order) self.assertFalse('root' in order) #don't include the root!
def test_graph_nodes_with_duplicate_key_in_lfl_and_derived(self): # Test that LFL nodes are used in place of Derived where available. # Tests a few of the colours class One(DerivedParameterNode): # Hack to allow objects rather than classes to be added to the tree. __base__ = DerivedParameterNode __bases__ = [__base__] def derive(self, dep=P('DepOne')): pass class Four(DerivedParameterNode): # Hack to allow objects rather than classes to be added to the tree. __base__ = DerivedParameterNode __bases__ = [__base__] def derive(self, dep=P('DepFour')): pass one = One('overridden') four = Four('used') mgr1 = NodeManager({'Start Datetime': datetime.now()}, 10, ['1', '2'], ['2', '4'], [], {'1':one, '4':four},{}, {}) gr = graph_nodes(mgr1) self.assertEqual(len(gr), 5) # LFL self.assertEqual(list(gr.edges('1')), []) # as it's in LFL, it shouldn't have any edges self.assertEqual(gr.node['1'], {'node_type': 'HDFNode'}) # Derived self.assertEqual(list(gr.edges('4')), [('4','DepFour')]) self.assertEqual(gr.node['4'], {'node_type': 'DerivedParameterNode'}) # Root self.assertEqual(list(gr.successors('root')), ['2','4']) # only the two requested are linked self.assertEqual(gr.node['root'], {})
def test_graph_middle_level_depenency_builds_partial_tree(self): requested = ['P5'] mgr = NodeManager({'Start Datetime': datetime.now()}, 1, self.lfl_params, requested, [], self.derived_nodes, {}, {}) gr = graph_nodes(mgr) # should only be linked to P5 self.assertEqual(gr.neighbors('root'), ['P5'])
def test_graph_nodes_using_sample_tree(self): requested = ['P7', 'P8'] mgr2 = NodeManager({'Start Datetime': datetime.now()}, 10, self.lfl_params, requested, [], self.derived_nodes, {}, {}) gr = graph_nodes(mgr2) self.assertEqual(len(gr), 11) self.assertEqual(gr.neighbors('root'), ['P8', 'P7'])
def derived_trimmer(hdf_path, node_names, dest): ''' Trims an HDF file of parameters which are not dependencies of nodes in node_names. :param hdf_path: file path of hdf file. :type hdf_path: str :param node_names: A list of Node names which are required. :type node_names: list of str :param dest: destination path for trimmed output file :type dest: str :return: parameters in stripped hdf file :rtype: [str] ''' params = [] with hdf_file(hdf_path) as hdf: derived_nodes = get_derived_nodes(settings.NODE_MODULES) node_mgr = NodeManager( datetime.now(), hdf.duration, hdf.valid_param_names(), [], [], derived_nodes, {}, {}) _graph = graph_nodes(node_mgr) for node_name in node_names: deps = dependencies3(_graph, node_name, node_mgr) params.extend(filter(lambda d: d in node_mgr.hdf_keys, deps)) return strip_hdf(hdf_path, params, dest)
def test_graph_nodes_using_sample_tree(self): requested = ['P7', 'P8'] mgr2 = NodeManager(datetime.now(), 10, self.lfl_params, requested, [], self.derived_nodes, {}, {}) gr = graph_nodes(mgr2) self.assertEqual(len(gr), 11) self.assertEqual(gr.neighbors('root'), ['P8', 'P7'])
def test_graph_nodes_with_duplicate_key_in_lfl_and_derived(self): # Test that LFL nodes are used in place of Derived where available. # Tests a few of the colours class One(DerivedParameterNode): # Hack to allow objects rather than classes to be added to the tree. __base__ = DerivedParameterNode __bases__ = [__base__] def derive(self, dep=P('DepOne')): pass class Four(DerivedParameterNode): # Hack to allow objects rather than classes to be added to the tree. __base__ = DerivedParameterNode __bases__ = [__base__] def derive(self, dep=P('DepFour')): pass one = One('overridden') four = Four('used') mgr1 = NodeManager({'Start Datetime': datetime.now()}, 10, [1, 2], [2, 4], [], {1:one, 4:four},{}, {}) gr = graph_nodes(mgr1) self.assertEqual(len(gr), 5) # LFL self.assertEqual(gr.edges(1), []) # as it's in LFL, it shouldn't have any edges self.assertEqual(gr.node[1], {'color': '#72f4eb', 'node_type': 'HDFNode'}) # Derived self.assertEqual(gr.edges(4), [(4,'DepFour')]) self.assertEqual(gr.node[4], {'color': '#72cdf4', 'node_type': 'DerivedParameterNode'}) # Root from analysis_engine.dependency_graph import draw_graph draw_graph(gr, 'test_graph_nodes_with_duplicate_key_in_lfl_and_derived') self.assertEqual(gr.successors('root'), [2,4]) # only the two requested are linked self.assertEqual(gr.node['root'], {'color': '#ffffff'})
def test_indent_tree(self): requested = ["P7", "P8"] mgr2 = NodeManager( {"Start Datetime": datetime.now()}, 10, self.lfl_params, requested, [], self.derived_nodes, {}, {} ) gr = graph_nodes(mgr2) gr.node["Raw1"]["active"] = True gr.node["Raw2"]["active"] = False gr.node["P4"]["active"] = False self.assertEqual( indent_tree(gr, "P7", space="__", delim=" ", label=False), [" P7", "__ [P4]", "____ Raw1", "____ [Raw2]", "__ P5", "____ Raw3", "____ Raw4", "__ P6", "____ Raw3"], ) # don't recurse valid parameters... self.assertEqual(indent_tree(gr, "P5", label=False, recurse_active=False), []) self.assertEqual(indent_tree(gr, "P4", label=False, recurse_active=False), ["- [P4]", " - [Raw2]"]) self.assertEqual( indent_tree(gr, "root"), [ "- root", " - P7 (DerivedParameterNode)", " - [P4] (DerivedParameterNode)", " - Raw1 (HDFNode)", " - [Raw2] (HDFNode)", " - P5 (DerivedParameterNode)", " - Raw3 (HDFNode)", " - Raw4 (HDFNode)", " - P6 (DerivedParameterNode)", " - Raw3 (HDFNode)", " - P8 (DerivedParameterNode)", " - Raw5 (HDFNode)", ], )
def test_dependency_with_lowlevel_dependencies_requested(self): """ Simulate requesting a Raw Parameter as a dependency. This requires the requested node to be removed when it is not at the top of the dependency tree. """ requested = ["P7", "P8", "P4", "P5", "P6", "Raw3"] # top level nodes # middle level node # bottom level node mgr = NodeManager( {"Start Datetime": datetime.now()}, 10, self.lfl_params + ["Floating"], requested, [], self.derived_nodes, {}, {}, ) gr = graph_nodes(mgr) gr_all, gr_st, order = process_order(gr, mgr) self.assertEqual(len(gr_st), 11) pos = order.index self.assertTrue(pos("P8") > pos("Raw5")) self.assertTrue(pos("P7") > pos("P4")) self.assertTrue(pos("P7") > pos("P5")) self.assertTrue(pos("P7") > pos("P6")) self.assertTrue(pos("P6") > pos("Raw3")) self.assertTrue(pos("P5") > pos("Raw3")) self.assertTrue(pos("P5") > pos("Raw4")) self.assertTrue(pos("P4") > pos("Raw1")) self.assertTrue(pos("P4") > pos("Raw2")) self.assertFalse("Floating" in order) self.assertFalse("root" in order) # don't include the root!
def test_graph_middle_level_depenency_builds_partial_tree(self): requested = ['P5'] mgr = NodeManager(datetime.now(), 1, self.lfl_params, requested, [], self.derived_nodes, {}, {}) gr = graph_nodes(mgr) # should only be linked to P5 self.assertEqual(gr.neighbors('root'), ['P5'])
def test_dependency_with_lowlevel_dependencies_requested(self): """ Simulate requesting a Raw Parameter as a dependency. This requires the requested node to be removed when it is not at the top of the dependency tree. """ requested = ['P7', 'P8', # top level nodes 'P4', 'P5', 'P6', # middle level node 'Raw3', # bottom level node ] mgr = NodeManager({'Start Datetime': datetime.now()}, 10, self.lfl_params + ['Floating'], requested, [], self.derived_nodes, {}, {}) gr = graph_nodes(mgr) gr_all, gr_st, order = process_order(gr, mgr) self.assertEqual(len(gr_st), 11) pos = order.index self.assertTrue(pos('P8') > pos('Raw5')) self.assertTrue(pos('P7') > pos('P4')) self.assertTrue(pos('P7') > pos('P5')) self.assertTrue(pos('P7') > pos('P6')) self.assertTrue(pos('P6') > pos('Raw3')) self.assertTrue(pos('P5') > pos('Raw3')) self.assertTrue(pos('P5') > pos('Raw4')) self.assertTrue(pos('P4') > pos('Raw1')) self.assertTrue(pos('P4') > pos('Raw2')) self.assertFalse('Floating' in order) self.assertFalse('root' in order) #don't include the root!
def test_required_available(self): nodes = ["a", "b", "c"] required = ["a", "c"] mgr = NodeManager({"Start Datetime": datetime.now()}, 10, nodes, nodes, required, {}, {}, {}) _graph = graph_nodes(mgr) gr_all, gr_st, order = process_order(_graph, mgr) self.assertEqual(set(required) - set(order), set())
def test_graph_requesting_all_dependencies_links_root_to_end_leafs(self): # build list of all nodes as required requested = self.lfl_params + self.derived_nodes.keys() mgr = NodeManager({'Start Datetime': datetime.now()}, 1, self.lfl_params, requested, [], self.derived_nodes, {}, {}) gr = graph_nodes(mgr) # should only be linked to end leafs self.assertEqual(gr.neighbors('root'), ['P8', 'P7'])
def test_required_available(self): nodes = ['a', 'b', 'c'] required = ['a', 'c'] mgr = NodeManager(datetime.now(), 10, nodes, nodes, required, {}, {}, {}) _graph = graph_nodes(mgr) gr_all, gr_st, order = process_order(_graph, mgr) self.assertEqual(set(required) - set(order), set())
def test_graph_nodes_using_sample_tree(self): requested = ["P7", "P8"] mgr2 = NodeManager( {"Start Datetime": datetime.now()}, 10, self.lfl_params, requested, [], self.derived_nodes, {}, {} ) gr = graph_nodes(mgr2) self.assertEqual(len(gr), 11) self.assertEqual(gr.neighbors("root"), ["P8", "P7"])
def test_graph_requesting_all_dependencies_links_root_to_end_leafs(self): # build list of all nodes as required requested = self.lfl_params + self.derived_nodes.keys() mgr = NodeManager(datetime.now(), 1, self.lfl_params, requested, [], self.derived_nodes, {}, {}) gr = graph_nodes(mgr) # should only be linked to end leafs self.assertEqual(gr.neighbors('root'), ['P8', 'P7'])
def test_graph_requesting_all_dependencies_links_root_to_all_requests(self): # build list of all nodes as required requested = self.lfl_params + list(self.derived_nodes.keys()) mgr = NodeManager({'Start Datetime': datetime.now()}, 1, self.lfl_params, requested, [], self.derived_nodes, {}, {}) gr = graph_nodes(mgr) # should be linked to all requested nodes self.assertEqual(sorted(gr.neighbors('root')), sorted(requested))
def _generate_json(self, lfl_params): ''' Returns list of parameters used in the spanning tree. Note: LFL parameters not used will not be returned! ''' print "Establishing Node dependencies from Analysis Engine" # Ensure file is a valid HDF file before continuing: derived_nodes = get_derived_nodes(settings.NODE_MODULES) required_params = derived_nodes.keys() # TODO: Update ac_info with keys from provided fields: ac_info = { 'Family': u'B737 NG', 'Frame': u'737-3C', 'Identifier': u'15', 'Main Gear To Lowest Point Of Tail': None, 'Main Gear To Radio Altimeter Antenna': None, 'Manufacturer Serial Number': u'39009', 'Manufacturer': u'Boeing', 'Model': u'B737-8JP', 'Precise Positioning': True, 'Series': u'B737-800', 'Tail Number': 'G-ABCD', } # TODO: Option to populate an AFR: achieved_flight_record = {} # Generate the dependency tree: node_mgr = NodeManager( {}, 1000, lfl_params, required_params, [], derived_nodes, ac_info, achieved_flight_record, ) _graph = graph_nodes(node_mgr) gr_all, gr_st, order = process_order(_graph, node_mgr) # Save the dependency tree to tree.json: tree = os.path.join(AJAX_DIR, 'tree.json') with open(tree, 'w') as fh: json.dump(graph_adjacencies(gr_st), fh, indent=4) # Save the list of nodes to node_list.json: node_list = os.path.join(AJAX_DIR, 'node_list.json') spanning_tree_params = sorted(gr_st.nodes()) with open(node_list, 'w') as fh: json.dump(spanning_tree_params, fh, indent=4) return
def test_indent_tree(self): requested = ['P7', 'P8'] mgr2 = NodeManager({'Start Datetime': datetime.now()}, 10, self.lfl_params, requested, [], self.derived_nodes, {}, {}) gr = graph_nodes(mgr2) gr.node['Raw1']['active'] = True gr.node['Raw2']['active'] = False gr.node['P4']['active'] = False self.assertEqual( indent_tree(gr, 'P7', space='__', delim=' ', label=False), [' P7', '__ [P4]', '____ Raw1', '____ [Raw2]', '__ P5', '____ Raw3', '____ Raw4', '__ P6', '____ Raw3', ]) # don't recurse valid parameters... self.assertEqual( indent_tree(gr, 'P5', label=False, recurse_active=False), []) self.assertEqual( indent_tree(gr, 'P4', label=False, recurse_active=False), ['- [P4]', ' - [Raw2]', ]) self.assertEqual( indent_tree(gr, 'root'), ['- root', ' - P7 (DerivedParameterNode)', ' - [P4] (DerivedParameterNode)', ' - Raw1 (HDFNode)', ' - [Raw2] (HDFNode)', ' - P5 (DerivedParameterNode)', ' - Raw3 (HDFNode)', ' - Raw4 (HDFNode)', ' - P6 (DerivedParameterNode)', ' - Raw3 (HDFNode)', ' - P8 (DerivedParameterNode)', ' - Raw5 (HDFNode)', ])
def test_dependency(self): requested = ['P7', 'P8'] mgr = NodeManager(datetime.now(), 10, self.lfl_params, requested, [], self.derived_nodes, {}, {}) gr = graph_nodes(mgr) gr_all, gr_st, order = process_order(gr, mgr) self.assertEqual(len(gr_st), 11) pos = order.index self.assertTrue(pos('P8') > pos('Raw5')) self.assertTrue(pos('P7') > pos('P4')) self.assertTrue(pos('P7') > pos('P5')) self.assertTrue(pos('P7') > pos('P6')) self.assertTrue(pos('P6') > pos('Raw3')) self.assertTrue(pos('P5') > pos('Raw3')) self.assertTrue(pos('P5') > pos('Raw4')) self.assertTrue(pos('P4') > pos('Raw1')) self.assertTrue(pos('P4') > pos('Raw2')) self.assertFalse('root' in order) #don't include the root! """
def test_dependency(self): requested = ['P7', 'P8'] mgr = NodeManager({'Start Datetime': datetime.now()}, 10, self.lfl_params, requested, [], self.derived_nodes, {}, {}) gr = graph_nodes(mgr) gr_all, gr_st, order = process_order(gr, mgr) self.assertEqual(len(gr_st), 11) pos = order.index self.assertTrue(pos('P8') > pos('Raw5')) self.assertTrue(pos('P7') > pos('P4')) self.assertTrue(pos('P7') > pos('P5')) self.assertTrue(pos('P7') > pos('P6')) self.assertTrue(pos('P6') > pos('Raw3')) self.assertTrue(pos('P5') > pos('Raw3')) self.assertTrue(pos('P5') > pos('Raw4')) self.assertTrue(pos('P4') > pos('Raw1')) self.assertTrue(pos('P4') > pos('Raw2')) self.assertFalse('root' in order) #don't include the root! """
def test_dependency(self): requested = ["P7", "P8"] mgr = NodeManager( {"Start Datetime": datetime.now()}, 10, self.lfl_params, requested, [], self.derived_nodes, {}, {} ) gr = graph_nodes(mgr) gr_all, gr_st, order = process_order(gr, mgr) self.assertEqual(len(gr_st), 11) pos = order.index self.assertTrue(pos("P8") > pos("Raw5")) self.assertTrue(pos("P7") > pos("P4")) self.assertTrue(pos("P7") > pos("P5")) self.assertTrue(pos("P7") > pos("P6")) self.assertTrue(pos("P6") > pos("Raw3")) self.assertTrue(pos("P5") > pos("Raw3")) self.assertTrue(pos("P5") > pos("Raw4")) self.assertTrue(pos("P4") > pos("Raw1")) self.assertTrue(pos("P4") > pos("Raw2")) self.assertFalse("root" in order) # don't include the root! """
def test_graph_nodes_with_duplicate_key_in_lfl_and_derived(self): """ Test that LFL nodes are used in place of Derived where available. Tests a few of the colours """ class One(DerivedParameterNode): # Hack to allow objects rather than classes to be added to the tree. __base__ = DerivedParameterNode __bases__ = [__base__] def derive(self, dep=P("DepOne")): pass class Four(DerivedParameterNode): # Hack to allow objects rather than classes to be added to the tree. __base__ = DerivedParameterNode __bases__ = [__base__] def derive(self, dep=P("DepFour")): pass one = One("overridden") four = Four("used") mgr1 = NodeManager({"Start Datetime": datetime.now()}, 10, [1, 2], [2, 4], [], {1: one, 4: four}, {}, {}) gr = graph_nodes(mgr1) self.assertEqual(len(gr), 5) # LFL self.assertEqual(gr.edges(1), []) # as it's in LFL, it shouldn't have any edges self.assertEqual(gr.node[1], {"color": "#72f4eb", "node_type": "HDFNode"}) # Derived self.assertEqual(gr.edges(4), [(4, "DepFour")]) self.assertEqual(gr.node[4], {"color": "#72cdf4", "node_type": "DerivedParameterNode"}) # Root from analysis_engine.dependency_graph import draw_graph draw_graph(gr, "test_graph_nodes_with_duplicate_key_in_lfl_and_derived") self.assertEqual(gr.successors("root"), [2, 4]) # only the two requested are linked self.assertEqual(gr.node["root"], {"color": "#ffffff"})
def test_required_unavailable(self): nodes = ['a', 'b', 'c'] required = ['a', 'c', 'd'] mgr = NodeManager(datetime.now(), 10, nodes, nodes, required, {}, {}, {}) gr = self.assertRaises(graph_nodes(mgr))
def test_required_unavailable(self): nodes = ['a', 'b', 'c'] required = ['a', 'c', 'd'] mgr = NodeManager({'Start Datetime': datetime.now()}, 10, nodes, nodes, required, {}, {}, {}) gr = self.assertRaises(graph_nodes(mgr))
def test_required_unavailable(self): nodes = ["a", "b", "c"] required = ["a", "c", "d"] mgr = NodeManager({"Start Datetime": datetime.now()}, 10, nodes, nodes, required, {}, {}, {}) gr = self.assertRaises(graph_nodes(mgr))