def test_filter_inputs_dynamic_keys(self): mapping = {'node1': {'out11': 'inp11', 'out12': 'inp12'}, 'node2': {'out22': 'inp22', 'out21': '_'}} nucleotide = Nucleotide( inbound_nodes=['node1', 'node2', 'node3'], incoming_keys_mapping=mapping) nucleotide.incoming_keys = ['inp11'] nucleotide.dynamic_incoming_keys = True data = {'node1': {'out11': 'value11', 'out12': 'value12'}, 'node2': {'out21': 'value21', 'out22': 'value22', 'out': 'value23'}, 'node3': {'out31': 'value31', 'out32': 'value32', 'out': 'value33'}, 'node4': {'out41': 'value41'}} data_remapped_must = {'inp11': 'value11', 'inp12': 'value12', 'inp22': 'value22', 'out31': 'value31', 'out32': 'value32', 'out': ['value23', 'value33']} data_remapped = nucleotide.filter_inputs(data) self.assertDictEqual(data_remapped_must, data_remapped)
def test_check_node2node_connection(self): dataset_node = Nucleotide(name='data', inbound_nodes=[]) dataset_node.generated_keys = ['image', 'labels', 'temp'] node_cnn1 = Nucleotide(name='cnn', inbound_nodes=['data'], incoming_keys_mapping={ 'data': { 'image': 'inputs_cnn', 'temp': 'inputs_optional' } }) node_cnn1.incoming_keys = [ 'inputs_cnn', '_inputs_optional', '_inputs_optional_2' ] node_cnn1.generated_keys = ['predictions'] node_loss = Nucleotide( name='loss', inbound_nodes=['data', 'cnn'], incoming_keys_mapping={'cnn': { 'predictions:0': 'logits:0' }}) node_loss.incoming_keys = ['logits', 'labels'] node_loss_wrong = Nucleotide( name='loss', inbound_nodes=['data', 'cnn'], incoming_keys_mapping={'cnn': { 'predictions': 'logits2' }}) node_loss_wrong.incoming_keys = ['logits', 'labels'] nodes_all = { 'right': [dataset_node, node_cnn1, node_loss], 'wrong': [dataset_node, node_cnn1, node_loss_wrong] } for t, nodes in nodes_all.items(): nodes = {n.name: n for n in nodes} if t == 'right': self.assertTrue( graph_utils.check_node2node_connection( nodes['loss'], [nodes[k] for k in ['data', 'cnn']])) else: with self.assertRaises(ValueError): graph_utils.check_node2node_connection( nodes['loss'], [nodes[k] for k in ['data', 'cnn']])
def test_represent_predictor_through_nucleotides(self): class PredictorMock(object): def __init__(self_): self_._fetch_tensors = None @property def fetch_tensors(self_): return self_._fetch_tensors @property def feed_tensors(self_): return self_._feed_tensors tf.reset_default_graph() predictor = PredictorMock() fetch_tensors = { 'nucleotide1': { 'output1': 'value1', 'output2': 'value2' }, 'nucleotide2': { 'output3': 'value3', 'output4': 'value4' } } feed_tensors = { "data1": tf.placeholder(tf.float32), "data2": tf.placeholder(tf.float32), "parameter1": tf.placeholder_with_default(10, []) } fetch_tensors_flatten = nest_utils.flatten_nested_struct(fetch_tensors) predictor._fetch_tensors = fetch_tensors_flatten predictor._feed_tensors = feed_tensors nucleotides = represent_predictor_through_nucleotides(predictor) nucleotide1_must = Nucleotide(name='nucleotide1') nucleotide1_must.generated_keys = ['output1', 'output2'] nucleotide1_must.incoming_keys = ['data1', 'data2'] nucleotide2_must = Nucleotide(name='nucleotide2') nucleotide2_must.generated_keys = ['output3', 'output4'] nucleotide2_must.incoming_keys = ['data1', 'data2'] nucleotides_must = [nucleotide1_must, nucleotide2_must] for nucleotide, nucleotide_must in zip(nucleotides, nucleotides_must): self.assertEqual(nucleotide_must.name, nucleotide.name) self.assertSetEqual(set(nucleotide_must.generated_keys), set(nucleotide.generated_keys)) self.assertSetEqual(set(nucleotide_must.incoming_keys), set(nucleotide.incoming_keys))
def test_filter_inputs_empty(self): mapping = {'node1': {'out11': 'inp11', 'out12': 'inp12'}, 'node2': {'out22': 'inp22'}} nucleotide = Nucleotide( inbound_nodes=['node1', 'node2', 'node3'], incoming_keys_mapping=mapping) nucleotide.incoming_keys = ['inp11', 'inp12', 'inp22', 'out31'] data = {'node1': {'out11': 'value11', 'out12': 'value12'}, 'node2': {'out21': 'value21', 'out22': 'value22'}, 'node3': None, 'node4': {'out41': 'value41'}} self.assertIsNone(nucleotide.filter_inputs(data))
def test_filter_inputs_nested_input_mapping(self): mapping = {'node1': {'out11:out111': 'inp11', 'out12': 'inp12'}, 'node2': {'out22:2': 'inp22'}} nucleotide = Nucleotide( inbound_nodes=['node1', 'node2', 'node3'], incoming_keys_mapping=mapping) nucleotide.incoming_keys = ['inp11', 'inp12', 'inp22', 'out31'] data = {'node1': {'out11': {'out111': 'value11', 'out112': 'value112'}, 'out12': 'value12'}, 'node2': {'out21': 'value21', 'out22': ['value220', 'value221', 'value22', 'value223']}, 'node3': {'out31': 'value31'}, 'node4': {'out41': 'value41'}} data_remapped_must = {'inp11': 'value11', 'inp12': 'value12', 'inp22': 'value22', 'out31': 'value31'} data_remapped = nucleotide.filter_inputs(data) self.assertDictEqual(data_remapped_must, data_remapped)
def test_check_graph_connections(self): dataset_node = Nucleotide(name='data', inbound_nodes=[]) dataset_node.generated_keys = ['image', 'labels', 'temp'] node_cnn = Nucleotide( name='cnn', inbound_nodes=['data'], incoming_keys_mapping={'data': { 'image': 'inputs_cnn' }}) node_cnn.incoming_keys = ['inputs_cnn'] node_cnn.generated_keys = ['predictions'] node_flatten = Nucleotide( name='flatten', inbound_nodes=['cnn'], incoming_keys_mapping={'cnn': { 'predictions': 'inputs_flatten' }}) node_flatten.incoming_keys = ['inputs_flatten'] node_flatten.generated_keys = ['predictions'] node_mlp = Nucleotide( name='mlp', inbound_nodes=['flatten'], incoming_keys_mapping={'flatten': { 'predictions': 'inputs_mlp' }}) node_mlp.incoming_keys = ['inputs_mlp'] node_mlp.generated_keys = ['predictions'] node_mlp_wrong = Nucleotide( name='mlp', inbound_nodes=['flatten'], incoming_keys_mapping={'flatten': { 'predictions': 'inputs' }}) node_mlp_wrong.incoming_keys = ['inputs_mlp'] node_mlp_wrong.generated_keys = ['predictions'] node_loss = Nucleotide( name='loss', inbound_nodes=['data', 'mlp'], incoming_keys_mapping={'mlp': { 'predictions': 'logits' }}) node_loss.incoming_keys = ['labels', 'logits'] node_loss_wrong = Nucleotide(name='loss', inbound_nodes=['data', 'mlp']) node_loss_wrong.incoming_keys = ['labels', 'logits'] node_pp = Nucleotide( name='pp', inbound_nodes=['mlp'], incoming_keys_mapping={'mlp': { 'predictions': 'inputs_pp' }}) node_pp.incoming_keys = ['inputs_pp'] nodes_all = { 'right': [[ dataset_node, node_cnn, node_flatten, node_mlp, node_loss, node_pp ]], 'wrong': [[ dataset_node, node_cnn, node_flatten, node_mlp_wrong, node_loss, node_pp ], [ dataset_node, node_cnn, node_flatten, node_mlp_wrong, node_loss_wrong, node_pp ]] } for test_mode, nodes_case in nodes_all.items(): for nodes in nodes_case: graph = graph_utils.construct_graph_from_nucleotides(nodes) if test_mode == 'right': self.assertTrue(graph_utils.check_graph_connections(graph)) else: with self.assertRaises(ValueError): graph_utils.check_graph_connections(graph)
def test_get_nucleotide_signature(self): node = Nucleotide( name='cnn', inbound_nodes=['data'], incoming_keys_mapping={'data': { 'image': 'inputs_cnn' }}) node.incoming_keys = ['inputs1', 'inputs2', '_input_optional'] node.generated_keys = ['predictions', '_predictions_optional'] node.__doc__ = """ Attributes ---------- incoming_keys : list * inputs1 : inputs1 to node * inputs2 : inputs2 to node * inputs_wrong : wrong description * input_optional : optional inputs generated_keys : list * predictions : predictions 1 * predictions_optional : optional predictions """ args_must = { 'inputs1': 'inputs1 to node', 'inputs2': 'inputs2 to node', 'input_optional': 'optional inputs' } returns_must = { 'predictions': 'predictions 1', 'predictions_optional': 'optional predictions' } args, returns = nucleotide_utils.get_nucleotide_signature(node) self.assertSetEqual({'inputs1', 'inputs2', 'input_optional'}, set(args)) self.assertSetEqual({'predictions', 'predictions_optional'}, set(returns)) self.assertDictEqual(args_must, args) self.assertDictEqual(returns_must, returns) node.__doc__ = None node.process = MagicMock(return_value=0) node.process.__doc__ = """ Parameters ---------- inputs1 inputs1 to node inputs2 inputs2 to node inputs_wrong wrong description input_optional optional inputs Returns ------- predictions predictions 1 predictions_optional optional predictions """ args, returns = nucleotide_utils.get_nucleotide_signature(node) self.assertDictEqual(args_must, args) self.assertDictEqual(returns_must, returns)
def _get_dna(): nucleotide1 = Nucleotide(name="nucleotide1") nucleotide1.generated_keys = ["y"] nucleotide2 = Nucleotide(name="nucleotide2", inbound_nodes=["nucleotide1"]) nucleotide2.incoming_keys = ["y"] nucleotide2.generated_keys = ["z"] nucleotide3 = Nucleotide(name="nucleotide3", inbound_nodes=["nucleotide1", "nucleotide2"]) nucleotide3.dynamic_incoming_keys = True nucleotide3.generated_keys = ["w"] nucleotide3.dynamic_generated_keys = True nucleotide4 = Nucleotide(name="nucleotide4", inbound_nodes=["nucleotide3"]) nucleotide4.incoming_keys = ["y", "z", "w"] nucleotides_list = [nucleotide2, nucleotide3, nucleotide4] incoming_nucleotides_list = [nucleotide1] dna_must = { 'nucleotide2': DNAConnection(incoming={'nucleotide1'}, outgoing={'nucleotide3'}), 'nucleotide3': DNAConnection(incoming={'nucleotide1', 'nucleotide2'}, outgoing={'nucleotide4'}), 'nucleotide4': DNAConnection(incoming={'nucleotide3'}, outgoing=set()), } topological_sort_must = ["nucleotide2", "nucleotide3", "nucleotide4"] raise_build_error = False yield (nucleotides_list, incoming_nucleotides_list, dna_must, topological_sort_must, raise_build_error) nucleotide3 = Nucleotide(name="nucleotide3", inbound_nodes=["nucleotide1", "nucleotide2"]) nucleotide3.incoming_keys = [] nucleotide3.generated_keys = ["w"] nucleotides_list = [nucleotide2, nucleotide3, nucleotide4] raise_build_error = True yield (nucleotides_list, incoming_nucleotides_list, dna_must, topological_sort_must, raise_build_error) nucleotides_list = [ Nucleotide(name='first', inbound_nodes=['input_node1']), Nucleotide(name='second', inbound_nodes=['first', 'input_node2']), Nucleotide(name='fourth', inbound_nodes=['first', 'third']), Nucleotide(name='fifth', inbound_nodes=['fourth']), Nucleotide(name='third', inbound_nodes=['first', 'second']) ] incoming_nucleotides_list = [ Nucleotide(name='input_node1'), Nucleotide(name='input_node2') ] dna_must = { 'first': DNAConnection(incoming={'input_node1'}, outgoing={'second', 'fourth', 'third'}), 'second': DNAConnection(incoming={'first', 'input_node2'}, outgoing={'third'}), 'third': DNAConnection(incoming={'first', 'second'}, outgoing={'fourth'}), 'fourth': DNAConnection(incoming={'first', 'third'}, outgoing={'fifth'}), 'fifth': DNAConnection(incoming={'fourth'}, outgoing=set()) } topological_sort_must = ["first", "second", "third", "fourth", "fifth"] raise_build_error = False yield (nucleotides_list, incoming_nucleotides_list, dna_must, topological_sort_must, raise_build_error) nucleotides_list = [ Nucleotide(name='first', inbound_nodes=['input_node1']), Nucleotide(name='second', inbound_nodes=[]), Nucleotide(name='fourth', inbound_nodes=['first', 'third']), Nucleotide(name='fifth', inbound_nodes=['fourth']), Nucleotide(name='third', inbound_nodes=['first', 'second']) ] incoming_nucleotides_list = [Nucleotide(name='input_node1')] dna_must = { 'first': DNAConnection(incoming={'input_node1'}, outgoing={'fourth', 'third'}), 'second': DNAConnection(incoming=set(), outgoing={'third'}), 'third': DNAConnection(incoming={'first', 'second'}, outgoing={'fourth'}), 'fourth': DNAConnection(incoming={'first', 'third'}, outgoing={'fifth'}), 'fifth': DNAConnection(incoming={'fourth'}, outgoing=set()) } topological_sort_must = ["second", "first", "third", "fourth", "fifth"] raise_build_error = False yield (nucleotides_list, incoming_nucleotides_list, dna_must, topological_sort_must, raise_build_error) nucleotides_list = [ Nucleotide(name='first', inbound_nodes=[]), Nucleotide(name='second', inbound_nodes=[]), Nucleotide(name='fourth', inbound_nodes=['first', 'third']), Nucleotide(name='fifth', inbound_nodes=['fourth']), Nucleotide(name='third', inbound_nodes=['first', 'second']) ] incoming_nucleotides_list = [] dna_must = { 'first': DNAConnection(incoming=set(), outgoing={'fourth', 'third'}), 'second': DNAConnection(incoming=set(), outgoing={'third'}), 'third': DNAConnection(incoming={'first', 'second'}, outgoing={'fourth'}), 'fourth': DNAConnection(incoming={'first', 'third'}, outgoing={'fifth'}), 'fifth': DNAConnection(incoming={'fourth'}, outgoing=set()) } topological_sort_must = ["first", "second", "third", "fourth", "fifth"] raise_build_error = False yield (nucleotides_list, incoming_nucleotides_list, dna_must, topological_sort_must, raise_build_error)