def test_filter_inputs_dynamic_keys(self):
mapping = {'node1': {'out11': 'inp11',
'out12': 'inp12'},
'node2': {'out22': 'inp22',
'out21': '_'}}
nucleotide = Nucleotide(
inbound_nodes=['node1', 'node2', 'node3'],
incoming_keys_mapping=mapping)
nucleotide.incoming_keys = ['inp11']
nucleotide.dynamic_incoming_keys = True
data = {'node1': {'out11': 'value11',
'out12': 'value12'},
'node2': {'out21': 'value21',
'out22': 'value22',
'out': 'value23'},
'node3': {'out31': 'value31',
'out32': 'value32',
'out': 'value33'},
'node4': {'out41': 'value41'}}
data_remapped_must = {'inp11': 'value11',
'inp12': 'value12',
'inp22': 'value22',
'out31': 'value31',
'out32': 'value32',
'out': ['value23', 'value33']}
data_remapped = nucleotide.filter_inputs(data)
self.assertDictEqual(data_remapped_must, data_remapped)
def test_check_node2node_connection(self):
dataset_node = Nucleotide(name='data', inbound_nodes=[])
dataset_node.generated_keys = ['image', 'labels', 'temp']
node_cnn1 = Nucleotide(name='cnn',
inbound_nodes=['data'],
incoming_keys_mapping={
'data': {
'image': 'inputs_cnn',
'temp': 'inputs_optional'
}
})
node_cnn1.incoming_keys = [
'inputs_cnn', '_inputs_optional', '_inputs_optional_2'
]
node_cnn1.generated_keys = ['predictions']
node_loss = Nucleotide(
name='loss',
inbound_nodes=['data', 'cnn'],
incoming_keys_mapping={'cnn': {
'predictions:0': 'logits:0'
}})
node_loss.incoming_keys = ['logits', 'labels']
node_loss_wrong = Nucleotide(
name='loss',
inbound_nodes=['data', 'cnn'],
incoming_keys_mapping={'cnn': {
'predictions': 'logits2'
}})
node_loss_wrong.incoming_keys = ['logits', 'labels']
nodes_all = {
'right': [dataset_node, node_cnn1, node_loss],
'wrong': [dataset_node, node_cnn1, node_loss_wrong]
}
for t, nodes in nodes_all.items():
nodes = {n.name: n for n in nodes}
if t == 'right':
self.assertTrue(
graph_utils.check_node2node_connection(
nodes['loss'], [nodes[k] for k in ['data', 'cnn']]))
else:
with self.assertRaises(ValueError):
graph_utils.check_node2node_connection(
nodes['loss'], [nodes[k] for k in ['data', 'cnn']])
def test_represent_predictor_through_nucleotides(self):
class PredictorMock(object):
def __init__(self_):
self_._fetch_tensors = None
@property
def fetch_tensors(self_):
return self_._fetch_tensors
@property
def feed_tensors(self_):
return self_._feed_tensors
tf.reset_default_graph()
predictor = PredictorMock()
fetch_tensors = {
'nucleotide1': {
'output1': 'value1',
'output2': 'value2'
},
'nucleotide2': {
'output3': 'value3',
'output4': 'value4'
}
}
feed_tensors = {
"data1": tf.placeholder(tf.float32),
"data2": tf.placeholder(tf.float32),
"parameter1": tf.placeholder_with_default(10, [])
}
fetch_tensors_flatten = nest_utils.flatten_nested_struct(fetch_tensors)
predictor._fetch_tensors = fetch_tensors_flatten
predictor._feed_tensors = feed_tensors
nucleotides = represent_predictor_through_nucleotides(predictor)
nucleotide1_must = Nucleotide(name='nucleotide1')
nucleotide1_must.generated_keys = ['output1', 'output2']
nucleotide1_must.incoming_keys = ['data1', 'data2']
nucleotide2_must = Nucleotide(name='nucleotide2')
nucleotide2_must.generated_keys = ['output3', 'output4']
nucleotide2_must.incoming_keys = ['data1', 'data2']
nucleotides_must = [nucleotide1_must, nucleotide2_must]
for nucleotide, nucleotide_must in zip(nucleotides, nucleotides_must):
self.assertEqual(nucleotide_must.name, nucleotide.name)
self.assertSetEqual(set(nucleotide_must.generated_keys),
set(nucleotide.generated_keys))
self.assertSetEqual(set(nucleotide_must.incoming_keys),
set(nucleotide.incoming_keys))
def test_filter_inputs_empty(self):
mapping = {'node1': {'out11': 'inp11',
'out12': 'inp12'},
'node2': {'out22': 'inp22'}}
nucleotide = Nucleotide(
inbound_nodes=['node1', 'node2', 'node3'],
incoming_keys_mapping=mapping)
nucleotide.incoming_keys = ['inp11', 'inp12', 'inp22', 'out31']
data = {'node1': {'out11': 'value11',
'out12': 'value12'},
'node2': {'out21': 'value21',
'out22': 'value22'},
'node3': None,
'node4': {'out41': 'value41'}}
self.assertIsNone(nucleotide.filter_inputs(data))
def test_filter_inputs_nested_input_mapping(self):
mapping = {'node1': {'out11:out111': 'inp11',
'out12': 'inp12'},
'node2': {'out22:2': 'inp22'}}
nucleotide = Nucleotide(
inbound_nodes=['node1', 'node2', 'node3'],
incoming_keys_mapping=mapping)
nucleotide.incoming_keys = ['inp11', 'inp12', 'inp22', 'out31']
data = {'node1': {'out11': {'out111': 'value11',
'out112': 'value112'},
'out12': 'value12'},
'node2': {'out21': 'value21',
'out22': ['value220', 'value221',
'value22', 'value223']},
'node3': {'out31': 'value31'},
'node4': {'out41': 'value41'}}
data_remapped_must = {'inp11': 'value11',
'inp12': 'value12',
'inp22': 'value22',
'out31': 'value31'}
data_remapped = nucleotide.filter_inputs(data)
self.assertDictEqual(data_remapped_must, data_remapped)
def test_check_graph_connections(self):
dataset_node = Nucleotide(name='data', inbound_nodes=[])
dataset_node.generated_keys = ['image', 'labels', 'temp']
node_cnn = Nucleotide(
name='cnn',
inbound_nodes=['data'],
incoming_keys_mapping={'data': {
'image': 'inputs_cnn'
}})
node_cnn.incoming_keys = ['inputs_cnn']
node_cnn.generated_keys = ['predictions']
node_flatten = Nucleotide(
name='flatten',
inbound_nodes=['cnn'],
incoming_keys_mapping={'cnn': {
'predictions': 'inputs_flatten'
}})
node_flatten.incoming_keys = ['inputs_flatten']
node_flatten.generated_keys = ['predictions']
node_mlp = Nucleotide(
name='mlp',
inbound_nodes=['flatten'],
incoming_keys_mapping={'flatten': {
'predictions': 'inputs_mlp'
}})
node_mlp.incoming_keys = ['inputs_mlp']
node_mlp.generated_keys = ['predictions']
node_mlp_wrong = Nucleotide(
name='mlp',
inbound_nodes=['flatten'],
incoming_keys_mapping={'flatten': {
'predictions': 'inputs'
}})
node_mlp_wrong.incoming_keys = ['inputs_mlp']
node_mlp_wrong.generated_keys = ['predictions']
node_loss = Nucleotide(
name='loss',
inbound_nodes=['data', 'mlp'],
incoming_keys_mapping={'mlp': {
'predictions': 'logits'
}})
node_loss.incoming_keys = ['labels', 'logits']
node_loss_wrong = Nucleotide(name='loss',
inbound_nodes=['data', 'mlp'])
node_loss_wrong.incoming_keys = ['labels', 'logits']
node_pp = Nucleotide(
name='pp',
inbound_nodes=['mlp'],
incoming_keys_mapping={'mlp': {
'predictions': 'inputs_pp'
}})
node_pp.incoming_keys = ['inputs_pp']
nodes_all = {
'right': [[
dataset_node, node_cnn, node_flatten, node_mlp, node_loss,
node_pp
]],
'wrong': [[
dataset_node, node_cnn, node_flatten, node_mlp_wrong,
node_loss, node_pp
],
[
dataset_node, node_cnn, node_flatten, node_mlp_wrong,
node_loss_wrong, node_pp
]]
}
for test_mode, nodes_case in nodes_all.items():
for nodes in nodes_case:
graph = graph_utils.construct_graph_from_nucleotides(nodes)
if test_mode == 'right':
self.assertTrue(graph_utils.check_graph_connections(graph))
else:
with self.assertRaises(ValueError):
graph_utils.check_graph_connections(graph)
def test_get_nucleotide_signature(self):
node = Nucleotide(
name='cnn',
inbound_nodes=['data'],
incoming_keys_mapping={'data': {
'image': 'inputs_cnn'
}})
node.incoming_keys = ['inputs1', 'inputs2', '_input_optional']
node.generated_keys = ['predictions', '_predictions_optional']
node.__doc__ = """
Attributes
----------
incoming_keys : list
* inputs1 : inputs1 to node
* inputs2 : inputs2 to node
* inputs_wrong : wrong description
* input_optional : optional inputs
generated_keys : list
* predictions : predictions 1
* predictions_optional : optional predictions
"""
args_must = {
'inputs1': 'inputs1 to node',
'inputs2': 'inputs2 to node',
'input_optional': 'optional inputs'
}
returns_must = {
'predictions': 'predictions 1',
'predictions_optional': 'optional predictions'
}
args, returns = nucleotide_utils.get_nucleotide_signature(node)
self.assertSetEqual({'inputs1', 'inputs2', 'input_optional'},
set(args))
self.assertSetEqual({'predictions', 'predictions_optional'},
set(returns))
self.assertDictEqual(args_must, args)
self.assertDictEqual(returns_must, returns)
node.__doc__ = None
node.process = MagicMock(return_value=0)
node.process.__doc__ = """
Parameters
----------
inputs1
inputs1 to node
inputs2
inputs2 to node
inputs_wrong
wrong description
input_optional
optional inputs
Returns
-------
predictions
predictions 1
predictions_optional
optional predictions
"""
args, returns = nucleotide_utils.get_nucleotide_signature(node)
self.assertDictEqual(args_must, args)
self.assertDictEqual(returns_must, returns)
def _get_dna():
nucleotide1 = Nucleotide(name="nucleotide1")
nucleotide1.generated_keys = ["y"]
nucleotide2 = Nucleotide(name="nucleotide2", inbound_nodes=["nucleotide1"])
nucleotide2.incoming_keys = ["y"]
nucleotide2.generated_keys = ["z"]
nucleotide3 = Nucleotide(name="nucleotide3",
inbound_nodes=["nucleotide1", "nucleotide2"])
nucleotide3.dynamic_incoming_keys = True
nucleotide3.generated_keys = ["w"]
nucleotide3.dynamic_generated_keys = True
nucleotide4 = Nucleotide(name="nucleotide4", inbound_nodes=["nucleotide3"])
nucleotide4.incoming_keys = ["y", "z", "w"]
nucleotides_list = [nucleotide2, nucleotide3, nucleotide4]
incoming_nucleotides_list = [nucleotide1]
dna_must = {
'nucleotide2':
DNAConnection(incoming={'nucleotide1'}, outgoing={'nucleotide3'}),
'nucleotide3':
DNAConnection(incoming={'nucleotide1', 'nucleotide2'},
outgoing={'nucleotide4'}),
'nucleotide4':
DNAConnection(incoming={'nucleotide3'}, outgoing=set()),
}
topological_sort_must = ["nucleotide2", "nucleotide3", "nucleotide4"]
raise_build_error = False
yield (nucleotides_list, incoming_nucleotides_list, dna_must,
topological_sort_must, raise_build_error)
nucleotide3 = Nucleotide(name="nucleotide3",
inbound_nodes=["nucleotide1", "nucleotide2"])
nucleotide3.incoming_keys = []
nucleotide3.generated_keys = ["w"]
nucleotides_list = [nucleotide2, nucleotide3, nucleotide4]
raise_build_error = True
yield (nucleotides_list, incoming_nucleotides_list, dna_must,
topological_sort_must, raise_build_error)
nucleotides_list = [
Nucleotide(name='first', inbound_nodes=['input_node1']),
Nucleotide(name='second', inbound_nodes=['first', 'input_node2']),
Nucleotide(name='fourth', inbound_nodes=['first', 'third']),
Nucleotide(name='fifth', inbound_nodes=['fourth']),
Nucleotide(name='third', inbound_nodes=['first', 'second'])
]
incoming_nucleotides_list = [
Nucleotide(name='input_node1'),
Nucleotide(name='input_node2')
]
dna_must = {
'first':
DNAConnection(incoming={'input_node1'},
outgoing={'second', 'fourth', 'third'}),
'second':
DNAConnection(incoming={'first', 'input_node2'}, outgoing={'third'}),
'third':
DNAConnection(incoming={'first', 'second'}, outgoing={'fourth'}),
'fourth':
DNAConnection(incoming={'first', 'third'}, outgoing={'fifth'}),
'fifth':
DNAConnection(incoming={'fourth'}, outgoing=set())
}
topological_sort_must = ["first", "second", "third", "fourth", "fifth"]
raise_build_error = False
yield (nucleotides_list, incoming_nucleotides_list, dna_must,
topological_sort_must, raise_build_error)
nucleotides_list = [
Nucleotide(name='first', inbound_nodes=['input_node1']),
Nucleotide(name='second', inbound_nodes=[]),
Nucleotide(name='fourth', inbound_nodes=['first', 'third']),
Nucleotide(name='fifth', inbound_nodes=['fourth']),
Nucleotide(name='third', inbound_nodes=['first', 'second'])
]
incoming_nucleotides_list = [Nucleotide(name='input_node1')]
dna_must = {
'first':
DNAConnection(incoming={'input_node1'}, outgoing={'fourth', 'third'}),
'second':
DNAConnection(incoming=set(), outgoing={'third'}),
'third':
DNAConnection(incoming={'first', 'second'}, outgoing={'fourth'}),
'fourth':
DNAConnection(incoming={'first', 'third'}, outgoing={'fifth'}),
'fifth':
DNAConnection(incoming={'fourth'}, outgoing=set())
}
topological_sort_must = ["second", "first", "third", "fourth", "fifth"]
raise_build_error = False
yield (nucleotides_list, incoming_nucleotides_list, dna_must,
topological_sort_must, raise_build_error)
nucleotides_list = [
Nucleotide(name='first', inbound_nodes=[]),
Nucleotide(name='second', inbound_nodes=[]),
Nucleotide(name='fourth', inbound_nodes=['first', 'third']),
Nucleotide(name='fifth', inbound_nodes=['fourth']),
Nucleotide(name='third', inbound_nodes=['first', 'second'])
]
incoming_nucleotides_list = []
dna_must = {
'first': DNAConnection(incoming=set(), outgoing={'fourth', 'third'}),
'second': DNAConnection(incoming=set(), outgoing={'third'}),
'third': DNAConnection(incoming={'first', 'second'},
outgoing={'fourth'}),
'fourth': DNAConnection(incoming={'first', 'third'},
outgoing={'fifth'}),
'fifth': DNAConnection(incoming={'fourth'}, outgoing=set())
}
topological_sort_must = ["first", "second", "third", "fourth", "fifth"]
raise_build_error = False
yield (nucleotides_list, incoming_nucleotides_list, dna_must,
topological_sort_must, raise_build_error)