def test_constructor(self):
inbound_nodes = ['node1', 'node2']
incoming_keys_mapping = {'node1': {"out1": "in1"}}
_ = Nucleotide(inbound_nodes=inbound_nodes,
incoming_keys_mapping=incoming_keys_mapping)
incoming_keys_mapping_wrong = {'node1': {"out1": "in1"},
'node3': {"out1": "in2"}}
with self.assertRaises(AttributeError):
_ = Nucleotide(inbound_nodes=inbound_nodes,
incoming_keys_mapping=incoming_keys_mapping_wrong)
inbound_nodes_with_mapping = {'node1': {"out1": "in1"},
'node2': {"out2": "in2"}}
nucleotide = Nucleotide(
inbound_nodes=inbound_nodes_with_mapping)
self.assertSetEqual(set(nucleotide.inbound_nodes),
set(inbound_nodes_with_mapping.keys()))
self.assertDictEqual(nucleotide.incoming_keys_mapping,
inbound_nodes_with_mapping)
self.assertEqual(nucleotide.name, nucleotide.__class__.__name__)
nucleotide_name = "nucleotide_name"
nucleotide_with_name = Nucleotide(inbound_nodes=inbound_nodes,
name=nucleotide_name)
self.assertEqual(nucleotide_with_name.name, nucleotide_name)
def test_mode_setter(self):
mode = 'NEW_MODE'
class GeneHandlerDummy(GeneHandler):
gene_name_and_nucleotide_super_cls = {
'gene1': Nucleotide,
'gene2': Nucleotide,
'gene3': Nucleotide,
'gene4': Nucleotide
}
genes = {
'gene1': [
Nucleotide(name='gene1_nucleotide1', inbound_nodes=[]),
Nucleotide(name='gene1_nucleotide2', inbound_nodes=[]),
Nucleotide(name='gene1_nucleotide3', inbound_nodes=[])
],
'gene2': [
Nucleotide(name='gene2_nucleotide1', inbound_nodes=[]),
Nucleotide(name='gene2_nucleotide2', inbound_nodes=[])
]
}
gene_handler = GeneHandlerDummy(gene1=genes['gene1'],
gene2=genes['gene2']).build()
gene_handler.mode = mode
for gene in genes.values():
for nucleotide in gene:
self.assertEqual(nucleotide.mode, mode)
def test_check_class_topology_right(self):
nucleotides = [
_NucleotideType1(name='gene1_nucleotide1',
inbound_nodes=['input']),
_NucleotideType1(name='gene1_nucleotide2', inbound_nodes=[]),
_NucleotideType1(name='gene1_nucleotide3', inbound_nodes=[]),
_NucleotideType2(name='gene2_nucleotide1',
inbound_nodes=[
'input', 'gene1_nucleotide1',
'gene1_nucleotide2'
]),
_NucleotideType2(
name='gene2_nucleotide2',
inbound_nodes=['gene2_nucleotide1', 'gene1_nucleotide3']),
Nucleotide(name='gene3_nucleotide1',
inbound_nodes=['gene2_nucleotide1']),
Nucleotide(
name='gene3_nucleotide2',
inbound_nodes=['gene2_nucleotide2', 'gene3_nucleotide1'])
]
dna_helix = DNAHelix(nucleotides, self.incoming_nucleotides)
dna_helix.build()
self.assertTrue(
dna_helix.check_class_topology(
self.nucleotide_type_dependency_map))
def test_assert_all_nucleotides_have_same_mode(self):
mode = 'NEW_MODE'
class GeneHandlerDummy(GeneHandler):
gene_name_and_nucleotide_super_cls = {
'gene1': Nucleotide,
'gene2': Nucleotide,
'gene3': Nucleotide,
'gene4': Nucleotide
}
genes = {
'gene1': [
Nucleotide(name='gene1_nucleotide1', inbound_nodes=[]),
Nucleotide(name='gene1_nucleotide2', inbound_nodes=[]),
Nucleotide(name='gene1_nucleotide3', inbound_nodes=[])
],
'gene2': [
Nucleotide(name='gene2_nucleotide1', inbound_nodes=[]),
Nucleotide(name='gene2_nucleotide2', inbound_nodes=[])
]
}
gene_handler = GeneHandlerDummy(gene1=genes['gene1'],
gene2=genes['gene2']).build()
gene_handler.mode = mode
gene_handler.assert_all_nucleotides_have_same_mode()
gene_handler.gene1['gene1_nucleotide1'].mode = 'WrongMode'
with self.assertRaises(ValueError):
gene_handler.assert_all_nucleotides_have_same_mode()
def test_filter_inputs_dynamic_keys(self):
mapping = {'node1': {'out11': 'inp11',
'out12': 'inp12'},
'node2': {'out22': 'inp22',
'out21': '_'}}
nucleotide = Nucleotide(
inbound_nodes=['node1', 'node2', 'node3'],
incoming_keys_mapping=mapping)
nucleotide.incoming_keys = ['inp11']
nucleotide.dynamic_incoming_keys = True
data = {'node1': {'out11': 'value11',
'out12': 'value12'},
'node2': {'out21': 'value21',
'out22': 'value22',
'out': 'value23'},
'node3': {'out31': 'value31',
'out32': 'value32',
'out': 'value33'},
'node4': {'out41': 'value41'}}
data_remapped_must = {'inp11': 'value11',
'inp12': 'value12',
'inp22': 'value22',
'out31': 'value31',
'out32': 'value32',
'out': ['value23', 'value33']}
data_remapped = nucleotide.filter_inputs(data)
self.assertDictEqual(data_remapped_must, data_remapped)
def test_construct_graph_from_nucleotides_with_cycle(self):
nucleotides = [
Nucleotide(name='first'),
Nucleotide(name='second', inbound_nodes=['first']),
Nucleotide(name='third',
inbound_nodes=['first', 'second', 'third']),
Nucleotide(name='fourth', inbound_nodes=['third', 'second']),
]
with self.assertRaises(ValueError):
_ = graph_utils.construct_graph_from_nucleotides(nucleotides)
def test_kpi_call(self, is_last_iteration):
temp_dir = self.get_temp_dir()
os.mkdir(os.path.join(temp_dir, "save"))
os.mkdir(os.path.join(temp_dir, "cache"))
cacher1 = KPIMD5Cacher().build()
kpi_plugin = DummyTpFpTnFnKPIPlugin(name="kpi_plugin1",
inbound_nodes=["dataset"],
cachers=[cacher1]).build()
saver2 = KPIJsonSaver().build()
cacher2 = KPIMD5Cacher().build()
kpi_accumulator1 = DummyF1KPIAccumulator(
name="kpi_acc1",
inbound_nodes=["kpi_plugin1", "dataset"],
cachers=[cacher2],
savers=[saver2]).build()
saver3 = KPIJsonSaver().build()
cacher3 = KPIMD5Cacher().build()
kpi_accumulator2 = _MeanKPIAccumulator(
name="kpi_acc2",
inbound_nodes=["kpi_acc1", "dataset"],
incoming_keys_mapping={
"dataset": {
"evaluate": "_"
}
},
cachers=[cacher3],
savers=[saver3]).build()
kpi_evaluator = KPIEvaluator(
plugins=kpi_plugin,
accumulators=[kpi_accumulator1, kpi_accumulator2]).build()
kpi_evaluator.save_target = os.path.join(temp_dir, "save")
kpi_evaluator.cache_target = os.path.join(temp_dir, "cache")
dataset_nucleotide = Nucleotide(name="dataset").build()
dataset_nucleotide.generated_keys = [
"labels", "predictions", "evaluate", "prefix"
]
incoming_nucleotides = {'dataset': dataset_nucleotide}
kpi_evaluator.build_dna(incoming_nucleotides)
data_batch = nest_utils.combine_nested(self.data, np.array)
kpi_evaluator.is_last_iteration = is_last_iteration
_ = kpi_evaluator(dataset=data_batch)
if is_last_iteration:
last_kpi_must = {
"kpi_acc1": self.kpis1_must[-1],
"kpi_acc2": self.kpis2_must[-1]
}
else:
last_kpi_must = {"kpi_acc1": self.kpis1_must[3]}
self.assertAllClose(last_kpi_must, kpi_evaluator.last_kpi)
def test_get_repeated_node_names(self):
unique_node_names = ['first', 'second', 'third']
repeated_node_names = ['first', 'second', 'third', 'second']
nodes_unique_names = [
Nucleotide([], name=name) for name in unique_node_names
]
nodes_repeated_names = [
Nucleotide([], name=name) for name in repeated_node_names
]
self.assertDictEqual(
graph_utils.get_repeated_node_names(nodes_unique_names), {})
self.assertDictEqual(
graph_utils.get_repeated_node_names(nodes_repeated_names),
{'second': [Nucleotide.__name__] * 2})
def test_build(self):
class GeneHandlerDummy(GeneHandler):
gene_name_and_nucleotide_super_cls = {
'gene1': Nucleotide,
'gene2': Nucleotide,
'gene3': Nucleotide,
'gene4': Nucleotide
}
genes = {
'gene1': [
Nucleotide(name='gene1_nucleotide1', inbound_nodes=["data"]),
Nucleotide(name='gene1_nucleotide2',
inbound_nodes=["data", "gene1_nucleotide1"]),
Nucleotide(name='gene1_nucleotide3',
inbound_nodes=["data2", "gene1_nucleotide1"])
],
'gene2': [
Nucleotide(name='gene2_nucleotide1', inbound_nodes=["data"]),
Nucleotide(name='gene2_nucleotide2',
inbound_nodes=[
"gene2_nucleotide1", "data3",
"gene1_nucleotide1"
])
]
}
gene3 = {
'gene3_nucleotide1':
Nucleotide(name='gene3_nucleotide1', inbound_nodes=[]),
'gene3_nucleotide2':
Nucleotide(name='gene3_nucleotide2', inbound_nodes=[])
}
gene4 = Nucleotide(name='gene4_nucleotide1', inbound_nodes=[])
gene_handler = GeneHandlerDummy(gene1=genes['gene1'],
gene2=genes['gene2'],
gene3=gene3,
gene4=gene4).build()
genes1_dict_representation = {n.name: n for n in genes['gene1']}
genes2_dict_representation = {n.name: n for n in genes['gene2']}
genes3_dict_representation = gene3
genes4_dict_representation = {"gene4_nucleotide1": gene4}
self.assertDictEqual(genes1_dict_representation, gene_handler.gene1)
self.assertDictEqual(genes2_dict_representation, gene_handler.gene2)
self.assertDictEqual(genes3_dict_representation, gene_handler.gene3)
self.assertDictEqual(genes4_dict_representation, gene_handler.gene4)
self.assertSetEqual({"data", "data2", "data3"},
set(gene_handler.inbound_nodes))
self.assertEqual(3, len(gene_handler.inbound_nodes))
gene2_repeated = Nucleotide(name='gene1_nucleotide1', inbound_nodes=[])
with self.assertRaises(AssertionError):
_ = GeneHandlerDummy(gene1=genes['gene1'],
gene2=gene2_repeated).build()
def test_filter_inputs_empty(self):
mapping = {'node1': {'out11': 'inp11',
'out12': 'inp12'},
'node2': {'out22': 'inp22'}}
nucleotide = Nucleotide(
inbound_nodes=['node1', 'node2', 'node3'],
incoming_keys_mapping=mapping)
nucleotide.incoming_keys = ['inp11', 'inp12', 'inp22', 'out31']
data = {'node1': {'out11': 'value11',
'out12': 'value12'},
'node2': {'out21': 'value21',
'out22': 'value22'},
'node3': None,
'node4': {'out41': 'value41'}}
self.assertIsNone(nucleotide.filter_inputs(data))
def _draw_key_connections_for_nucleotide_pair(
nucleotide: Nucleotide,
inbound_nucleotide: Nucleotide,
nucleotide_plots: Dict[Nucleotide, _NUCLEOTIDE_PLOT]
) -> List[patches.FancyArrowPatch]:
nucleotide_plot = nucleotide_plots[nucleotide]
incoming_nucleotide_plot = nucleotide_plots[inbound_nucleotide]
inputs_to_nucleotide = {each_inbound_node: {"": None}
for each_inbound_node in nucleotide.inbound_nodes}
inputs_to_nucleotide[inbound_nucleotide.name] = {
k: k for k in inbound_nucleotide.generated_keys_all}
inputs_to_nucleotide_filtered = nucleotide.filter_inputs(
inputs_to_nucleotide)
if "" in inputs_to_nucleotide_filtered:
del inputs_to_nucleotide_filtered[""]
edge_patches = []
for each_incoming_key, each_generated_key in (
inputs_to_nucleotide_filtered.items()):
edge_patch = _draw_edge_between_keys(
nucleotide, inbound_nucleotide,
each_incoming_key, each_generated_key,
nucleotide_plot, incoming_nucleotide_plot)
edge_patches.append(edge_patch)
return edge_patches
def test_topological_sort_of_nucleotides_multiple_inputs(self):
nucleotides = [
Nucleotide(name='input_node'),
Nucleotide(name='first', inbound_nodes=['input_node']),
Nucleotide(name='second', inbound_nodes=[]),
Nucleotide(name='fourth', inbound_nodes=['first', 'third']),
Nucleotide(name='fifth', inbound_nodes=['fourth']),
Nucleotide(name='third', inbound_nodes=[])
]
graph = graph_utils.construct_graph_from_nucleotides(nucleotides)
nodes_sorted = graph_utils.topological_sort_of_nucleotides(graph)
self.assertGreater(nodes_sorted.index(nucleotides[1]),
nodes_sorted.index(nucleotides[0]))
self.assertGreater(nodes_sorted.index(nucleotides[3]),
nodes_sorted.index(nucleotides[1]))
self.assertGreater(nodes_sorted.index(nucleotides[3]),
nodes_sorted.index(nucleotides[5]))
self.assertGreater(nodes_sorted.index(nucleotides[4]),
nodes_sorted.index(nucleotides[3]))
node_names_sorted_must = [
'input_node', 'second', 'third', 'first', 'fourth', 'fifth'
]
node_names_sorted = [
each_nucleotide.name for each_nucleotide in nodes_sorted
]
self.assertListEqual(node_names_sorted_must, node_names_sorted)
def test_filter_inputs_nested_input_mapping(self):
mapping = {'node1': {'out11:out111': 'inp11',
'out12': 'inp12'},
'node2': {'out22:2': 'inp22'}}
nucleotide = Nucleotide(
inbound_nodes=['node1', 'node2', 'node3'],
incoming_keys_mapping=mapping)
nucleotide.incoming_keys = ['inp11', 'inp12', 'inp22', 'out31']
data = {'node1': {'out11': {'out111': 'value11',
'out112': 'value112'},
'out12': 'value12'},
'node2': {'out21': 'value21',
'out22': ['value220', 'value221',
'value22', 'value223']},
'node3': {'out31': 'value31'},
'node4': {'out41': 'value41'}}
data_remapped_must = {'inp11': 'value11',
'inp12': 'value12',
'inp22': 'value22',
'out31': 'value31'}
data_remapped = nucleotide.filter_inputs(data)
self.assertDictEqual(data_remapped_must, data_remapped)
def _get_model(self,
with_metric=True,
inputs=None,
with_keras_layers=False,
regularization_l1=0,
regularization_l2=0):
plugins = self._get_model_plugins(with_keras_layers=with_keras_layers)
losses = self._get_loss()
postprocessors = self._get_postprocessors()
summary = self._get_summary()
metric = self._get_metric() if with_metric else None
model = Model(plugins=plugins,
losses=losses,
postprocessors=postprocessors,
summaries=summary,
metrics=metric,
regularization_l1=regularization_l1,
regularization_l2=regularization_l2).build()
if inputs is not None:
dataset_nucleotide = Nucleotide(name='dataset')
dataset_nucleotide.generated_keys = list(inputs.keys())
model.build_dna(dataset_nucleotide)
return model
def test_represent_predictor_through_nucleotides(self):
class PredictorMock(object):
def __init__(self_):
self_._fetch_tensors = None
@property
def fetch_tensors(self_):
return self_._fetch_tensors
@property
def feed_tensors(self_):
return self_._feed_tensors
tf.reset_default_graph()
predictor = PredictorMock()
fetch_tensors = {
'nucleotide1': {
'output1': 'value1',
'output2': 'value2'
},
'nucleotide2': {
'output3': 'value3',
'output4': 'value4'
}
}
feed_tensors = {
"data1": tf.placeholder(tf.float32),
"data2": tf.placeholder(tf.float32),
"parameter1": tf.placeholder_with_default(10, [])
}
fetch_tensors_flatten = nest_utils.flatten_nested_struct(fetch_tensors)
predictor._fetch_tensors = fetch_tensors_flatten
predictor._feed_tensors = feed_tensors
nucleotides = represent_predictor_through_nucleotides(predictor)
nucleotide1_must = Nucleotide(name='nucleotide1')
nucleotide1_must.generated_keys = ['output1', 'output2']
nucleotide1_must.incoming_keys = ['data1', 'data2']
nucleotide2_must = Nucleotide(name='nucleotide2')
nucleotide2_must.generated_keys = ['output3', 'output4']
nucleotide2_must.incoming_keys = ['data1', 'data2']
nucleotides_must = [nucleotide1_must, nucleotide2_must]
for nucleotide, nucleotide_must in zip(nucleotides, nucleotides_must):
self.assertEqual(nucleotide_must.name, nucleotide.name)
self.assertSetEqual(set(nucleotide_must.generated_keys),
set(nucleotide.generated_keys))
self.assertSetEqual(set(nucleotide_must.incoming_keys),
set(nucleotide.incoming_keys))
def _get_gene_dummy_handler():
nucleotide1 = _NucleotideWithProcess(
name='nucleotide1',
inbound_nodes=['input_node1'],
incoming_keys_mapping={'input_node1': {
'output1': 'input11'
}})
nucleotide1.incoming_keys = ['input11']
nucleotide1.generated_keys = ['output11', 'output12']
nucleotide2 = _NucleotideWithProcess(
name='nucleotide2',
inbound_nodes=['nucleotide1', 'input_node2'],
incoming_keys_mapping={
'nucleotide1': {
'output11': 'input22',
},
'input_node2': {
'output1': 'input21',
}
})
nucleotide2.incoming_keys = ['input21', '_input22']
nucleotide2.generated_keys = ['output21', 'output22']
nucleotide3 = _NucleotideWithProcess(
name='nucleotide3',
inbound_nodes=['nucleotide1', 'nucleotide2'],
incoming_keys_mapping={
'nucleotide1': {
'output11': 'input31',
},
'nucleotide2': {
'output22': 'input32',
}
})
nucleotide3.incoming_keys = ['input31', 'input32']
nucleotide3.generated_keys = ['output31']
input_node1 = Nucleotide(name='input_node1')
input_node1.generated_keys = ['output1', 'output2']
input_node2 = Nucleotide(name='input_node2')
input_node2.generated_keys = ['output1']
incoming_nucleotides = [input_node1, input_node2]
gene_inputs = {
'input_node1': {
'output1': 'value11',
'output2': 'value12'
},
'input_node2': {
'output1': 'value21'
}
}
gene = [nucleotide3, nucleotide1, nucleotide2]
gene_handler = GeneHandler(gene=gene)
gene_handler.build()
gene_handler.build_dna(incoming_nucleotides)
return (gene_handler, gene_inputs, input_node1, input_node2, nucleotide1,
nucleotide2, nucleotide3)
def test_check_class_topology(self):
class Node1(Nucleotide):
def __init__(self, **kwargs):
super(Node1, self).__init__(**kwargs)
class Node2(Nucleotide):
def __init__(self, **kwargs):
super(Node2, self).__init__(**kwargs)
class Node3(Nucleotide):
def __init__(self, **kwargs):
super(Node3, self).__init__(**kwargs)
dependency_map = {
Node1: [Node1],
Node2: [Node1, Node2],
Node3: [Node2, Node3]
}
nucleotides_right = [
Nucleotide(name='input_node', inbound_nodes=[]),
Node1(name='n1', inbound_nodes=['input_node']),
Node1(name='n2', inbound_nodes=['n1']),
Node1(name='n3', inbound_nodes=['n2']),
Node2(name='n4', inbound_nodes=['n1']),
Node2(name='n5', inbound_nodes=['n2']),
Node2(name='n6', inbound_nodes=['n4', 'n5']),
Node2(name='n7', inbound_nodes=['n6'])
]
incoming_nucleotides = [nucleotides_right[0]]
nucleotides_wrong = copy.deepcopy(nucleotides_right)
nucleotides_wrong[2] = Node2(name='n2', inbound_nodes=['n1'])
nucleotides_all = {
'right': nucleotides_right,
'wrong': nucleotides_wrong
}
for t, nodes in nucleotides_all.items():
graph = graph_utils.construct_graph_from_nucleotides(nodes)
if t == 'right':
self.assertTrue(
graph_utils.check_class_topology(graph, dependency_map,
incoming_nucleotides))
else:
with self.assertRaises(ValueError):
graph_utils.check_class_topology(graph, dependency_map,
incoming_nucleotides)
def test_build_inference_meta_graph(self):
model = ModelMock(self.num_classes).build()
model.reset_tf_graph = MagicMock(wraps=model.reset_tf_graph)
_, inputs = model.get_test_inputs(self.batch_size, self.data_dim)
model_keys = {
'predictions_raw': ['predictions_raw'],
'losses': ['loss', 'total_loss'],
'predictions': ['classes'],
'summary': ['scalar_labels', 'scalar_classes'],
'metric': ['metric']
}
input_shapes = {k: v.get_shape() for k, v in inputs.items()}
with self.assertRaises(ValueError):
# because ModelMock does not have postprocessors set,
# it should raise error
_ = model.build_inference_graph(inputs)
# lets set postprocessord to some mock value
model.postprocessors = {"postprocessor1": Nucleotide()}
_ = model.build_inference_graph(inputs)
self.assertEqual(tf.estimator.ModeKeys.PREDICT, model.mode)
model.reset_tf_graph.assert_called_once_with()
inputs_from_coll = tf_collections_utils.collection2nested(
CollectionNames.INPUTS)
inputs_connected_names_must = ['data']
self.assertSetEqual(set(inputs_from_coll),
set(inputs_connected_names_must))
predictions_from_coll = tf_collections_utils.collection2nested(
CollectionNames.PREDICTIONS)
self.assertSetEqual(set(predictions_from_coll),
set(model_keys['predictions']))
for k in inputs_connected_names_must:
shape = input_shapes[k]
shape_res = inputs_from_coll[k].shape
shape_must = shape
self.assertTrue(
np.all([
a == b
for a, b in zip(shape_res.as_list(), shape_must.as_list())
]))
def test_add(self):
nucleotide1 = Nucleotide(name="nucleotide1")
nucleotide2 = Nucleotide(name="nucleotide2",
inbound_nodes=["nucleotide1"])
nucleotide3 = Nucleotide(name="nucleotide3")
nucleotide4 = Nucleotide(name="nucleotide4",
inbound_nodes=["nucleotide2", "nucleotide3"])
nucleotide5 = Nucleotide(name="nucleotide5")
nucleotide6 = Nucleotide(name="nucleotide6",
inbound_nodes=["nucleotide4"])
nucleotide7 = Nucleotide(
name="nucleotide7",
inbound_nodes=["nucleotide5", "nucleotide3", "nucleotide6"])
incoming_nucleotides1 = [nucleotide1]
incoming_nucleotides2 = [nucleotide3, nucleotide4, nucleotide5]
nucleotides1 = [nucleotide2, nucleotide3, nucleotide4]
nucleotides2 = [nucleotide6, nucleotide7]
incoming_nucleotides12 = [nucleotide1, nucleotide5]
nucleotides12 = [
nucleotide2, nucleotide3, nucleotide4, nucleotide6, nucleotide7
]
dna_helix1 = DNAHelix(nucleotides1, incoming_nucleotides1).build()
dna_helix2 = DNAHelix(nucleotides2, incoming_nucleotides2).build()
dna_helix12 = dna_helix1 + dna_helix2
dna_helix12_must = DNAHelix(nucleotides12,
incoming_nucleotides12).build()
self.assertEqual(7, len(dna_helix12_must.get()))
self.assertEqual(5, len(dna_helix12_must.get(False)))
self.assertTrue(dna_helix12.built)
self.assertDictEqual(dna_helix12_must.as_dict(), dna_helix12.as_dict())
self.assertSetEqual(set(dna_helix12_must.incoming_nucleotides),
set(dna_helix12.incoming_nucleotides))
self.assertSetEqual(set(dna_helix12_must.nucleotides),
set(dna_helix12.nucleotides))
self.assertListEqual(dna_helix12_must.topological_order,
dna_helix12.topological_order)
def test_construct_graph_from_nucleotides(self):
nucleotides = [
Nucleotide(name="input_node1"),
Nucleotide(name="input_node2"),
Nucleotide(name='first', inbound_nodes=['input_node1']),
Nucleotide(name='second', inbound_nodes=['first', 'input_node2']),
Nucleotide(name='fourth', inbound_nodes=['first', 'third']),
Nucleotide(name='fifth', inbound_nodes=['fourth']),
Nucleotide(name='third', inbound_nodes=['first', 'second']),
Nucleotide(name='sixth')
]
graph = graph_utils.construct_graph_from_nucleotides(nucleotides)
edges_must = {(nucleotides[0], nucleotides[2]),
(nucleotides[1], nucleotides[3]),
(nucleotides[2], nucleotides[3]),
(nucleotides[2], nucleotides[4]),
(nucleotides[2], nucleotides[6]),
(nucleotides[3], nucleotides[6]),
(nucleotides[4], nucleotides[5]),
(nucleotides[6], nucleotides[4])}
self.assertSetEqual(set(nucleotides), set(graph.nodes))
self.assertSetEqual(edges_must, set(graph.edges))
def test_build_dna_and_properties(self):
class _NucleotideType1(Nucleotide):
pass
class _NucleotideType2(Nucleotide):
pass
class _NucleotideType3(Nucleotide):
pass
class _InputNucleotide(Nucleotide):
pass
class _GeneHandlerDummy(GeneHandler):
gene_name_and_nucleotide_super_cls = {
'gene1': _NucleotideType1,
'gene2': _NucleotideType2,
'gene3': _NucleotideType3
}
nucleotide_type_dependency_map = {
_NucleotideType1: [_NucleotideType1],
_NucleotideType2: [_NucleotideType1, _NucleotideType2],
_NucleotideType3: [_NucleotideType2, _NucleotideType3]
}
incoming_nucleotides = [
_InputNucleotide(name='input', inbound_nodes=[])
]
genes_right_type_dependency = {
'gene1': [
_NucleotideType1(name='gene1_nucleotide1',
inbound_nodes=['input']),
_NucleotideType1(name='gene1_nucleotide2', inbound_nodes=[]),
_NucleotideType1(name='gene1_nucleotide3', inbound_nodes=[])
],
'gene2': [
_NucleotideType2(name='gene2_nucleotide1',
inbound_nodes=[
'input', 'gene1_nucleotide1',
'gene1_nucleotide2'
]),
_NucleotideType2(
name='gene2_nucleotide2',
inbound_nodes=['gene2_nucleotide1', 'gene1_nucleotide3'])
],
'gene3': [
Nucleotide(name='gene3_nucleotide1',
inbound_nodes=['gene2_nucleotide1']),
Nucleotide(
name='gene3_nucleotide2',
inbound_nodes=['gene2_nucleotide2', 'gene3_nucleotide1'])
]
}
genes_wrong_type_dependency = {
'gene1': [
_NucleotideType1(name='gene1_nucleotide1',
inbound_nodes=['input']),
_NucleotideType1(name='gene1_nucleotide2',
inbound_nodes=['input']),
_NucleotideType1(name='gene1_nucleotide3',
inbound_nodes=['input'])
],
'gene2': [
_NucleotideType2(
name='gene2_nucleotide1',
inbound_nodes=['gene1_nucleotide1', 'gene1_nucleotide2']),
_NucleotideType2(
name='gene2_nucleotide2',
inbound_nodes=['gene2_nucleotide1', 'gene1_nucleotide3'])
],
'gene3': [
_NucleotideType3(name='gene3_nucleotide1',
inbound_nodes=['gene2_nucleotide1']),
_NucleotideType3(
name='gene3_nucleotide2',
inbound_nodes=[
'gene1_nucleotide2',
# not allowed
'gene3_nucleotide1'
])
]
}
gene_handler_wrong_dependency = _GeneHandlerDummy(
**genes_wrong_type_dependency).build()
with self.assertRaises(ValueError):
gene_handler_wrong_dependency.build_dna(incoming_nucleotides)
gene_handler = _GeneHandlerDummy(**genes_right_type_dependency).build()
gene_handler.build_dna(incoming_nucleotides)
dna_must = {
'gene1_nucleotide1':
DNAConnection(incoming={'input'}, outgoing={'gene2_nucleotide1'}),
'gene1_nucleotide2':
DNAConnection(incoming=set(), outgoing={'gene2_nucleotide1'}),
'gene1_nucleotide3':
DNAConnection(incoming=set(), outgoing={'gene2_nucleotide2'}),
'gene2_nucleotide1':
DNAConnection(
incoming={'gene1_nucleotide1', 'gene1_nucleotide2', 'input'},
outgoing={'gene2_nucleotide2', 'gene3_nucleotide1'}),
'gene2_nucleotide2':
DNAConnection(incoming={'gene1_nucleotide3', 'gene2_nucleotide1'},
outgoing={'gene3_nucleotide2'}),
'gene3_nucleotide1':
DNAConnection(incoming={'gene2_nucleotide1'},
outgoing={'gene3_nucleotide2'}),
'gene3_nucleotide2':
DNAConnection(incoming={'gene2_nucleotide2', 'gene3_nucleotide1'},
outgoing=set())
}
sorted_nucleotide_names_must = {
'gene1':
['gene1_nucleotide2', 'gene1_nucleotide3', 'gene1_nucleotide1'],
'gene2': ['gene2_nucleotide1', 'gene2_nucleotide2'],
'gene3': ['gene3_nucleotide1', 'gene3_nucleotide2']
}
self.assertDictEqual(dna_must, gene_handler.dna_helix.as_dict(False))
self.assertDictEqual(sorted_nucleotide_names_must,
gene_handler.sorted_nucleotide_names)
inbound_nodes_must = ['input']
self.assertListEqual(inbound_nodes_must, gene_handler.inbound_nodes)
inbound_nodes_all_nucleotides_must = {
'gene1_nucleotide1': ['input'],
'gene1_nucleotide2': [],
'gene1_nucleotide3': [],
'gene2_nucleotide1':
['input', 'gene1_nucleotide1', 'gene1_nucleotide2'],
'gene2_nucleotide2': ['gene2_nucleotide1', 'gene1_nucleotide3'],
'gene3_nucleotide1': ['gene2_nucleotide1'],
'gene3_nucleotide2': ['gene2_nucleotide2', 'gene3_nucleotide1']
}
inbound_nodes_all_nucleotides = (
gene_handler.get_property_from_all_genes('inbound_nodes'))
self.assertDictEqual(inbound_nodes_all_nucleotides_must,
inbound_nodes_all_nucleotides)
def test_check_node2node_connection(self):
dataset_node = Nucleotide(name='data', inbound_nodes=[])
dataset_node.generated_keys = ['image', 'labels', 'temp']
node_cnn1 = Nucleotide(name='cnn',
inbound_nodes=['data'],
incoming_keys_mapping={
'data': {
'image': 'inputs_cnn',
'temp': 'inputs_optional'
}
})
node_cnn1.incoming_keys = [
'inputs_cnn', '_inputs_optional', '_inputs_optional_2'
]
node_cnn1.generated_keys = ['predictions']
node_loss = Nucleotide(
name='loss',
inbound_nodes=['data', 'cnn'],
incoming_keys_mapping={'cnn': {
'predictions:0': 'logits:0'
}})
node_loss.incoming_keys = ['logits', 'labels']
node_loss_wrong = Nucleotide(
name='loss',
inbound_nodes=['data', 'cnn'],
incoming_keys_mapping={'cnn': {
'predictions': 'logits2'
}})
node_loss_wrong.incoming_keys = ['logits', 'labels']
nodes_all = {
'right': [dataset_node, node_cnn1, node_loss],
'wrong': [dataset_node, node_cnn1, node_loss_wrong]
}
for t, nodes in nodes_all.items():
nodes = {n.name: n for n in nodes}
if t == 'right':
self.assertTrue(
graph_utils.check_node2node_connection(
nodes['loss'], [nodes[k] for k in ['data', 'cnn']]))
else:
with self.assertRaises(ValueError):
graph_utils.check_node2node_connection(
nodes['loss'], [nodes[k] for k in ['data', 'cnn']])
def test_check_graph_connections(self):
dataset_node = Nucleotide(name='data', inbound_nodes=[])
dataset_node.generated_keys = ['image', 'labels', 'temp']
node_cnn = Nucleotide(
name='cnn',
inbound_nodes=['data'],
incoming_keys_mapping={'data': {
'image': 'inputs_cnn'
}})
node_cnn.incoming_keys = ['inputs_cnn']
node_cnn.generated_keys = ['predictions']
node_flatten = Nucleotide(
name='flatten',
inbound_nodes=['cnn'],
incoming_keys_mapping={'cnn': {
'predictions': 'inputs_flatten'
}})
node_flatten.incoming_keys = ['inputs_flatten']
node_flatten.generated_keys = ['predictions']
node_mlp = Nucleotide(
name='mlp',
inbound_nodes=['flatten'],
incoming_keys_mapping={'flatten': {
'predictions': 'inputs_mlp'
}})
node_mlp.incoming_keys = ['inputs_mlp']
node_mlp.generated_keys = ['predictions']
node_mlp_wrong = Nucleotide(
name='mlp',
inbound_nodes=['flatten'],
incoming_keys_mapping={'flatten': {
'predictions': 'inputs'
}})
node_mlp_wrong.incoming_keys = ['inputs_mlp']
node_mlp_wrong.generated_keys = ['predictions']
node_loss = Nucleotide(
name='loss',
inbound_nodes=['data', 'mlp'],
incoming_keys_mapping={'mlp': {
'predictions': 'logits'
}})
node_loss.incoming_keys = ['labels', 'logits']
node_loss_wrong = Nucleotide(name='loss',
inbound_nodes=['data', 'mlp'])
node_loss_wrong.incoming_keys = ['labels', 'logits']
node_pp = Nucleotide(
name='pp',
inbound_nodes=['mlp'],
incoming_keys_mapping={'mlp': {
'predictions': 'inputs_pp'
}})
node_pp.incoming_keys = ['inputs_pp']
nodes_all = {
'right': [[
dataset_node, node_cnn, node_flatten, node_mlp, node_loss,
node_pp
]],
'wrong': [[
dataset_node, node_cnn, node_flatten, node_mlp_wrong,
node_loss, node_pp
],
[
dataset_node, node_cnn, node_flatten, node_mlp_wrong,
node_loss_wrong, node_pp
]]
}
for test_mode, nodes_case in nodes_all.items():
for nodes in nodes_case:
graph = graph_utils.construct_graph_from_nucleotides(nodes)
if test_mode == 'right':
self.assertTrue(graph_utils.check_graph_connections(graph))
else:
with self.assertRaises(ValueError):
graph_utils.check_graph_connections(graph)
def _get_callbacks_and_incoming_nucleotides():
def side_effect_callback_on_iteration_end(nucleotide: Nucleotide,
**inputs):
return {
k: '_'.join([nucleotide.name, k])
for k in nucleotide.generated_keys_all
}
input_node1 = Nucleotide(name='input_node1')
input_node1.generated_keys = ['output1', 'output2']
input_node2 = Nucleotide(name='input_node2')
input_node2.generated_keys = ['output1']
input_node1.process = MagicMock(return_value=None)
input_node2.process = MagicMock(return_value=None)
callback1 = CoordinatorCallback(
name='callback1',
inbound_nodes=['input_node1'],
incoming_keys_mapping={'input_node1': {
'output1': 'input11'
}})
callback1.incoming_keys = ['input11']
callback1.generated_keys = ['output11', 'output12']
callback2 = CoordinatorCallback(
name='callback2',
inbound_nodes=['callback1', 'input_node2'],
incoming_keys_mapping={
'callback1': {
'output11': 'input22',
},
'input_node2': {
'output1': 'input21',
}
})
callback2.incoming_keys = ['input21', '_input22']
callback2.generated_keys = ['output21', 'output22']
callback3 = CoordinatorCallback(
name='callback3',
inbound_nodes=['callback1', 'callback2'],
incoming_keys_mapping={
'callback1': {
'output11': 'input31',
},
'callback2': {
'output22': 'input32',
}
})
callback3.incoming_keys = ['input31', 'input32']
callback3.generated_keys = ['output31']
callback1.on_iteration_end = MagicMock(side_effect=partial(
side_effect_callback_on_iteration_end, nucleotide=callback1))
callback2.on_iteration_end = MagicMock(side_effect=partial(
side_effect_callback_on_iteration_end, nucleotide=callback2))
callback3.on_iteration_end = MagicMock(side_effect=partial(
side_effect_callback_on_iteration_end, nucleotide=callback3))
callbacks = [callback1.build(), callback2.build(), callback3.build()]
incoming_nucleotides = {
'input_node1': input_node1.build(),
'input_node2': input_node2.build()
}
return callbacks, incoming_nucleotides
def _get_dna():
nucleotide1 = Nucleotide(name="nucleotide1")
nucleotide1.generated_keys = ["y"]
nucleotide2 = Nucleotide(name="nucleotide2", inbound_nodes=["nucleotide1"])
nucleotide2.incoming_keys = ["y"]
nucleotide2.generated_keys = ["z"]
nucleotide3 = Nucleotide(name="nucleotide3",
inbound_nodes=["nucleotide1", "nucleotide2"])
nucleotide3.dynamic_incoming_keys = True
nucleotide3.generated_keys = ["w"]
nucleotide3.dynamic_generated_keys = True
nucleotide4 = Nucleotide(name="nucleotide4", inbound_nodes=["nucleotide3"])
nucleotide4.incoming_keys = ["y", "z", "w"]
nucleotides_list = [nucleotide2, nucleotide3, nucleotide4]
incoming_nucleotides_list = [nucleotide1]
dna_must = {
'nucleotide2':
DNAConnection(incoming={'nucleotide1'}, outgoing={'nucleotide3'}),
'nucleotide3':
DNAConnection(incoming={'nucleotide1', 'nucleotide2'},
outgoing={'nucleotide4'}),
'nucleotide4':
DNAConnection(incoming={'nucleotide3'}, outgoing=set()),
}
topological_sort_must = ["nucleotide2", "nucleotide3", "nucleotide4"]
raise_build_error = False
yield (nucleotides_list, incoming_nucleotides_list, dna_must,
topological_sort_must, raise_build_error)
nucleotide3 = Nucleotide(name="nucleotide3",
inbound_nodes=["nucleotide1", "nucleotide2"])
nucleotide3.incoming_keys = []
nucleotide3.generated_keys = ["w"]
nucleotides_list = [nucleotide2, nucleotide3, nucleotide4]
raise_build_error = True
yield (nucleotides_list, incoming_nucleotides_list, dna_must,
topological_sort_must, raise_build_error)
nucleotides_list = [
Nucleotide(name='first', inbound_nodes=['input_node1']),
Nucleotide(name='second', inbound_nodes=['first', 'input_node2']),
Nucleotide(name='fourth', inbound_nodes=['first', 'third']),
Nucleotide(name='fifth', inbound_nodes=['fourth']),
Nucleotide(name='third', inbound_nodes=['first', 'second'])
]
incoming_nucleotides_list = [
Nucleotide(name='input_node1'),
Nucleotide(name='input_node2')
]
dna_must = {
'first':
DNAConnection(incoming={'input_node1'},
outgoing={'second', 'fourth', 'third'}),
'second':
DNAConnection(incoming={'first', 'input_node2'}, outgoing={'third'}),
'third':
DNAConnection(incoming={'first', 'second'}, outgoing={'fourth'}),
'fourth':
DNAConnection(incoming={'first', 'third'}, outgoing={'fifth'}),
'fifth':
DNAConnection(incoming={'fourth'}, outgoing=set())
}
topological_sort_must = ["first", "second", "third", "fourth", "fifth"]
raise_build_error = False
yield (nucleotides_list, incoming_nucleotides_list, dna_must,
topological_sort_must, raise_build_error)
nucleotides_list = [
Nucleotide(name='first', inbound_nodes=['input_node1']),
Nucleotide(name='second', inbound_nodes=[]),
Nucleotide(name='fourth', inbound_nodes=['first', 'third']),
Nucleotide(name='fifth', inbound_nodes=['fourth']),
Nucleotide(name='third', inbound_nodes=['first', 'second'])
]
incoming_nucleotides_list = [Nucleotide(name='input_node1')]
dna_must = {
'first':
DNAConnection(incoming={'input_node1'}, outgoing={'fourth', 'third'}),
'second':
DNAConnection(incoming=set(), outgoing={'third'}),
'third':
DNAConnection(incoming={'first', 'second'}, outgoing={'fourth'}),
'fourth':
DNAConnection(incoming={'first', 'third'}, outgoing={'fifth'}),
'fifth':
DNAConnection(incoming={'fourth'}, outgoing=set())
}
topological_sort_must = ["second", "first", "third", "fourth", "fifth"]
raise_build_error = False
yield (nucleotides_list, incoming_nucleotides_list, dna_must,
topological_sort_must, raise_build_error)
nucleotides_list = [
Nucleotide(name='first', inbound_nodes=[]),
Nucleotide(name='second', inbound_nodes=[]),
Nucleotide(name='fourth', inbound_nodes=['first', 'third']),
Nucleotide(name='fifth', inbound_nodes=['fourth']),
Nucleotide(name='third', inbound_nodes=['first', 'second'])
]
incoming_nucleotides_list = []
dna_must = {
'first': DNAConnection(incoming=set(), outgoing={'fourth', 'third'}),
'second': DNAConnection(incoming=set(), outgoing={'third'}),
'third': DNAConnection(incoming={'first', 'second'},
outgoing={'fourth'}),
'fourth': DNAConnection(incoming={'first', 'third'},
outgoing={'fifth'}),
'fifth': DNAConnection(incoming={'fourth'}, outgoing=set())
}
topological_sort_must = ["first", "second", "third", "fourth", "fifth"]
raise_build_error = False
yield (nucleotides_list, incoming_nucleotides_list, dna_must,
topological_sort_must, raise_build_error)
def test_get_nucleotide_signature(self):
node = Nucleotide(
name='cnn',
inbound_nodes=['data'],
incoming_keys_mapping={'data': {
'image': 'inputs_cnn'
}})
node.incoming_keys = ['inputs1', 'inputs2', '_input_optional']
node.generated_keys = ['predictions', '_predictions_optional']
node.__doc__ = """
Attributes
----------
incoming_keys : list
* inputs1 : inputs1 to node
* inputs2 : inputs2 to node
* inputs_wrong : wrong description
* input_optional : optional inputs
generated_keys : list
* predictions : predictions 1
* predictions_optional : optional predictions
"""
args_must = {
'inputs1': 'inputs1 to node',
'inputs2': 'inputs2 to node',
'input_optional': 'optional inputs'
}
returns_must = {
'predictions': 'predictions 1',
'predictions_optional': 'optional predictions'
}
args, returns = nucleotide_utils.get_nucleotide_signature(node)
self.assertSetEqual({'inputs1', 'inputs2', 'input_optional'},
set(args))
self.assertSetEqual({'predictions', 'predictions_optional'},
set(returns))
self.assertDictEqual(args_must, args)
self.assertDictEqual(returns_must, returns)
node.__doc__ = None
node.process = MagicMock(return_value=0)
node.process.__doc__ = """
Parameters
----------
inputs1
inputs1 to node
inputs2
inputs2 to node
inputs_wrong
wrong description
input_optional
optional inputs
Returns
-------
predictions
predictions 1
predictions_optional
optional predictions
"""
args, returns = nucleotide_utils.get_nucleotide_signature(node)
self.assertDictEqual(args_must, args)
self.assertDictEqual(returns_must, returns)