def test_get_marker_tree_leaves(self):
"""
Make sure all the leaves of a marker are returned
as expected from marker._list_of_leaf_markers()
"""
from furious.marker_tree.identity_utils import leaf_persistence_id_from_group_id
from furious.marker_tree.marker import Marker
root_marker = Marker(id="polly")
for x in xrange(3):
root_marker.children.append(Marker(id=leaf_persistence_id_from_group_id(root_marker.id, x)))
originally_a_leaf_marker = root_marker.children[0]
sub_tree_marker = Marker(
id=originally_a_leaf_marker.id,
children=[Marker(id=leaf_persistence_id_from_group_id(originally_a_leaf_marker.id, i)) for i in xrange(3)],
)
root_marker.children[0] = sub_tree_marker
root_marker.persist()
leaves = root_marker._list_of_leaf_markers()
self.assertEqual(len(leaves), 5)
reloaded_root_marker = Marker.get(root_marker.id)
self.assertIsNotNone(reloaded_root_marker)
leaves = reloaded_root_marker._list_of_leaf_markers()
# no jobs run and updated, so there should be no
# leaves persisted yet
self.assertEqual(len(leaves), 0)
def test_persist_marker_tree_graph(self):
"""
Make sure when a marker tree is persisted,
it only saves the non-leaf nodes and the
children properties contains only IDs
"""
from furious.marker_tree.identity_utils import leaf_persistence_id_from_group_id
from furious.marker_tree.marker import Marker
root_marker = Marker(id="peanut")
for x in xrange(3):
root_marker.children.append(Marker(id=leaf_persistence_id_from_group_id(root_marker.id, x)))
root_marker.persist()
loaded_marker = Marker.get(root_marker.id)
self.assertIsNotNone(loaded_marker)
self.assertEqual(root_marker.id, loaded_marker.id)
self.assertEqual(root_marker.done, loaded_marker.done)
self.assertEqual(len(root_marker.children), len(loaded_marker.children))
children_ids = [child.id for child in root_marker.children]
loaded_children_ids = loaded_marker.children
for index, idx in enumerate(children_ids):
self.assertEqual(idx, loaded_children_ids[index])
loaded_child_marker = Marker.get(root_marker.children[0].id)
self.assertIsNone(loaded_child_marker)
def test_persist_internal_node_marker(self):
"""
Make sure internal nodes are saved during the persistence
of a marker tree graph.
"""
from furious.marker_tree.identity_utils import leaf_persistence_id_from_group_id
from furious.marker_tree.marker import Marker
root_marker = Marker(id="cracker")
for x in xrange(2):
root_marker.children.append(
Marker(
id=str(x),
group_id=root_marker.id,
children=[Marker(id=leaf_persistence_id_from_group_id(str(x), i)) for i in xrange(3)],
)
)
root_marker.persist()
internal_node1 = root_marker.children[0]
leaf_node2 = internal_node1.children[1]
loaded_internal = Marker.get(internal_node1.id)
self.assertIsNotNone(loaded_internal)
loaded_leaf = Marker.get(leaf_node2.id)
self.assertIsNone(loaded_leaf)
def test_combiner_results(self):
"""
Make sure the expected results are in the root_marker.results
after the job is done.
"""
from furious.marker_tree.identity_utils import leaf_persistence_id_from_group_id
from furious.marker_tree.marker import Marker
from furious.tests.marker_tree import dummy_success_callback
from furious.tests.marker_tree import dummy_internal_vertex_combiner
from furious.tests.marker_tree import dummy_leaf_combiner
context_callbacks = {
'success': dummy_success_callback,
'internal_vertex_combiner': dummy_internal_vertex_combiner,
'leaf_combiner': dummy_leaf_combiner
}
root_marker = Marker(id="big_job",
callbacks=context_callbacks)
for x in xrange(2):
root_marker.children.append(Marker(
id=str(x),
group_id=root_marker.id,
callbacks=context_callbacks,
children=[
Marker(id=leaf_persistence_id_from_group_id(
str(x), i)) for i in xrange(3)]
))
originally_a_leaf_marker = root_marker.children[1].children[1]
sub_tree_marker = Marker(id=originally_a_leaf_marker.id,
children=[Marker(id=
leaf_persistence_id_from_group_id(originally_a_leaf_marker.id, i))
for i in xrange(3)],
callbacks=context_callbacks)
root_marker.children[1].children[1] = sub_tree_marker
#persist marker tree
root_marker.persist()
#similate running all the jobs
leaf_markers = root_marker._list_of_leaf_markers()
for marker in leaf_markers:
marker.done = True
marker.result = 1
marker.update_done(persist_first=True)
loaded_root_marker = Marker.get("big_job")
self.assertTrue(loaded_root_marker.done)
self.assertEqual(loaded_root_marker.result,
[[[1, 1, 1]], [[1], [[1, 1, 1]], [1]]])
def test_update_done_of_leaf_travels_to_root_when_last(self, mock_count_update, mock_count_marked_as_done):
"""
Make sure when all but one marker is done and it
runs an update_done, the processes will bubble
up to the root marker and it's update done will be called.
How many times is Marker.update_done called?
2
\
------
\ \
3 3
---- ----
\ \ \ \ \ \
1 1 1 1 1 1
Each leaf node calls itself once.
Each internal vertex is called during bubble_up_done
of each child.
The first time the root marker.update_done is run, the
right child node is not done
"""
mock_count_update.return_value = None
from furious.marker_tree.identity_utils import leaf_persistence_id_from_group_id
from furious.marker_tree.marker import Marker
from furious.tests.marker_tree import dummy_success_callback
from furious.tests.marker_tree import dummy_internal_vertex_combiner
from furious.tests.marker_tree import dummy_leaf_combiner
context_callbacks = {
"success": dummy_success_callback,
"internal_vertex_combiner": dummy_internal_vertex_combiner,
"leaf_combiner": dummy_leaf_combiner,
}
root_marker = Marker(id="delve", callbacks=context_callbacks)
for x in xrange(2):
root_marker.children.append(
Marker(
id=str(x),
group_id=root_marker.id,
callbacks=context_callbacks,
children=[Marker(id=leaf_persistence_id_from_group_id(str(x), i)) for i in xrange(3)],
)
)
root_marker.persist()
with patch("furious.tests.marker_tree." "dummy_success_callback", auto_spec=True) as mock_success_callback:
with patch("furious.tests.marker_tree." "dummy_leaf_combiner", auto_spec=True) as mock_leaf_combiner:
with patch(
"furious.tests.marker_tree." "dummy_internal_vertex_combiner", auto_spec=True
) as mock_internal_vertex_combiner:
mock_leaf_combiner.return_value = ["1"]
mock_internal_vertex_combiner.return_value = ["2"]
for internal_node in root_marker.children:
for leaf_node in internal_node.children:
leaf_node.done = True
leaf_node.result = 1
leaf_node.update_done(persist_first=True)
loaded_root_marker = Marker.get("delve")
self.assertTrue(loaded_root_marker.done)
self.assertEqual(mock_count_update.call_count, 14)
# 9 is the number of nodes in the graph
self.assertEqual(mock_count_marked_as_done.call_count, 9)
# pretend a task was run again later on after
# the root had succeeded, it should only
# reach it's parent node and that should
# not bubble up
leaf_node = root_marker.children[0].children[1]
leaf_node.update_done(persist_first=True)
self.assertEqual(mock_count_update.call_count, 16)
mock_success_callback.assert_called_once_with("delve", ["2"])
# one for each non-leaf node
self.assertEqual(mock_internal_vertex_combiner.call_count, 3)
self.assertEqual(mock_leaf_combiner.call_count, 2)