def test_persist_internal_node_marker(self):
"""
Make sure internal nodes are saved during the persistence
of a marker tree graph.
"""
from furious.marker_tree.identity_utils import leaf_persistence_id_from_group_id
from furious.marker_tree.marker import Marker
root_marker = Marker(id="cracker")
for x in xrange(2):
root_marker.children.append(
Marker(
id=str(x),
group_id=root_marker.id,
children=[Marker(id=leaf_persistence_id_from_group_id(str(x), i)) for i in xrange(3)],
)
)
root_marker.persist()
internal_node1 = root_marker.children[0]
leaf_node2 = internal_node1.children[1]
loaded_internal = Marker.get(internal_node1.id)
self.assertIsNotNone(loaded_internal)
loaded_leaf = Marker.get(leaf_node2.id)
self.assertIsNone(loaded_leaf)
def test_persist_marker_tree_graph(self):
"""
Make sure when a marker tree is persisted,
it only saves the non-leaf nodes and the
children properties contains only IDs
"""
from furious.marker_tree.identity_utils import leaf_persistence_id_from_group_id
from furious.marker_tree.marker import Marker
root_marker = Marker(id="peanut")
for x in xrange(3):
root_marker.children.append(Marker(id=leaf_persistence_id_from_group_id(root_marker.id, x)))
root_marker.persist()
loaded_marker = Marker.get(root_marker.id)
self.assertIsNotNone(loaded_marker)
self.assertEqual(root_marker.id, loaded_marker.id)
self.assertEqual(root_marker.done, loaded_marker.done)
self.assertEqual(len(root_marker.children), len(loaded_marker.children))
children_ids = [child.id for child in root_marker.children]
loaded_children_ids = loaded_marker.children
for index, idx in enumerate(children_ids):
self.assertEqual(idx, loaded_children_ids[index])
loaded_child_marker = Marker.get(root_marker.children[0].id)
self.assertIsNone(loaded_child_marker)
def test_get_marker_tree_leaves(self):
"""
Make sure all the leaves of a marker are returned
as expected from marker._list_of_leaf_markers()
"""
from furious.marker_tree.identity_utils import leaf_persistence_id_from_group_id
from furious.marker_tree.marker import Marker
root_marker = Marker(id="polly")
for x in xrange(3):
root_marker.children.append(Marker(id=leaf_persistence_id_from_group_id(root_marker.id, x)))
originally_a_leaf_marker = root_marker.children[0]
sub_tree_marker = Marker(
id=originally_a_leaf_marker.id,
children=[Marker(id=leaf_persistence_id_from_group_id(originally_a_leaf_marker.id, i)) for i in xrange(3)],
)
root_marker.children[0] = sub_tree_marker
root_marker.persist()
leaves = root_marker._list_of_leaf_markers()
self.assertEqual(len(leaves), 5)
reloaded_root_marker = Marker.get(root_marker.id)
self.assertIsNotNone(reloaded_root_marker)
leaves = reloaded_root_marker._list_of_leaf_markers()
# no jobs run and updated, so there should be no
# leaves persisted yet
self.assertEqual(len(leaves), 0)
def process(self, idx):
"""
returns the result of the job
"""
root_marker = Marker.get(idx)
result = root_marker.result_to_dict()
self.response.content_type = "text/json"
self.response.write(json.dumps(result))
def test_combiner_results(self):
"""
Make sure the expected results are in the root_marker.results
after the job is done.
"""
from furious.marker_tree.identity_utils import leaf_persistence_id_from_group_id
from furious.marker_tree.marker import Marker
from furious.tests.marker_tree import dummy_success_callback
from furious.tests.marker_tree import dummy_internal_vertex_combiner
from furious.tests.marker_tree import dummy_leaf_combiner
context_callbacks = {
'success': dummy_success_callback,
'internal_vertex_combiner': dummy_internal_vertex_combiner,
'leaf_combiner': dummy_leaf_combiner
}
root_marker = Marker(id="big_job",
callbacks=context_callbacks)
for x in xrange(2):
root_marker.children.append(Marker(
id=str(x),
group_id=root_marker.id,
callbacks=context_callbacks,
children=[
Marker(id=leaf_persistence_id_from_group_id(
str(x), i)) for i in xrange(3)]
))
originally_a_leaf_marker = root_marker.children[1].children[1]
sub_tree_marker = Marker(id=originally_a_leaf_marker.id,
children=[Marker(id=
leaf_persistence_id_from_group_id(originally_a_leaf_marker.id, i))
for i in xrange(3)],
callbacks=context_callbacks)
root_marker.children[1].children[1] = sub_tree_marker
#persist marker tree
root_marker.persist()
#similate running all the jobs
leaf_markers = root_marker._list_of_leaf_markers()
for marker in leaf_markers:
marker.done = True
marker.result = 1
marker.update_done(persist_first=True)
loaded_root_marker = Marker.get("big_job")
self.assertTrue(loaded_root_marker.done)
self.assertEqual(loaded_root_marker.result,
[[[1, 1, 1]], [[1], [[1, 1, 1]], [1]]])
def handle_async_done(async):
"""
Args:
an Async instance
This will mark and async as done and will
begin the process to see if all the other asyncs
in it's context, if it has one, are done
"""
if async.id:
marker = Marker.get(async.id)
if not marker:
marker = Marker.from_async(async)
marker.done = True
marker.result = async.result
marker.update_done(persist_first=True)
def get_status(idx):
"""Helper function that gets the status of a job
Args:
idx: the id of a Context job
Returns:
a dictionary indicating the status of the job
"""
status = {
'progress': 0,
'aborted': False,
'complete': False,
'warnings': False,
'errors': False
}
root_marker = Marker.get(idx)
status.update({'complete': root_marker.done})
return status
def test_persist_leaf_marker(self):
"""
Make sure a leaf marker is saved when it persists
itself but only during the update_done process.
Make sure that loaded leaf marker can id it's parent
marker
"""
from furious.marker_tree.identity_utils import leaf_persistence_id_from_group_id
from furious.marker_tree.marker import Marker
from furious.marker_tree.exceptions import NotSafeToSave
root_marker = Marker(id="heart")
for x in xrange(3):
root_marker.children.append(Marker(id=leaf_persistence_id_from_group_id(root_marker.id, x)))
leaf_marker = root_marker.children[0]
self.assertRaises(NotSafeToSave, leaf_marker.persist)
leaf_marker._update_done_in_progress = True
leaf_marker.persist()
loaded_marker = Marker.get(leaf_marker.id)
self.assertIsNotNone(loaded_marker)
self.assertIsInstance(loaded_marker, Marker)
self.assertTrue(loaded_marker.get_group_id(), "heart")
def test_update_done_of_leaf_travels_to_root_when_last(self, mock_count_update, mock_count_marked_as_done):
"""
Make sure when all but one marker is done and it
runs an update_done, the processes will bubble
up to the root marker and it's update done will be called.
How many times is Marker.update_done called?
2
\
------
\ \
3 3
---- ----
\ \ \ \ \ \
1 1 1 1 1 1
Each leaf node calls itself once.
Each internal vertex is called during bubble_up_done
of each child.
The first time the root marker.update_done is run, the
right child node is not done
"""
mock_count_update.return_value = None
from furious.marker_tree.identity_utils import leaf_persistence_id_from_group_id
from furious.marker_tree.marker import Marker
from furious.tests.marker_tree import dummy_success_callback
from furious.tests.marker_tree import dummy_internal_vertex_combiner
from furious.tests.marker_tree import dummy_leaf_combiner
context_callbacks = {
"success": dummy_success_callback,
"internal_vertex_combiner": dummy_internal_vertex_combiner,
"leaf_combiner": dummy_leaf_combiner,
}
root_marker = Marker(id="delve", callbacks=context_callbacks)
for x in xrange(2):
root_marker.children.append(
Marker(
id=str(x),
group_id=root_marker.id,
callbacks=context_callbacks,
children=[Marker(id=leaf_persistence_id_from_group_id(str(x), i)) for i in xrange(3)],
)
)
root_marker.persist()
with patch("furious.tests.marker_tree." "dummy_success_callback", auto_spec=True) as mock_success_callback:
with patch("furious.tests.marker_tree." "dummy_leaf_combiner", auto_spec=True) as mock_leaf_combiner:
with patch(
"furious.tests.marker_tree." "dummy_internal_vertex_combiner", auto_spec=True
) as mock_internal_vertex_combiner:
mock_leaf_combiner.return_value = ["1"]
mock_internal_vertex_combiner.return_value = ["2"]
for internal_node in root_marker.children:
for leaf_node in internal_node.children:
leaf_node.done = True
leaf_node.result = 1
leaf_node.update_done(persist_first=True)
loaded_root_marker = Marker.get("delve")
self.assertTrue(loaded_root_marker.done)
self.assertEqual(mock_count_update.call_count, 14)
# 9 is the number of nodes in the graph
self.assertEqual(mock_count_marked_as_done.call_count, 9)
# pretend a task was run again later on after
# the root had succeeded, it should only
# reach it's parent node and that should
# not bubble up
leaf_node = root_marker.children[0].children[1]
leaf_node.update_done(persist_first=True)
self.assertEqual(mock_count_update.call_count, 16)
mock_success_callback.assert_called_once_with("delve", ["2"])
# one for each non-leaf node
self.assertEqual(mock_internal_vertex_combiner.call_count, 3)
self.assertEqual(mock_leaf_combiner.call_count, 2)