def test_persist_internal_node_marker(self): """ Make sure internal nodes are saved during the persistence of a marker tree graph. """ from furious.marker_tree.identity_utils import leaf_persistence_id_from_group_id from furious.marker_tree.marker import Marker root_marker = Marker(id="cracker") for x in xrange(2): root_marker.children.append( Marker( id=str(x), group_id=root_marker.id, children=[Marker(id=leaf_persistence_id_from_group_id(str(x), i)) for i in xrange(3)], ) ) root_marker.persist() internal_node1 = root_marker.children[0] leaf_node2 = internal_node1.children[1] loaded_internal = Marker.get(internal_node1.id) self.assertIsNotNone(loaded_internal) loaded_leaf = Marker.get(leaf_node2.id) self.assertIsNone(loaded_leaf)
def test_persist_marker_tree_graph(self): """ Make sure when a marker tree is persisted, it only saves the non-leaf nodes and the children properties contains only IDs """ from furious.marker_tree.identity_utils import leaf_persistence_id_from_group_id from furious.marker_tree.marker import Marker root_marker = Marker(id="peanut") for x in xrange(3): root_marker.children.append(Marker(id=leaf_persistence_id_from_group_id(root_marker.id, x))) root_marker.persist() loaded_marker = Marker.get(root_marker.id) self.assertIsNotNone(loaded_marker) self.assertEqual(root_marker.id, loaded_marker.id) self.assertEqual(root_marker.done, loaded_marker.done) self.assertEqual(len(root_marker.children), len(loaded_marker.children)) children_ids = [child.id for child in root_marker.children] loaded_children_ids = loaded_marker.children for index, idx in enumerate(children_ids): self.assertEqual(idx, loaded_children_ids[index]) loaded_child_marker = Marker.get(root_marker.children[0].id) self.assertIsNone(loaded_child_marker)
def test_get_marker_tree_leaves(self): """ Make sure all the leaves of a marker are returned as expected from marker._list_of_leaf_markers() """ from furious.marker_tree.identity_utils import leaf_persistence_id_from_group_id from furious.marker_tree.marker import Marker root_marker = Marker(id="polly") for x in xrange(3): root_marker.children.append(Marker(id=leaf_persistence_id_from_group_id(root_marker.id, x))) originally_a_leaf_marker = root_marker.children[0] sub_tree_marker = Marker( id=originally_a_leaf_marker.id, children=[Marker(id=leaf_persistence_id_from_group_id(originally_a_leaf_marker.id, i)) for i in xrange(3)], ) root_marker.children[0] = sub_tree_marker root_marker.persist() leaves = root_marker._list_of_leaf_markers() self.assertEqual(len(leaves), 5) reloaded_root_marker = Marker.get(root_marker.id) self.assertIsNotNone(reloaded_root_marker) leaves = reloaded_root_marker._list_of_leaf_markers() # no jobs run and updated, so there should be no # leaves persisted yet self.assertEqual(len(leaves), 0)
def process(self, idx): """ returns the result of the job """ root_marker = Marker.get(idx) result = root_marker.result_to_dict() self.response.content_type = "text/json" self.response.write(json.dumps(result))
def test_combiner_results(self): """ Make sure the expected results are in the root_marker.results after the job is done. """ from furious.marker_tree.identity_utils import leaf_persistence_id_from_group_id from furious.marker_tree.marker import Marker from furious.tests.marker_tree import dummy_success_callback from furious.tests.marker_tree import dummy_internal_vertex_combiner from furious.tests.marker_tree import dummy_leaf_combiner context_callbacks = { 'success': dummy_success_callback, 'internal_vertex_combiner': dummy_internal_vertex_combiner, 'leaf_combiner': dummy_leaf_combiner } root_marker = Marker(id="big_job", callbacks=context_callbacks) for x in xrange(2): root_marker.children.append(Marker( id=str(x), group_id=root_marker.id, callbacks=context_callbacks, children=[ Marker(id=leaf_persistence_id_from_group_id( str(x), i)) for i in xrange(3)] )) originally_a_leaf_marker = root_marker.children[1].children[1] sub_tree_marker = Marker(id=originally_a_leaf_marker.id, children=[Marker(id= leaf_persistence_id_from_group_id(originally_a_leaf_marker.id, i)) for i in xrange(3)], callbacks=context_callbacks) root_marker.children[1].children[1] = sub_tree_marker #persist marker tree root_marker.persist() #similate running all the jobs leaf_markers = root_marker._list_of_leaf_markers() for marker in leaf_markers: marker.done = True marker.result = 1 marker.update_done(persist_first=True) loaded_root_marker = Marker.get("big_job") self.assertTrue(loaded_root_marker.done) self.assertEqual(loaded_root_marker.result, [[[1, 1, 1]], [[1], [[1, 1, 1]], [1]]])
def handle_async_done(async): """ Args: an Async instance This will mark and async as done and will begin the process to see if all the other asyncs in it's context, if it has one, are done """ if async.id: marker = Marker.get(async.id) if not marker: marker = Marker.from_async(async) marker.done = True marker.result = async.result marker.update_done(persist_first=True)
def get_status(idx): """Helper function that gets the status of a job Args: idx: the id of a Context job Returns: a dictionary indicating the status of the job """ status = { 'progress': 0, 'aborted': False, 'complete': False, 'warnings': False, 'errors': False } root_marker = Marker.get(idx) status.update({'complete': root_marker.done}) return status
def test_persist_leaf_marker(self): """ Make sure a leaf marker is saved when it persists itself but only during the update_done process. Make sure that loaded leaf marker can id it's parent marker """ from furious.marker_tree.identity_utils import leaf_persistence_id_from_group_id from furious.marker_tree.marker import Marker from furious.marker_tree.exceptions import NotSafeToSave root_marker = Marker(id="heart") for x in xrange(3): root_marker.children.append(Marker(id=leaf_persistence_id_from_group_id(root_marker.id, x))) leaf_marker = root_marker.children[0] self.assertRaises(NotSafeToSave, leaf_marker.persist) leaf_marker._update_done_in_progress = True leaf_marker.persist() loaded_marker = Marker.get(leaf_marker.id) self.assertIsNotNone(loaded_marker) self.assertIsInstance(loaded_marker, Marker) self.assertTrue(loaded_marker.get_group_id(), "heart")
def test_update_done_of_leaf_travels_to_root_when_last(self, mock_count_update, mock_count_marked_as_done): """ Make sure when all but one marker is done and it runs an update_done, the processes will bubble up to the root marker and it's update done will be called. How many times is Marker.update_done called? 2 \ ------ \ \ 3 3 ---- ---- \ \ \ \ \ \ 1 1 1 1 1 1 Each leaf node calls itself once. Each internal vertex is called during bubble_up_done of each child. The first time the root marker.update_done is run, the right child node is not done """ mock_count_update.return_value = None from furious.marker_tree.identity_utils import leaf_persistence_id_from_group_id from furious.marker_tree.marker import Marker from furious.tests.marker_tree import dummy_success_callback from furious.tests.marker_tree import dummy_internal_vertex_combiner from furious.tests.marker_tree import dummy_leaf_combiner context_callbacks = { "success": dummy_success_callback, "internal_vertex_combiner": dummy_internal_vertex_combiner, "leaf_combiner": dummy_leaf_combiner, } root_marker = Marker(id="delve", callbacks=context_callbacks) for x in xrange(2): root_marker.children.append( Marker( id=str(x), group_id=root_marker.id, callbacks=context_callbacks, children=[Marker(id=leaf_persistence_id_from_group_id(str(x), i)) for i in xrange(3)], ) ) root_marker.persist() with patch("furious.tests.marker_tree." "dummy_success_callback", auto_spec=True) as mock_success_callback: with patch("furious.tests.marker_tree." "dummy_leaf_combiner", auto_spec=True) as mock_leaf_combiner: with patch( "furious.tests.marker_tree." "dummy_internal_vertex_combiner", auto_spec=True ) as mock_internal_vertex_combiner: mock_leaf_combiner.return_value = ["1"] mock_internal_vertex_combiner.return_value = ["2"] for internal_node in root_marker.children: for leaf_node in internal_node.children: leaf_node.done = True leaf_node.result = 1 leaf_node.update_done(persist_first=True) loaded_root_marker = Marker.get("delve") self.assertTrue(loaded_root_marker.done) self.assertEqual(mock_count_update.call_count, 14) # 9 is the number of nodes in the graph self.assertEqual(mock_count_marked_as_done.call_count, 9) # pretend a task was run again later on after # the root had succeeded, it should only # reach it's parent node and that should # not bubble up leaf_node = root_marker.children[0].children[1] leaf_node.update_done(persist_first=True) self.assertEqual(mock_count_update.call_count, 16) mock_success_callback.assert_called_once_with("delve", ["2"]) # one for each non-leaf node self.assertEqual(mock_internal_vertex_combiner.call_count, 3) self.assertEqual(mock_leaf_combiner.call_count, 2)