def test_ordered_grouping_results(self): """ Ensure results will be combined into groups of combined leaf results to maintain the result order is the same as the tasks inserted. When a single task fans out, it's results will pulled in position """ from furious.marker_tree.identity_utils import leaf_persistence_id_from_group_id from furious.marker_tree.result_sorter import first_iv_markers from furious.marker_tree.result_sorter import group_into_internal_vertex_results from furious.marker_tree.marker import Marker from furious.tests.marker_tree import dummy_leaf_combiner root_marker = Marker(id="little_job") for x in xrange(3): root_marker.children.append(Marker( id=str(x), group_id=root_marker.id, result=[2, 2, 2], children=[Marker(id= leaf_persistence_id_from_group_id(str(x), i), result=2) for i in xrange(3)] )) for x in xrange(2): root_marker.children.append(Marker( id=leaf_persistence_id_from_group_id(root_marker.id, x + 3), result=1 )) markers = first_iv_markers(root_marker.children) self.assertEqual(len(markers), 3) iv_results = group_into_internal_vertex_results(root_marker.children, dummy_leaf_combiner) self.assertEqual(len(iv_results), 4) self.assertEqual(iv_results, [[2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 1]]) #shuffle children a few times chlin = root_marker.children children = [chlin[3], chlin[4], chlin[0], chlin[1], chlin[2]] iv_results = group_into_internal_vertex_results(children, dummy_leaf_combiner) self.assertEqual(iv_results, [[1, 1], [2, 2, 2], [2, 2, 2], [2, 2, 2]]) children = [chlin[3], chlin[0], chlin[4], chlin[1], chlin[2]] iv_results = group_into_internal_vertex_results(children, dummy_leaf_combiner) self.assertEqual(iv_results, [[1], [2, 2, 2], [1], [2, 2, 2], [2, 2, 2]]) iv_results = group_into_internal_vertex_results(children, None) self.assertEqual(iv_results, [[2, 2, 2], [2, 2, 2], [2, 2, 2]])
def update_done(self, persist_first=False): """ Args: persist_first: save any changes before bubbling up the tree. Used after a leaf task has been set as done. Returns: Boolean: True if done illustration of a way results are handled Marker tree o \ o-----------o \ \ -------- ----- \ \ \ \ \ \ \ \ \ o o o o o o o o o \ --- \ \ \ o o o ================ ints are the order of task results o \ o-----------o \ \ -------- ------- \ \ \ \ \ \ \ \ \ 0 1 2 o 6 7 8 9 10 \ --- \ \ \ 3 4 5 the leaf combiner would combine each contiguous group of leaf results and the internal vertex combiner will combine each group [[[0,1,2],[[3,4,5]],[6]],[[7,8,9,10]]] """ count_update(self.id) self._update_done_in_progress = True # If a marker has just been changed # it must persist itself before checking if it's children # are all done and bubbling up. Doing so will allow it's # parent to know it's changed. logger.debug("update done for id: %s" % self.id) if persist_first: count_marked_as_done(self.id) self.persist() leaf = self.is_leaf() if leaf and self.done: logger.debug("leaf and done id: %s" % self.id) self.bubble_up_done() self._update_done_in_progress = False return True elif not leaf and not self.done: logger.debug("not leaf and not done yet id: %s" % self.id) children_markers = self.get_persisted_children() done_markers = [] for marker in children_markers: if marker and marker.done: done_markers.append(marker) if len(done_markers) == len(self.children): self.done = True logger.debug("done now") if self.callbacks: callbacks = decode_callbacks(self.callbacks) leaf_combiner = callbacks.get('leaf_combiner') internal_vertex_combiner = callbacks.get( 'internal_vertex_combiner') if leaf_combiner or internal_vertex_combiner: # If results are going to be worked with, # reload children markers with results attached. done_markers = self.get_persisted_children( load_results=True) internal_vertex_results = group_into_internal_vertex_results( done_markers, leaf_combiner) if internal_vertex_combiner: result_of_combined_internal_vertexes = \ internal_vertex_combiner( [result for result in internal_vertex_results]) self.result = result_of_combined_internal_vertexes count_marked_as_done(self.id) self.persist() self._update_done_in_progress = False # Bubble up to tell the group marker to update done. self.bubble_up_done() return True self._update_done_in_progress = False return False elif self.done: logger.debug("already done id: %s" % self.id) self._update_done_in_progress = False # No need to bubble up, it would have been done already. return True