def test_ordered_grouping_results(self):
        """
        Ensure results will be combined into groups of
        combined leaf results to maintain the result
        order is the same as the tasks inserted.
        When a single task fans out, it's results
        will pulled in position

        """
        from furious.marker_tree.identity_utils import leaf_persistence_id_from_group_id
        from furious.marker_tree.result_sorter import first_iv_markers
        from furious.marker_tree.result_sorter import group_into_internal_vertex_results
        from furious.marker_tree.marker import Marker

        from furious.tests.marker_tree import dummy_leaf_combiner

        root_marker = Marker(id="little_job")
        for x in xrange(3):
            root_marker.children.append(Marker(
                id=str(x),
                group_id=root_marker.id,
                result=[2, 2, 2],
                children=[Marker(id=
                                 leaf_persistence_id_from_group_id(str(x), i),
                                 result=2)
                          for i in xrange(3)]
            ))

        for x in xrange(2):
            root_marker.children.append(Marker(
                id=leaf_persistence_id_from_group_id(root_marker.id, x + 3),
                result=1
            ))

        markers = first_iv_markers(root_marker.children)

        self.assertEqual(len(markers), 3)

        iv_results = group_into_internal_vertex_results(root_marker.children,
                                                        dummy_leaf_combiner)

        self.assertEqual(len(iv_results), 4)
        self.assertEqual(iv_results, [[2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 1]])

        #shuffle children a few times
        chlin = root_marker.children
        children = [chlin[3], chlin[4], chlin[0], chlin[1], chlin[2]]
        iv_results = group_into_internal_vertex_results(children,
                                                        dummy_leaf_combiner)
        self.assertEqual(iv_results, [[1, 1], [2, 2, 2], [2, 2, 2], [2, 2, 2]])
        children = [chlin[3], chlin[0], chlin[4], chlin[1], chlin[2]]
        iv_results = group_into_internal_vertex_results(children,
                                                        dummy_leaf_combiner)
        self.assertEqual(iv_results, [[1], [2, 2, 2], [1], [2, 2, 2], [2, 2, 2]])
        iv_results = group_into_internal_vertex_results(children,
                                                        None)
        self.assertEqual(iv_results, [[2, 2, 2], [2, 2, 2], [2, 2, 2]])
Example #2
0
    def update_done(self, persist_first=False):
        """
        Args:
            persist_first: save any changes before bubbling
            up the tree. Used after a leaf task has been
            set as done.
        Returns:
            Boolean: True if done

        illustration of a way results are handled
        Marker tree
            o
            \
        o-----------o
        \           \
        --------   -----
        \ \ \ \ \  \ \ \ \
        o o o o o   o o o o
              \
              ---
              \ \ \
              o o o

        ================
        ints are the order of task results
            o
            \
        o-----------o
        \           \
        --------   -------
        \ \ \ \ \  \ \ \  \
        0 1 2 o 6  7 8 9  10
              \
              ---
              \ \ \
              3 4 5

        the leaf combiner would combine each contiguous
        group of leaf results
        and the internal vertex combiner will
        combine each group
        [[[0,1,2],[[3,4,5]],[6]],[[7,8,9,10]]]
        """
        count_update(self.id)
        self._update_done_in_progress = True

        # If a marker has just been changed
        # it must persist itself before checking if it's children
        # are all done and bubbling up. Doing so will allow it's
        # parent to know it's changed.
        logger.debug("update done for id: %s" % self.id)
        if persist_first:
            count_marked_as_done(self.id)
            self.persist()

        leaf = self.is_leaf()
        if leaf and self.done:
            logger.debug("leaf and done id: %s" % self.id)
            self.bubble_up_done()
            self._update_done_in_progress = False
            return True
        elif not leaf and not self.done:
            logger.debug("not leaf and not done yet id: %s" % self.id)
            children_markers = self.get_persisted_children()
            done_markers = []
            for marker in children_markers:
                if marker and marker.done:
                    done_markers.append(marker)
            if len(done_markers) == len(self.children):
                self.done = True
                logger.debug("done now")
                if self.callbacks:
                    callbacks = decode_callbacks(self.callbacks)
                    leaf_combiner = callbacks.get('leaf_combiner')
                    internal_vertex_combiner = callbacks.get(
                        'internal_vertex_combiner')

                    if leaf_combiner or internal_vertex_combiner:
                        # If results are going to be worked with,
                        # reload children markers with results attached.
                        done_markers = self.get_persisted_children(
                            load_results=True)

                        internal_vertex_results = group_into_internal_vertex_results(
                            done_markers, leaf_combiner)

                        if internal_vertex_combiner:
                            result_of_combined_internal_vertexes = \
                                internal_vertex_combiner(
                                    [result for result in
                                     internal_vertex_results])

                            self.result = result_of_combined_internal_vertexes

                count_marked_as_done(self.id)
                self.persist()
                self._update_done_in_progress = False

                # Bubble up to tell the group marker to update done.
                self.bubble_up_done()
                return True

            self._update_done_in_progress = False
            return False
        elif self.done:
            logger.debug("already done id: %s" % self.id)
            self._update_done_in_progress = False

            # No need to bubble up, it would have been done already.
            return True