def from_dict(cls, context_options_dict): """Return a context job from a dict output by Context.to_dict.""" import copy context_options = copy.deepcopy(context_options_dict) tasks_inserted = context_options.pop('_tasks_inserted', False) insert_tasks = context_options.pop('insert_tasks', None) if insert_tasks: context_options['insert_tasks'] = path_to_reference(insert_tasks) # The constructor expects a reference to the persistence engine. persistence_engine = context_options.pop('persistence_engine', None) if persistence_engine: context_options['persistence_engine'] = path_to_reference( persistence_engine) # If there are callbacks, reconstitute them. callbacks = context_options.pop('callbacks', None) if callbacks: context_options['callbacks'] = decode_callbacks(callbacks) context = cls(**context_options) context._tasks_inserted = tasks_inserted return context
def bubble_up_done(self): """ If this marker has a group_id(the ID of it's parent node) load that parent marker and call update_done. If not, this is the root marker and call it's success callback """ logger.debug("start bubble up") group_id = self.get_group_id() if group_id: parent_marker = Marker.get(group_id) if parent_marker: logger.debug("has parent bubble up") return parent_marker.update_done() logger.error("group marker %s did not load" % group_id) else: logger.debug("top level reached, job complete") success_callback = None if self.callbacks: callbacks = decode_callbacks(self.callbacks) success_callback = callbacks.get('success') if success_callback and callable(success_callback): # Load a version of Marker with results. marker = Marker.get(self.id, load_results=True) if marker: success_callback(marker.id, marker.result) return True
def from_dict(cls, marker_dict): import copy marker_options = copy.deepcopy(marker_dict) # If there is are callbacks, reconstitute them. callbacks = marker_options.pop('callbacks', None) if callbacks: marker_options['callbacks'] = decode_callbacks(callbacks) marker_options['children'] = cls.children_from_dict( marker_dict.get('children', [])) return cls(**marker_options)
def decode_async_options(options): """Decode Async options from JSON decoding.""" async_options = copy.deepcopy(options) # JSON don't like datetimes. eta = async_options.get('task_args', {}).get('eta') if eta: from datetime import datetime async_options['task_args']['eta'] = datetime.fromtimestamp(eta) # If there are callbacks, reconstitute them. callbacks = async_options.get('callbacks', {}) if callbacks: async_options['callbacks'] = decode_callbacks(callbacks) return async_options
def decode_async_options(options): """Decode Async options from JSON decoding.""" async_options = copy.deepcopy(options) # JSON don't like datetimes. eta = async_options.get('task_args', {}).get('eta') if eta: from datetime import datetime async_options['task_args']['eta'] = datetime.fromtimestamp(eta) # If there are callbacks, reconstitute them. callbacks = async_options.get('callbacks', {}) if callbacks: async_options['callbacks'] = decode_callbacks(callbacks) if '__context_checker' in options: _checker = options['__context_checker'] async_options['_context_checker'] = path_to_reference(_checker) if '__process_results' in options: _processor = options['__process_results'] async_options['_process_results'] = path_to_reference(_processor) return async_options
def update_done(self, persist_first=False): """ Args: persist_first: save any changes before bubbling up the tree. Used after a leaf task has been set as done. Returns: Boolean: True if done illustration of a way results are handled Marker tree o \ o-----------o \ \ -------- ----- \ \ \ \ \ \ \ \ \ o o o o o o o o o \ --- \ \ \ o o o ================ ints are the order of task results o \ o-----------o \ \ -------- ------- \ \ \ \ \ \ \ \ \ 0 1 2 o 6 7 8 9 10 \ --- \ \ \ 3 4 5 the leaf combiner would combine each contiguous group of leaf results and the internal vertex combiner will combine each group [[[0,1,2],[[3,4,5]],[6]],[[7,8,9,10]]] """ count_update(self.id) self._update_done_in_progress = True # If a marker has just been changed # it must persist itself before checking if it's children # are all done and bubbling up. Doing so will allow it's # parent to know it's changed. logger.debug("update done for id: %s" % self.id) if persist_first: count_marked_as_done(self.id) self.persist() leaf = self.is_leaf() if leaf and self.done: logger.debug("leaf and done id: %s" % self.id) self.bubble_up_done() self._update_done_in_progress = False return True elif not leaf and not self.done: logger.debug("not leaf and not done yet id: %s" % self.id) children_markers = self.get_persisted_children() done_markers = [] for marker in children_markers: if marker and marker.done: done_markers.append(marker) if len(done_markers) == len(self.children): self.done = True logger.debug("done now") if self.callbacks: callbacks = decode_callbacks(self.callbacks) leaf_combiner = callbacks.get('leaf_combiner') internal_vertex_combiner = callbacks.get( 'internal_vertex_combiner') if leaf_combiner or internal_vertex_combiner: # If results are going to be worked with, # reload children markers with results attached. done_markers = self.get_persisted_children( load_results=True) internal_vertex_results = group_into_internal_vertex_results( done_markers, leaf_combiner) if internal_vertex_combiner: result_of_combined_internal_vertexes = \ internal_vertex_combiner( [result for result in internal_vertex_results]) self.result = result_of_combined_internal_vertexes count_marked_as_done(self.id) self.persist() self._update_done_in_progress = False # Bubble up to tell the group marker to update done. self.bubble_up_done() return True self._update_done_in_progress = False return False elif self.done: logger.debug("already done id: %s" % self.id) self._update_done_in_progress = False # No need to bubble up, it would have been done already. return True