def from_dict(cls, context_options_dict):
"""Return a context job from a dict output by Context.to_dict."""
import copy
context_options = copy.deepcopy(context_options_dict)
tasks_inserted = context_options.pop('_tasks_inserted', False)
insert_tasks = context_options.pop('insert_tasks', None)
if insert_tasks:
context_options['insert_tasks'] = path_to_reference(insert_tasks)
# The constructor expects a reference to the persistence engine.
persistence_engine = context_options.pop('persistence_engine', None)
if persistence_engine:
context_options['persistence_engine'] = path_to_reference(
persistence_engine)
# If there are callbacks, reconstitute them.
callbacks = context_options.pop('callbacks', None)
if callbacks:
context_options['callbacks'] = decode_callbacks(callbacks)
context = cls(**context_options)
context._tasks_inserted = tasks_inserted
return context
def bubble_up_done(self):
"""
If this marker has a group_id(the ID of it's parent node)
load that parent marker and call update_done.
If not, this is the root marker and call it's success
callback
"""
logger.debug("start bubble up")
group_id = self.get_group_id()
if group_id:
parent_marker = Marker.get(group_id)
if parent_marker:
logger.debug("has parent bubble up")
return parent_marker.update_done()
logger.error("group marker %s did not load" % group_id)
else:
logger.debug("top level reached, job complete")
success_callback = None
if self.callbacks:
callbacks = decode_callbacks(self.callbacks)
success_callback = callbacks.get('success')
if success_callback and callable(success_callback):
# Load a version of Marker with results.
marker = Marker.get(self.id, load_results=True)
if marker:
success_callback(marker.id, marker.result)
return True
def from_dict(cls, context_options_dict):
"""Return a context job from a dict output by Context.to_dict."""
import copy
context_options = copy.deepcopy(context_options_dict)
tasks_inserted = context_options.pop('_tasks_inserted', False)
insert_tasks = context_options.pop('insert_tasks', None)
if insert_tasks:
context_options['insert_tasks'] = path_to_reference(insert_tasks)
# The constructor expects a reference to the persistence engine.
persistence_engine = context_options.pop('persistence_engine', None)
if persistence_engine:
context_options['persistence_engine'] = path_to_reference(
persistence_engine)
# If there are callbacks, reconstitute them.
callbacks = context_options.pop('callbacks', None)
if callbacks:
context_options['callbacks'] = decode_callbacks(callbacks)
context = cls(**context_options)
context._tasks_inserted = tasks_inserted
return context
def from_dict(cls, marker_dict):
import copy
marker_options = copy.deepcopy(marker_dict)
# If there is are callbacks, reconstitute them.
callbacks = marker_options.pop('callbacks', None)
if callbacks:
marker_options['callbacks'] = decode_callbacks(callbacks)
marker_options['children'] = cls.children_from_dict(
marker_dict.get('children', []))
return cls(**marker_options)
def decode_async_options(options):
"""Decode Async options from JSON decoding."""
async_options = copy.deepcopy(options)
# JSON don't like datetimes.
eta = async_options.get('task_args', {}).get('eta')
if eta:
from datetime import datetime
async_options['task_args']['eta'] = datetime.fromtimestamp(eta)
# If there are callbacks, reconstitute them.
callbacks = async_options.get('callbacks', {})
if callbacks:
async_options['callbacks'] = decode_callbacks(callbacks)
return async_options
def decode_async_options(options):
"""Decode Async options from JSON decoding."""
async_options = copy.deepcopy(options)
# JSON don't like datetimes.
eta = async_options.get('task_args', {}).get('eta')
if eta:
from datetime import datetime
async_options['task_args']['eta'] = datetime.fromtimestamp(eta)
# If there are callbacks, reconstitute them.
callbacks = async_options.get('callbacks', {})
if callbacks:
async_options['callbacks'] = decode_callbacks(callbacks)
return async_options
def decode_async_options(options):
"""Decode Async options from JSON decoding."""
async_options = copy.deepcopy(options)
# JSON don't like datetimes.
eta = async_options.get('task_args', {}).get('eta')
if eta:
from datetime import datetime
async_options['task_args']['eta'] = datetime.fromtimestamp(eta)
# If there are callbacks, reconstitute them.
callbacks = async_options.get('callbacks', {})
if callbacks:
async_options['callbacks'] = decode_callbacks(callbacks)
if '__context_checker' in options:
_checker = options['__context_checker']
async_options['_context_checker'] = path_to_reference(_checker)
if '__process_results' in options:
_processor = options['__process_results']
async_options['_process_results'] = path_to_reference(_processor)
return async_options
def update_done(self, persist_first=False):
"""
Args:
persist_first: save any changes before bubbling
up the tree. Used after a leaf task has been
set as done.
Returns:
Boolean: True if done
illustration of a way results are handled
Marker tree
o
\
o-----------o
\ \
-------- -----
\ \ \ \ \ \ \ \ \
o o o o o o o o o
\
---
\ \ \
o o o
================
ints are the order of task results
o
\
o-----------o
\ \
-------- -------
\ \ \ \ \ \ \ \ \
0 1 2 o 6 7 8 9 10
\
---
\ \ \
3 4 5
the leaf combiner would combine each contiguous
group of leaf results
and the internal vertex combiner will
combine each group
[[[0,1,2],[[3,4,5]],[6]],[[7,8,9,10]]]
"""
count_update(self.id)
self._update_done_in_progress = True
# If a marker has just been changed
# it must persist itself before checking if it's children
# are all done and bubbling up. Doing so will allow it's
# parent to know it's changed.
logger.debug("update done for id: %s" % self.id)
if persist_first:
count_marked_as_done(self.id)
self.persist()
leaf = self.is_leaf()
if leaf and self.done:
logger.debug("leaf and done id: %s" % self.id)
self.bubble_up_done()
self._update_done_in_progress = False
return True
elif not leaf and not self.done:
logger.debug("not leaf and not done yet id: %s" % self.id)
children_markers = self.get_persisted_children()
done_markers = []
for marker in children_markers:
if marker and marker.done:
done_markers.append(marker)
if len(done_markers) == len(self.children):
self.done = True
logger.debug("done now")
if self.callbacks:
callbacks = decode_callbacks(self.callbacks)
leaf_combiner = callbacks.get('leaf_combiner')
internal_vertex_combiner = callbacks.get(
'internal_vertex_combiner')
if leaf_combiner or internal_vertex_combiner:
# If results are going to be worked with,
# reload children markers with results attached.
done_markers = self.get_persisted_children(
load_results=True)
internal_vertex_results = group_into_internal_vertex_results(
done_markers, leaf_combiner)
if internal_vertex_combiner:
result_of_combined_internal_vertexes = \
internal_vertex_combiner(
[result for result in
internal_vertex_results])
self.result = result_of_combined_internal_vertexes
count_marked_as_done(self.id)
self.persist()
self._update_done_in_progress = False
# Bubble up to tell the group marker to update done.
self.bubble_up_done()
return True
self._update_done_in_progress = False
return False
elif self.done:
logger.debug("already done id: %s" % self.id)
self._update_done_in_progress = False
# No need to bubble up, it would have been done already.
return True
