def load(flow,
store=None,
flow_detail=None,
book=None,
engine_conf=None,
backend=None,
namespace=ENGINES_NAMESPACE):
"""Load flow into engine
This function creates and prepares engine to run the
flow. All that is left is to run the engine with 'run()' method.
Which engine to load is specified in 'engine_conf' parameter. It
can be a string that names engine type or a dictionary which holds
engine type (with 'engine' key) and additional engine-specific
configuration (for example, executor for multithreaded engine).
Which storage backend to use is defined by backend parameter. It
can be backend itself, or a dictionary that is passed to
taskflow.persistence.backends.fetch to obtain backend.
:param flow: flow to load
:param store: dict -- data to put to storage to satisfy flow requirements
:param flow_detail: FlowDetail that holds state of the flow
:param book: LogBook to create flow detail in if flow_detail is None
:param engine_conf: engine type and configuration configuration
:param backend: storage backend to use or configuration
:param namespace: driver namespace for stevedore (default is fine
if you don't know what is it)
:returns: engine
"""
if engine_conf is None:
engine_conf = {'engine': 'default'}
# NOTE(imelnikov): this allows simpler syntax
if isinstance(engine_conf, six.string_types):
engine_conf = {'engine': engine_conf}
if isinstance(backend, dict):
backend = p_backends.fetch(backend)
if flow_detail is None:
flow_detail = p_utils.create_flow_detail(flow,
book=book,
backend=backend)
mgr = stevedore.driver.DriverManager(namespace,
engine_conf['engine'],
invoke_on_load=True,
invoke_kwds={
'conf': engine_conf.copy(),
'flow': flow,
'flow_detail': flow_detail,
'backend': backend
})
engine = mgr.driver
if store:
engine.storage.inject(store)
return engine
def test_graph_translate(self):
wf = graph_flow.Flow("graph")
one = self.someTask(provides=['data'])
two = self.someTask(requires=['data'])
three = self.someTask()
wf.add(one, two, three)
wf.link(two, three)
backend = impl_memory.MemoryBackend({})
lb = logbook.LogBook("my-log-book")
fd = persistence_utils.create_flow_detail(wf, book=lb, backend=backend)
storage = task_storage.Storage(fd)
engine = distributed_engine.DistributedEngine(wf, storage)
trans = dist_translator.DistributedTranslator(engine)
trans.translate(wf)
self.assertEqual(3, len(engine.tasks))
self.assertEqual(1, len(engine.roots))
dict_listeners = engine.client._listeners
actual = set([])
for listeners in dict_listeners.values():
for listener in listeners:
actual.add(listener[1].flow_task)
self.assertEquals(actual, set([two, three]))
def _create_engine(**kwargs):
flow = lf.Flow('test-flow').add(utils.DummyTask())
backend = backends.fetch({'connection': 'memory'})
flow_detail = pu.create_flow_detail(flow, backend=backend)
options = kwargs.copy()
return engine.WorkerBasedActionEngine(flow, flow_detail, backend,
options)
def load_from_factory(flow_factory, factory_args=None, factory_kwargs=None,
store=None, book=None, engine_conf=None, backend=None,
namespace=ENGINES_NAMESPACE, engine=ENGINE_DEFAULT,
**kwargs):
"""Loads a flow from a factory function into an engine.
Gets flow factory function (or name of it) and creates flow with
it. Then, the flow is loaded into an engine with the :func:`load() <load>`
function, and the factory function fully qualified name is saved to flow
metadata so that it can be later resumed.
:param flow_factory: function or string: function that creates the flow
:param factory_args: list or tuple of factory positional arguments
:param factory_kwargs: dict of factory keyword arguments
Further arguments are interpreted as for :func:`load() <load>`.
:returns: engine
"""
_factory_name, factory_fun = _fetch_validate_factory(flow_factory)
if not factory_args:
factory_args = []
if not factory_kwargs:
factory_kwargs = {}
flow = factory_fun(*factory_args, **factory_kwargs)
if isinstance(backend, dict):
backend = p_backends.fetch(backend)
flow_detail = p_utils.create_flow_detail(flow, book=book, backend=backend)
save_factory_details(flow_detail,
flow_factory, factory_args, factory_kwargs,
backend=backend)
return load(flow=flow, store=store, flow_detail=flow_detail, book=book,
engine_conf=engine_conf, backend=backend, namespace=namespace,
engine=engine, **kwargs)
def load(flow, store=None, flow_detail=None, book=None,
engine_conf=None, backend=None,
namespace=ENGINES_NAMESPACE, engine=ENGINE_DEFAULT, **kwargs):
"""Load a flow into an engine.
This function creates and prepares an engine to run the provided flow. All
that is left after this returns is to run the engine with the
engines :py:meth:`~taskflow.engines.base.Engine.run` method.
Which engine to load is specified via the ``engine`` parameter. It
can be a string that names the engine type to use, or a string that
is a URI with a scheme that names the engine type to use and further
options contained in the URI's host, port, and query parameters...
Which storage backend to use is defined by the backend parameter. It
can be backend itself, or a dictionary that is passed to
:py:func:`~taskflow.persistence.backends.fetch` to obtain a
viable backend.
:param flow: flow to load
:param store: dict -- data to put to storage to satisfy flow requirements
:param flow_detail: FlowDetail that holds the state of the flow (if one is
not provided then one will be created for you in the provided backend)
:param book: LogBook to create flow detail in if flow_detail is None
:param engine_conf: engine type or URI and options (**deprecated**)
:param backend: storage backend to use or configuration that defines it
:param namespace: driver namespace for stevedore (or empty for default)
:param engine: string engine type or URI string with scheme that contains
the engine type and any URI specific components that will
become part of the engine options.
:param kwargs: arbitrary keyword arguments passed as options (merged with
any extracted ``engine`` and ``engine_conf`` options),
typically used for any engine specific options that do not
fit as any of the existing arguments.
:returns: engine
"""
kind, options = _extract_engine(engine_conf=engine_conf,
engine=engine, **kwargs)
if isinstance(backend, dict):
backend = p_backends.fetch(backend)
if flow_detail is None:
flow_detail = p_utils.create_flow_detail(flow, book=book,
backend=backend)
LOG.debug('Looking for %r engine driver in %r', kind, namespace)
try:
mgr = stevedore.driver.DriverManager(
namespace, kind,
invoke_on_load=True,
invoke_args=(flow, flow_detail, backend, options))
engine = mgr.driver
except RuntimeError as e:
raise exc.NotFound("Could not find engine '%s'" % (kind), e)
else:
if store:
engine.storage.inject(store)
return engine
def __init__(self, flow, flow_detail=None, book=None, backend=None):
if flow_detail is None:
flow_detail = p_utils.create_flow_detail(flow,
book=book,
backend=backend)
ActionEngine.__init__(self, flow,
storage=t_storage.Storage(flow_detail, backend))
def load_from_factory(flow_factory, factory_args=None, factory_kwargs=None,
store=None, book=None, engine_conf=None, backend=None,
namespace=ENGINES_NAMESPACE, engine=ENGINE_DEFAULT,
**kwargs):
"""Loads a flow from a factory function into an engine.
Gets flow factory function (or name of it) and creates flow with
it. Then, the flow is loaded into an engine with the :func:`load() <load>`
function, and the factory function fully qualified name is saved to flow
metadata so that it can be later resumed.
:param flow_factory: function or string: function that creates the flow
:param factory_args: list or tuple of factory positional arguments
:param factory_kwargs: dict of factory keyword arguments
Further arguments are interpreted as for :func:`load() <load>`.
:returns: engine
"""
_factory_name, factory_fun = _fetch_validate_factory(flow_factory)
if not factory_args:
factory_args = []
if not factory_kwargs:
factory_kwargs = {}
flow = factory_fun(*factory_args, **factory_kwargs)
if isinstance(backend, dict):
backend = p_backends.fetch(backend)
flow_detail = p_utils.create_flow_detail(flow, book=book, backend=backend)
save_factory_details(flow_detail,
flow_factory, factory_args, factory_kwargs,
backend=backend)
return load(flow=flow, store=store, flow_detail=flow_detail, book=book,
engine_conf=engine_conf, backend=backend, namespace=namespace,
engine=engine, **kwargs)
def _create_engine(**kwargs):
flow = lf.Flow('test-flow').add(utils.DummyTask())
backend = backends.fetch({'connection': 'memory'})
flow_detail = pu.create_flow_detail(flow, backend=backend)
options = kwargs.copy()
return engine.WorkerBasedActionEngine(flow, flow_detail,
backend, options)
def test_broken_flow(self):
backend = impl_memory.MemoryBackend({})
flow = self.make_broken_flow()
lb = logbook.LogBook("my-log-book")
fd = persistence_utils.create_flow_detail(flow,
book=lb, backend=backend)
storage = task_storage.Storage(fd)
distributed_engine.DistributedEngine(flow, storage)
def load(flow, store=None, flow_detail=None, book=None,
engine_conf=None, backend=None,
namespace=ENGINES_NAMESPACE, engine=ENGINE_DEFAULT, **kwargs):
"""Load a flow into an engine.
This function creates and prepares an engine to run the provided flow. All
that is left after this returns is to run the engine with the
engines ``run()`` method.
Which engine to load is specified via the ``engine`` parameter. It
can be a string that names the engine type to use, or a string that
is a URI with a scheme that names the engine type to use and further
options contained in the URI's host, port, and query parameters...
Which storage backend to use is defined by the backend parameter. It
can be backend itself, or a dictionary that is passed to
``taskflow.persistence.backends.fetch()`` to obtain a viable backend.
:param flow: flow to load
:param store: dict -- data to put to storage to satisfy flow requirements
:param flow_detail: FlowDetail that holds the state of the flow (if one is
not provided then one will be created for you in the provided backend)
:param book: LogBook to create flow detail in if flow_detail is None
:param engine_conf: engine type or URI and options (**deprecated**)
:param backend: storage backend to use or configuration that defines it
:param namespace: driver namespace for stevedore (or empty for default)
:param engine: string engine type or URI string with scheme that contains
the engine type and any URI specific components that will
become part of the engine options.
:param kwargs: arbitrary keyword arguments passed as options (merged with
any extracted ``engine`` and ``engine_conf`` options),
typically used for any engine specific options that do not
fit as any of the existing arguments.
:returns: engine
"""
kind, options = _extract_engine(engine_conf=engine_conf,
engine=engine, **kwargs)
if isinstance(backend, dict):
backend = p_backends.fetch(backend)
if flow_detail is None:
flow_detail = p_utils.create_flow_detail(flow, book=book,
backend=backend)
try:
mgr = stevedore.driver.DriverManager(
namespace, kind,
invoke_on_load=True,
invoke_args=(flow, flow_detail, backend, options))
engine = mgr.driver
except RuntimeError as e:
raise exc.NotFound("Could not find engine '%s'" % (kind), e)
else:
if store:
engine.storage.inject(store)
return engine
def load(flow, store=None, flow_detail=None, book=None,
engine_conf=None, backend=None, namespace=ENGINES_NAMESPACE):
"""Load flow into engine.
This function creates and prepares engine to run the
flow. All that is left is to run the engine with 'run()' method.
Which engine to load is specified in 'engine_conf' parameter. It
can be a string that names engine type or a dictionary which holds
engine type (with 'engine' key) and additional engine-specific
configuration (for example, executor for multithreaded engine).
Which storage backend to use is defined by backend parameter. It
can be backend itself, or a dictionary that is passed to
taskflow.persistence.backends.fetch to obtain backend.
:param flow: flow to load
:param store: dict -- data to put to storage to satisfy flow requirements
:param flow_detail: FlowDetail that holds the state of the flow (if one is
not provided then one will be created for you in the provided backend)
:param book: LogBook to create flow detail in if flow_detail is None
:param engine_conf: engine type and configuration configuration
:param backend: storage backend to use or configuration
:param namespace: driver namespace for stevedore (default is fine
if you don't know what is it)
:returns: engine
"""
if engine_conf is None:
engine_conf = {'engine': 'default'}
# NOTE(imelnikov): this allows simpler syntax.
if isinstance(engine_conf, six.string_types):
engine_conf = {'engine': engine_conf}
if isinstance(backend, dict):
backend = p_backends.fetch(backend)
if flow_detail is None:
flow_detail = p_utils.create_flow_detail(flow, book=book,
backend=backend)
mgr = stevedore.driver.DriverManager(
namespace, engine_conf['engine'],
invoke_on_load=True,
invoke_kwds={
'conf': engine_conf.copy(),
'flow': flow,
'flow_detail': flow_detail,
'backend': backend
})
engine = mgr.driver
if store:
engine.storage.inject(store)
return engine
def load_from_factory(flow_factory,
factory_args=None,
factory_kwargs=None,
store=None,
book=None,
engine_conf=None,
backend=None):
"""Load flow from factory function into engine
Gets flow factory function (or name of it) and creates flow with
it. Then, flow is loaded into engine with load(), and factory
function fully qualified name is saved to flow metadata so that
it can be later resumed with resume.
:param flow_factory: function or string: function that creates the flow
:param factory_args: list or tuple of factory positional arguments
:param factory_kwargs: dict of factory keyword arguments
:param store: dict -- data to put to storage to satisfy flow requirements
:param book: LogBook to create flow detail in
:param engine_conf: engine type and configuration configuration
:param backend: storage backend to use or configuration
:returns: engine
"""
if isinstance(flow_factory, six.string_types):
factory_fun = importutils.import_class(flow_factory)
factory_name = flow_factory
else:
factory_fun = flow_factory
factory_name = reflection.get_callable_name(flow_factory)
try:
reimported = importutils.import_class(factory_name)
assert reimported == factory_fun
except (ImportError, AssertionError):
raise ValueError('Flow factory %r is not reimportable by name %s' %
(factory_fun, factory_name))
args = factory_args or []
kwargs = factory_kwargs or {}
flow = factory_fun(*args, **kwargs)
factory_data = dict(name=factory_name, args=args, kwargs=kwargs)
if isinstance(backend, dict):
backend = p_backends.fetch(backend)
flow_detail = p_utils.create_flow_detail(flow,
book=book,
backend=backend,
meta={'factory': factory_data})
return load(flow=flow,
flow_detail=flow_detail,
store=store,
book=book,
engine_conf=engine_conf,
backend=backend)
def _make_runtime(self, flow, initial_state=None):
compilation = compiler.PatternCompiler(flow).compile()
flow_detail = pu.create_flow_detail(flow)
store = storage.SingleThreadedStorage(flow_detail)
# This ensures the tasks exist in storage...
for task in compilation.execution_graph:
store.ensure_atom(task)
if initial_state:
store.set_flow_state(initial_state)
task_notifier = misc.Notifier()
task_executor = executor.SerialTaskExecutor()
task_executor.start()
self.addCleanup(task_executor.stop)
return runtime.Runtime(compilation, store, task_notifier, task_executor)
def _make_runtime(self, flow, initial_state=None):
compilation = compiler.PatternCompiler(flow).compile()
flow_detail = pu.create_flow_detail(flow)
store = storage.SingleThreadedStorage(flow_detail)
# This ensures the tasks exist in storage...
for task in compilation.execution_graph:
store.ensure_atom(task)
if initial_state:
store.set_flow_state(initial_state)
task_notifier = misc.Notifier()
task_executor = executor.SerialTaskExecutor()
task_executor.start()
self.addCleanup(task_executor.stop)
return runtime.Runtime(compilation, store,
task_notifier, task_executor)
def load_from_factory(flow_factory,
factory_args=None,
factory_kwargs=None,
store=None,
book=None,
engine_conf=None,
backend=None,
namespace=ENGINES_NAMESPACE):
"""Loads a flow from a factory function into an engine.
Gets flow factory function (or name of it) and creates flow with
it. Then, flow is loaded into engine with load(), and factory
function fully qualified name is saved to flow metadata so that
it can be later resumed with resume.
:param flow_factory: function or string: function that creates the flow
:param factory_args: list or tuple of factory positional arguments
:param factory_kwargs: dict of factory keyword arguments
:param store: dict -- data to put to storage to satisfy flow requirements
:param book: LogBook to create flow detail in
:param engine_conf: engine type and configuration configuration
:param backend: storage backend to use or configuration
:param namespace: driver namespace for stevedore (default is fine
if you don't know what is it)
:returns: engine
"""
_factory_name, factory_fun = _fetch_validate_factory(flow_factory)
if not factory_args:
factory_args = []
if not factory_kwargs:
factory_kwargs = {}
flow = factory_fun(*factory_args, **factory_kwargs)
if isinstance(backend, dict):
backend = p_backends.fetch(backend)
flow_detail = p_utils.create_flow_detail(flow, book=book, backend=backend)
save_factory_details(flow_detail,
flow_factory,
factory_args,
factory_kwargs,
backend=backend)
return load(flow=flow,
store=store,
flow_detail=flow_detail,
book=book,
engine_conf=engine_conf,
backend=backend,
namespace=namespace)
def _make_runtime(self, flow, initial_state=None):
compilation = compiler.PatternCompiler(flow).compile()
flow_detail = pu.create_flow_detail(flow)
store = storage.Storage(flow_detail)
nodes_iter = compilation.execution_graph.nodes_iter(data=True)
for node, node_attrs in nodes_iter:
if node_attrs['kind'] in ('task', 'retry'):
store.ensure_atom(node)
if initial_state:
store.set_flow_state(initial_state)
atom_notifier = notifier.Notifier()
task_executor = executor.SerialTaskExecutor()
retry_executor = executor.SerialRetryExecutor()
task_executor.start()
self.addCleanup(task_executor.stop)
r = runtime.Runtime(compilation, store, atom_notifier, task_executor,
retry_executor)
r.compile()
return r
def test_multiple_root_translate(self):
wf = graph_flow.Flow("mult-root")
one = self.someTask()
two = self.someTask()
three = self.someOtherTask()
wf.add(one, two, three)
wf.link(one, three)
wf.link(two, three)
backend = impl_memory.MemoryBackend({})
lb = logbook.LogBook("my-log-book")
fd = persistence_utils.create_flow_detail(wf, book=lb, backend=backend)
storage = task_storage.Storage(fd)
engine = distributed_engine.DistributedEngine(wf, storage)
trans = dist_translator.DistributedTranslator(engine)
trans.translate(wf)
self.assertEqual(2, len(engine.roots))
def _make_runtime(self, flow, initial_state=None):
compilation = compiler.PatternCompiler(flow).compile()
flow_detail = pu.create_flow_detail(flow)
store = storage.Storage(flow_detail)
nodes_iter = compilation.execution_graph.nodes_iter(data=True)
for node, node_attrs in nodes_iter:
if node_attrs['kind'] in ('task', 'retry'):
store.ensure_atom(node)
if initial_state:
store.set_flow_state(initial_state)
atom_notifier = notifier.Notifier()
task_executor = executor.SerialTaskExecutor()
retry_executor = executor.SerialRetryExecutor()
task_executor.start()
self.addCleanup(task_executor.stop)
r = runtime.Runtime(compilation, store,
atom_notifier, task_executor,
retry_executor)
r.compile()
return r
def load_from_factory(
flow_factory, factory_args=None, factory_kwargs=None, store=None, book=None, engine_conf=None, backend=None
):
"""Load flow from factory function into engine
Gets flow factory function (or name of it) and creates flow with
it. Then, flow is loaded into engine with load(), and factory
function fully qualified name is saved to flow metadata so that
it can be later resumed with resume.
:param flow_factory: function or string: function that creates the flow
:param factory_args: list or tuple of factory positional arguments
:param factory_kwargs: dict of factory keyword arguments
:param store: dict -- data to put to storage to satisfy flow requirements
:param book: LogBook to create flow detail in
:param engine_conf: engine type and configuration configuration
:param backend: storage backend to use or configuration
:returns: engine
"""
if isinstance(flow_factory, six.string_types):
factory_fun = importutils.import_class(flow_factory)
factory_name = flow_factory
else:
factory_fun = flow_factory
factory_name = reflection.get_callable_name(flow_factory)
try:
reimported = importutils.import_class(factory_name)
assert reimported == factory_fun
except (ImportError, AssertionError):
raise ValueError("Flow factory %r is not reimportable by name %s" % (factory_fun, factory_name))
args = factory_args or []
kwargs = factory_kwargs or {}
flow = factory_fun(*args, **kwargs)
factory_data = dict(name=factory_name, args=args, kwargs=kwargs)
if isinstance(backend, dict):
backend = p_backends.fetch(backend)
flow_detail = p_utils.create_flow_detail(flow, book=book, backend=backend, meta={"factory": factory_data})
return load(flow=flow, flow_detail=flow_detail, store=store, book=book, engine_conf=engine_conf, backend=backend)
def test_linear_translate(self):
wf = linear_flow.Flow("linear")
one = self.someTask()
two = self.someOtherTask()
wf.add(one, two)
backend = impl_memory.MemoryBackend({})
lb = logbook.LogBook("my-log-book")
fd = persistence_utils.create_flow_detail(wf, book=lb, backend=backend)
storage = task_storage.Storage(fd)
engine = distributed_engine.DistributedEngine(wf, storage)
trans = dist_translator.DistributedTranslator(engine)
trans.translate(wf)
listeners = engine.client._listeners
hybrid = listeners[listeners.keys()[0]][0][1]
self.assertEqual(1, len(engine.roots))
self.assertEqual(hybrid.name,
"taskflow.tests.unit.test_distributed_translator"
".someOtherTask")
def load_from_factory(flow_factory, factory_args=None, factory_kwargs=None,
store=None, book=None, engine_conf=None, backend=None,
namespace=ENGINES_NAMESPACE, **kwargs):
"""Loads a flow from a factory function into an engine.
Gets flow factory function (or name of it) and creates flow with
it. Then, flow is loaded into engine with load(), and factory
function fully qualified name is saved to flow metadata so that
it can be later resumed with resume.
:param flow_factory: function or string: function that creates the flow
:param factory_args: list or tuple of factory positional arguments
:param factory_kwargs: dict of factory keyword arguments
:param store: dict -- data to put to storage to satisfy flow requirements
:param book: LogBook to create flow detail in
:param engine_conf: engine type and configuration configuration
:param backend: storage backend to use or configuration
:param namespace: driver namespace for stevedore (default is fine
if you don't know what is it)
:returns: engine
"""
_factory_name, factory_fun = _fetch_validate_factory(flow_factory)
if not factory_args:
factory_args = []
if not factory_kwargs:
factory_kwargs = {}
flow = factory_fun(*factory_args, **factory_kwargs)
if isinstance(backend, dict):
backend = p_backends.fetch(backend)
flow_detail = p_utils.create_flow_detail(flow, book=book, backend=backend)
save_factory_details(flow_detail,
flow_factory, factory_args, factory_kwargs,
backend=backend)
return load(flow=flow, store=store, flow_detail=flow_detail, book=book,
engine_conf=engine_conf, backend=backend, namespace=namespace,
**kwargs)
def __init__(self, flow, flow_detail=None, book=None, backend=None,
executor=None):
if flow_detail is None:
flow_detail = p_utils.create_flow_detail(flow,
book=book,
backend=backend)
ActionEngine.__init__(self, flow,
storage=t_storage.ThreadSafeStorage(flow_detail,
backend))
if executor is not None:
self._executor = executor
self._owns_executor = False
self._thread_count = -1
else:
self._executor = None
self._owns_executor = True
# TODO(harlowja): allow this to be configurable??
try:
self._thread_count = multiprocessing.cpu_count() + 1
except NotImplementedError:
# NOTE(harlowja): apparently may raise so in this case we will
# just setup two threads since its hard to know what else we
# should do in this situation.
self._thread_count = 2
return lf.Flow('resume from backend example').add(
TestTask(name='first'),
InterruptTask(name='boom'),
TestTask(name='second'))
### INITIALIZE PERSISTENCE ####################################
backend = get_backend()
logbook = p_utils.temporary_log_book(backend)
### CREATE AND RUN THE FLOW: FIRST ATTEMPT ####################
flow = flow_factory()
flowdetail = p_utils.create_flow_detail(flow, logbook, backend)
engine = taskflow.engines.load(flow, flow_detail=flowdetail,
backend=backend)
print_task_states(flowdetail, "At the beginning, there is no state")
print_wrapped("Running")
engine.run()
print_task_states(flowdetail, "After running")
### RE-CREATE, RESUME, RUN ####################################
print_wrapped("Resuming and running again")
# NOTE(harlowja): reload the flow detail from backend, this will allow us to
# resume the flow from its suspended state, but first we need to search for
from taskflow.engines.action_engine import engine
from taskflow.patterns import linear_flow as lf
from taskflow.persistence.backends import impl_memory
from taskflow import task
from taskflow.utils import persistence_utils
# INTRO: This examples shows how to run a engine using the engine iteration
# capability, in between iterations other activities occur (in this case a
# value is output to stdout); but more complicated actions can occur at the
# boundary when a engine yields its current state back to the caller.
class EchoNameTask(task.Task):
def execute(self):
print(self.name)
f = lf.Flow("counter")
for i in range(0, 10):
f.add(EchoNameTask("echo_%s" % (i + 1)))
be = impl_memory.MemoryBackend()
book = persistence_utils.temporary_log_book(be)
fd = persistence_utils.create_flow_detail(f, book, be)
e = engine.SingleThreadedActionEngine(f, fd, be, {})
e.compile()
e.prepare()
for i, st in enumerate(e.run_iter(), 1):
print("Transition %s: %s" % (i, st))
def flow_factory():
return lf.Flow('resume from backend example').add(
TestTask(name='first'), InterruptTask(name='boom'),
TestTask(name='second'))
### INITIALIZE PERSISTENCE ####################################
backend = get_backend()
logbook = p_utils.temporary_log_book(backend)
### CREATE AND RUN THE FLOW: FIRST ATTEMPT ####################
flow = flow_factory()
flowdetail = p_utils.create_flow_detail(flow, logbook, backend)
engine = taskflow.engines.load(flow, flow_detail=flowdetail, backend=backend)
print_task_states(flowdetail, "At the beginning, there is no state")
print_wrapped("Running")
engine.run()
print_task_states(flowdetail, "After running")
### RE-CREATE, RESUME, RUN ####################################
print_wrapped("Resuming and running again")
# NOTE(harlowja): reload the flow detail from backend, this will allow us to
# resume the flow from its suspended state, but first we need to search for
# the right flow details in the correct logbook where things are stored.
#
def _make_engine(self, flow, flow_detail=None):
if flow_detail is None:
flow_detail = p_utils.create_flow_detail(flow, self.book, self.backend)
return eng.SingleThreadedActionEngine(flow, backend=self.backend, flow_detail=flow_detail)
def _create_engine(**kwargs):
flow = lf.Flow("test-flow").add(utils.DummyTask())
backend = backends.fetch({"connection": "memory"})
flow_detail = pu.create_flow_detail(flow, backend=backend)
options = kwargs.copy()
return engine.ParallelActionEngine(flow, flow_detail, backend, options)
else:
f.add(
EchoTask(name="echoer_%s" % curr_value,
rebind={'value': curr_value}))
curr_value = next_value
return f
# Adjust this number to change how many engines/flows run at once.
flow_count = 1
flows = []
for i in range(0, flow_count):
f = make_alphabet_flow(i + 1)
flows.append(make_alphabet_flow(i + 1))
be = persistence_backends.fetch(conf={'connection': 'memory'})
book = persistence_utils.temporary_log_book(be)
engine_iters = []
for f in flows:
fd = persistence_utils.create_flow_detail(f, book, be)
e = engines.load(f, flow_detail=fd, backend=be, book=book)
e.compile()
e.storage.inject({'A': 'A'})
e.prepare()
engine_iters.append(e.run_iter())
while engine_iters:
for it in list(engine_iters):
try:
print(six.next(it))
except StopIteration:
engine_iters.remove(it)
def load(flow, store=None, flow_detail=None, book=None,
engine_conf=None, backend=None, namespace=ENGINES_NAMESPACE,
**kwargs):
"""Load a flow into an engine.
This function creates and prepares engine to run the
flow. All that is left is to run the engine with 'run()' method.
Which engine to load is specified in 'engine_conf' parameter. It
can be a string that names engine type or a dictionary which holds
engine type (with 'engine' key) and additional engine-specific
configuration.
Which storage backend to use is defined by backend parameter. It
can be backend itself, or a dictionary that is passed to
taskflow.persistence.backends.fetch to obtain backend.
:param flow: flow to load
:param store: dict -- data to put to storage to satisfy flow requirements
:param flow_detail: FlowDetail that holds the state of the flow (if one is
not provided then one will be created for you in the provided backend)
:param book: LogBook to create flow detail in if flow_detail is None
:param engine_conf: engine type and configuration configuration
:param backend: storage backend to use or configuration
:param namespace: driver namespace for stevedore (default is fine
if you don't know what is it)
:returns: engine
"""
if engine_conf is None:
engine_conf = {'engine': 'default'}
# NOTE(imelnikov): this allows simpler syntax.
if isinstance(engine_conf, six.string_types):
engine_conf = {'engine': engine_conf}
engine_name = engine_conf['engine']
try:
pieces = misc.parse_uri(engine_name)
except (TypeError, ValueError):
pass
else:
engine_name = pieces['scheme']
engine_conf = misc.merge_uri(pieces, engine_conf.copy())
if isinstance(backend, dict):
backend = p_backends.fetch(backend)
if flow_detail is None:
flow_detail = p_utils.create_flow_detail(flow, book=book,
backend=backend)
try:
mgr = stevedore.driver.DriverManager(
namespace, engine_name,
invoke_on_load=True,
invoke_args=(flow, flow_detail, backend, engine_conf),
invoke_kwds=kwargs)
engine = mgr.driver
except RuntimeError as e:
raise exc.NotFound("Could not find engine %s" % (engine_name), e)
else:
if store:
engine.storage.inject(store)
return engine
def _make_engine(self, flow, flow_detail=None, executor=None):
if flow_detail is None:
flow_detail = p_utils.create_flow_detail(flow, self.book, self.backend)
return eng.MultiThreadedActionEngine(flow, backend=self.backend, flow_detail=flow_detail, executor=executor)
